diff --git a/.cargo/config.toml b/.cargo/config.toml index 9768afc5..ddbe09f3 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,67 +1,2 @@ -# Linux x86_64 -[target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "target-cpu=native"] - -# Linux ARM64 -[target.aarch64-unknown-linux-gnu] -rustflags = ["-C", "target-feature=+lse", "-C", "target-cpu=native"] - -# Windows x86_64 -[target.x86_64-pc-windows-msvc] - -# Windows ARM64 -[target.aarch64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+lse"] - -# Tier B -# Linux x86_64 MUSL -[target.x86_64-unknown-linux-musl] - -# Linux ARM64 MUSL -[target.aarch64-unknown-linux-musl] -rustflags = ["-C", "target-feature=+lse"] - -# Tier C - WASM -# WASM - Browser -[target.wasm32-unknown-unknown] - -# WASM/WASI - Server -[target.wasm32-wasip1] - -# Tier D - Enterprise -# IBM s390x (Z mainframes) -[target.s390x-unknown-linux-gnu] - -# IBM PowerPC64 (little-endian, modern POWER systems) -[target.powerpc64le-unknown-linux-gnu] - -# RISC-V 64-bit Linux (RV64GC: general + compressed) -[target.riscv64gc-unknown-linux-gnu] - -# LoongArch 64-bit Linux -[target.loongarch64-unknown-linux-gnu] - -# Tier E - no_std Bare Metal Sentinels -# Cortex-M0/M0+ (minimal no_std sentinel - no FPU, no DSP) -# If it compiles here, it compiles everywhere -[target.thumbv6m-none-eabi] - -# ARM64 bare metal (modern no_std sentinel - hypervisors, TEEs) -# No LSE: target hardware capabilities vary -[target.aarch64-unknown-none] - -# RISC-V 64-bit bare metal (RV64IMAFDC) -[target.riscv64gc-unknown-none-elf] - -# RISC-V 32-bit bare metal (RV32IMAC) -[target.riscv32imac-unknown-none-elf] - -# x86_64 bare metal (kernels, bootloaders, hypervisors) -[target.x86_64-unknown-none] - -# Build Configuration [build] rustdocflags = ["-D", "warnings"] - -[env] -RUST_LOG = "info" diff --git a/.changes/add-debug-output-for-4cec.md b/.changes/add-debug-output-for-4cec.md new file mode 100644 index 00000000..7f386f2c --- /dev/null +++ b/.changes/add-debug-output-for-4cec.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Add Debug output for Ed25519 verification diagnostic snapshots. diff --git a/.changes/add-debug-output-for-6a2e.md b/.changes/add-debug-output-for-6a2e.md new file mode 100644 index 00000000..52f38010 --- /dev/null +++ b/.changes/add-debug-output-for-6a2e.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Add Debug output for AES-GCM nonce-counter state. diff --git a/.changes/add-redacted-debug-output-4c0e.md b/.changes/add-redacted-debug-output-4c0e.md new file mode 100644 index 00000000..5a8a312e --- /dev/null +++ b/.changes/add-redacted-debug-output-4c0e.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Group caller-supplied RSA blinding inputs in `RsaBlindingPair` and add redacted `Debug` output for borrowed RSA private-key inputs. diff --git a/.changes/document-checksum-and-digest-e536.md b/.changes/document-checksum-and-digest-e536.md new file mode 100644 index 00000000..35942df7 --- /dev/null +++ b/.changes/document-checksum-and-digest-e536.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Document checksum and digest I/O adapter state and ownership methods. diff --git a/.changes/document-shake128-and-shake256-2928.md b/.changes/document-shake128-and-shake256-2928.md new file mode 100644 index 00000000..a6c482da --- /dev/null +++ b/.changes/document-shake128-and-shake256-2928.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Document SHAKE128 and SHAKE256 state and output-reader APIs. diff --git a/.changes/harden-aegis256-aesni-a37b.md b/.changes/harden-aegis256-aesni-a37b.md new file mode 100644 index 00000000..46d4b9b3 --- /dev/null +++ b/.changes/harden-aegis256-aesni-a37b.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden the x86 and POWER8 AEGIS-256 backends against out-of-bounds pointer formation while preserving oracle-compatible ciphertext and tags. diff --git a/.changes/harden-blake2b-length-handling-4ab7.md b/.changes/harden-blake2b-length-handling-4ab7.md new file mode 100644 index 00000000..3c2dfee8 --- /dev/null +++ b/.changes/harden-blake2b-length-handling-4ab7.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden Blake2b length handling, state cleanup, and portable diagnostics. diff --git a/.changes/harden-blake2s-length-handling-454e.md b/.changes/harden-blake2s-length-handling-454e.md new file mode 100644 index 00000000..694dba0a --- /dev/null +++ b/.changes/harden-blake2s-length-handling-454e.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden Blake2s length handling, state cleanup, and portable diagnostics. diff --git a/.changes/harden-blake3-tree-arithmetic-32c1.md b/.changes/harden-blake3-tree-arithmetic-32c1.md new file mode 100644 index 00000000..193fb694 --- /dev/null +++ b/.changes/harden-blake3-tree-arithmetic-32c1.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden BLAKE3 tree and SIMD kernel arithmetic, bounds, and assembly ABI validation. diff --git a/.changes/harden-ecdsa-limb-boundaries-6503.md b/.changes/harden-ecdsa-limb-boundaries-6503.md new file mode 100644 index 00000000..6dcdd27f --- /dev/null +++ b/.changes/harden-ecdsa-limb-boundaries-6503.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden ECDSA limb arithmetic, endian conversion, diagnostic documentation, and fixed-work RISC-V and s390x multiplication while preserving P-256 and P-384 signature semantics. diff --git a/.changes/harden-ed25519-simd-boundaries-de52.md b/.changes/harden-ed25519-simd-boundaries-de52.md new file mode 100644 index 00000000..221f8db8 --- /dev/null +++ b/.changes/harden-ed25519-simd-boundaries-de52.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden Ed25519 scalar encoding, fixed-base table validation, table selection, and AVX2/AVX-512 IFMA safety boundaries while preserving signature and verification results. diff --git a/.changes/harden-ml-kem-encoding-2483.md b/.changes/harden-ml-kem-encoding-2483.md new file mode 100644 index 00000000..4c082bbe --- /dev/null +++ b/.changes/harden-ml-kem-encoding-2483.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden ML-KEM encoding, arithmetic bounds, and key-layout validation. diff --git a/.changes/harden-pbkdf2-block-indexing-54fe.md b/.changes/harden-pbkdf2-block-indexing-54fe.md new file mode 100644 index 00000000..2074cb96 --- /dev/null +++ b/.changes/harden-pbkdf2-block-indexing-54fe.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Harden PBKDF2 block indexing and document portable verification diagnostics. diff --git a/.changes/remove-unused-riscv-crc64-surfac-b7f9.md b/.changes/remove-unused-riscv-crc64-surfac-b7f9.md new file mode 100644 index 00000000..b18285c7 --- /dev/null +++ b/.changes/remove-unused-riscv-crc64-surfac-b7f9.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Remove the unused RISC-V CRC-64 Zbc/Zvbc force modes and dead accelerated backend; RISC-V CRC-64 continues to use the portable slice-by-16 implementation until target evidence justifies acceleration. diff --git a/.changes/report-unavailable-argon2-backe-8b41.md b/.changes/report-unavailable-argon2-backe-8b41.md new file mode 100644 index 00000000..7d02e777 --- /dev/null +++ b/.changes/report-unavailable-argon2-backe-8b41.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "patch" +--- + +Return `Argon2Error::BackendUnavailable` when a forced diagnostic backend is unsupported. diff --git a/.changes/unsafe-chacha20-diagnostic-backe-51d5.md b/.changes/unsafe-chacha20-diagnostic-backe-51d5.md new file mode 100644 index 00000000..2c8fa02f --- /dev/null +++ b/.changes/unsafe-chacha20-diagnostic-backe-51d5.md @@ -0,0 +1,5 @@ +--- +"rscrypto" = "major" +--- + +Make the forced hardware ChaCha20 diagnostic functions `unsafe`; callers must establish the documented CPU capabilities before invoking them. diff --git a/.config/ci-tool-archives.tsv b/.config/ci-tool-archives.tsv index 75987b9d..fcda12a9 100644 --- a/.config/ci-tool-archives.tsv +++ b/.config/ci-tool-archives.tsv @@ -1,6 +1,6 @@ # tool version os architecture filename url sha256 -wasmtime v46.0.1 linux x86_64 wasmtime-v46.0.1-x86_64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-x86_64-linux.tar.xz 9ae0b17ea298bcc52277a8208d6ab7fae8e1a89579672f9d82f9d86c116edb62 -wasmtime v46.0.1 linux aarch64 wasmtime-v46.0.1-aarch64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-aarch64-linux.tar.xz 071c4def2a08f0ebc95c52dfd4f2886eb697ba495804217cf76e13b09d70a1be -wasmtime v46.0.1 macos x86_64 wasmtime-v46.0.1-x86_64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-x86_64-macos.tar.xz 0513db67e7089c7e5f743a01427782bc4def83854222f4bc9b1d75f0b925240b -wasmtime v46.0.1 macos aarch64 wasmtime-v46.0.1-aarch64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v46.0.1/wasmtime-v46.0.1-aarch64-macos.tar.xz acee50be70dbe90b0ab2ac7db1321fc44715153a1b1cc58291c97b6d7cffc558 +wasmtime v47.0.3 linux x86_64 wasmtime-v47.0.3-x86_64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-x86_64-linux.tar.xz ca1fc56d1afc40c8782e96c297fd182a0da162f9a8f52a1e7b094e1dd648e178 +wasmtime v47.0.3 linux aarch64 wasmtime-v47.0.3-aarch64-linux.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-aarch64-linux.tar.xz 497b518db00ae585f04390758eaa99ad555bee50612dce7d102602778fb46ff0 +wasmtime v47.0.3 macos x86_64 wasmtime-v47.0.3-x86_64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-x86_64-macos.tar.xz 424a50f76a9dcf4d02dab326b2374be1ad404030576ee915866e4af106058b35 +wasmtime v47.0.3 macos aarch64 wasmtime-v47.0.3-aarch64-macos.tar.xz https://github.com/bytecodealliance/wasmtime/releases/download/v47.0.3/wasmtime-v47.0.3-aarch64-macos.tar.xz c2684249e5d9ef9351942cf2d315982cf201fe0300f05d63bc1527446f0cd37f codecov v11.3.1 linux x86_64 codecov https://cli.codecov.io/v11.3.1/linux/codecov ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d diff --git a/.config/nextest.toml b/.config/nextest.toml index 2635257e..f139574f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,3 +1,5 @@ +nextest-version = { required = "0.9.143", recommended = "0.9.143" } + [profile.default] status-level = "pass" success-output = "never" @@ -15,6 +17,10 @@ slow-timeout = { period = "60s", terminate-after = 8 } filter = 'binary(=rsa_wycheproof) & test(=wycheproof_rsaes_pkcs1v15_3072_decrypt_vectors_match_expected_results)' slow-timeout = { period = "60s", terminate-after = 8 } +[[profile.default.overrides]] +filter = 'binary(=rsa_wycheproof) & (test(=wycheproof_oaep_mgf1sha1_vectors_are_rejected_by_sha2_mgf1sha2_profiles) | test(=wycheproof_pkcs1v15_sha2_sig_gen_vectors_match_expected_signatures))' +slow-timeout = { period = "60s", terminate-after = 4 } + [profile.commit] status-level = "fail" success-output = "never" @@ -22,7 +28,7 @@ failure-output = "immediate-final" fail-fast = false test-threads = "num-cpus" slow-timeout = { period = "60s", terminate-after = 4 } -retries = { backoff = "exponential", count = 2, delay = "1s", jitter = true } +retries = 0 [profile.commit.junit] path = "junit.xml" diff --git a/.config/rail.toml b/.config/rail.toml index 41f90f4e..ef09c902 100644 --- a/.config/rail.toml +++ b/.config/rail.toml @@ -4,23 +4,21 @@ # Targets targets = [ "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", "aarch64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-musl", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "x86_64-unknown-linux-musl", - "aarch64-pc-windows-msvc", "aarch64-unknown-none", - "loongarch64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "riscv32imac-unknown-none-elf", "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-none-elf", "s390x-unknown-linux-gnu", "thumbv6m-none-eabi", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", "x86_64-unknown-none", ] diff --git a/.config/target-matrix.json b/.config/target-matrix.json index 938a59ca..9294bc28 100644 --- a/.config/target-matrix.json +++ b/.config/target-matrix.json @@ -1,28 +1,28 @@ { "groups": { "win": [ - "x86_64-pc-windows-msvc", - "aarch64-pc-windows-msvc" + "aarch64-pc-windows-msvc", + "x86_64-pc-windows-msvc" ], "macos": [ "aarch64-apple-darwin", "x86_64-apple-darwin" ], "linux": [ - "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", "riscv64gc-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl" + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl" ], "ibm": [ - "s390x-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu" + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu" ], "no_std": [ - "thumbv6m-none-eabi", - "riscv32imac-unknown-none-elf", "aarch64-unknown-none", + "riscv32imac-unknown-none-elf", + "thumbv6m-none-eabi", "x86_64-unknown-none" ], "wasm": [ @@ -31,9 +31,9 @@ ] }, "ci": [ - { "name": "x86_64-unknown-linux-gnu", "type": "runson", "pool": "linux-x64-ci" }, + { "name": "aarch64-pc-windows-msvc", "type": "gha", "runner": "windows-11-arm" }, { "name": "aarch64-unknown-linux-gnu", "type": "runson", "pool": "linux-arm64-ci" }, { "name": "x86_64-pc-windows-msvc", "type": "gha", "runner": "windows-latest" }, - { "name": "aarch64-pc-windows-msvc", "type": "gha", "runner": "windows-11-arm" } + { "name": "x86_64-unknown-linux-gnu", "type": "runson", "pool": "linux-x64-ci" } ] } diff --git a/.config/toolchains.toml b/.config/toolchains.toml new file mode 100644 index 00000000..034d6d27 --- /dev/null +++ b/.config/toolchains.toml @@ -0,0 +1 @@ +nightly = "nightly-2026-08-12" diff --git a/.github/actions/setup-toolchain/action.yaml b/.github/actions/setup-toolchain/action.yaml index 9f3f968c..67e7ae40 100644 --- a/.github/actions/setup-toolchain/action.yaml +++ b/.github/actions/setup-toolchain/action.yaml @@ -1,9 +1,12 @@ -name: Setup Rust Toolchain (Dynamic) +name: Setup Rust Toolchain description: | - Install Rust toolchain reading version from rust-toolchain.toml. - Single source of truth for toolchain version. + Install the stable development toolchain, MSRV, or pinned nightly contract. inputs: + contract: + description: Toolchain contract to install (development, msrv, or nightly) + required: false + default: "development" components: description: Comma-separated list of components to install required: false @@ -11,7 +14,7 @@ inputs: outputs: toolchain: - description: The toolchain version that was installed + description: Exact installed toolchain version value: ${{ steps.read-toolchain.outputs.toolchain }} runs: @@ -20,8 +23,15 @@ runs: - name: Read Toolchain Version id: read-toolchain shell: bash + env: + TOOLCHAIN_CONTRACT: ${{ inputs.contract }} run: | - TOOLCHAIN=$(bash scripts/lib/toolchain.sh) + case "$TOOLCHAIN_CONTRACT" in + development) TOOLCHAIN=$(bash scripts/lib/toolchain.sh) ;; + msrv) TOOLCHAIN=$(bash scripts/lib/toolchain.sh --msrv) ;; + nightly) TOOLCHAIN=$(bash scripts/lib/toolchain.sh --nightly) ;; + *) echo "Unsupported toolchain contract: $TOOLCHAIN_CONTRACT" >&2; exit 2 ;; + esac echo "toolchain=$TOOLCHAIN" >> "$GITHUB_OUTPUT" echo "Using toolchain: $TOOLCHAIN" @@ -30,4 +40,6 @@ runs: env: TOOLCHAIN: ${{ steps.read-toolchain.outputs.toolchain }} TOOLCHAIN_COMPONENTS: ${{ inputs.components }} - run: scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" + run: | + scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" "$GITHUB_ENV" + RUSTUP_TOOLCHAIN="$TOOLCHAIN" rustc --version --verbose diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index a9f05709..99a2b35d 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -4,6 +4,10 @@ description: | Unified host setup for GHA + runs-on.com + IBM/RISC-V runners. inputs: + toolchain-contract: + description: Toolchain contract to install (development, msrv, or nightly). + required: false + default: "development" cache-key: description: Suffix for the rust-cache key (workflow + target). required: false @@ -31,11 +35,12 @@ runs: # runs-on.com S3-backed cache intercept. No-op on non-runs-on pools. - name: Setup MagicCache if: inputs.enable-magic-cache == 'true' - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0 + uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0 - name: Setup Toolchain uses: ./.github/actions/setup-toolchain with: + contract: ${{ inputs.toolchain-contract }} components: ${{ inputs.toolchain-components }} - name: Install Cargo Tools @@ -47,7 +52,7 @@ runs: - name: Setup Rust Cache if: inputs.enable-rust-cache == 'true' - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: shared-key: "rscrypto-v1-${{ inputs.cache-key }}" cache-directories: ${{ inputs.tools-mode == 'rail' && 'target/cargo-rail/cache' || '' }} diff --git a/.github/workflows/_ci-suite.yaml b/.github/workflows/_ci-suite.yaml index aecc3dd7..f4c7357c 100644 --- a/.github/workflows/_ci-suite.yaml +++ b/.github/workflows/_ci-suite.yaml @@ -67,6 +67,20 @@ jobs: enable_rust_cache: true operation: quality + msrv: + name: MSRV + uses: ./.github/workflows/_rust-job.yaml + with: + runner: runs-on=${{ github.run_id }}/runner=linux-x64-ci + timeout_minutes: 30 + cache_key: ${{ inputs.cache_key_prefix }}-msrv + tools_mode: none + toolchain_contract: msrv + toolchain_components: "" + enable_magic_cache: true + enable_rust_cache: true + operation: msrv + cargo-graph: if: ${{ inputs.include_cargo_graph }} name: Cargo Graph Assurance @@ -126,6 +140,7 @@ jobs: timeout_minutes: 120 cache_key: ${{ inputs.cache_key_prefix }}-ibm-s390x tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true @@ -140,6 +155,7 @@ jobs: timeout_minutes: 120 cache_key: ${{ inputs.cache_key_prefix }}-ibm-power10 tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true @@ -154,6 +170,7 @@ jobs: timeout_minutes: 30 cache_key: ${{ inputs.cache_key_prefix }}-platform-amx-test-nodebug tools_mode: none + toolchain_contract: nightly toolchain_components: "rustfmt" enable_magic_cache: true enable_rust_cache: true @@ -167,6 +184,7 @@ jobs: timeout_minutes: 90 cache_key: ${{ inputs.cache_key_prefix }}-cross-targets tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rust-src" enable_magic_cache: true enable_rust_cache: true diff --git a/.github/workflows/_rust-job.yaml b/.github/workflows/_rust-job.yaml index 8508c2de..e0ef686c 100644 --- a/.github/workflows/_rust-job.yaml +++ b/.github/workflows/_rust-job.yaml @@ -25,6 +25,10 @@ on: required: false type: string default: standard + toolchain_contract: + required: false + type: string + default: development toolchain_components: required: false type: string @@ -128,6 +132,7 @@ jobs: - name: Setup uses: ./.github/actions/setup with: + toolchain-contract: ${{ inputs.toolchain_contract }} cache-key: ${{ inputs.cache_key }} tools-mode: ${{ inputs.tools_mode }} toolchain-components: ${{ inputs.toolchain_components }} diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 26417e10..a3ebd8fe 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -71,6 +71,7 @@ jobs: timeout_minutes: ${{ matrix.timeout_minutes }} cache_key: bench-${{ matrix.artifact_suffix }} tools_mode: ${{ matrix.tools_mode }} + toolchain_contract: ${{ (matrix.platform == 'ibm-s390x' || matrix.platform == 'ibm-power10') && 'nightly' || 'development' }} toolchain_components: ${{ matrix.toolchain_components }} enable_magic_cache: false enable_rust_cache: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6d00ab7f..6e890908 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: id: rail uses: loadingalias/cargo-rail-action@ebc8ef2740efefe2cd5c42826344a8f1213113ae # v6.1.2 with: - version: "0.20.0" + version: "0.21.0" checksum: required since: ${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/ct.yaml b/.github/workflows/ct.yaml index 569c55ea..0c7fabbf 100644 --- a/.github/workflows/ct.yaml +++ b/.github/workflows/ct.yaml @@ -175,6 +175,7 @@ jobs: timeout_minutes: ${{ matrix.timeout_minutes }} cache_key: ct-${{ matrix.artifact_suffix }} tools_mode: ${{ matrix.tools_mode }} + toolchain_contract: ${{ (matrix.platform == 'ibm-s390x' || matrix.platform == 'ibm-power10' || matrix.platform == 'rise-riscv') && 'nightly' || 'development' }} toolchain_components: ${{ matrix.toolchain_components }} enable_magic_cache: ${{ matrix.enable_magic_cache }} enable_rust_cache: ${{ matrix.enable_rust_cache }} diff --git a/.github/workflows/riscv.yaml b/.github/workflows/riscv.yaml index f6ac5a5d..3f8ae671 100644 --- a/.github/workflows/riscv.yaml +++ b/.github/workflows/riscv.yaml @@ -52,6 +52,7 @@ jobs: timeout_minutes: 240 cache_key: weekly-riscv64 tools_mode: none + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true @@ -83,6 +84,7 @@ jobs: timeout_minutes: 240 cache_key: bench-rise-riscv tools_mode: ibm + toolchain_contract: nightly toolchain_components: "clippy, rustfmt" enable_magic_cache: false enable_rust_cache: true diff --git a/.github/workflows/rsa.yaml b/.github/workflows/rsa.yaml index 4bf3fa1b..b3b0d714 100644 --- a/.github/workflows/rsa.yaml +++ b/.github/workflows/rsa.yaml @@ -38,6 +38,7 @@ jobs: timeout_minutes: 60 cache_key: rsa-miri-linux-x64 tools_mode: minimal + toolchain_contract: nightly toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true diff --git a/.github/workflows/weekly.yaml b/.github/workflows/weekly.yaml index 29b95b1b..cb9de23e 100644 --- a/.github/workflows/weekly.yaml +++ b/.github/workflows/weekly.yaml @@ -90,6 +90,7 @@ jobs: timeout_minutes: 60 cache_key: weekly-miri-${{ matrix.target.name }} tools_mode: minimal + toolchain_contract: nightly toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true @@ -104,6 +105,7 @@ jobs: timeout_minutes: 60 cache_key: weekly-miri-tree-borrows tools_mode: minimal + toolchain_contract: nightly toolchain_components: "miri, rust-src" enable_magic_cache: true enable_rust_cache: true @@ -128,6 +130,7 @@ jobs: timeout_minutes: 120 cache_key: weekly-fuzz-${{ matrix.target.name }} tools_mode: fuzz + toolchain_contract: nightly enable_magic_cache: true enable_rust_cache: true operation: fuzz @@ -155,6 +158,7 @@ jobs: timeout_minutes: 120 cache_key: weekly-asan-fuzz-${{ matrix.target.name }} tools_mode: fuzz + toolchain_contract: nightly toolchain_components: "rust-src" enable_magic_cache: true enable_rust_cache: true diff --git a/.gitignore b/.gitignore index d9aa2a11..cac10004 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,9 @@ Thumbs.db AGENTS.md # Zed/IDE -.zed +.zed/* +!.zed/settings.json +!.zed/tasks.json # Keys/Creds *.pem diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 00000000..e68e209b --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,13 @@ +{ + "lsp": { + "rust-analyzer": { + "enable_lsp_tasks": false, + "initialization_options": { + "cargo": { + "extraArgs": ["--locked"], + "features": "all" + } + } + } + } +} diff --git a/.zed/tasks.json b/.zed/tasks.json new file mode 100644 index 00000000..048cc3df --- /dev/null +++ b/.zed/tasks.json @@ -0,0 +1,18 @@ +[ + { + "label": "rscrypto: check", + "command": "just", + "args": ["check"], + "cwd": "$ZED_WORKTREE_ROOT", + "allow_concurrent_runs": false, + "save": "all" + }, + { + "label": "rscrypto: test", + "command": "just", + "args": ["test"], + "cwd": "$ZED_WORKTREE_ROOT", + "allow_concurrent_runs": false, + "save": "all" + } +] diff --git a/Cargo.lock b/Cargo.lock index 133c9ad3..46569232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,16 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common 0.1.7", - "generic-array", -] - [[package]] name = "aead" version = "0.6.1" @@ -19,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -32,24 +22,13 @@ dependencies = [ "softaes", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", -] - [[package]] name = "aes" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cipher 0.5.2", + "cipher", "cpubits", "cpufeatures 0.3.0", ] @@ -60,34 +39,33 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ - "aead 0.6.1", - "aes 0.9.2", - "cipher 0.5.2", - "ctr 0.10.1", + "aead", + "aes", + "cipher", + "ctr", "ghash", "subtle", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ - "aead 0.5.2", - "aes 0.8.4", - "cipher 0.4.4", - "ctr 0.9.2", - "polyval 0.6.2", + "aead", + "aes", + "cipher", + "ctr", + "polyval", "subtle", - "zeroize", ] [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -115,13 +93,13 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "argon2" -version = "0.6.0-rc.8" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures 0.3.0", + "cpufeatures 0.2.17", "password-hash", ] @@ -143,9 +121,9 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cba922132fed390c20be89a16fa713999c46b2ce84ac4b0ecee0f3586a39d88" dependencies = [ - "aead 0.6.1", + "aead", "ascon-core", - "inout 0.2.2", + "inout", "subtle", ] @@ -173,32 +151,20 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ - "aws-lc-sys 0.43.0", + "aws-lc-sys", "untrusted 0.7.1", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "aws-lc-sys" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", @@ -207,12 +173,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -268,18 +228,18 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.11.3", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -329,9 +289,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -361,7 +321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", "cpufeatures 0.3.0", "rand_core 0.10.1", "zeroize", @@ -373,9 +333,9 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" dependencies = [ - "aead 0.6.1", + "aead", "chacha20", - "cipher 0.5.2", + "cipher", "poly1305", ] @@ -406,16 +366,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.2" @@ -424,7 +374,7 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.2", + "inout", "zeroize", ] @@ -441,18 +391,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstyle", "clap_lex", @@ -657,18 +607,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.7.5" @@ -692,7 +630,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", - "rand_core 0.6.4", "typenum", ] @@ -714,26 +651,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6250a2d96a09edbe8e75ed29c87d05512ee2cbb24c7e8c684657f7930ffd3c6" dependencies = [ "digest 0.11.3", - "keccak 0.2.0", + "keccak 0.2.1", "sponge-cursor", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher 0.4.4", -] - [[package]] name = "ctr" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher 0.5.2", + "cipher", ] [[package]] @@ -755,8 +683,23 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest 0.10.7", - "fiat-crypto", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", "rustc_version", "subtle", "zeroize", @@ -816,22 +759,23 @@ dependencies = [ "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", + "zeroize", ] [[package]] name = "dryoc" -version = "0.8.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09fac1c1cf30622ccd33105965abb8ef17dd1c3aae34a489350dd0b8c3fae4" +checksum = "2a9268c01fc6ec1f2a104ae9e40343c255fbcafea97a48f334237206c96838b2" dependencies = [ - "bitflags", "chacha20", - "curve25519-dalek", - "lazy_static", + "curve25519-dalek 4.1.3", + "getrandom 0.4.3", "libc", "rand 0.10.2", "salsa20", "sha2 0.11.0", + "sha3 0.12.0", "subtle", "winapi", "zeroize", @@ -843,20 +787,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", -] - [[package]] name = "ecdsa" version = "0.17.0" @@ -865,8 +795,8 @@ checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ "der 0.8.1", "digest 0.11.3", - "elliptic-curve 0.14.1", - "rfc6979 0.6.0", + "elliptic-curve", + "rfc6979", "signature 3.0.0", "spki 0.8.0", "zeroize", @@ -874,24 +804,22 @@ dependencies = [ [[package]] name = "ed25519" -version = "2.2.3" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "signature 3.0.0", ] [[package]] name = "ed25519-dalek" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 5.0.0", "ed25519", - "serde", - "sha2 0.10.9", + "sha2 0.11.0", "subtle", "zeroize", ] @@ -902,41 +830,22 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff 0.13.1", - "generic-array", - "group 0.13.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1 0.7.3", - "subtle", - "zeroize", -] - [[package]] name = "elliptic-curve" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ - "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "base16ct", + "crypto-bigint", "crypto-common 0.2.2", "digest 0.11.3", - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", "pkcs8 0.11.0", "rand_core 0.10.1", - "sec1 0.8.1", + "sec1", "subtle", "zeroize", ] @@ -957,16 +866,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "ff" version = "0.14.0" @@ -983,11 +882,17 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "fips203" @@ -1015,21 +920,21 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -1045,7 +950,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -1078,9 +982,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1089,7 +995,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "polyval 0.7.3", + "polyval", ] [[package]] @@ -1098,24 +1004,13 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff 0.13.1", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "group" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", ] @@ -1174,16 +1069,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac 0.13.0", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", + "hmac", ] [[package]] @@ -1207,15 +1093,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "inout" version = "0.2.2" @@ -1252,9 +1129,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -1272,9 +1149,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -1472,9 +1349,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -1511,35 +1388,31 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "p256" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ - "ecdsa 0.17.0", - "elliptic-curve 0.14.1", + "ecdsa", + "elliptic-curve", "primefield", - "primeorder 0.14.0", + "primeorder", "sha2 0.11.0", ] [[package]] name = "p384" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", - "primeorder 0.13.6", - "sha2 0.10.9", + "ecdsa", + "elliptic-curve", + "fiat-crypto 0.3.0", + "primefield", + "primeorder", + "sha2 0.11.0", ] [[package]] @@ -1554,11 +1427,13 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ - "phc", + "base64ct", + "rand_core 0.6.4", + "subtle", ] [[package]] @@ -1574,17 +1449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest 0.11.3", - "hmac 0.13.0", -] - -[[package]] -name = "phc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" -dependencies = [ - "base64ct", - "ctutils", + "hmac", ] [[package]] @@ -1665,19 +1530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" dependencies = [ "cpufeatures 0.3.0", - "universal-hash 0.6.1", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", - "universal-hash 0.5.1", + "universal-hash", ] [[package]] @@ -1688,7 +1541,7 @@ checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures 0.3.0", - "universal-hash 0.6.1", + "universal-hash", ] [[package]] @@ -1716,30 +1569,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-common 0.2.2", - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", "zeroize", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve 0.13.8", -] - [[package]] name = "primeorder" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.1", + "elliptic-curve", "once_cell", "primefield", "serdect", @@ -1950,9 +1794,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1965,24 +1809,14 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "rfc6979" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "crypto-bigint 0.7.5", - "hmac 0.13.0", + "crypto-bigint", + "hmac", ] [[package]] @@ -2031,7 +1865,7 @@ dependencies = [ "ascon-aead", "ascon-hash", "aws-lc-rs", - "aws-lc-sys 0.41.0", + "aws-lc-sys", "blake2", "blake3", "chacha20poly1305", @@ -2048,8 +1882,8 @@ dependencies = [ "fips203", "getrandom 0.4.3", "hkdf", - "hmac 0.13.0", - "keccak 0.2.0", + "hmac", + "keccak 0.2.1", "libcrux-ml-kem", "ml-kem", "p256", @@ -2124,7 +1958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", ] [[package]] @@ -2148,27 +1982,13 @@ dependencies = [ "sha2 0.11.0", ] -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", -] - [[package]] name = "sec1" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 1.0.0", + "base16ct", "ctutils", "der 0.8.1", "hybrid-array", @@ -2231,7 +2051,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] @@ -2274,7 +2094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" dependencies = [ "digest 0.11.3", - "keccak 0.2.0", + "keccak 0.2.1", ] [[package]] @@ -2284,7 +2104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" dependencies = [ "digest 0.11.3", - "keccak 0.2.0", + "keccak 0.2.1", "sponge-cursor", ] @@ -2375,6 +2195,9 @@ name = "sponge-cursor" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" +dependencies = [ + "zeroize", +] [[package]] name = "subtle" @@ -2454,16 +2277,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common 0.1.7", - "subtle", -] - [[package]] name = "universal-hash" version = "0.6.1" @@ -2539,9 +2352,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -2552,9 +2365,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2562,9 +2375,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -2575,18 +2388,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -2723,20 +2536,19 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" dependencies = [ - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", ] [[package]] name = "x25519-dalek" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ - "curve25519-dalek", - "rand_core 0.6.4", - "serde", + "curve25519-dalek 5.0.0", + "rand_core 0.10.1", "zeroize", ] @@ -2748,18 +2560,18 @@ checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 2d718ff7..108993d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,36 +47,101 @@ rustdoc-args = ["--cfg", "docsrs"] resolver = "3" members = ["."] -[workspace.lints.rust] -unsafe_code = "allow" +[lints.rust] +warnings = { level = "deny", priority = -1 } +ambiguous_negative_literals = "deny" unsafe_op_in_unsafe_fn = "deny" -dangling_pointers_from_locals = "deny" -integer_to_ptr_transmutes = "deny" -unexpected_cfgs = { level = "warn", check-cfg = [ +ffi_unwind_calls = "deny" +let_underscore_drop = "deny" +unreachable_pub = "deny" +unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(miri)', 'cfg(fuzzing)', 'cfg(rscrypto_internal_fuzzing)', 'cfg(target_feature, values("movdiri", "movdir64b", "serialize"))', ] } -[workspace.lints.clippy] -correctness = { level = "deny", priority = -1 } -suspicious = { level = "deny", priority = -1 } -perf = { level = "deny", priority = -1 } +[lints.clippy] +all = { level = "deny", priority = -1 } + +non_send_fields_in_send_ty = "deny" +significant_drop_in_scrutinee = "deny" + +panic = "deny" +panic_in_result_fn = "deny" +string_slice = "deny" +todo = "deny" +unchecked_time_subtraction = "deny" +unimplemented = "deny" +unreachable = "deny" +unwrap_used = "deny" + +assertions_on_result_states = "deny" +let_underscore_must_use = "deny" +unused_result_ok = "deny" + +cast_possible_truncation = "deny" +cast_possible_wrap = "deny" +cast_sign_loss = "deny" +checked_conversions = "deny" +fn_to_numeric_cast_any = "deny" +invalid_upcast_comparisons = "deny" +lossy_float_literal = "deny" + +allow_attributes = "deny" +allow_attributes_without_reason = "deny" +dbg_macro = "deny" +enum_glob_use = "deny" +expl_impl_clone_on_copy = "deny" +iter_not_returning_iterator = "deny" +same_functions_in_if_condition = "deny" + +negative_feature_names = "deny" +wildcard_dependencies = "deny" + +coerce_container_to_any = "deny" +collection_is_never_read = "deny" +debug_assert_with_mut_call = "deny" +default_union_representation = "deny" +fallible_impl_from = "deny" +ignore_without_reason = "deny" +literal_string_with_formatting_args = "deny" +mixed_read_write_in_expression = "deny" +needless_collect = "deny" +or_fun_call = "deny" +path_buf_push_overwrite = "deny" +read_zero_byte_vec = "deny" +redundant_clone = "deny" +should_panic_without_expect = "deny" +suspicious_operation_groupings = "deny" +trait_duplication_in_bounds = "deny" +transmute_undefined_repr = "deny" +uninhabited_references = "deny" +unsafe_derive_deserialize = "deny" +volatile_composites = "deny" + +as_ptr_cast_mut = "deny" +as_pointer_underscore = "deny" +borrow_as_ptr = "deny" +cast_ptr_alignment = "deny" +mem_forget = "deny" +ptr_as_ptr = "deny" +ptr_cast_constness = "deny" +ref_as_ptr = "deny" +undocumented_unsafe_blocks = "deny" +unnecessary_safety_comment = "deny" +unnecessary_safety_doc = "deny" -std_instead_of_core = "warn" -std_instead_of_alloc = "warn" -unwrap_used = "allow" -expect_used = "allow" -indexing_slicing = "allow" +alloc_instead_of_core = "deny" +std_instead_of_alloc = "deny" +std_instead_of_core = "deny" -undocumented_unsafe_blocks = "deny" -missing_safety_doc = "deny" -chunks_exact_to_as_chunks = "allow" +arithmetic_side_effects = "deny" +cargo_common_metadata = "deny" +cfg_not_test = "deny" -[workspace.lints.rustdoc] -broken_intra_doc_links = "deny" -private_intra_doc_links = "warn" +[lints.rustdoc] +all = "deny" [features] default = ["std"] @@ -164,36 +229,36 @@ crc32c = "0.6.8" crc32fast = "1.5.0" crc64fast = "1.1.0" ascon-hash = "0.4.0" -argon2 = { version = "0.6.0-rc.8", default-features = false, features = ["alloc"] } -blake3 = { version = "1.8.5", default-features = false, features = ["std", "rayon"] } -blake2 = "0.11.0-rc.6" +argon2 = { version = "0.5.3", default-features = false, features = ["alloc"] } +blake3 = { version = "1.8.6", default-features = false, features = ["std", "rayon"] } +blake2 = "0.10.6" pbkdf2 = "0.13.0" scrypt = { version = "0.12.0", default-features = false } sha2 = "0.11.0" sha2_010 = { package = "sha2", version = "0.10.9" } sha3 = "0.12.0" -keccak = "0.2.0" +keccak = "0.2.1" xxhash-rust = { version = "0.8.18", features = ["xxh3"] } rapidhash = "=4.5.1" tiny-keccak = { version = "2.0.2", features = ["kmac", "shake"] } digest = { version = "0.11.3", features = ["dev"] } p256 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } -p384 = { version = "0.13.1", default-features = false, features = ["ecdsa", "pkcs8", "std"] } -ed25519-dalek = "2.2.0" -x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +p384 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } +ed25519-dalek = "3.0.0" +x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } fips203 = { version = "0.4.3", default-features = false, features = ["ml-kem-512", "ml-kem-768", "ml-kem-1024"] } rustcrypto-ml-kem = { package = "ml-kem", version = "0.3.2", default-features = false, features = ["hazmat"] } hmac = "0.13.0" hkdf = "0.13.0" chacha20poly1305 = "0.11.0" aes-gcm = "0.11.0" -aes-gcm-siv = "0.11.1" +aes-gcm-siv = "0.12.0" aegis = "0.9.15" ascon-aead = { version = "0.6", default-features = false, features = ["alloc"] } serde_json = "1" ring = "0.17.14" rsa = { version = "0.9.10", default-features = false, features = ["std", "u64_digit", "sha2"] } -dryoc = "0.8.0" +dryoc = "1.0.0" [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dev-dependencies] libcrux-ml-kem = { version = "0.0.10", default-features = false, features = [ @@ -219,8 +284,8 @@ libcrux-ml-kem = { version = "0.0.10", default-features = false, features = [ ] } [target.'cfg(all(any(unix, windows), not(target_arch = "wasm32"), not(any(target_arch = "s390x", target_arch = "powerpc64"))))'.dev-dependencies] -aws-lc-rs = "1.17.3" -aws-lc-sys = "0.41.0" +aws-lc-rs = "1.18.0" +aws-lc-sys = "0.44.0" [lib] bench = false @@ -334,9 +399,6 @@ name = "kmac_cshake" harness = false required-features = ["sha3", "kmac"] -[lints] -workspace = true - [profile.dev] opt-level = 0 debug = true diff --git a/README.md b/README.md index d7271097..26b1cb72 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,14 @@ Minimal `no_std` SHA-2 build: ```toml [dependencies] -rscrypto = { version = "0.8.0", default-features = false, features = ["sha2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } ``` Full primitive stack with OS randomness enabled: ```toml [dependencies] -rscrypto = { version = "0.8.0", features = ["full", "getrandom"] } +rscrypto = { version = "0.8.1", features = ["full", "getrandom"] } ``` Use `default-features = false` for `no_std` builds. Enable `getrandom` only for @@ -156,24 +156,28 @@ code from a binary. ## Performance -The published 2026-07-04 benchmark snapshot is historical. Its aggregate -geomeans are not equivalent-work performance claims because the historical -RustCrypto HMAC-SHA-256 rows included key setup while the compared rscrypto, -`ring`, and AWS-LC rows reused keyed state. The benchmark source now aligns -that setup, but a new aggregate requires a complete regenerated artifact. +The 2026-08-18 snapshot covers eight Linux CI runners at commit `7eb44e9`. The +RustCrypto HMAC-SHA-256 key setup is now hoisted out of the timed loop, matching +the reusable keyed state given to rscrypto, `ring`, and AWS-LC, so these +aggregates are equivalent-work claims. Ratios are `external / rscrypto`; higher +is better. -
-Historical 2026-07-04 scorecard (not an equivalent-work aggregate)

- rscrypto benchmark chart: 1.59x Linux and 1.37x Apple Silicon fastest-matched geomeans, checksums at 5.18x against crc-fast, crc, crc32fast, crc32c, and crc64fast, plus primitive geomean bars and M1 MBP Apple Silicon notes.

-
+ +3,780 of 6,144 fastest-external comparisons are wins and 5,475 are wins or ties, +for a 1.62x Linux geomean. Known losses: ECDSA P-256/P-384 regressed sharply on +IBM z16/s390x in this run and drags every ECDSA aggregate below parity (0.87x +across 128 rows; 1.19x-1.53x excluding that one runner), and +`rapidhash-stream/one-write` trails the `rapidhash` crate at 0.87x, mostly on +x86_64. Use individual shape-compatible rows for investigation and benchmark the -deployment workload on its target hardware. The correction, raw results, -methodology, and known losses are in +deployment workload on its target hardware. Raw results, methodology, and the +full loss list are in [`benchmark_results/OVERVIEW.md`](benchmark_results/OVERVIEW.md) and [`docs/benchmarking.md`](docs/benchmarking.md). @@ -195,9 +199,10 @@ methodology, and known losses are in The minimum supported Rust version is **1.91.0**. -The pinned development nightly in -[`rust-toolchain.toml`](rust-toolchain.toml) is separate from the MSRV and is -used for Miri, fuzzing, and architecture-specific checks. +The pinned stable development toolchain in +[`rust-toolchain.toml`](rust-toolchain.toml) is separate from the MSRV. +Nightly-only Miri, fuzzing, and architecture checks use the dated exception in +[`toolchains.toml`](.config/toolchains.toml). ## License diff --git a/assets/readme/perf.svg b/assets/readme/perf.svg index 1030bba0..40a68c56 100644 --- a/assets/readme/perf.svg +++ b/assets/readme/perf.svg @@ -2,4 +2,4 @@ text.sans { font-family: ui-sans-serif, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; } text.mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, 'Liberation Mono', monospace; } text.hero { letter-spacing: -2px; paint-order: stroke fill; stroke: #0090FF; stroke-width: 0.7px; } -rscryptoGeomean speedups vs fastest matched competitors. Higher is better.Linux1.59x4,052 wins | 6,101 wins/ties | 6,750 casesApple Silicon1.37x382 wins | 708 wins/ties | 774 casesChecksums5.18x- Competitor Crates/Libs: crc-fast, crc, crc32fast, crc32c, crc64fastPrimitive GeomeansAEAD1.56xRSA1.55xECDSA1.45xHashes/MAC/XOF1.35xAuth/KDF1.25xPassword Hashing1.07xML-KEM1.49x- Linux Runners: AMD Zen 4/5; Intel Sapphire Rapids/Ice Lake; AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x); Rise RISC-V- macOS: MBP M1 10-Core, 16GB RAM - Local Dev Box- Fastest External: aws-lc-rs, ring, RustCrypto, BLAKE3, libcrux, crc-fast, etc. \ No newline at end of file +rscryptoGeomean speedups vs fastest matched competitors. Higher is better.Linux1.62x3,780 wins | 5,475 wins/ties | 6,144 casesApple Silicon1.37x382 wins | 708 wins/ties | 774 casesChecksums6.18x- Competitor Crates/Libs: crc-fast, crc, crc32fast, crc32c, crc64fastPrimitive GeomeansAEAD1.61xRSA1.65xECDSA0.87xHashes/MAC/XOF1.35xAuth/KDF1.28xPassword Hashing1.07xML-KEM1.55x1.00x- Linux Runners: AMD Zen 4/5; Intel Sapphire Rapids/Ice Lake; AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x)- macOS: MBP M1 10-Core, 16GB RAM - Local Dev Box- Fastest External: aws-lc-rs, ring, RustCrypto, BLAKE3, libcrux, crc-fast, etc. \ No newline at end of file diff --git a/benches/aead.rs b/benches/aead.rs index 8b40ff6c..743c9dc0 100644 --- a/benches/aead.rs +++ b/benches/aead.rs @@ -9,7 +9,6 @@ mod common; use core::hint::black_box; use aes_gcm::aead::{AeadInOut as _, KeyInit as _}; -use aes_gcm_siv::aead::{AeadInPlace as _, KeyInit as _}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use rscrypto::aead::expert::AeadWithNonce; @@ -78,7 +77,7 @@ fn xchacha20_poly1305_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -100,13 +99,15 @@ fn xchacha20_poly1305_decrypt(c: &mut Criterion) { // Pre-encrypt with rscrypto to get valid ciphertext + tag. let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); // Pre-encrypt with RustCrypto to get its tag format. let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -120,7 +121,7 @@ fn xchacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -137,7 +138,7 @@ fn xchacha20_poly1305_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -161,10 +162,12 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { let nonce_rc = chacha20poly1305::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = - ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32) + .expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("chacha20-poly1305/encrypt"); for (len, data) in &inputs { @@ -174,7 +177,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { let mut buf_owned = data.clone(); #[cfg(all(feature = "diag", target_arch = "x86_64", target_os = "linux"))] let mut buf_x86_asm = data.clone(); - let mut buf_combined: Vec = Vec::with_capacity(data.len() + 16); + let mut buf_combined: Vec = Vec::with_capacity(data.len().strict_add(16)); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { @@ -224,7 +227,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -240,7 +243,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_combined), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_combined); }) }); @@ -256,7 +259,7 @@ fn chacha20_poly1305_encrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_combined), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_combined); }) }); @@ -278,22 +281,26 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { let nonce_rc = chacha20poly1305::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::CHACHA20_POLY1305, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = - ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::CHACHA20_POLY1305, &KEY_32) + .expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("chacha20-poly1305/decrypt"); for (len, data) in &inputs { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws: Vec = data.clone(); @@ -303,7 +310,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); } let mut ct_ring: Vec = data.clone(); @@ -313,7 +320,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), &mut ct_ring, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); #[cfg(feature = "diag")] @@ -331,7 +338,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -347,7 +354,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&mut buf_owned), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_owned); }) }); @@ -365,7 +372,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(&tag_rs), ) .expect("x86 asm path must apply to benchmarked non-empty sizes") - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_x86_asm); }) }); @@ -383,7 +390,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -400,7 +407,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -417,7 +424,7 @@ fn chacha20_poly1305_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_ring), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ring); }) }); @@ -437,10 +444,10 @@ fn aes256_gcm_siv_encrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes256GcmSiv::new(&rscrypto::Aes256GcmSivKey::from_bytes(KEY_32)); let cipher_rc = aes_gcm_siv::Aes256GcmSiv::new(&KEY_32.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-256-gcm-siv/encrypt"); @@ -460,8 +467,12 @@ fn aes256_gcm_siv_encrypt(c: &mut Criterion) { buf.copy_from_slice(d); black_box( cipher_rc - .encrypt_in_place_detached(black_box(nonce_rc), black_box(AAD), black_box(&mut buf)) - .unwrap(), + .encrypt_inout_detached( + black_box(&nonce_rc), + black_box(AAD), + black_box(buf.as_mut_slice()).into(), + ) + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -477,7 +488,7 @@ fn aes256_gcm_siv_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -497,10 +508,10 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes256GcmSiv::new(&rscrypto::Aes256GcmSivKey::from_bytes(KEY_32)); let cipher_rc = aes_gcm_siv::Aes256GcmSiv::new(&KEY_32.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM_SIV, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-256-gcm-siv/decrypt"); @@ -508,10 +519,14 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); - let tag_rc = cipher_rc.encrypt_in_place_detached(nonce_rc, AAD, &mut ct_rc).unwrap(); + let tag_rc = cipher_rc + .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -521,7 +536,7 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -538,7 +553,7 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -549,13 +564,13 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { b.iter(|| { buf_rc.copy_from_slice(ct); cipher_rc - .decrypt_in_place_detached( - black_box(nonce_rc), + .decrypt_inout_detached( + black_box(&nonce_rc), black_box(AAD), - black_box(&mut buf_rc), + black_box(buf_rc.as_mut_slice()).into(), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -572,7 +587,7 @@ fn aes256_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -593,10 +608,10 @@ fn aes128_gcm_siv_encrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes128GcmSiv::new(&rscrypto::Aes128GcmSivKey::from_bytes(KEY_16)); let cipher_rc = aes_gcm_siv::Aes128GcmSiv::new(&KEY_16.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-128-gcm-siv/encrypt"); @@ -616,8 +631,12 @@ fn aes128_gcm_siv_encrypt(c: &mut Criterion) { buf.copy_from_slice(d); black_box( cipher_rc - .encrypt_in_place_detached(black_box(nonce_rc), black_box(AAD), black_box(&mut buf)) - .unwrap(), + .encrypt_inout_detached( + black_box(&nonce_rc), + black_box(AAD), + black_box(buf.as_mut_slice()).into(), + ) + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -633,7 +652,7 @@ fn aes128_gcm_siv_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -653,10 +672,10 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { let nonce_rs = rscrypto::aead::Nonce96::from_bytes(NONCE_12); let cipher_rs = rscrypto::Aes128GcmSiv::new(&rscrypto::Aes128GcmSivKey::from_bytes(KEY_16)); let cipher_rc = aes_gcm_siv::Aes128GcmSiv::new(&KEY_16.into()); - let nonce_rc = aes_gcm_siv::Nonce::from_slice(&NONCE_12); + let nonce_rc = aes_gcm_siv::Nonce::from(NONCE_12); aws_lc_bench! { let aws_key = - aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).unwrap()); + aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM_SIV, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } let mut g = c.benchmark_group("aes-128-gcm-siv/decrypt"); @@ -664,10 +683,14 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); - let tag_rc = cipher_rc.encrypt_in_place_detached(nonce_rc, AAD, &mut ct_rc).unwrap(); + let tag_rc = cipher_rc + .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -677,7 +700,7 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -694,7 +717,7 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -705,13 +728,13 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { b.iter(|| { buf_rc.copy_from_slice(ct); cipher_rc - .decrypt_in_place_detached( - black_box(nonce_rc), + .decrypt_inout_detached( + black_box(&nonce_rc), black_box(AAD), - black_box(&mut buf_rc), + black_box(buf_rc.as_mut_slice()).into(), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -728,7 +751,7 @@ fn aes128_gcm_siv_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -752,9 +775,11 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes256Gcm::new(&KEY_32.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-256-gcm/encrypt"); for (len, data) in &inputs { @@ -778,7 +803,7 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -794,7 +819,7 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -811,7 +836,7 @@ fn aes256_gcm_encrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_ring), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_ring); }) @@ -833,21 +858,25 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes256Gcm::new(&KEY_32.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_256_GCM, &KEY_32).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-256-gcm/decrypt"); for (len, data) in &inputs { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -857,7 +886,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -869,7 +898,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), &mut ct_ring, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -883,7 +912,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -900,7 +929,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -917,7 +946,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -936,7 +965,7 @@ fn aes256_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf_ring), 0.., ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ring); }) }); @@ -959,9 +988,11 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes128Gcm::new(&KEY_16.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-128-gcm/encrypt"); for (len, data) in &inputs { @@ -985,7 +1016,7 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -1001,7 +1032,7 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_aws); }) @@ -1018,7 +1049,7 @@ fn aes128_gcm_encrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), black_box(&mut buf_ring), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(tag.as_ref()); black_box(&buf_ring); }) @@ -1040,21 +1071,25 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { let cipher_rc = aes_gcm::Aes128Gcm::new(&KEY_16.into()); let nonce_rc = aes_gcm::Nonce::from(NONCE_12); aws_lc_bench! { - let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).unwrap()); + let aws_key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(&aws_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed")); } - let ring_key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).unwrap()); + let ring_key = ring_aead::LessSafeKey::new( + ring_aead::UnboundKey::new(&ring_aead::AES_128_GCM, &KEY_16).expect("valid AEAD benchmark operation must succeed"), + ); let mut g = c.benchmark_group("aes-128-gcm/decrypt"); for (len, data) in &inputs { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_rc = data.clone(); let tag_rc = cipher_rc .encrypt_inout_detached(&nonce_rc, AAD, ct_rc.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); aws_lc_bench! { let mut ct_aws = data.clone(); @@ -1064,7 +1099,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), &mut ct_aws, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); // AWS-LC exposes detached tags for seal, but its in-place open API takes ct||tag. ct_aws.extend_from_slice(tag_aws.as_ref()); } @@ -1076,7 +1111,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { ring_aead::Aad::from(AAD), &mut ct_ring, ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -1090,7 +1125,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -1107,7 +1142,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { black_box(buf_rc.as_mut_slice().into()), black_box(&tag_rc), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_rc); }) }); @@ -1124,7 +1159,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { aws_aead::Aad::from(AAD), black_box(&mut buf_aws), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_aws); }) }); @@ -1143,7 +1178,7 @@ fn aes128_gcm_decrypt(c: &mut Criterion) { black_box(&mut buf_ring), 0.., ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ring); }) }); @@ -1194,7 +1229,9 @@ fn aegis256_decrypt(c: &mut Criterion) { // Pre-encrypt with rscrypto to get valid ciphertext + tag. let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); // Pre-encrypt with aegis crate to get its tag format. let mut ct_ac = data.clone(); @@ -1213,7 +1250,7 @@ fn aegis256_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -1225,7 +1262,7 @@ fn aegis256_decrypt(c: &mut Criterion) { buf_ac.copy_from_slice(ct); cipher_ac .decrypt_in_place(black_box(&mut buf_ac), black_box(&tag_ac), black_box(AAD)) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ac); }) }); @@ -1267,7 +1304,7 @@ fn ascon_aead128_encrypt(c: &mut Criterion) { black_box(AAD), black_box(buf.as_mut_slice().into()), ) - .unwrap(), + .expect("valid AEAD benchmark operation must succeed"), ) }) }); @@ -1290,12 +1327,14 @@ fn ascon_aead128_decrypt(c: &mut Criterion) { common::set_throughput(&mut g, *len); let mut ciphertext = data.clone(); - let tag_rs = cipher_rs.encrypt_in_place(&nonce_rs, AAD, &mut ciphertext).unwrap(); + let tag_rs = cipher_rs + .encrypt_in_place(&nonce_rs, AAD, &mut ciphertext) + .expect("valid AEAD benchmark operation must succeed"); let mut ct_ac = data.clone(); let tag_ac = cipher_ac .encrypt_inout_detached(&nonce_ac, AAD, ct_ac.as_mut_slice().into()) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); let mut buf = ciphertext.clone(); @@ -1309,7 +1348,7 @@ fn ascon_aead128_decrypt(c: &mut Criterion) { black_box(&mut buf), black_box(&tag_rs), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf); }) }); @@ -1326,7 +1365,7 @@ fn ascon_aead128_decrypt(c: &mut Criterion) { black_box(buf_ac.as_mut_slice().into()), black_box(&tag_ac), ) - .unwrap(); + .expect("valid AEAD benchmark operation must succeed"); black_box(&buf_ac); }) }); diff --git a/benches/aead_kernels.rs b/benches/aead_kernels.rs index 4ecc827f..17108f67 100644 --- a/benches/aead_kernels.rs +++ b/benches/aead_kernels.rs @@ -31,12 +31,16 @@ fn chacha20_xor_kernel(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("aarch64-neon", len), data, |b, d| { b.iter(|| { buf.copy_from_slice(d); - rscrypto::aead::diag_chacha20_xor_keystream_aarch64_neon( - black_box(&KEY_32), - black_box(1), - black_box(&NONCE_12), - black_box(&mut buf), - ); + // SAFETY: AArch64 NEON is part of this target's compile-time baseline. `comp_sizes` tops out at 1,048,576 + // bytes (16,384 blocks), which fits the `u32` counter range starting at 1. + unsafe { + rscrypto::aead::diag_chacha20_xor_keystream_aarch64_neon( + black_box(&KEY_32), + black_box(1), + black_box(&NONCE_12), + black_box(&mut buf), + ); + } black_box(buf.as_ptr()) }) }); @@ -59,7 +63,7 @@ fn poly1305_auth_kernel(c: &mut Criterion) { b.iter(|| { black_box( rscrypto::aead::diag_chacha20poly1305_authenticate_aead(black_box(AAD), black_box(d), black_box(&POLY_KEY)) - .unwrap(), + .expect("selected AEAD benchmark kernel must be available"), ) }) }); @@ -73,7 +77,7 @@ fn poly1305_auth_kernel(c: &mut Criterion) { black_box(d), black_box(&POLY_KEY), ) - .unwrap(), + .expect("selected AEAD benchmark kernel must be available"), ) }) }); diff --git a/benches/ascon.rs b/benches/ascon.rs index 12de7ea3..bdf789a6 100644 --- a/benches/ascon.rs +++ b/benches/ascon.rs @@ -140,8 +140,8 @@ fn ascon_xof128_many(c: &mut Criterion) { let mut hasher = rscrypto::AsconXof::new(); hasher.update(black_box(input)); let mut reader = hasher.finalize_xof(); - let base = index * OUT_LEN; - reader.squeeze(&mut out[base..base + OUT_LEN]); + let base = index.strict_mul(OUT_LEN); + reader.squeeze(&mut out[base..base.strict_add(OUT_LEN)]); } black_box(out[0]) }) @@ -163,7 +163,8 @@ fn ascon_cxof128(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { let mut out = [0u8; OUT_LEN]; - rscrypto::AsconCxof128::hash_into(black_box(CUSTOMIZATION), black_box(d), &mut out).unwrap(); + rscrypto::AsconCxof128::hash_into(black_box(CUSTOMIZATION), black_box(d), &mut out) + .expect("valid Ascon benchmark operation must succeed"); black_box(out) }) }); diff --git a/benches/auth.rs b/benches/auth.rs index 9a37ac57..d16956c4 100644 --- a/benches/auth.rs +++ b/benches/auth.rs @@ -43,7 +43,8 @@ fn array_from_slice(slice: &[u8]) -> [u8; N] { fn deterministic_bytes(offset: u8) -> [u8; N] { let mut out = [0u8; N]; for (i, byte) in out.iter_mut().enumerate() { - *byte = offset.wrapping_add(i as u8); + let low_index = u8::try_from(i & usize::from(u8::MAX)).expect("masked deterministic-byte index must fit u8"); + *byte = offset.wrapping_add(low_index); } out } @@ -147,7 +148,8 @@ fn hmac_sha256(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { - let base_mac = RustCryptoHmacSha256::new_from_slice(&key).unwrap(); + let base_mac = + RustCryptoHmacSha256::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); b.iter(|| { use hmac::Mac as _; @@ -196,7 +198,8 @@ fn hmac_sha384(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { - let base_mac = RustCryptoHmacSha384::new_from_slice(&key).unwrap(); + let base_mac = + RustCryptoHmacSha384::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); b.iter(|| { use hmac::Mac as _; @@ -245,7 +248,8 @@ fn hmac_sha512(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto", len), data, |b, d| { - let base_mac = RustCryptoHmacSha512::new_from_slice(&key).unwrap(); + let base_mac = + RustCryptoHmacSha512::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); b.iter(|| { use hmac::Mac as _; @@ -292,7 +296,8 @@ fn hmac_sha256_streaming(c: &mut Criterion) { b.iter(|| { use hmac::Mac as _; - let mut mac = RustCryptoHmacSha256::new_from_slice(&key).unwrap(); + let mut mac = + RustCryptoHmacSha256::new_from_slice(&key).expect("valid authentication benchmark operation must succeed"); for chunk in data.chunks(chunk_size) { mac.update(black_box(chunk)); } @@ -345,7 +350,8 @@ fn hmac_sha256_internal(c: &mut Criterion) { b.iter(|| { use hmac::Mac as _; - let mut mac = RustCryptoHmacSha256::new_from_slice(black_box(&key)).unwrap(); + let mut mac = RustCryptoHmacSha256::new_from_slice(black_box(&key)) + .expect("valid authentication benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -384,7 +390,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - hkdf.expand(black_box(&info), black_box(&mut out)).unwrap(); + hkdf + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -392,7 +400,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - rustcrypto.expand(black_box(&info), black_box(&mut out)).unwrap(); + rustcrypto + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -403,9 +413,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { b.iter(|| { aws_prk .expand(&[black_box(&info)], AwsHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -416,9 +426,9 @@ fn hkdf_sha256_expand(c: &mut Criterion) { b.iter(|| { ring_prk .expand(&[black_box(&info)], RingHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -446,7 +456,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - hkdf.expand(black_box(&info), black_box(&mut out)).unwrap(); + hkdf + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -454,7 +466,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - rustcrypto.expand(black_box(&info), black_box(&mut out)).unwrap(); + rustcrypto + .expand(black_box(&info), black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -465,9 +479,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { b.iter(|| { aws_prk .expand(&[black_box(&info)], AwsHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -478,9 +492,9 @@ fn hkdf_sha384_expand(c: &mut Criterion) { b.iter(|| { ring_prk .expand(&[black_box(&info)], RingHkdfLen(len)) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .fill(black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -497,7 +511,8 @@ fn pbkdf2_sha256_derive(c: &mut Criterion) { let state = Pbkdf2Sha256::new(&password); for &iterations in &[1u32, 100, 1000] { - let nz_iters = core::num::NonZeroU32::new(iterations).unwrap(); + let nz_iters = + core::num::NonZeroU32::new(iterations).expect("valid authentication benchmark operation must succeed"); let mut g = c.benchmark_group(format!("pbkdf2-sha256/iters={iterations}")); for &out_len in &[32usize, 64] { @@ -507,7 +522,7 @@ fn pbkdf2_sha256_derive(c: &mut Criterion) { let mut out = vec![0u8; len]; b.iter(|| { Pbkdf2Sha256::derive_key_primitive(black_box(&password), black_box(&salt), iterations, black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -559,7 +574,9 @@ fn pbkdf2_sha256_derive(c: &mut Criterion) { g_state.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - state.derive(black_box(&salt), iterations, black_box(&mut out)).unwrap(); + state + .derive(black_box(&salt), iterations, black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -576,7 +593,8 @@ fn pbkdf2_sha256_internal(c: &mut Criterion) { let state = Pbkdf2Sha256::new(&password); for &iterations in &[1u32, 100, 1000] { - let nz_iters = core::num::NonZeroU32::new(iterations).unwrap(); + let nz_iters = + core::num::NonZeroU32::new(iterations).expect("valid authentication benchmark operation must succeed"); let mut g = c.benchmark_group(format!("pbkdf2-sha256/internal/iters={iterations}")); for &out_len in &[32usize, 64] { @@ -586,7 +604,7 @@ fn pbkdf2_sha256_internal(c: &mut Criterion) { let mut out = vec![0u8; len]; b.iter(|| { Pbkdf2Sha256::derive_key_primitive(black_box(&password), black_box(&salt), iterations, black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -594,7 +612,9 @@ fn pbkdf2_sha256_internal(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rscrypto-state", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - state.derive(black_box(&salt), iterations, black_box(&mut out)).unwrap(); + state + .derive(black_box(&salt), iterations, black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -650,7 +670,8 @@ fn pbkdf2_sha512_derive(c: &mut Criterion) { let state = Pbkdf2Sha512::new(&password); for &iterations in &[1u32, 100, 1000] { - let nz_iters = core::num::NonZeroU32::new(iterations).unwrap(); + let nz_iters = + core::num::NonZeroU32::new(iterations).expect("valid authentication benchmark operation must succeed"); let mut g = c.benchmark_group(format!("pbkdf2-sha512/iters={iterations}")); for &out_len in &[64usize, 128] { @@ -660,7 +681,7 @@ fn pbkdf2_sha512_derive(c: &mut Criterion) { let mut out = vec![0u8; len]; b.iter(|| { Pbkdf2Sha512::derive_key_primitive(black_box(&password), black_box(&salt), iterations, black_box(&mut out)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -712,7 +733,9 @@ fn pbkdf2_sha512_derive(c: &mut Criterion) { g_state.bench_with_input(BenchmarkId::new("rscrypto", out_len), &out_len, |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - state.derive(black_box(&salt), iterations, black_box(&mut out)).unwrap(); + state + .derive(black_box(&salt), iterations, black_box(&mut out)) + .expect("valid authentication benchmark operation must succeed"); black_box(out[0]) }) }); @@ -744,13 +767,15 @@ fn ed25519_public_key(c: &mut Criterion) { fn ecdsa_p256_verify(c: &mut Criterion) { let secret_bytes = [0x11u8; 32]; - let signing_key = P256OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P256OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let verifying_key = signing_key.verifying_key(); let sec1 = EcdsaP256SecretKey::from_bytes(secret_bytes) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .public_key() .to_sec1_bytes(); - let public = EcdsaP256PublicKey::from_sec1_bytes(sec1.as_slice()).unwrap(); + let public = EcdsaP256PublicKey::from_sec1_bytes(sec1.as_slice()) + .expect("valid authentication benchmark operation must succeed"); let ring_upk = ring::signature::UnparsedPublicKey::new(&ring::signature::ECDSA_P256_SHA256_FIXED, sec1.as_slice()); aws_lc_bench! { let aws_upk = @@ -766,11 +791,14 @@ fn ecdsa_p256_verify(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); let oracle_signature: P256OracleSignature = p256::ecdsa::signature::Signer::sign(&signing_key, data); - let signature = EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap(); + let signature = EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { - black_box(&public).verify(black_box(d), black_box(&signature)).unwrap(); + black_box(&public) + .verify(black_box(d), black_box(&signature)) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -778,14 +806,16 @@ fn ecdsa_p256_verify(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto-p256", len), data, |b, d| { b.iter(|| { p256::ecdsa::signature::Verifier::verify(black_box(verifying_key), black_box(d), black_box(&oracle_signature)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { b.iter(|| { - ring_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + ring_upk + .verify(black_box(d), black_box(signature.as_bytes())) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -793,7 +823,7 @@ fn ecdsa_p256_verify(c: &mut Criterion) { aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { b.iter(|| { - aws_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + aws_upk.verify(black_box(d), black_box(signature.as_bytes())).expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -805,10 +835,12 @@ fn ecdsa_p256_verify(c: &mut Criterion) { fn ecdsa_p256_sign(c: &mut Criterion) { let secret_bytes = [0x11u8; 32]; - let secret = EcdsaP256SecretKey::from_bytes(secret_bytes).unwrap(); + let secret = + EcdsaP256SecretKey::from_bytes(secret_bytes).expect("valid authentication benchmark operation must succeed"); let keypair = EcdsaP256Keypair::from_secret_key(secret); let blind = [0x5cu8; 64]; - let signing_key = P256OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P256OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let sec1 = keypair.public_key().to_sec1_bytes(); let ring_rng = ring::rand::SystemRandom::new(); let ring_key = ring::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -817,7 +849,7 @@ fn ecdsa_p256_sign(c: &mut Criterion) { &sec1, &ring_rng, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); aws_lc_bench! { let aws_rng = aws_lc_rs::rand::SystemRandom::new(); let aws_key = aws_lc_rs::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -825,7 +857,7 @@ fn ecdsa_p256_sign(c: &mut Criterion) { &secret_bytes, &sec1, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); } let inputs = [0usize, 32, 1024, 16384] @@ -838,7 +870,13 @@ fn ecdsa_p256_sign(c: &mut Criterion) { common::set_throughput(&mut g, *len); g.bench_with_input(BenchmarkId::new("rscrypto-deterministic", len), data, |b, d| { - b.iter(|| black_box(black_box(&keypair).try_sign(black_box(d)).unwrap())) + b.iter(|| { + black_box( + black_box(&keypair) + .try_sign(black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); g.bench_with_input(BenchmarkId::new("rscrypto-blinded", len), data, |b, d| { @@ -846,7 +884,7 @@ fn ecdsa_p256_sign(c: &mut Criterion) { black_box( black_box(&keypair) .try_sign_blinded(black_box(d), |out| out.copy_from_slice(black_box(&blind))) - .unwrap(), + .expect("valid authentication benchmark operation must succeed"), ) }) }); @@ -860,12 +898,18 @@ fn ecdsa_p256_sign(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { - b.iter(|| black_box(ring_key.sign(&ring_rng, black_box(d)).unwrap())) + b.iter(|| { + black_box( + ring_key + .sign(&ring_rng, black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { - b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).unwrap())) + b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).expect("valid authentication benchmark operation must succeed"))) }); } } @@ -875,13 +919,15 @@ fn ecdsa_p256_sign(c: &mut Criterion) { fn ecdsa_p384_verify(c: &mut Criterion) { let secret_bytes = [0x31u8; 48]; - let signing_key = P384OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P384OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let verifying_key = signing_key.verifying_key(); let sec1 = EcdsaP384SecretKey::from_bytes(secret_bytes) - .unwrap() + .expect("valid authentication benchmark operation must succeed") .public_key() .to_sec1_bytes(); - let public = EcdsaP384PublicKey::from_sec1_bytes(sec1.as_slice()).unwrap(); + let public = EcdsaP384PublicKey::from_sec1_bytes(sec1.as_slice()) + .expect("valid authentication benchmark operation must succeed"); let ring_upk = ring::signature::UnparsedPublicKey::new(&ring::signature::ECDSA_P384_SHA384_FIXED, sec1.as_slice()); aws_lc_bench! { let aws_upk = @@ -897,11 +943,14 @@ fn ecdsa_p384_verify(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); let oracle_signature: P384OracleSignature = p384::ecdsa::signature::Signer::sign(&signing_key, data); - let signature = EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap(); + let signature = EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { - black_box(&public).verify(black_box(d), black_box(&signature)).unwrap(); + black_box(&public) + .verify(black_box(d), black_box(&signature)) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -909,14 +958,16 @@ fn ecdsa_p384_verify(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("rustcrypto-p384", len), data, |b, d| { b.iter(|| { p384::ecdsa::signature::Verifier::verify(black_box(verifying_key), black_box(d), black_box(&oracle_signature)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { b.iter(|| { - ring_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + ring_upk + .verify(black_box(d), black_box(signature.as_bytes())) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -924,7 +975,7 @@ fn ecdsa_p384_verify(c: &mut Criterion) { aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { b.iter(|| { - aws_upk.verify(black_box(d), black_box(signature.as_bytes())).unwrap(); + aws_upk.verify(black_box(d), black_box(signature.as_bytes())).expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -936,10 +987,12 @@ fn ecdsa_p384_verify(c: &mut Criterion) { fn ecdsa_p384_sign(c: &mut Criterion) { let secret_bytes = [0x31u8; 48]; - let secret = EcdsaP384SecretKey::from_bytes(secret_bytes).unwrap(); + let secret = + EcdsaP384SecretKey::from_bytes(secret_bytes).expect("valid authentication benchmark operation must succeed"); let keypair = EcdsaP384Keypair::from_secret_key(secret); let blind = [0xa3u8; 96]; - let signing_key = P384OracleSigningKey::from_slice(&secret_bytes).unwrap(); + let signing_key = + P384OracleSigningKey::from_slice(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let sec1 = keypair.public_key().to_sec1_bytes(); let ring_rng = ring::rand::SystemRandom::new(); let ring_key = ring::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -948,7 +1001,7 @@ fn ecdsa_p384_sign(c: &mut Criterion) { &sec1, &ring_rng, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); aws_lc_bench! { let aws_rng = aws_lc_rs::rand::SystemRandom::new(); let aws_key = aws_lc_rs::signature::EcdsaKeyPair::from_private_key_and_public_key( @@ -956,7 +1009,7 @@ fn ecdsa_p384_sign(c: &mut Criterion) { &secret_bytes, &sec1, ) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); } let inputs = [0usize, 32, 1024, 16384] @@ -969,7 +1022,13 @@ fn ecdsa_p384_sign(c: &mut Criterion) { common::set_throughput(&mut g, *len); g.bench_with_input(BenchmarkId::new("rscrypto-deterministic", len), data, |b, d| { - b.iter(|| black_box(black_box(&keypair).try_sign(black_box(d)).unwrap())) + b.iter(|| { + black_box( + black_box(&keypair) + .try_sign(black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); g.bench_with_input(BenchmarkId::new("rscrypto-blinded", len), data, |b, d| { @@ -977,7 +1036,7 @@ fn ecdsa_p384_sign(c: &mut Criterion) { black_box( black_box(&keypair) .try_sign_blinded(black_box(d), |out| out.copy_from_slice(black_box(&blind))) - .unwrap(), + .expect("valid authentication benchmark operation must succeed"), ) }) }); @@ -991,12 +1050,18 @@ fn ecdsa_p384_sign(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { - b.iter(|| black_box(ring_key.sign(&ring_rng, black_box(d)).unwrap())) + b.iter(|| { + black_box( + ring_key + .sign(&ring_rng, black_box(d)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { - b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).unwrap())) + b.iter(|| black_box(aws_key.sign(&aws_rng, black_box(d)).expect("valid authentication benchmark operation must succeed"))) }); } } @@ -1118,9 +1183,10 @@ fn ed25519_sign(c: &mut Criterion) { let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); let signing_key = SigningKey::from_bytes(&secret_bytes); aws_lc_bench! { - let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).expect("valid authentication benchmark operation must succeed"); } - let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes) + .expect("valid authentication benchmark operation must succeed"); let (_dryoc_pk, dryoc_sk) = crypto_sign_seed_keypair(&secret_bytes); let mut dryoc_sig: [u8; 64] = [0u8; 64]; let inputs = [0usize, 32, 1024, 16384] @@ -1156,7 +1222,8 @@ fn ed25519_sign(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc", len), data, |b, d| { b.iter(|| { - crypto_sign_detached(&mut dryoc_sig, black_box(d), &dryoc_sk).unwrap(); + crypto_sign_detached(&mut dryoc_sig, black_box(d), &dryoc_sk) + .expect("valid authentication benchmark operation must succeed"); black_box(&dryoc_sig); }) }); @@ -1179,11 +1246,12 @@ fn ed25519_verify(c: &mut Criterion) { let signing_key = SigningKey::from_bytes(&secret_bytes); let verifying_key = signing_key.verifying_key(); aws_lc_bench! { - let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let aws_kp = aws_lc_rs::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).expect("valid authentication benchmark operation must succeed"); let aws_pubkey: Vec = aws_kp.public_key().as_ref().to_vec(); let aws_upk = aws_lc_rs::signature::UnparsedPublicKey::new(&aws_lc_rs::signature::ED25519, aws_pubkey); } - let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes).unwrap(); + let ring_kp = ring::signature::Ed25519KeyPair::from_seed_unchecked(&secret_bytes) + .expect("valid authentication benchmark operation must succeed"); let ring_pubkey: Vec = ring_kp.public_key().as_ref().to_vec(); let ring_upk = ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, ring_pubkey); let (dryoc_pk, dryoc_sk) = crypto_sign_seed_keypair(&secret_bytes); @@ -1202,11 +1270,14 @@ fn ed25519_verify(c: &mut Criterion) { } let ring_sig = ring_kp.sign(data); let mut dryoc_sig: [u8; 64] = [0u8; 64]; - crypto_sign_detached(&mut dryoc_sig, data, &dryoc_sk).unwrap(); + crypto_sign_detached(&mut dryoc_sig, data, &dryoc_sk) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("rscrypto", len), data, |b, d| { b.iter(|| { - black_box(&public).verify(black_box(d), black_box(&ours)).unwrap(); + black_box(&public) + .verify(black_box(d), black_box(&ours)) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1215,7 +1286,7 @@ fn ed25519_verify(c: &mut Criterion) { b.iter(|| { black_box(&verifying_key) .verify_strict(black_box(d), black_box(&dalek)) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1223,7 +1294,7 @@ fn ed25519_verify(c: &mut Criterion) { aws_lc_bench! { g.bench_with_input(BenchmarkId::new("aws-lc-rs", len), data, |b, d| { b.iter(|| { - aws_upk.verify(black_box(d), aws_sig.as_ref()).unwrap(); + aws_upk.verify(black_box(d), aws_sig.as_ref()).expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1231,14 +1302,17 @@ fn ed25519_verify(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("ring", len), data, |b, d| { b.iter(|| { - ring_upk.verify(black_box(d), ring_sig.as_ref()).unwrap(); + ring_upk + .verify(black_box(d), ring_sig.as_ref()) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); g.bench_with_input(BenchmarkId::new("dryoc", len), data, |b, d| { b.iter(|| { - crypto_sign_verify_detached(&dryoc_sig, black_box(d), &dryoc_pk).unwrap(); + crypto_sign_verify_detached(&dryoc_sig, black_box(d), &dryoc_pk) + .expect("valid authentication benchmark operation must succeed"); black_box(()) }) }); @@ -1269,7 +1343,8 @@ fn ed25519_verify_phase(c: &mut Criterion) { for (len, data) in &inputs { common::set_throughput(&mut g, *len); let signature = keypair.sign(data); - let scalars = diag_ed25519_verify_scalars(&public, &signature, data).unwrap(); + let scalars = diag_ed25519_verify_scalars(&public, &signature, data) + .expect("valid authentication benchmark operation must succeed"); g.bench_with_input(BenchmarkId::new("challenge-reduce", len), data, |b, d| { b.iter(|| { @@ -1352,8 +1427,8 @@ fn x25519_public_key(c: &mut Criterion) { b.iter(|| { let priv_key = aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, black_box(&secret_bytes)) - .unwrap(); - black_box(priv_key.compute_public_key().unwrap()) + .expect("valid authentication benchmark operation must succeed"); + black_box(priv_key.compute_public_key().expect("valid authentication benchmark operation must succeed")) }) }); } @@ -1381,11 +1456,11 @@ fn x25519_diffie_hellman(c: &mut Criterion) { let dalek_bob_public = DalekX25519PublicKey::from(&DalekX25519Secret::from(bob_bytes)); aws_lc_bench! { let aws_alice = - aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &alice_bytes).unwrap(); + aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &alice_bytes).expect("valid authentication benchmark operation must succeed"); let aws_bob_pub_bytes: [u8; 32] = { let bob_priv = - aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &bob_bytes).unwrap(); - let pk = bob_priv.compute_public_key().unwrap(); + aws_lc_rs::agreement::PrivateKey::from_private_key(&aws_lc_rs::agreement::X25519, &bob_bytes).expect("valid authentication benchmark operation must succeed"); + let pk = bob_priv.compute_public_key().expect("valid authentication benchmark operation must succeed"); let mut out = [0u8; 32]; out.copy_from_slice(pk.as_ref()); out @@ -1398,7 +1473,13 @@ fn x25519_diffie_hellman(c: &mut Criterion) { let mut g = c.benchmark_group("x25519/diffie-hellman"); g.bench_function("rscrypto", |b| { - b.iter(|| black_box(black_box(&alice).diffie_hellman(black_box(&bob_public)).unwrap())) + b.iter(|| { + black_box( + black_box(&alice) + .diffie_hellman(black_box(&bob_public)) + .expect("valid authentication benchmark operation must succeed"), + ) + }) }); g.bench_function("dalek", |b| { @@ -1413,7 +1494,7 @@ fn x25519_diffie_hellman(c: &mut Criterion) { out.copy_from_slice(bytes); Ok::<[u8; 32], ()>(out) }) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); black_box(shared) }) }); @@ -1422,7 +1503,8 @@ fn x25519_diffie_hellman(c: &mut Criterion) { g.bench_function("dryoc", |b| { let mut shared = [0u8; 32]; b.iter(|| { - crypto_scalarmult(&mut shared, black_box(&alice_bytes), black_box(&dryoc_bob_pub)); + crypto_scalarmult(&mut shared, black_box(&alice_bytes), black_box(&dryoc_bob_pub)) + .expect("valid authentication benchmark operation must succeed"); black_box(shared) }) }); @@ -1454,7 +1536,7 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(black_box(&key_random)); Ok::<(), MlKemError>(()) }) - .unwrap() + .expect("valid authentication benchmark operation must succeed") }) }); @@ -1464,7 +1546,7 @@ macro_rules! mlkem_profile_benches { aws_lc_bench! { g.bench_function("aws-lc-rs", |b| { - b.iter(|| black_box(AwsMlKemDecapsulationKey::generate(&$aws_algorithm).unwrap())) + b.iter(|| black_box(AwsMlKemDecapsulationKey::generate(&$aws_algorithm).expect("valid authentication benchmark operation must succeed"))) }); } @@ -1490,8 +1572,8 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(&key_random); Ok::<(), MlKemError>(()) }) - .unwrap(); - let prepared_ek = ek.prepare().unwrap(); + .expect("valid authentication benchmark operation must succeed"); + let prepared_ek = ek.prepare().expect("valid authentication benchmark operation must succeed"); let (fips_ek, _) = $fips::KG::keygen_from_seed( array_from_slice::<32>(&key_random[..32]), array_from_slice::<32>(&key_random[32..]), @@ -1502,8 +1584,8 @@ macro_rules! mlkem_profile_benches { let libcrux_keypair = $libcrux::generate_key_pair(key_random); let libcrux_ek = libcrux_keypair.public_key().clone(); aws_lc_bench! { - let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).unwrap(); - let aws_ek = aws_dk.encapsulation_key().unwrap(); + let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).expect("valid authentication benchmark operation must succeed"); + let aws_ek = aws_dk.encapsulation_key().expect("valid authentication benchmark operation must succeed"); } let mut g = c.benchmark_group(concat!($group, "/encapsulate")); @@ -1514,7 +1596,7 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(black_box(&encaps_random)); Ok::<(), MlKemError>(()) }) - .unwrap() + .expect("valid authentication benchmark operation must succeed") }) }); @@ -1524,7 +1606,7 @@ macro_rules! mlkem_profile_benches { aws_lc_bench! { g.bench_function("aws-lc-rs", |b| { - b.iter(|| black_box(aws_ek.encapsulate().unwrap())) + b.iter(|| black_box(aws_ek.encapsulate().expect("valid authentication benchmark operation must succeed"))) }); } @@ -1548,15 +1630,15 @@ macro_rules! mlkem_profile_benches { out.copy_from_slice(&key_random); Ok::<(), MlKemError>(()) }) - .unwrap(); - let prepared_ek = ek.prepare().unwrap(); - let prepared_dk = dk.prepare().unwrap(); + .expect("valid authentication benchmark operation must succeed"); + let prepared_ek = ek.prepare().expect("valid authentication benchmark operation must succeed"); + let prepared_dk = dk.prepare().expect("valid authentication benchmark operation must succeed"); let (ciphertext, _) = prepared_ek .encapsulate(|out| { out.copy_from_slice(&encaps_random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("valid authentication benchmark operation must succeed"); let (fips_ek, fips_dk) = $fips::KG::keygen_from_seed( array_from_slice::<32>(&key_random[..32]), array_from_slice::<32>(&key_random[32..]), @@ -1572,14 +1654,14 @@ macro_rules! mlkem_profile_benches { let libcrux_dk = libcrux_keypair.private_key().clone(); let (libcrux_ciphertext, _) = $libcrux::encapsulate(&libcrux_ek, encaps_random); aws_lc_bench! { - let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).unwrap(); - let aws_ek = aws_dk.encapsulation_key().unwrap(); - let (aws_ciphertext, _) = aws_ek.encapsulate().unwrap(); + let aws_dk = AwsMlKemDecapsulationKey::generate(&$aws_algorithm).expect("valid authentication benchmark operation must succeed"); + let aws_ek = aws_dk.encapsulation_key().expect("valid authentication benchmark operation must succeed"); + let (aws_ciphertext, _) = aws_ek.encapsulate().expect("valid authentication benchmark operation must succeed"); } let mut g = c.benchmark_group(concat!($group, "/decapsulate")); g.bench_function("rscrypto", |b| { - b.iter(|| black_box(&prepared_dk).decapsulate(black_box(&ciphertext)).unwrap()) + b.iter(|| black_box(&prepared_dk).decapsulate(black_box(&ciphertext)).expect("valid authentication benchmark operation must succeed")) }); g.bench_function("libcrux", |b| { @@ -1597,14 +1679,14 @@ macro_rules! mlkem_profile_benches { black_box( aws_dk .decapsulate(AwsMlKemCiphertext::from(black_box(aws_ciphertext.as_ref()))) - .unwrap(), + .expect("valid authentication benchmark operation must succeed"), ) }) }); } g.bench_function("fips203", |b| { - b.iter(|| black_box(fips_dk.try_decaps(black_box(&fips_ciphertext)).unwrap())) + b.iter(|| black_box(fips_dk.try_decaps(black_box(&fips_ciphertext)).expect("valid authentication benchmark operation must succeed"))) }); g.bench_function("rustcrypto", |b| { diff --git a/benches/blake2.rs b/benches/blake2.rs index 7e851d9e..bf31d82b 100644 --- a/benches/blake2.rs +++ b/benches/blake2.rs @@ -5,12 +5,14 @@ mod common; use core::hint::black_box; use blake2::{ - Blake2b256 as RustCryptoBlake2b256, Blake2b512 as RustCryptoBlake2b512, Blake2bMac, - Blake2s128 as RustCryptoBlake2s128, Blake2s256 as RustCryptoBlake2s256, Blake2sMac, Digest as _, + Blake2b as RustCryptoBlake2b, Blake2b512 as RustCryptoBlake2b512, Blake2bMac, Blake2s as RustCryptoBlake2s, + Blake2s256 as RustCryptoBlake2s256, Blake2sMac, + digest::{ + Digest as _, Mac as _, + consts::{U16, U32, U64}, + }, }; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use digest::typenum::{U16, U32, U64}; -use hmac::{Mac as _, digest::KeyInit}; use rscrypto::{ Blake2b256, Blake2b512, Blake2bKey, Blake2bParams, Blake2s128, Blake2s256, Blake2sKey, Blake2sParams, Digest, }; @@ -19,6 +21,8 @@ type RustCryptoBlake2bMac256 = Blake2bMac; type RustCryptoBlake2bMac512 = Blake2bMac; type RustCryptoBlake2sMac128 = Blake2sMac; type RustCryptoBlake2sMac256 = Blake2sMac; +type RustCryptoBlake2b256 = RustCryptoBlake2b; +type RustCryptoBlake2s128 = RustCryptoBlake2s; fn oneshot(c: &mut Criterion) { use dryoc::classic::crypto_generichash::crypto_generichash; @@ -38,7 +42,8 @@ fn oneshot(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b256", len), data, |b, d| { let mut out = [0u8; 32]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), None).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), None) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -52,7 +57,8 @@ fn oneshot(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b512", len), data, |b, d| { let mut out = [0u8; 64]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), None).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), None) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -86,8 +92,8 @@ fn host_overhead(c: &mut Criterion) { let inputs = tiny_inputs(); let key_b = [0x42u8; 64]; let key_s = [0x24u8; 32]; - let key_b_typed = Blake2bKey::new(black_box(&key_b[..32])).unwrap(); - let key_s_typed = Blake2sKey::new(black_box(&key_s)).unwrap(); + let key_b_typed = Blake2bKey::new(black_box(&key_b[..32])).expect("valid BLAKE2 benchmark operation must succeed"); + let key_s_typed = Blake2sKey::new(black_box(&key_s)).expect("valid BLAKE2 benchmark operation must succeed"); let mut oneshot = c.benchmark_group("blake2/host-overhead"); for (len, data) in &inputs { @@ -118,7 +124,8 @@ fn host_overhead(c: &mut Criterion) { }); keyed.bench_with_input(BenchmarkId::new("rustcrypto/blake2b256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])).unwrap(); + let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -129,7 +136,8 @@ fn host_overhead(c: &mut Criterion) { }); keyed.bench_with_input(BenchmarkId::new("rustcrypto/blake2s256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)).unwrap(); + let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -180,10 +188,10 @@ fn keyed(c: &mut Criterion) { let inputs = common::comp_sizes(); let key_b = [0x42u8; 64]; let key_s = [0x24u8; 32]; - let key_b_256 = Blake2bKey::new(black_box(&key_b[..32])).unwrap(); - let key_b_512 = Blake2bKey::new(black_box(&key_b)).unwrap(); - let key_s_128 = Blake2sKey::new(black_box(&key_s[..16])).unwrap(); - let key_s_256 = Blake2sKey::new(black_box(&key_s)).unwrap(); + let key_b_256 = Blake2bKey::new(black_box(&key_b[..32])).expect("valid BLAKE2 benchmark operation must succeed"); + let key_b_512 = Blake2bKey::new(black_box(&key_b)).expect("valid BLAKE2 benchmark operation must succeed"); + let key_s_128 = Blake2sKey::new(black_box(&key_s[..16])).expect("valid BLAKE2 benchmark operation must succeed"); + let key_s_256 = Blake2sKey::new(black_box(&key_s)).expect("valid BLAKE2 benchmark operation must succeed"); let mut g = c.benchmark_group("blake2/keyed"); for (len, data) in &inputs { @@ -194,7 +202,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2b256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])).unwrap(); + let mut mac = RustCryptoBlake2bMac256::new_from_slice(black_box(&key_b[..32])) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -202,7 +211,8 @@ fn keyed(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b256", len), data, |b, d| { let mut out = [0u8; 32]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..32]))).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..32]))) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -212,7 +222,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2b512", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2bMac512::new_from_slice(black_box(&key_b)).unwrap(); + let mut mac = RustCryptoBlake2bMac512::new_from_slice(black_box(&key_b)) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -220,7 +231,8 @@ fn keyed(c: &mut Criterion) { g.bench_with_input(BenchmarkId::new("dryoc/blake2b512", len), data, |b, d| { let mut out = [0u8; 64]; b.iter(|| { - crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..]))).unwrap(); + crypto_generichash(black_box(&mut out), black_box(d), Some(black_box(&key_b[..]))) + .expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); @@ -230,7 +242,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2s128", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2sMac128::new_from_slice(black_box(&key_s[..16])).unwrap(); + let mut mac = RustCryptoBlake2sMac128::new_from_slice(black_box(&key_s[..16])) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -241,7 +254,8 @@ fn keyed(c: &mut Criterion) { }); g.bench_with_input(BenchmarkId::new("rustcrypto/blake2s256", len), data, |b, d| { b.iter(|| { - let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)).unwrap(); + let mut mac = RustCryptoBlake2sMac256::new_from_slice(black_box(&key_s)) + .expect("valid BLAKE2 benchmark operation must succeed"); mac.update(black_box(d)); black_box(mac.finalize().into_bytes()) }) @@ -281,12 +295,12 @@ fn streaming(c: &mut Criterion) { }); g.bench_function(format!("dryoc/blake2b256/{chunk_size}B"), |b| { b.iter(|| { - let mut state = crypto_generichash_init(None, 32).unwrap(); + let mut state = crypto_generichash_init(None, 32).expect("valid BLAKE2 benchmark operation must succeed"); for chunk in data.chunks(chunk_size) { crypto_generichash_update(&mut state, black_box(chunk)); } let mut out = [0u8; 32]; - crypto_generichash_final(state, &mut out).unwrap(); + crypto_generichash_final(state, &mut out).expect("valid BLAKE2 benchmark operation must succeed"); black_box(out) }) }); diff --git a/benches/blake3.rs b/benches/blake3.rs index 34d87f48..0cc81a25 100644 --- a/benches/blake3.rs +++ b/benches/blake3.rs @@ -132,7 +132,14 @@ fn oneshot(c: &mut Criterion) { g.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), len), data, - |b, d| b.iter(|| black_box(diag_blake3_digest_with_kernel(kernel, black_box(d)).unwrap())), + |b, d| { + b.iter(|| { + black_box( + diag_blake3_digest_with_kernel(kernel, black_box(d)) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) + }, ); } @@ -167,7 +174,12 @@ fn keyed(c: &mut Criterion) { BenchmarkId::new(format!("rscrypto-{}", kernel.label()), len), data, |b, d| { - b.iter(|| black_box(diag_blake3_keyed_digest_with_kernel(kernel, black_box(&key), black_box(d)).unwrap())) + b.iter(|| { + black_box( + diag_blake3_keyed_digest_with_kernel(kernel, black_box(&key), black_box(d)) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) }, ); } @@ -228,7 +240,12 @@ fn streaming(c: &mut Criterion) { continue; } g.bench_function(format!("rscrypto-{}/{chunk_size}B", kernel.label()), |b| { - b.iter(|| black_box(diag_blake3_streaming_digest_with_kernel(kernel, black_box(&data), chunk_size).unwrap())) + b.iter(|| { + black_box( + diag_blake3_streaming_digest_with_kernel(kernel, black_box(&data), chunk_size) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) }); } @@ -279,7 +296,8 @@ fn xof(c: &mut Criterion) { |b, d| { b.iter(|| { let mut out = [0u8; OUT_LEN]; - diag_blake3_xof_with_kernel(kernel, black_box(d), &mut out).unwrap(); + diag_blake3_xof_with_kernel(kernel, black_box(d), &mut out) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out) }) }, @@ -335,7 +353,8 @@ fn xof_output(c: &mut Criterion) { |b, &len| { let mut out = vec![0u8; len]; b.iter(|| { - diag_blake3_xof_with_kernel(kernel, black_box(&data), black_box(out.as_mut_slice())).unwrap(); + diag_blake3_xof_with_kernel(kernel, black_box(&data), black_box(out.as_mut_slice())) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out[0]) }) }, @@ -368,7 +387,7 @@ fn tail_diagnostics(c: &mut Criterion) { let mut digest_group = c.benchmark_group("blake3/chunk-tail-digest"); for chunks in tail_counts.iter().copied() { - let data = common::random_bytes(chunks * BLAKE3_CHUNK_LEN); + let data = common::random_bytes(chunks.strict_mul(BLAKE3_CHUNK_LEN)); common::set_throughput(&mut digest_group, data.len()); digest_group.bench_with_input(BenchmarkId::new("rscrypto", chunks), &data, |b, d| { @@ -382,7 +401,14 @@ fn tail_diagnostics(c: &mut Criterion) { digest_group.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), chunks), &data, - |b, d| b.iter(|| black_box(diag_blake3_digest_with_kernel(kernel, black_box(d)).unwrap())), + |b, d| { + b.iter(|| { + black_box( + diag_blake3_digest_with_kernel(kernel, black_box(d)) + .expect("selected BLAKE3 benchmark kernel must be available"), + ) + }) + }, ); } @@ -394,20 +420,21 @@ fn tail_diagnostics(c: &mut Criterion) { let mut chunk_group = c.benchmark_group("blake3/chunk-tail-cvs"); for chunks in tail_counts.iter().copied() { - let data = common::random_bytes(chunks * BLAKE3_CHUNK_LEN); + let data = common::random_bytes(chunks.strict_mul(BLAKE3_CHUNK_LEN)); common::set_throughput(&mut chunk_group, data.len()); for &kernel in chunk_tail_diag_kernels() { if !diag_blake3_kernel_available(kernel) { continue; } - let mut out = vec![0u8; chunks * BLAKE3_OUT_LEN]; + let mut out = vec![0u8; chunks.strict_mul(BLAKE3_OUT_LEN)]; chunk_group.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), chunks), &data, |b, d| { b.iter(|| { - diag_blake3_chunk_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())).unwrap(); + diag_blake3_chunk_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out[0]) }) }, @@ -418,20 +445,21 @@ fn tail_diagnostics(c: &mut Criterion) { let mut parent_group = c.benchmark_group("blake3/parent-tail-cvs"); for parents in tail_counts { - let children = common::random_bytes(parents * 2 * BLAKE3_OUT_LEN); + let children = common::random_bytes(parents.strict_mul(2).strict_mul(BLAKE3_OUT_LEN)); common::set_throughput(&mut parent_group, children.len()); for &kernel in parent_tail_diag_kernels() { if !diag_blake3_kernel_available(kernel) { continue; } - let mut out = vec![0u8; parents * BLAKE3_OUT_LEN]; + let mut out = vec![0u8; parents.strict_mul(BLAKE3_OUT_LEN)]; parent_group.bench_with_input( BenchmarkId::new(format!("rscrypto-{}", kernel.label()), parents), &children, |b, d| { b.iter(|| { - diag_blake3_parent_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())).unwrap(); + diag_blake3_parent_cvs_with_kernel(kernel, black_box(d), black_box(out.as_mut_slice())) + .expect("selected BLAKE3 benchmark kernel must be available"); black_box(out[0]) }) }, diff --git a/benches/common/mod.rs b/benches/common/mod.rs index 1acda7aa..2cb1011e 100644 --- a/benches/common/mod.rs +++ b/benches/common/mod.rs @@ -1,7 +1,7 @@ use criterion::{BenchmarkGroup, Throughput, measurement::WallTime}; /// Deterministic pseudo-random bytes for reproducible benchmarks. -pub fn random_bytes(len: usize) -> Vec { +pub(crate) fn random_bytes(len: usize) -> Vec { let mut state: u64 = (len as u64) ^ 0x517c_c1b7_2722_0a95; (0..len) .map(|_| { @@ -19,7 +19,7 @@ pub fn random_bytes(len: usize) -> Vec { /// Covers: overhead (0B, 1B), small (32B), block boundary (64B), /// medium (256B, 1 KiB), page-aligned (4 KiB, 16 KiB), in-cache /// (64 KiB, 256 KiB), throughput (1 MiB). -pub fn comp_sizes() -> Vec<(usize, Vec)> { +pub(crate) fn comp_sizes() -> Vec<(usize, Vec)> { [0, 1, 32, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576] .into_iter() .map(|len| (len, random_bytes(len))) @@ -27,7 +27,7 @@ pub fn comp_sizes() -> Vec<(usize, Vec)> { } /// Set criterion throughput for a benchmark group. -pub fn set_throughput(group: &mut BenchmarkGroup<'_, WallTime>, len: usize) { +pub(crate) fn set_throughput(group: &mut BenchmarkGroup<'_, WallTime>, len: usize) { if len > 0 { group.throughput(Throughput::Bytes(len as u64)); } diff --git a/benches/password_hashing.rs b/benches/password_hashing.rs index c6037893..8b7ea8e2 100644 --- a/benches/password_hashing.rs +++ b/benches/password_hashing.rs @@ -4,12 +4,13 @@ //! Organised by cost-parameter classes so CI can run the fast groups on //! every push and reserve the OWASP-scale group for dedicated perf runs. -#![allow(clippy::unwrap_used)] - use core::{hint::black_box, time::Duration}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use dryoc::classic::crypto_pwhash::{PasswordHashAlgorithm, crypto_pwhash}; +use dryoc::{ + classic::crypto_pwhash::{PasswordHashAlgorithm, crypto_pwhash}, + constants::{CRYPTO_PWHASH_ARGON2I_OPSLIMIT_MIN, CRYPTO_PWHASH_ARGON2ID_OPSLIMIT_MIN}, +}; use rscrypto::{ Argon2Error, Argon2Params, Argon2d, Argon2i, Argon2id, Argon2idPassword, Scrypt, ScryptParams, ScryptPassword, }; @@ -22,12 +23,13 @@ const SALT: &[u8] = b"rscrypto-bench-salt-16bytes!"; /// Build rscrypto params. fn rs_params(m_kib: u32, t: u32, p: u32, _out_len: u32) -> Argon2Params { - Argon2Params::new(m_kib, t, p).unwrap() + Argon2Params::new(m_kib, t, p).expect("supported password-hashing benchmark parameters must succeed") } /// Build RustCrypto oracle context. fn oracle_ctx(algo: argon2::Algorithm, m_kib: u32, t: u32, p: u32, out_len: usize) -> argon2::Argon2<'static> { - let params = argon2::Params::new(m_kib, t, p, Some(out_len)).unwrap(); + let params = argon2::Params::new(m_kib, t, p, Some(out_len)) + .expect("supported password-hashing benchmark parameters must succeed"); argon2::Argon2::new(algo, argon2::Version::V0x13, params) } @@ -40,6 +42,15 @@ const SMALL_MATRIX: &[(u32, u32, u32)] = &[ (64, 3, 2), ]; +fn dryoc_supports_small_row(algorithm: PasswordHashAlgorithm, time_cost: u32, parallelism: u32) -> bool { + let minimum_time_cost = match algorithm { + PasswordHashAlgorithm::Argon2i13 => CRYPTO_PWHASH_ARGON2I_OPSLIMIT_MIN, + PasswordHashAlgorithm::Argon2id13 => CRYPTO_PWHASH_ARGON2ID_OPSLIMIT_MIN, + }; + + parallelism == 1 && u64::from(time_cost) >= minimum_time_cost +} + fn bench_small_variant( c: &mut Criterion, group_name: &str, @@ -53,7 +64,12 @@ fn bench_small_variant( for &(m, t, p) in SMALL_MATRIX { let out_len = 32usize; let param_id = format!("m={m}_t={t}_p={p}"); - let rs_params = rs_params(m, t, p, out_len as u32); + let rs_params = rs_params( + m, + t, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_ctx(oracle_algo, m, t, p, out_len); g.bench_with_input(BenchmarkId::new("rscrypto", ¶m_id), &rs_params, |b, params| { @@ -65,7 +81,7 @@ fn bench_small_variant( black_box(SALT), black_box(&mut out), ) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -74,28 +90,26 @@ fn bench_small_variant( b.iter(|| { ctx .hash_password_into(black_box(PASSWORD), black_box(SALT), black_box(&mut out)) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); - // dryoc wraps libsodium's `crypto_pwhash`, which fixes parallelism at p=1. - // Skip the row whenever the matrix asks for p>1; that case is rscrypto+rustcrypto only. - if p == 1 - && let Some(ref alg) = dryoc_algo - { - let memlimit_bytes = (m as usize).saturating_mul(1024); - g.bench_with_input(BenchmarkId::new("dryoc", ¶m_id), alg, |b, algorithm| { + if let Some(algorithm) = dryoc_algo.filter(|&algorithm| dryoc_supports_small_row(algorithm, t, p)) { + let memlimit_bytes = usize::try_from(m) + .expect("benchmark memory cost must fit usize") + .strict_mul(1024); + g.bench_with_input(BenchmarkId::new("dryoc", ¶m_id), &algorithm, |b, algorithm| { let mut out = [0u8; 32]; b.iter(|| { crypto_pwhash( black_box(&mut out), black_box(PASSWORD), black_box(&SALT[..16]), - t as u64, + u64::from(t), memlimit_bytes, - algorithm.clone(), + *algorithm, ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } @@ -139,7 +153,12 @@ fn argon2id_owasp(c: &mut Criterion) { let out_len = 32usize; // OWASP 2024: m=19MiB, t=2, p=1 - let rs_params = rs_params(19 * 1024, 2, 1, out_len as u32); + let rs_params = rs_params( + 19 * 1024, + 2, + 1, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_ctx(argon2::Algorithm::Argon2id, 19 * 1024, 2, 1, out_len); g.bench_function(BenchmarkId::new("rscrypto", "m=19MiB_t=2_p=1"), |b| { @@ -151,7 +170,7 @@ fn argon2id_owasp(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -160,12 +179,12 @@ fn argon2id_owasp(c: &mut Criterion) { b.iter(|| { oracle .hash_password_into(black_box(PASSWORD), black_box(SALT), black_box(&mut out)) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); // dryoc / libsodium-classic Argon2id at OWASP parameters (memlimit in bytes). - let dryoc_memlimit = (19 * 1024usize).saturating_mul(1024); + let dryoc_memlimit = 19usize.strict_mul(1024).strict_mul(1024); g.bench_function(BenchmarkId::new("dryoc", "m=19MiB_t=2_p=1"), |b| { let mut out = [0u8; 32]; b.iter(|| { @@ -177,7 +196,7 @@ fn argon2id_owasp(c: &mut Criterion) { dryoc_memlimit, PasswordHashAlgorithm::Argon2id13, ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); @@ -186,12 +205,12 @@ fn argon2id_owasp(c: &mut Criterion) { /// Build rscrypto scrypt params. fn rs_scrypt_params(log_n: u8, r: u32, p: u32, _out_len: u32) -> ScryptParams { - ScryptParams::new(log_n, r, p).unwrap() + ScryptParams::new(log_n, r, p).expect("supported password-hashing benchmark parameters must succeed") } /// Build RustCrypto scrypt oracle params. fn oracle_scrypt_params(log_n: u8, r: u32, p: u32, _out_len: usize) -> scrypt::Params { - scrypt::Params::new(log_n, r, p).unwrap() + scrypt::Params::new(log_n, r, p).expect("supported password-hashing benchmark parameters must succeed") } /// Small / CI-friendly scrypt matrix: (log_n, r, p). @@ -204,7 +223,12 @@ fn scrypt_small(c: &mut Criterion) { for &(log_n, r, p) in SCRYPT_SMALL_MATRIX { let out_len = 32usize; let id = format!("log_n={log_n}_r={r}_p={p}"); - let rs = rs_scrypt_params(log_n, r, p, out_len as u32); + let rs = rs_scrypt_params( + log_n, + r, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_scrypt_params(log_n, r, p, out_len); g.bench_with_input(BenchmarkId::new("rscrypto", &id), &rs, |b, params| { @@ -216,14 +240,15 @@ fn scrypt_small(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); g.bench_with_input(BenchmarkId::new("rustcrypto", &id), &oracle, |b, params| { let mut out = [0u8; 32]; b.iter(|| { - scrypt::scrypt(black_box(PASSWORD), black_box(SALT), params, black_box(&mut out)).unwrap(); + scrypt::scrypt(black_box(PASSWORD), black_box(SALT), params, black_box(&mut out)) + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } @@ -238,7 +263,12 @@ fn scrypt_owasp(c: &mut Criterion) { g.measurement_time(Duration::from_secs(30)); let out_len = 32usize; - let rs = rs_scrypt_params(17, 8, 1, out_len as u32); + let rs = rs_scrypt_params( + 17, + 8, + 1, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); let oracle = oracle_scrypt_params(17, 8, 1, out_len); g.bench_function(BenchmarkId::new("rscrypto", "log_n=17_r=8_p=1"), |b| { @@ -250,14 +280,15 @@ fn scrypt_owasp(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); g.bench_function(BenchmarkId::new("rustcrypto", "log_n=17_r=8_p=1"), |b| { let mut out = [0u8; 32]; b.iter(|| { - scrypt::scrypt(black_box(PASSWORD), black_box(SALT), &oracle, black_box(&mut out)).unwrap(); + scrypt::scrypt(black_box(PASSWORD), black_box(SALT), &oracle, black_box(&mut out)) + .expect("supported password-hashing benchmark parameters must succeed"); }); }); @@ -270,17 +301,23 @@ fn scrypt_phc_roundtrip(c: &mut Criterion) { g.sample_size(20); let params = rs_scrypt_params(10, 8, 1, 32); - let password = ScryptPassword::new(params).unwrap(); + let password = ScryptPassword::new(params).expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("hash_password", |b| { - b.iter(|| password.hash_password(black_box(PASSWORD)).unwrap()); + b.iter(|| { + password + .hash_password(black_box(PASSWORD)) + .expect("supported password-hashing benchmark parameters must succeed") + }); }); - let encoded = password.hash_password(PASSWORD).unwrap(); + let encoded = password + .hash_password(PASSWORD) + .expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("verify_password", |b| { b.iter(|| { password .verify_password(black_box(PASSWORD), black_box(&encoded)) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -298,17 +335,23 @@ fn argon2id_phc_roundtrip(c: &mut Criterion) { g.sample_size(30); let params = rs_params(32, 2, 1, 32); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("hash_password", |b| { - b.iter(|| password.hash_password(black_box(PASSWORD)).unwrap()); + b.iter(|| { + password + .hash_password(black_box(PASSWORD)) + .expect("supported password-hashing benchmark parameters must succeed") + }); }); - let encoded = password.hash_password(PASSWORD).unwrap(); + let encoded = password + .hash_password(PASSWORD) + .expect("supported password-hashing benchmark parameters must succeed"); g.bench_function("verify_password", |b| { b.iter(|| { password .verify_password(black_box(PASSWORD), black_box(&encoded)) - .unwrap() + .expect("supported password-hashing benchmark parameters must succeed") }); }); @@ -336,7 +379,12 @@ fn argon2id_parallel_scaling(c: &mut Criterion) { for &p in &[1u32, 4, 8, 16] { let id = format!("p={p}"); - let params = rs_params(m_kib, t, p, out_len as u32); + let params = rs_params( + m_kib, + t, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); g.bench_with_input(BenchmarkId::new("rscrypto", &id), ¶ms, |b, params| { let mut out = [0u8; 32]; @@ -347,7 +395,7 @@ fn argon2id_parallel_scaling(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } @@ -369,7 +417,12 @@ fn argon2id_parallel_owasp(c: &mut Criterion) { for &p in &[1u32, 4, 8, 16] { let id = format!("p={p}"); - let params = rs_params(m_kib, t, p, out_len as u32); + let params = rs_params( + m_kib, + t, + p, + u32::try_from(out_len).expect("benchmark output length must fit u32"), + ); g.bench_with_input(BenchmarkId::new("rscrypto", &id), ¶ms, |b, params| { let mut out = [0u8; 32]; @@ -380,7 +433,7 @@ fn argon2id_parallel_owasp(c: &mut Criterion) { black_box(SALT), black_box(&mut out), ) - .unwrap(); + .expect("supported password-hashing benchmark parameters must succeed"); }); }); } diff --git a/benches/rsa.rs b/benches/rsa.rs index 14ec2cd5..831f52b7 100644 --- a/benches/rsa.rs +++ b/benches/rsa.rs @@ -61,38 +61,45 @@ const RSA8192_PKCS1V15_SHA256: &[u8] = include_bytes!("rsa_fixtures/rsa8192_pkcs fn hex_to_vec(hex: &str) -> Vec { let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("RSA benchmark fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("RSA benchmark fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long DER length must contain a non-zero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("DER length-of-length must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -104,7 +111,7 @@ fn sequence(value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -114,7 +121,10 @@ fn integer_unsigned(value: &[u8]) -> Vec { fn exponent_bytes(exponent: u64) -> Vec { let bytes = exponent.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); bytes[first_nonzero..].to_vec() } @@ -240,9 +250,11 @@ fn rsa_components_for_size( pkcs1_sig: &[u8], import_policy: &RsaPublicKeyPolicy, ) { - let pss_key = RsaPublicKey::from_spki_der_with_policy(pss_spki, import_policy).unwrap(); + let pss_key = + RsaPublicKey::from_spki_der_with_policy(pss_spki, import_policy).expect("valid RSA benchmark fixture must succeed"); let mut pss_scratch = pss_key.public_scratch(); - let pkcs1_key = RsaPublicKey::from_spki_der_with_policy(pkcs1_spki, import_policy).unwrap(); + let pkcs1_key = RsaPublicKey::from_spki_der_with_policy(pkcs1_spki, import_policy) + .expect("valid RSA benchmark fixture must succeed"); let mut pkcs1_scratch = pkcs1_key.public_scratch(); let pss_pkcs1 = pkcs1_der_from_key(&pss_key); let pkcs1_pkcs1 = pkcs1_der_from_key(&pkcs1_key); @@ -254,7 +266,7 @@ fn rsa_components_for_size( let mut pss_encoded = vec![0u8; pss_key.modulus().len()]; pss_key .public_operation_with_scratch(pss_sig, &mut pss_encoded, &mut pss_scratch) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let pss_em_bits = pss_key.modulus_bits().strict_sub(1); let pss_em_len = pss_em_bits.strict_add(7) / 8; let leading = pss_encoded.len().strict_sub(pss_em_len); @@ -263,7 +275,7 @@ fn rsa_components_for_size( let mut pkcs1_encoded = vec![0u8; pkcs1_key.modulus().len()]; pkcs1_key .public_operation_with_scratch(pkcs1_sig, &mut pkcs1_encoded, &mut pkcs1_scratch) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); ( pss_encoded, @@ -286,20 +298,31 @@ fn rsa_components_for_size( let rustcrypto_pss_key = RustCryptoRsaPublicKey::from_public_key_der(pss_spki) .ok() .map(RustCryptoPssVerifyingKey::::new); - let rustcrypto_pss_sig = RustCryptoPssSignature::try_from(pss_sig).unwrap(); + let rustcrypto_pss_sig = RustCryptoPssSignature::try_from(pss_sig).expect("valid RSA benchmark fixture must succeed"); let rustcrypto_pkcs1_key = RustCryptoRsaPublicKey::from_public_key_der(pkcs1_spki) .ok() .map(RustCryptoPkcs1v15VerifyingKey::::new); - let rustcrypto_pkcs1_sig = RustCryptoPkcs1v15Signature::try_from(pkcs1_sig).unwrap(); + let rustcrypto_pkcs1_sig = + RustCryptoPkcs1v15Signature::try_from(pkcs1_sig).expect("valid RSA benchmark fixture must succeed"); let mut group = c.benchmark_group(name); group.bench_function("parse-spki-rscrypto", |b| { - b.iter(|| black_box(RsaPublicKey::from_spki_der_with_policy(black_box(pss_spki), import_policy).unwrap())) + b.iter(|| { + black_box( + RsaPublicKey::from_spki_der_with_policy(black_box(pss_spki), import_policy) + .expect("valid RSA benchmark fixture must succeed"), + ) + }) }); if rustcrypto_pss_key.is_some() { group.bench_function("parse-spki-rustcrypto-rsa", |b| { - b.iter(|| black_box(RustCryptoRsaPublicKey::from_public_key_der(black_box(pss_spki)).unwrap())) + b.iter(|| { + black_box( + RustCryptoRsaPublicKey::from_public_key_der(black_box(pss_spki)) + .expect("valid RSA benchmark fixture must succeed"), + ) + }) }); } group.bench_function("scratch-setup-rscrypto", |b| { @@ -313,7 +336,7 @@ fn rsa_components_for_size( black_box(&mut out), black_box(&mut pss_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -328,7 +351,7 @@ fn rsa_components_for_size( black_box(&mut out), black_box(&mut product_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e65537-cios-candidate", |b| { @@ -339,7 +362,7 @@ fn rsa_components_for_size( black_box(&mut out), black_box(&mut cios_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } @@ -354,7 +377,7 @@ fn rsa_components_for_size( black_box(&mut pss_db), black_box(&mut pss_db_mask), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -365,7 +388,7 @@ fn rsa_components_for_size( black_box(MESSAGE_PKCS1V15), black_box(&pkcs1_encoded), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pss-sha256-rscrypto", |b| { @@ -377,22 +400,22 @@ fn rsa_components_for_size( black_box(pss_sig), black_box(&mut pss_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pss-sha256-rscrypto-oneshot", |b| { b.iter(|| { pss_key .verify_pss(RsaPssProfile::Sha256, black_box(MESSAGE_PSS), black_box(pss_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pss-sha256-rscrypto-cold", |b| { b.iter(|| { RsaPublicKey::from_spki_der_with_policy(black_box(pss_spki), import_policy) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") .verify_pss(RsaPssProfile::Sha256, black_box(MESSAGE_PSS), black_box(pss_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); if let Some(rustcrypto_pss_key) = rustcrypto_pss_key { @@ -400,16 +423,20 @@ fn rsa_components_for_size( b.iter(|| { rustcrypto_pss_key .verify(black_box(MESSAGE_PSS), black_box(&rustcrypto_pss_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } group.bench_function("verify-pss-sha256-ring", |b| { - b.iter(|| ring_pss_key.verify(black_box(MESSAGE_PSS), black_box(pss_sig)).unwrap()) + b.iter(|| { + ring_pss_key + .verify(black_box(MESSAGE_PSS), black_box(pss_sig)) + .expect("valid RSA benchmark fixture must succeed") + }) }); aws_lc_bench! { group.bench_function("verify-pss-sha256-aws-lc-rs", |b| { - b.iter(|| aws_pss_key.verify(black_box(MESSAGE_PSS), black_box(pss_sig)).unwrap()) + b.iter(|| aws_pss_key.verify(black_box(MESSAGE_PSS), black_box(pss_sig)).expect("valid RSA benchmark fixture must succeed")) }); } group.bench_function("verify-pkcs1v15-sha256-rscrypto", |b| { @@ -421,7 +448,7 @@ fn rsa_components_for_size( black_box(pkcs1_sig), black_box(&mut pkcs1_scratch), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pkcs1v15-sha256-rscrypto-oneshot", |b| { @@ -432,19 +459,19 @@ fn rsa_components_for_size( black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("verify-pkcs1v15-sha256-rscrypto-cold", |b| { b.iter(|| { RsaPublicKey::from_spki_der_with_policy(black_box(pkcs1_spki), import_policy) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") .verify_pkcs1v15( RsaPkcs1v15Profile::Sha256, black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); if let Some(rustcrypto_pkcs1_key) = rustcrypto_pkcs1_key { @@ -452,7 +479,7 @@ fn rsa_components_for_size( b.iter(|| { rustcrypto_pkcs1_key .verify(black_box(MESSAGE_PKCS1V15), black_box(&rustcrypto_pkcs1_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } @@ -460,7 +487,7 @@ fn rsa_components_for_size( b.iter(|| { ring_pkcs1_key .verify(black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); aws_lc_bench! { @@ -468,7 +495,7 @@ fn rsa_components_for_size( b.iter(|| { aws_pkcs1_key .verify(black_box(MESSAGE_PKCS1V15), black_box(pkcs1_sig)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } @@ -481,18 +508,18 @@ fn rsa_public_exponents(c: &mut Criterion) { let input = legacy_exponent_input(); let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); - let key_e3 = - RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x03]), &policy).unwrap(); - let key_e17 = - RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x11]), &policy).unwrap(); + let key_e3 = RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x03]), &policy) + .expect("valid RSA benchmark fixture must succeed"); + let key_e17 = RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x11]), &policy) + .expect("valid RSA benchmark fixture must succeed"); let key_e65537 = RsaPublicKey::from_pkcs1_der_with_policy(&pkcs1_der_from_modulus_exponent(&modulus, &[0x01, 0x00, 0x01]), &policy) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let key_generic = RsaPublicKey::from_pkcs1_der_with_policy( &pkcs1_der_from_modulus_exponent(&modulus, &[0x49, 0xd2, 0xa1]), &policy.allow_legacy_odd_exponents(), ) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let mut scratch_e3 = key_e3.public_scratch(); let mut scratch_e17 = key_e17.public_scratch(); @@ -527,12 +554,15 @@ fn rsa_public_exponents(c: &mut Criterion) { b.iter(|| { key_e3 .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_e3)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e3-bitserial-baseline", |b| { - b.iter(|| diag_rsa_public_operation_bitserial(black_box(&key_e3), black_box(&input), black_box(&mut out)).unwrap()) + b.iter(|| { + diag_rsa_public_operation_bitserial(black_box(&key_e3), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") + }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e3-product-montgomery", |b| { @@ -543,7 +573,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_e3), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -555,7 +585,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut generic_scratch_e3), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -567,19 +597,22 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_e3), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e17", |b| { b.iter(|| { key_e17 .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_e17)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e17-bitserial-baseline", |b| { - b.iter(|| diag_rsa_public_operation_bitserial(black_box(&key_e17), black_box(&input), black_box(&mut out)).unwrap()) + b.iter(|| { + diag_rsa_public_operation_bitserial(black_box(&key_e17), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") + }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e17-product-montgomery", |b| { @@ -590,7 +623,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_e17), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -602,7 +635,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut generic_scratch_e17), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -614,20 +647,21 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_e17), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e65537", |b| { b.iter(|| { key_e65537 .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_e65537)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e65537-bitserial-baseline", |b| { b.iter(|| { - diag_rsa_public_operation_bitserial(black_box(&key_e65537), black_box(&input), black_box(&mut out)).unwrap() + diag_rsa_public_operation_bitserial(black_box(&key_e65537), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -639,7 +673,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_e65537), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -651,7 +685,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut generic_scratch_e65537), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -663,20 +697,21 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_e65537), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function("public-op-e0x49d2a1-generic", |b| { b.iter(|| { key_generic .public_operation_with_scratch(black_box(&input), black_box(&mut out), black_box(&mut scratch_generic)) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] group.bench_function("public-op-e0x49d2a1-bitserial-baseline", |b| { b.iter(|| { - diag_rsa_public_operation_bitserial(black_box(&key_generic), black_box(&input), black_box(&mut out)).unwrap() + diag_rsa_public_operation_bitserial(black_box(&key_generic), black_box(&input), black_box(&mut out)) + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -688,7 +723,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut product_scratch_generic), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); #[cfg(feature = "diag")] @@ -700,7 +735,7 @@ fn rsa_public_exponents(c: &mut Criterion) { black_box(&mut out), black_box(&mut cios_scratch_generic), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.finish(); @@ -748,7 +783,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { &synthetic_pkcs1_der(modulus_len), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("valid RSA benchmark fixture must succeed"); let input = modulus_minus_one(&key); let mut out_auto = vec![0u8; key.modulus().len()]; let mut out_product = vec![0u8; key.modulus().len()]; @@ -767,7 +802,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_auto), black_box(&mut scratch_auto), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function(format!("{name}/product-montgomery"), |b| { @@ -778,7 +813,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_product), black_box(&mut scratch_product), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function(format!("{name}/cios-candidate"), |b| { @@ -789,7 +824,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_cios), black_box(&mut scratch_cios), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); group.bench_function(format!("{name}/cios-portable"), |b| { @@ -800,7 +835,7 @@ fn rsa_montgomery_thresholds(c: &mut Criterion) { black_box(&mut out_cios_portable), black_box(&mut scratch_cios_portable), ) - .unwrap() + .expect("valid RSA benchmark fixture must succeed") }) }); } diff --git a/benches/sha2.rs b/benches/sha2.rs index 293da66c..3205c8f4 100644 --- a/benches/sha2.rs +++ b/benches/sha2.rs @@ -200,7 +200,7 @@ fn sha256_internal(c: &mut Criterion) { let mut g = c.benchmark_group("sha256/internal/compress"); for block_count in [1usize, 2, 16] { - let len = block_count * 64; + let len = block_count.strict_mul(64); common::set_throughput(&mut g, len); g.bench_with_input(BenchmarkId::new("selected-kernel", format!("{len}B")), &len, |b, &n| { let blocks = &blocks[..n]; diff --git a/benchmark_results/OVERVIEW.md b/benchmark_results/OVERVIEW.md index a7fad017..f55021e3 100644 --- a/benchmark_results/OVERVIEW.md +++ b/benchmark_results/OVERVIEW.md @@ -2,25 +2,35 @@ Sources: -- Linux benchmark CI run [#28710784737](https://github.com/loadingalias/rscrypto/actions/runs/28710784737), created 2026-07-04 15:26:12 UTC. -- Linux commit: `596498f0e07e869eac71fd31c157aa1b22186239`. -- Linux artifacts: nine successful `benchmark-*` artifacts extracted into `benchmark_results/2026-07-04/linux/*/results.txt`. -- Local macOS run: `benchmark_results/2026-07-04/macos/aarch64/results.txt` at commit `596498f0e07e869eac71fd31c157aa1b22186239`. +- Linux benchmark CI run [#32185659553](https://github.com/loadingalias/rscrypto/actions/runs/32185659553), created 2026-08-18 21:03:07 UTC. +- Linux commit: `7eb44e9a38ef7a031d9181dc8c4c0fad38f46504`. +- Linux artifacts: eight successful `benchmark-*` artifacts extracted into `benchmark_results/2026-08-18/linux/*/results.txt`. +- Local macOS run: `benchmark_results/2026-07-04/macos/aarch64/results.txt` at commit `596498f0e07e869eac71fd31c157aa1b22186239`, carried forward unchanged. - Local Ed25519 direct-secret before/after diagnostic, recorded below. -Scope: the 2026-07-04 nine-runner Linux CI benchmark matrix for commit `596498f`. Ratios are `external_crate_time / rscrypto_time`; higher is better. Wins are `>1.05x`, ties are `0.95x..1.05x`, and losses are `<0.95x`. Fastest-external comparisons keep only the fastest external implementation for each platform, primitive, operation, and input shape. Internal kernel, scratch-buffer, padding-only, cold-path, PHC roundtrip, parallel-scaling, threshold-selection, public-overhead, and phase-attribution microbenches are parsed as raw rows but excluded from external win/loss claims. The macOS local run is listed separately and is not mixed into Linux CI claims. +Scope: the 2026-08-18 eight-runner Linux CI benchmark matrix for commit `7eb44e9`. Ratios are `external_crate_time / rscrypto_time`; higher is better. Wins are `>1.05x`, ties are `0.95x..1.05x`, and losses are `<0.95x`. Fastest-external comparisons keep only the fastest external implementation for each platform, primitive, operation, and input shape. Internal kernel, scratch-buffer, padding-only, cold-path, PHC roundtrip, parallel-scaling, threshold-selection, public-overhead, and phase-attribution microbenches are parsed as raw rows but excluded from external win/loss claims. The macOS local run is listed separately and is not mixed into Linux CI claims. -This is a historical snapshot of commit `596498f`, not an inventory of the +This is a historical snapshot of commit `7eb44e9`, not an inventory of the current public API. Primitive rows remain as measured even when a later commit changes or removes that surface. -Equivalence correction: the historical RustCrypto HMAC-SHA-256 rows include -key setup inside the timed loop, while rscrypto, ring, and AWS-LC use reusable -keyed state. Consequently, the HMAC-SHA-256 row and aggregate statistics that -include it are archival measurements, not equivalent-work performance claims. -The current benchmark source precomputes the RustCrypto state; publish new -HMAC-SHA-256 and aggregate ratios only after a complete benchmark artifact is -regenerated. +Runner coverage change: this run has eight Linux runners. The RISE RISC-V +runner did not execute in run #32185659553, so every aggregate below is over +eight platforms rather than the nine in the 2026-07-04 snapshot. Row counts are +therefore not directly comparable to that snapshot; ratios and geomeans are. + +Equivalence correction resolved: the historical RustCrypto HMAC-SHA-256 rows +included key setup inside the timed loop. The current benchmark source hoists +`RustCryptoHmacSha256::new_from_slice` out of the timed loop and clones the +keyed state per iteration, matching the reusable-keyed-state treatment given to +rscrypto, ring, and AWS-LC. This artifact is a complete regenerated benchmark +pass, so the HMAC-SHA-256 rows and the aggregates that include them are +equivalent-work performance claims. + +Surface change since 2026-07-04: the rapidhash benchmark surface was collapsed. +The former `rapidhash-64`, `rapidhash-128`, and `rapidhash-v3-128` primitives no +longer exist; `rapidhash-v3-64`, `rapidhash-stream`, `rapidhash-buildhasher`, +`rapidhash-hash-one`, and `rapidhash-hashmap` are the current rows. Coverage note: this is a full Linux CI public benchmark pass. It includes checksum, hash, XOF, MAC, KDF, password-hashing, BLAKE2/BLAKE3, RSA import/verification, ECDSA P-256/P-384 signing and verification, Ed25519, X25519, AEAD, and ML-KEM-512/768/1024 keygen, encapsulation, and decapsulation rows. ML-KEM phase/arithmetic microbenches are present in the raw artifacts and intentionally excluded from release-level competitor claims. @@ -60,117 +70,121 @@ curation: | Scope | Pairs | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | -| Linux CI: all matched performance pairs | 10,781 | 7,542/2,470/769 | 70% | 1.76x | 1.22x | -| Linux CI: fastest external per case | 6,750 | 4,052/2,049/649 | 60% | 1.59x | 1.11x | +| Linux CI: all matched performance pairs | 9,674 | 6,831/2,085/758 | 71% | 1.78x | 1.24x | +| Linux CI: fastest external per case | 6,144 | 3,780/1,695/669 | 62% | 1.62x | 1.12x | Snapshot summary: -- **Headline:** 4,052 of 6,750 matched Linux CI fastest-external comparisons are wins; 6,101 are wins or ties. Linux CI fastest-external geomean is 1.59x. -- **Checksums:** 5.18x geomean across 693 fastest-external rows; W/T/L is 517/115/61. -- **Hashes/MACs/XOFs:** 1.35x geomean across 3,726 fastest-external rows; W/T/L is 2,028/1,457/241. -- **Auth/KDF:** 1.25x geomean across 180 fastest-external rows; W/T/L is 159/20/1. -- **Password hashing:** 1.07x geomean across 135 fastest-external rows; W/T/L is 69/28/38. -- **Public-key:** 1.33x geomean across 333 fastest-external rows; W/T/L is 217/73/43. -- **RSA:** 1.55x geomean across 99 fastest-external rows; W/T/L is 89/2/8. -- **AEAD:** 1.56x geomean across 1,584 fastest-external rows; W/T/L is 973/354/257. -- **ML-KEM:** 1.49x geomean across 81 fastest-external rows; W/T/L is 68/4/9. -- **ECDSA P-256/P-384:** Linux CI 1.45x geomean across 144 fastest-external rows; W/T/L is 116/7/21. -- **Top current loss areas:** `argon2id-owasp` / `hash`: 0.97x geomean across 9 rows; W/T/L 4/1/4; pressure `rustcrypto` 4; `ed25519` / `verify`: 1.00x geomean across 36 rows; W/T/L 7/20/9; pressure `ring` 6, `dalek` 3; `mlkem1024` / `keygen`: 1.00x geomean across 9 rows; W/T/L 6/0/3; pressure `aws-lc-rs` 3; `blake2b256` / `streaming`: 1.02x geomean across 27 rows; W/T/L 10/16/1; pressure `rustcrypto` 1; `argon2id-small` / `hash`: 1.02x geomean across 27 rows; W/T/L 12/4/11; pressure `rustcrypto` 11. +- **Headline:** 3,780 of 6,144 matched Linux CI fastest-external comparisons are wins; 5,475 are wins or ties. Linux CI fastest-external geomean is 1.62x. +- **Checksums:** 6.18x geomean across 616 fastest-external rows; W/T/L is 476/118/22. +- **Hashes/MACs/XOFs:** 1.35x geomean across 3,456 fastest-external rows; W/T/L is 1,926/1,181/349. +- **Auth/KDF:** 1.28x geomean across 160 fastest-external rows; W/T/L is 140/20/0. +- **Password hashing:** 1.07x geomean across 120 fastest-external rows; W/T/L is 55/27/38. +- **Public-key:** 1.09x geomean across 296 fastest-external rows; W/T/L is 187/59/50. +- **RSA:** 1.65x geomean across 88 fastest-external rows; W/T/L is 86/2/0. +- **AEAD:** 1.61x geomean across 1,408 fastest-external rows; W/T/L is 910/288/210. +- **ML-KEM:** 1.55x geomean across 72 fastest-external rows; W/T/L is 64/0/8. +- **ECDSA P-256/P-384:** Linux CI 0.87x geomean across 128 fastest-external rows; W/T/L is 88/7/33. +- **Top current loss areas:** `ecdsa-p384` / `sign`: 0.70x geomean across 32 rows; W/T/L is 12/0/20; pressure `aws-lc-rs` 16, `rustcrypto-p384` 4; `ecdsa-p256` / `verify`: 0.84x geomean across 32 rows; W/T/L is 20/7/5; pressure `rustcrypto-p256` 4, `aws-lc-rs` 1; `rapidhash-stream` / `one-write`: 0.87x geomean across 88 rows; W/T/L is 27/25/36; pressure `rapidhash` 36; `ecdsa-p256` / `sign`: 0.91x geomean across 32 rows; W/T/L is 28/0/4; pressure `ring` 4; `argon2id-owasp` / `hash`: 0.98x geomean across 8 rows; W/T/L is 3/1/4; pressure `rustcrypto` 3, `dryoc` 1. ## Coverage Matrix | Platform | Raw Criterion rows | All pairs | Fastest rows | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | --- | --- | -| AMD Zen4 | 2,356 | 1,251 | 750 | 513/189/48 | 68% | 1.50x | 1.15x | -| AMD Zen5 | 2,356 | 1,251 | 750 | 434/249/67 | 58% | 1.49x | 1.09x | -| AWS Graviton3 | 2,367 | 1,251 | 750 | 360/298/92 | 48% | 1.37x | 1.04x | -| AWS Graviton4 | 2,367 | 1,251 | 750 | 362/332/56 | 48% | 1.37x | 1.05x | -| IBM Power10 | 2,107 | 1,012 | 750 | 389/324/37 | 52% | 1.89x | 1.06x | -| IBM z16/s390x | 2,107 | 1,012 | 750 | 603/87/60 | 80% | 3.15x | 2.34x | -| Intel Ice Lake | 2,356 | 1,251 | 750 | 523/154/73 | 70% | 1.49x | 1.19x | -| Intel Sapphire Rapids | 2,356 | 1,251 | 750 | 533/149/68 | 71% | 1.63x | 1.20x | -| RISE RISC-V | 2,356 | 1,251 | 750 | 335/267/148 | 45% | 1.09x | 1.03x | +| AMD Zen4 | 2,304 | 1,269 | 768 | 525/171/72 | 68% | 1.47x | 1.14x | +| AMD Zen5 | 2,304 | 1,269 | 768 | 447/245/76 | 58% | 1.47x | 1.10x | +| AWS Graviton3 | 2,308 | 1,269 | 768 | 367/287/114 | 48% | 1.36x | 1.04x | +| AWS Graviton4 | 2,308 | 1,269 | 768 | 366/337/65 | 48% | 1.37x | 1.04x | +| IBM Power10 | 2,055 | 1,030 | 768 | 400/302/66 | 52% | 1.83x | 1.06x | +| IBM z16/s390x | 2,055 | 1,030 | 768 | 620/67/81 | 81% | 2.77x | 2.19x | +| Intel Ice Lake | 2,304 | 1,269 | 768 | 517/137/114 | 67% | 1.45x | 1.17x | +| Intel Sapphire Rapids | 2,304 | 1,269 | 768 | 538/149/81 | 70% | 1.60x | 1.18x | ## Category Summary | Category | Rows | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | -| Checksums | 693 | 517/115/61 | 75% | 5.18x | 2.44x | -| Hashes/MACs/XOFs | 3,726 | 2,028/1,457/241 | 54% | 1.35x | 1.07x | -| Auth/KDF | 180 | 159/20/1 | 88% | 1.25x | 1.13x | -| Password hashing | 135 | 69/28/38 | 51% | 1.07x | 1.05x | -| Public-key | 333 | 217/73/43 | 65% | 1.33x | 1.17x | -| RSA | 99 | 89/2/8 | 90% | 1.55x | 1.18x | -| AEAD | 1,584 | 973/354/257 | 61% | 1.56x | 1.15x | +| Checksums | 616 | 476/118/22 | 77% | 6.18x | 3.17x | +| Hashes/MACs/XOFs | 3,456 | 1,926/1,181/349 | 56% | 1.35x | 1.08x | +| Auth/KDF | 160 | 140/20/0 | 88% | 1.28x | 1.13x | +| Password hashing | 120 | 55/27/38 | 46% | 1.07x | 1.02x | +| Public-key | 296 | 187/59/50 | 63% | 1.09x | 1.14x | +| RSA | 88 | 86/2/0 | 98% | 1.65x | 1.20x | +| AEAD | 1,408 | 910/288/210 | 65% | 1.61x | 1.21x | ## BLAKE3 Summary -BLAKE3 rows come from Linux CI run [#28710784737](https://github.com/loadingalias/rscrypto/actions/runs/28710784737). All-pair and fastest-external BLAKE3 metrics are identical because official `blake3` is the only external implementation in this bench. +BLAKE3 rows come from Linux CI run [#32185659553](https://github.com/loadingalias/rscrypto/actions/runs/32185659553). All-pair and fastest-external BLAKE3 metrics are identical because official `blake3` is the only external implementation in this bench. | Scope | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| All Linux BLAKE3 rows | 432 | 234/168/30 | 1.41x | 1.08x | -| x86_64 | 192 | 92/90/10 | 1.26x | 1.05x | -| AArch64 | 96 | 44/46/6 | 1.44x | 1.05x | +| All Linux BLAKE3 rows | 384 | 187/134/63 | 1.35x | 1.04x | +| x86_64 | 192 | 79/89/24 | 1.18x | 1.02x | +| AArch64 | 96 | 44/36/16 | 1.40x | 1.04x | | Platform | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| AMD Zen4 | 48 | 23/23/2 | 1.34x | 1.05x | -| AMD Zen5 | 48 | 21/25/2 | 1.32x | 1.03x | -| AWS Graviton3 | 48 | 22/21/5 | 1.40x | 0.99x | -| AWS Graviton4 | 48 | 22/25/1 | 1.47x | 1.05x | -| IBM Power10 | 48 | 39/9/0 | 1.98x | 1.85x | -| IBM z16/s390x | 48 | 36/5/7 | 1.81x | 2.10x | -| Intel Ice Lake | 48 | 19/26/3 | 1.17x | 1.02x | -| Intel Sapphire Rapids | 48 | 29/16/3 | 1.24x | 1.08x | -| RISE RISC-V | 48 | 23/18/7 | 1.15x | 1.02x | +| AMD Zen4 | 48 | 20/22/6 | 1.24x | 1.01x | +| AMD Zen5 | 48 | 18/27/3 | 1.27x | 1.02x | +| AWS Graviton3 | 48 | 22/15/11 | 1.36x | 0.98x | +| AWS Graviton4 | 48 | 22/21/5 | 1.44x | 1.04x | +| IBM Power10 | 48 | 32/6/10 | 1.76x | 1.12x | +| IBM z16/s390x | 48 | 32/3/13 | 1.69x | 1.69x | +| Intel Ice Lake | 48 | 19/21/8 | 1.09x | 1.00x | +| Intel Sapphire Rapids | 48 | 22/19/7 | 1.13x | 1.03x | | Operation | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| `oneshot` | 99 | 45/47/7 | 1.32x | 1.02x | -| `keyed` | 99 | 46/45/8 | 1.34x | 1.03x | -| `derive-key` | 99 | 77/20/2 | 1.76x | 1.86x | -| `streaming` | 36 | 11/21/4 | 1.16x | 1.01x | -| `xof` | 99 | 55/35/9 | 1.35x | 1.07x | +| `oneshot` | 88 | 35/35/18 | 1.33x | 1.00x | +| `keyed` | 88 | 27/21/40 | 1.20x | 0.95x | +| `derive-key` | 88 | 65/21/2 | 1.59x | 1.53x | +| `streaming` | 32 | 10/21/1 | 1.21x | 1.02x | +| `xof` | 88 | 50/36/2 | 1.37x | 1.07x | ## ML-KEM Summary -ML-KEM public coverage is complete for the CI-selected primitive set: ML-KEM-512, ML-KEM-768, and ML-KEM-1024 each include keygen, encapsulate, and decapsulate on all nine Linux platforms. POWER10 and s390x do not have `aws-lc-rs` ML-KEM rows in this artifact set, but still have rscrypto plus `libcrux`, `fips203`, and RustCrypto comparison rows for every public operation. +ML-KEM public coverage is complete for the CI-selected primitive set: ML-KEM-512, ML-KEM-768, and ML-KEM-1024 each include keygen, encapsulate, and decapsulate on all eight Linux platforms. POWER10 and s390x do not have `aws-lc-rs` ML-KEM rows in this artifact set, but still have rscrypto plus `libcrux`, `fips203`, and RustCrypto comparison rows for every public operation. | Platform | Raw ML-KEM rows | Fastest rows | W/T/L | Geomean | Median | Fastest external split | | --- | --- | --- | --- | --- | --- | --- | -| AMD Zen4 | 45 | 9 | 9/0/0 | 1.81x | 1.70x | `libcrux` 7, `aws-lc-rs` 2 | -| AMD Zen5 | 45 | 9 | 9/0/0 | 1.93x | 1.75x | `libcrux` 9 | -| AWS Graviton3 | 45 | 9 | 5/0/4 | 1.08x | 1.13x | `aws-lc-rs` 9 | -| AWS Graviton4 | 45 | 9 | 5/0/4 | 1.08x | 1.14x | `aws-lc-rs` 9 | -| IBM Power10 | 36 | 9 | 9/0/0 | 1.47x | 1.58x | `libcrux` 9 | -| IBM z16/s390x | 36 | 9 | 9/0/0 | 1.58x | 1.64x | `libcrux` 9 | -| Intel Ice Lake | 45 | 9 | 9/0/0 | 1.79x | 1.74x | `libcrux` 7, `aws-lc-rs` 2 | -| Intel Sapphire Rapids | 45 | 9 | 9/0/0 | 1.85x | 1.82x | `aws-lc-rs` 5, `libcrux` 4 | -| RISE RISC-V | 45 | 9 | 4/4/1 | 1.13x | 1.03x | `aws-lc-rs` 9 | +| AMD Zen4 | 45 | 9 | 9/0/0 | 1.83x | 1.82x | `libcrux` 7, `aws-lc-rs` 2 | +| AMD Zen5 | 45 | 9 | 9/0/0 | 1.95x | 1.91x | `libcrux` 9 | +| AWS Graviton3 | 45 | 9 | 5/0/4 | 1.09x | 1.12x | `aws-lc-rs` 9 | +| AWS Graviton4 | 45 | 9 | 5/0/4 | 1.08x | 1.18x | `aws-lc-rs` 9 | +| IBM Power10 | 36 | 9 | 9/0/0 | 1.41x | 1.53x | `libcrux` 9 | +| IBM z16/s390x | 36 | 9 | 9/0/0 | 1.68x | 1.74x | `libcrux` 9 | +| Intel Ice Lake | 45 | 9 | 9/0/0 | 1.80x | 1.75x | `libcrux` 7, `aws-lc-rs` 2 | +| Intel Sapphire Rapids | 45 | 9 | 9/0/0 | 1.84x | 1.80x | `aws-lc-rs` 7, `libcrux` 2 | | Primitive/op | Rows | W/T/L | Win % | Geomean | Median | Pressure | | --- | --- | --- | --- | --- | --- | --- | -| `mlkem1024` / `decapsulate` | 9 | 9/0/0 | 100% | 1.59x | 1.67x | none | -| `mlkem1024` / `encapsulate` | 9 | 9/0/0 | 100% | 2.39x | 1.98x | none | -| `mlkem1024` / `keygen` | 9 | 6/0/3 | 67% | 1.00x | 1.09x | `aws-lc-rs` 3 | -| `mlkem512` / `decapsulate` | 9 | 6/1/2 | 67% | 1.33x | 1.51x | `aws-lc-rs` 2 | -| `mlkem512` / `encapsulate` | 9 | 9/0/0 | 100% | 1.86x | 1.70x | none | -| `mlkem512` / `keygen` | 9 | 6/1/2 | 67% | 1.08x | 1.18x | `aws-lc-rs` 2 | -| `mlkem768` / `decapsulate` | 9 | 8/1/0 | 89% | 1.49x | 1.70x | none | -| `mlkem768` / `encapsulate` | 9 | 9/0/0 | 100% | 2.27x | 2.02x | none | -| `mlkem768` / `keygen` | 9 | 6/1/2 | 67% | 1.04x | 1.12x | `aws-lc-rs` 2 | +| `mlkem1024` / `decapsulate` | 8 | 8/0/0 | 100% | 1.70x | 1.86x | none | +| `mlkem1024` / `encapsulate` | 8 | 8/0/0 | 100% | 2.51x | 2.63x | none | +| `mlkem1024` / `keygen` | 8 | 6/0/2 | 75% | 1.02x | 1.13x | `aws-lc-rs` 2 | +| `mlkem512` / `decapsulate` | 8 | 6/0/2 | 75% | 1.41x | 1.59x | `aws-lc-rs` 2 | +| `mlkem512` / `encapsulate` | 8 | 8/0/0 | 100% | 1.94x | 2.17x | none | +| `mlkem512` / `keygen` | 8 | 6/0/2 | 75% | 1.09x | 1.22x | `aws-lc-rs` 2 | +| `mlkem768` / `decapsulate` | 8 | 8/0/0 | 100% | 1.58x | 1.75x | none | +| `mlkem768` / `encapsulate` | 8 | 8/0/0 | 100% | 2.33x | 2.54x | none | +| `mlkem768` / `keygen` | 8 | 6/0/2 | 75% | 1.06x | 1.13x | `aws-lc-rs` 2 | ## ECDSA Summary ECDSA signing includes both deterministic and blinded rscrypto rows in raw results; aggregate fastest-external comparisons use the fastest rscrypto row for the exact case. Constant-time release evidence is tracked separately by `ct.toml` and CT workflow artifacts. +Regression: every ECDSA aggregate in this snapshot is dominated by a single +platform. On IBM z16/s390x, P-256 signing went from 137.10 µs (2026-07-04) to +8,889.30 µs, and P-384 signing from 562.91 µs to 34,557.00 µs, while the +external crates on the same runner moved by less than 1.4x. Excluding s390x, the +seven-runner geomeans are `ecdsa-p256` / `sign` 1.33x, `ecdsa-p256` / `verify` +1.19x, `ecdsa-p384` / `sign` 1.01x, and `ecdsa-p384` / `verify` 1.53x. + | Operation | Rows | W/T/L | Geomean | Median | | --- | --- | --- | --- | --- | -| `ecdsa-p256` / `sign` | 36 | 36/0/0 | 1.45x | 1.35x | -| `ecdsa-p256` / `verify` | 36 | 28/7/1 | 1.59x | 1.15x | -| `ecdsa-p384` / `sign` | 36 | 16/0/20 | 1.07x | 0.90x | -| `ecdsa-p384` / `verify` | 36 | 36/0/0 | 1.81x | 1.39x | +| `ecdsa-p256` / `sign` | 32 | 28/0/4 | 0.91x | 1.30x | +| `ecdsa-p256` / `verify` | 32 | 20/7/5 | 0.84x | 1.08x | +| `ecdsa-p384` / `sign` | 32 | 12/0/20 | 0.70x | 0.83x | +| `ecdsa-p384` / `verify` | 32 | 28/0/4 | 1.08x | 1.36x | ## Primitive Summary @@ -178,146 +192,149 @@ Linux CI primitives with matched exact `rscrypto` comparisons. Fastest columns a | Primitive | Fastest rows | Fastest W/T/L | Fastest geomean | All pairs | All W/T/L | All geomean | | --- | --- | --- | --- | --- | --- | --- | -| `argon2id-owasp` | 9 | 4/1/4 | 0.97x | 18 | 11/2/5 | 1.27x | -| `argon2id-small` | 27 | 12/4/11 | 1.02x | 45 | 29/5/11 | 1.35x | -| `argon2d-small` | 27 | 13/6/8 | 1.03x | 27 | 13/6/8 | 1.03x | -| `x25519` | 18 | 2/16/0 | 1.03x | 50 | 33/17/0 | 1.54x | -| `argon2i-small` | 27 | 13/2/12 | 1.03x | 45 | 30/2/13 | 1.35x | -| `rapidhash-v3-64` | 99 | 27/53/19 | 1.05x | 99 | 27/53/19 | 1.05x | -| `blake2b256` | 225 | 120/100/5 | 1.07x | 351 | 238/108/5 | 1.31x | -| `rapidhash-v3-128` | 99 | 34/44/21 | 1.07x | 99 | 34/44/21 | 1.07x | -| `blake2b512` | 198 | 109/82/7 | 1.07x | 297 | 207/83/7 | 1.31x | -| `xxh3-128` | 99 | 29/51/19 | 1.08x | 99 | 29/51/19 | 1.08x | -| `scrypt-owasp` | 9 | 5/2/2 | 1.09x | 9 | 5/2/2 | 1.09x | -| `xxh3-64` | 99 | 33/47/19 | 1.10x | 99 | 33/47/19 | 1.10x | -| `rapidhash-64` | 99 | 32/51/16 | 1.10x | 99 | 32/51/16 | 1.10x | -| `chacha20-poly1305` | 198 | 83/108/7 | 1.10x | 550 | 344/198/8 | 1.29x | -| `blake2s256` | 225 | 117/107/1 | 1.10x | 225 | 117/107/1 | 1.10x | -| `ed25519` | 90 | 31/46/13 | 1.11x | 290 | 204/69/17 | 1.36x | -| `blake2s128` | 198 | 118/79/1 | 1.11x | 198 | 118/79/1 | 1.11x | -| `rapidhash-128` | 99 | 38/56/5 | 1.14x | 99 | 38/56/5 | 1.14x | -| `rsa-8192` | 18 | 14/2/2 | 1.19x | 32 | 28/2/2 | 1.25x | -| `scrypt-small` | 36 | 22/13/1 | 1.21x | 36 | 22/13/1 | 1.21x | -| `hmac-sha512` | 99 | 37/54/8 | 1.23x | 275 | 165/97/13 | 1.29x | -| `hmac-sha384` | 99 | 38/52/9 | 1.24x | 275 | 167/94/14 | 1.29x | -| `hkdf-sha384` | 36 | 33/2/1 | 1.24x | 100 | 96/3/1 | 1.53x | -| `sha256` | 117 | 41/59/17 | 1.24x | 293 | 164/99/30 | 1.54x | -| `sha512` | 117 | 45/61/11 | 1.25x | 293 | 172/104/17 | 1.27x | -| `pbkdf2-sha512` | 54 | 47/7/0 | 1.25x | 150 | 143/7/0 | 1.33x | -| `sha384` | 99 | 43/45/11 | 1.25x | 275 | 169/89/17 | 1.28x | -| `pbkdf2-sha256` | 54 | 50/4/0 | 1.25x | 150 | 146/4/0 | 1.65x | -| `hkdf-sha256` | 36 | 29/7/0 | 1.26x | 100 | 93/7/0 | 1.87x | -| `ascon-hash256` | 99 | 54/43/2 | 1.27x | 99 | 54/43/2 | 1.27x | -| `sha512-256` | 99 | 53/46/0 | 1.29x | 99 | 53/46/0 | 1.29x | -| `ascon-aead128` | 198 | 148/50/0 | 1.33x | 198 | 148/50/0 | 1.33x | -| `hmac-sha256` | 117 | 64/39/14 | 1.34x | 293 | 193/76/24 | 1.68x | -| `aegis-256` | 198 | 89/88/21 | 1.37x | 198 | 89/88/21 | 1.37x | -| `ascon-xof128` | 99 | 75/24/0 | 1.37x | 99 | 75/24/0 | 1.37x | -| `mlkem512` | 27 | 21/2/4 | 1.39x | 102 | 96/2/4 | 2.67x | -| `xchacha20-poly1305` | 198 | 182/16/0 | 1.39x | 198 | 182/16/0 | 1.39x | -| `ecdsa-p384` | 72 | 52/0/20 | 1.39x | 200 | 180/0/20 | 3.17x | -| `blake3` | 432 | 234/168/30 | 1.41x | 432 | 234/168/30 | 1.41x | -| `ecdsa-p256` | 72 | 64/7/1 | 1.52x | 200 | 188/11/1 | 2.40x | -| `mlkem768` | 27 | 23/2/2 | 1.52x | 102 | 98/2/2 | 3.11x | -| `mlkem1024` | 27 | 24/0/3 | 1.56x | 102 | 99/0/3 | 3.27x | -| `rsa-4096` | 27 | 25/0/2 | 1.56x | 59 | 57/0/2 | 2.52x | -| `crc32c` | 99 | 45/37/17 | 1.61x | 198 | 140/40/18 | 2.19x | -| `crc32` | 99 | 49/31/19 | 1.66x | 198 | 144/35/19 | 2.26x | -| `rsa-3072` | 27 | 25/0/2 | 1.66x | 59 | 57/0/2 | 2.59x | -| `aes-128-gcm` | 198 | 97/45/56 | 1.68x | 550 | 413/54/83 | 1.89x | -| `rsa-2048` | 27 | 25/0/2 | 1.71x | 59 | 57/0/2 | 2.63x | -| `aes-256-gcm` | 198 | 97/40/61 | 1.71x | 550 | 405/52/93 | 1.92x | -| `kmac256` | 99 | 65/18/16 | 1.75x | 99 | 65/18/16 | 1.75x | -| `cshake256` | 99 | 63/28/8 | 1.78x | 99 | 63/28/8 | 1.78x | -| `shake128` | 99 | 69/29/1 | 1.85x | 99 | 69/29/1 | 1.85x | -| `shake256` | 99 | 70/29/0 | 1.87x | 99 | 70/29/0 | 1.87x | -| `sha224` | 99 | 52/46/1 | 1.87x | 99 | 52/46/1 | 1.87x | -| `aes-128-gcm-siv` | 198 | 138/3/57 | 2.02x | 352 | 271/16/65 | 2.79x | -| `sha3-256` | 117 | 104/13/0 | 2.11x | 117 | 104/13/0 | 2.11x | -| `sha3-224` | 99 | 87/12/0 | 2.13x | 99 | 87/12/0 | 2.13x | -| `crc64-nvme` | 99 | 56/33/10 | 2.14x | 99 | 56/33/10 | 2.14x | -| `aes-256-gcm-siv` | 198 | 139/4/55 | 2.15x | 352 | 292/5/55 | 3.04x | -| `sha3-384` | 99 | 89/10/0 | 2.16x | 99 | 89/10/0 | 2.16x | -| `sha3-512` | 99 | 88/11/0 | 2.20x | 99 | 88/11/0 | 2.20x | -| `crc64-xz` | 99 | 76/12/11 | 2.39x | 99 | 76/12/11 | 2.39x | -| `crc24-openpgp` | 99 | 95/2/2 | 13.25x | 99 | 95/2/2 | 13.25x | -| `crc16-ccitt` | 99 | 98/0/1 | 23.12x | 99 | 98/0/1 | 23.12x | -| `crc16-ibm` | 99 | 98/0/1 | 24.01x | 99 | 98/0/1 | 24.01x | +| `ecdsa-p384` | 64 | 40/0/24 | 0.87x | 176 | 144/0/32 | 2.27x | +| `ecdsa-p256` | 64 | 48/7/9 | 0.87x | 176 | 148/11/17 | 1.57x | +| `rapidhash-stream` | 176 | 61/33/82 | 0.92x | 176 | 61/33/82 | 0.92x | +| `argon2id-owasp` | 8 | 3/1/4 | 0.98x | 16 | 7/4/5 | 1.25x | +| `xxh3-buildhasher` | 88 | 41/12/35 | 0.99x | 88 | 41/12/35 | 0.99x | +| `x25519` | 16 | 3/13/0 | 1.02x | 44 | 31/13/0 | 1.58x | +| `argon2i-small` | 24 | 10/3/11 | 1.03x | 40 | 26/3/11 | 1.34x | +| `argon2id-small` | 24 | 10/3/11 | 1.03x | 40 | 25/4/11 | 1.35x | +| `argon2d-small` | 24 | 10/5/9 | 1.04x | 24 | 10/5/9 | 1.04x | +| `rapidhash-v3-64` | 88 | 21/45/22 | 1.05x | 88 | 21/45/22 | 1.05x | +| `blake2b256` | 200 | 101/99/0 | 1.07x | 312 | 204/108/0 | 1.31x | +| `scrypt-owasp` | 8 | 4/2/2 | 1.08x | 8 | 4/2/2 | 1.08x | +| `blake2b512` | 176 | 106/69/1 | 1.08x | 264 | 194/69/1 | 1.33x | +| `blake2s256` | 200 | 114/86/0 | 1.11x | 200 | 114/86/0 | 1.11x | +| `chacha20-poly1305` | 176 | 75/101/0 | 1.12x | 484 | 304/180/0 | 1.32x | +| `xxh3-128` | 88 | 34/42/12 | 1.13x | 88 | 34/42/12 | 1.13x | +| `xxh3-64` | 88 | 34/34/20 | 1.13x | 88 | 34/34/20 | 1.13x | +| `blake2s128` | 176 | 113/63/0 | 1.13x | 176 | 113/63/0 | 1.13x | +| `ed25519` | 80 | 32/39/9 | 1.14x | 256 | 194/48/14 | 1.41x | +| `xxh3-hashmap` | 8 | 7/1/0 | 1.15x | 8 | 7/1/0 | 1.15x | +| `scrypt-small` | 32 | 18/13/1 | 1.18x | 32 | 18/13/1 | 1.18x | +| `rapidhash-buildhasher` | 88 | 44/29/15 | 1.19x | 88 | 44/29/15 | 1.19x | +| `aegis-256` | 176 | 81/65/30 | 1.23x | 176 | 81/65/30 | 1.23x | +| `hmac-sha256` | 104 | 42/36/26 | 1.24x | 258 | 144/78/36 | 1.60x | +| `hmac-sha384` | 88 | 28/49/11 | 1.24x | 242 | 133/93/16 | 1.29x | +| `hmac-sha512` | 88 | 32/44/12 | 1.27x | 242 | 137/88/17 | 1.31x | +| `sha256` | 104 | 44/46/14 | 1.27x | 258 | 143/89/26 | 1.60x | +| `hkdf-sha384` | 32 | 29/3/0 | 1.27x | 88 | 85/3/0 | 1.59x | +| `rsa-8192` | 16 | 14/2/0 | 1.28x | 28 | 26/2/0 | 1.33x | +| `hkdf-sha256` | 32 | 27/5/0 | 1.28x | 88 | 83/5/0 | 1.93x | +| `pbkdf2-sha256` | 48 | 43/5/0 | 1.28x | 132 | 127/5/0 | 1.71x | +| `pbkdf2-sha512` | 48 | 41/7/0 | 1.28x | 132 | 125/7/0 | 1.34x | +| `sha512` | 104 | 48/51/5 | 1.29x | 258 | 160/88/10 | 1.31x | +| `sha384` | 88 | 43/39/6 | 1.30x | 242 | 151/80/11 | 1.32x | +| `ascon-hash256` | 88 | 56/31/1 | 1.30x | 88 | 56/31/1 | 1.30x | +| `sha512-256` | 88 | 50/38/0 | 1.33x | 88 | 50/38/0 | 1.33x | +| `blake3` | 384 | 187/134/63 | 1.35x | 384 | 187/134/63 | 1.35x | +| `ascon-aead128` | 176 | 136/39/1 | 1.39x | 176 | 136/39/1 | 1.39x | +| `ascon-xof128` | 88 | 66/20/2 | 1.39x | 88 | 66/20/2 | 1.39x | +| `xchacha20-poly1305` | 176 | 173/3/0 | 1.43x | 176 | 173/3/0 | 1.43x | +| `mlkem512` | 24 | 20/0/4 | 1.44x | 90 | 86/0/4 | 2.90x | +| `rapidhash-hash-one` | 24 | 18/4/2 | 1.47x | 24 | 18/4/2 | 1.47x | +| `mlkem768` | 24 | 22/0/2 | 1.57x | 90 | 88/0/2 | 3.38x | +| `rapidhash-hashmap` | 24 | 24/0/0 | 1.61x | 24 | 24/0/0 | 1.61x | +| `mlkem1024` | 24 | 22/0/2 | 1.63x | 90 | 88/0/2 | 3.60x | +| `rsa-4096` | 24 | 24/0/0 | 1.70x | 52 | 52/0/0 | 2.69x | +| `crc32c` | 88 | 42/38/8 | 1.73x | 176 | 130/38/8 | 2.41x | +| `rsa-3072` | 24 | 24/0/0 | 1.75x | 52 | 52/0/0 | 2.73x | +| `rsa-2048` | 24 | 24/0/0 | 1.79x | 52 | 52/0/0 | 2.77x | +| `aes-128-gcm` | 176 | 96/42/38 | 1.80x | 484 | 390/50/44 | 2.01x | +| `crc32` | 88 | 47/33/8 | 1.80x | 176 | 133/35/8 | 2.51x | +| `aes-256-gcm` | 176 | 94/36/46 | 1.83x | 484 | 382/44/58 | 2.02x | +| `kmac256` | 88 | 58/19/11 | 1.86x | 88 | 58/19/11 | 1.86x | +| `cshake256` | 88 | 58/21/9 | 1.90x | 88 | 58/21/9 | 1.90x | +| `shake128` | 88 | 58/30/0 | 1.94x | 88 | 58/30/0 | 1.94x | +| `shake256` | 88 | 63/25/0 | 1.98x | 88 | 63/25/0 | 1.98x | +| `sha224` | 88 | 51/37/0 | 2.01x | 88 | 51/37/0 | 2.01x | +| `aes-128-gcm-siv` | 176 | 127/1/48 | 2.20x | 308 | 237/16/55 | 2.92x | +| `sha3-224` | 88 | 77/11/0 | 2.27x | 88 | 77/11/0 | 2.27x | +| `sha3-256` | 104 | 91/13/0 | 2.28x | 104 | 91/13/0 | 2.28x | +| `aes-256-gcm-siv` | 176 | 128/1/47 | 2.34x | 308 | 259/2/47 | 3.16x | +| `crc64-nvme` | 88 | 52/35/1 | 2.34x | 88 | 52/35/1 | 2.34x | +| `sha3-384` | 88 | 79/9/0 | 2.35x | 88 | 79/9/0 | 2.35x | +| `sha3-512` | 88 | 77/11/0 | 2.38x | 88 | 77/11/0 | 2.38x | +| `crc64-xz` | 88 | 73/12/3 | 2.78x | 88 | 73/12/3 | 2.78x | +| `crc24-openpgp` | 88 | 86/0/2 | 17.62x | 88 | 86/0/2 | 17.62x | +| `crc16-ccitt` | 88 | 88/0/0 | 30.24x | 88 | 88/0/0 | 30.24x | +| `crc16-ibm` | 88 | 88/0/0 | 32.07x | 88 | 88/0/0 | 32.07x | ## Linux Worst Individual Rows | Platform | Case | Fastest external | Ratio | | --- | --- | --- | --- | -| RISE RISC-V | `xxh3-64 / 0` | `xxhash-rust` | 0.43x | -| Intel Sapphire Rapids | `aes-256-gcm / encrypt / 32` | `rustcrypto` | 0.46x | -| Intel Ice Lake | `aes-256-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.48x | -| Intel Sapphire Rapids | `aes-128-gcm / encrypt / 32` | `rustcrypto` | 0.48x | -| Intel Ice Lake | `aes-256-gcm-siv / encrypt / 0` | `aws-lc-rs` | 0.48x | -| Intel Sapphire Rapids | `aes-256-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.49x | -| Intel Sapphire Rapids | `aes-128-gcm-siv / encrypt / 0` | `aws-lc-rs` | 0.49x | -| Intel Ice Lake | `aes-128-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.49x | -| Intel Sapphire Rapids | `aes-128-gcm-siv / encrypt / 32` | `aws-lc-rs` | 0.49x | -| AMD Zen4 | `aes-256-gcm / encrypt / 32` | `rustcrypto` | 0.49x | -| Intel Ice Lake | `aes-256-gcm / encrypt / 32` | `rustcrypto` | 0.49x | -| Intel Sapphire Rapids | `aes-256-gcm-siv / encrypt / 0` | `aws-lc-rs` | 0.49x | +| IBM z16/s390x | `ecdsa-p256 / sign / 1024` | `ring` | 0.05x | +| IBM z16/s390x | `ecdsa-p384 / sign / 16384` | `rustcrypto-p384` | 0.05x | +| IBM z16/s390x | `ecdsa-p384 / sign / 1024` | `rustcrypto-p384` | 0.05x | +| IBM z16/s390x | `ecdsa-p384 / sign / 0` | `rustcrypto-p384` | 0.06x | +| IBM z16/s390x | `ecdsa-p384 / sign / 32` | `rustcrypto-p384` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / sign / 0` | `ring` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / sign / 32` | `ring` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / verify / 32` | `rustcrypto-p256` | 0.06x | +| IBM z16/s390x | `ecdsa-p256 / verify / 1024` | `rustcrypto-p256` | 0.07x | +| IBM z16/s390x | `ecdsa-p256 / sign / 16384` | `ring` | 0.07x | +| IBM z16/s390x | `ecdsa-p256 / verify / 0` | `rustcrypto-p256` | 0.07x | +| IBM z16/s390x | `ecdsa-p384 / verify / 1024` | `rustcrypto-p384` | 0.09x | ## Linux Strongest Individual Rows | Platform | Case | Fastest external | Ratio | | --- | --- | --- | --- | -| Intel Sapphire Rapids | `crc16-ccitt / 262144` | `crc` | 214.89x | -| Intel Sapphire Rapids | `crc16-ibm / 262144` | `crc` | 209.87x | -| Intel Sapphire Rapids | `crc16-ibm / 16384` | `crc` | 208.50x | -| Intel Sapphire Rapids | `crc16-ccitt / 16384` | `crc` | 207.55x | -| Intel Sapphire Rapids | `crc16-ccitt / 4096` | `crc` | 184.38x | -| Intel Sapphire Rapids | `crc16-ccitt / 65536` | `crc` | 183.70x | -| Intel Sapphire Rapids | `crc16-ibm / 65536` | `crc` | 179.68x | -| Intel Sapphire Rapids | `crc16-ccitt / 1048576` | `crc` | 178.10x | -| IBM Power10 | `crc16-ibm / 262144` | `crc` | 176.65x | -| IBM Power10 | `crc16-ibm / 1048576` | `crc` | 176.64x | -| IBM Power10 | `crc16-ccitt / 262144` | `crc` | 176.33x | -| IBM Power10 | `crc16-ccitt / 1048576` | `crc` | 176.09x | +| Intel Sapphire Rapids | `crc16-ibm / 262144` | `crc` | 212.60x | +| Intel Sapphire Rapids | `crc16-ccitt / 262144` | `crc` | 209.27x | +| Intel Sapphire Rapids | `crc16-ccitt / 16384` | `crc` | 206.40x | +| Intel Sapphire Rapids | `crc16-ibm / 16384` | `crc` | 198.48x | +| Intel Sapphire Rapids | `crc16-ibm / 1048576` | `crc` | 187.52x | +| Intel Sapphire Rapids | `crc16-ibm / 4096` | `crc` | 178.55x | +| Intel Sapphire Rapids | `crc16-ibm / 65536` | `crc` | 178.28x | +| Intel Sapphire Rapids | `crc16-ccitt / 4096` | `crc` | 178.15x | +| IBM Power10 | `crc16-ccitt / 1048576` | `crc` | 176.67x | +| IBM Power10 | `crc16-ibm / 1048576` | `crc` | 176.60x | +| Intel Sapphire Rapids | `crc16-ccitt / 1048576` | `crc` | 176.46x | +| IBM Power10 | `crc16-ccitt / 262144` | `crc` | 175.61x | ## Top Five Loss Areas -- `argon2id-owasp` / `hash`: 0.97x geomean across 9 rows; W/T/L 4/1/4; pressure `rustcrypto` 4. -- `ed25519` / `verify`: 1.00x geomean across 36 rows; W/T/L 7/20/9; pressure `ring` 6, `dalek` 3. -- `mlkem1024` / `keygen`: 1.00x geomean across 9 rows; W/T/L 6/0/3; pressure `aws-lc-rs` 3. -- `blake2b256` / `streaming`: 1.02x geomean across 27 rows; W/T/L 10/16/1; pressure `rustcrypto` 1. -- `argon2id-small` / `hash`: 1.02x geomean across 27 rows; W/T/L 12/4/11; pressure `rustcrypto` 11. +- `ecdsa-p384` / `sign`: 0.70x geomean across 32 rows; W/T/L 12/0/20; pressure `aws-lc-rs` 16, `rustcrypto-p384` 4. +- `ecdsa-p256` / `verify`: 0.84x geomean across 32 rows; W/T/L 20/7/5; pressure `rustcrypto-p256` 4, `aws-lc-rs` 1. +- `rapidhash-stream` / `one-write`: 0.87x geomean across 88 rows; W/T/L 27/25/36; pressure `rapidhash` 36. +- `ecdsa-p256` / `sign`: 0.91x geomean across 32 rows; W/T/L 28/0/4; pressure `ring` 4. +- `argon2id-owasp` / `hash`: 0.98x geomean across 8 rows; W/T/L 3/1/4; pressure `rustcrypto` 3, `dryoc` 1. ## External Pressure | External | Pairs | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | -| `xxhash-rust` | 198 | 62/98/38 | 31% | 1.09x | 1.00x | -| `rapidhash` | 396 | 131/204/61 | 33% | 1.09x | 1.01x | -| `aws-lc-rs` | 1,673 | 1,044/384/245 | 62% | 1.25x | 1.15x | -| `ascon-hash` | 198 | 129/67/2 | 65% | 1.32x | 1.19x | -| `ascon-aead` | 198 | 148/50/0 | 75% | 1.33x | 1.15x | -| `aegis-crate` | 198 | 89/88/21 | 45% | 1.37x | 1.04x | -| `blake3` | 432 | 234/168/30 | 54% | 1.41x | 1.08x | -| `dalek` | 108 | 85/18/5 | 79% | 1.47x | 1.27x | -| `sha2` | 531 | 284/243/4 | 53% | 1.52x | 1.06x | -| `ring` | 1,656 | 1,333/228/95 | 80% | 1.66x | 1.29x | -| `rustcrypto` | 2,745 | 1,949/621/175 | 71% | 1.78x | 1.19x | -| `libcrux` | 81 | 81/0/0 | 100% | 1.79x | 1.70x | -| `dryoc` | 360 | 325/29/6 | 90% | 1.79x | 1.83x | -| `tiny-keccak` | 396 | 267/104/25 | 67% | 1.81x | 1.96x | -| `crc-fast` | 297 | 170/95/32 | 57% | 2.00x | 1.15x | -| `sha3` | 414 | 368/46/0 | 89% | 2.15x | 2.12x | -| `crc64fast` | 99 | 76/12/11 | 77% | 2.39x | 1.95x | -| `crc32fast` | 99 | 81/7/11 | 82% | 2.41x | 2.01x | -| `crc32c` | 99 | 89/6/4 | 90% | 2.71x | 2.19x | -| `fips203` | 81 | 81/0/0 | 100% | 4.66x | 5.11x | -| `rustcrypto-p384` | 72 | 72/0/0 | 100% | 4.79x | 5.26x | -| `rustcrypto-p256` | 72 | 72/0/0 | 100% | 5.65x | 4.91x | -| `rustcrypto-rsa` | 81 | 81/0/0 | 100% | 5.68x | 4.70x | -| `crc` | 297 | 291/2/4 | 98% | 19.45x | 28.96x | +| `rapidhash` | 400 | 168/111/121 | 42% | 1.07x | 1.01x | +| `xxhash-rust` | 272 | 116/89/67 | 43% | 1.08x | 1.00x | +| `aws-lc-rs` | 1,434 | 896/343/195 | 62% | 1.21x | 1.13x | +| `aegis-crate` | 176 | 81/65/30 | 46% | 1.23x | 1.04x | +| `ascon-hash` | 176 | 122/51/3 | 69% | 1.34x | 1.32x | +| `blake3` | 384 | 187/134/63 | 49% | 1.35x | 1.04x | +| `ascon-aead` | 176 | 136/39/1 | 77% | 1.39x | 1.38x | +| `dalek` | 96 | 80/12/4 | 83% | 1.52x | 1.49x | +| `sha2` | 472 | 276/194/2 | 58% | 1.60x | 1.07x | +| `ring` | 1,472 | 1,154/237/81 | 78% | 1.63x | 1.28x | +| `libcrux` | 72 | 72/0/0 | 100% | 1.79x | 1.72x | +| `dryoc` | 320 | 293/22/5 | 92% | 1.81x | 1.85x | +| `rustcrypto` | 2,440 | 1,783/529/128 | 73% | 1.87x | 1.21x | +| `tiny-keccak` | 352 | 237/95/20 | 67% | 1.92x | 2.10x | +| `crc-fast` | 264 | 153/101/10 | 58% | 2.15x | 1.20x | +| `sha3` | 368 | 324/44/0 | 88% | 2.32x | 2.15x | +| `crc32fast` | 88 | 79/4/5 | 90% | 2.75x | 2.06x | +| `crc64fast` | 88 | 73/12/3 | 83% | 2.78x | 2.49x | +| `rustcrypto-p256` | 64 | 56/0/8 | 88% | 3.03x | 3.10x | +| `rustcrypto-p384` | 64 | 56/0/8 | 88% | 3.06x | 5.50x | +| `crc32c` | 88 | 83/3/2 | 94% | 3.13x | 2.29x | +| `fips203` | 72 | 72/0/0 | 100% | 5.28x | 6.07x | +| `rustcrypto-rsa` | 72 | 72/0/0 | 100% | 6.07x | 6.50x | +| `crc` | 264 | 262/0/2 | 99% | 25.76x | 46.98x | ## macOS Local Snapshot -The macOS Apple Silicon run is local evidence from the 2026-07-04 full benchmark at commit `596498f`. It is useful for Apple Silicon planning but is not folded into Linux CI release claims. The ML-KEM row uses the same artifact's public ML-KEM rows. +The macOS Apple Silicon run is local evidence from the 2026-07-04 full benchmark at commit `596498f`, carried forward unchanged in this refresh. It is useful for Apple Silicon planning but is not folded into Linux CI release claims. The ML-KEM row uses the same artifact's public ML-KEM rows. | Scope | Pairs | W/T/L | Win % | Geomean | Median | | --- | --- | --- | --- | --- | --- | @@ -327,29 +344,28 @@ The macOS Apple Silicon run is local evidence from the 2026-07-04 full benchmark ## README Numbers -- **Headline:** 4,052 of 6,750 matched Linux CI fastest-external comparisons are wins; 6,101 are wins or ties. Linux CI geomean is 1.59x. -- **Checksums:** 5.18x geomean across 693 Linux CI fastest-external rows; W/T/L 517/115/61. -- **Hashes/MACs/XOFs:** 1.35x geomean across 3,726 Linux CI fastest-external rows; W/T/L 2,028/1,457/241. -- **Auth/KDF:** 1.25x geomean across 180 Linux CI fastest-external rows; W/T/L 159/20/1. -- **Password hashing:** 1.07x geomean across 135 Linux CI fastest-external rows; W/T/L 69/28/38. -- **Public-key:** 1.33x geomean across 333 Linux CI fastest-external rows; W/T/L 217/73/43. -- **RSA:** 1.55x geomean across 99 Linux CI fastest-external rows; W/T/L 89/2/8. -- **AEAD:** 1.56x geomean across 1,584 Linux CI fastest-external rows; W/T/L 973/354/257. -- **ML-KEM:** 1.49x geomean across 81 Linux CI fastest-external rows; W/T/L 68/4/9. -- **ECDSA P-256/P-384:** 1.45x Linux CI geomean across 144 fastest-external rows; W/T/L 116/7/21. -- **Current top losses:** `argon2id-owasp` / `hash`: 0.97x geomean across 9 rows; W/T/L 4/1/4; pressure `rustcrypto` 4; `ed25519` / `verify`: 1.00x geomean across 36 rows; W/T/L 7/20/9; pressure `ring` 6, `dalek` 3; `mlkem1024` / `keygen`: 1.00x geomean across 9 rows; W/T/L 6/0/3; pressure `aws-lc-rs` 3; `blake2b256` / `streaming`: 1.02x geomean across 27 rows; W/T/L 10/16/1; pressure `rustcrypto` 1; `argon2id-small` / `hash`: 1.02x geomean across 27 rows; W/T/L 12/4/11; pressure `rustcrypto` 11. +- **Headline:** 3,780 of 6,144 matched Linux CI fastest-external comparisons are wins; 5,475 are wins or ties. Linux CI geomean is 1.62x. +- **Checksums:** 6.18x geomean across 616 Linux CI fastest-external rows; W/T/L 476/118/22. +- **Hashes/MACs/XOFs:** 1.35x geomean across 3,456 Linux CI fastest-external rows; W/T/L 1,926/1,181/349. +- **Auth/KDF:** 1.28x geomean across 160 Linux CI fastest-external rows; W/T/L 140/20/0. +- **Password hashing:** 1.07x geomean across 120 Linux CI fastest-external rows; W/T/L 55/27/38. +- **Public-key:** 1.09x geomean across 296 Linux CI fastest-external rows; W/T/L 187/59/50. +- **RSA:** 1.65x geomean across 88 Linux CI fastest-external rows; W/T/L 86/2/0. +- **AEAD:** 1.61x geomean across 1,408 Linux CI fastest-external rows; W/T/L 910/288/210. +- **ML-KEM:** 1.55x geomean across 72 Linux CI fastest-external rows; W/T/L 64/0/8. +- **ECDSA P-256/P-384:** 0.87x Linux CI geomean across 128 fastest-external rows; W/T/L 88/7/33. +- **Current top losses:** `ecdsa-p384` / `sign`: 0.70x geomean across 32 rows; W/T/L 12/0/20; pressure `aws-lc-rs` 16, `rustcrypto-p384` 4; `ecdsa-p256` / `verify`: 0.84x geomean across 32 rows; W/T/L 20/7/5; pressure `rustcrypto-p256` 4, `aws-lc-rs` 1; `rapidhash-stream` / `one-write`: 0.87x geomean across 88 rows; W/T/L 27/25/36; pressure `rapidhash` 36; `ecdsa-p256` / `sign`: 0.91x geomean across 32 rows; W/T/L 28/0/4; pressure `ring` 4; `argon2id-owasp` / `hash`: 0.98x geomean across 8 rows; W/T/L 3/1/4; pressure `rustcrypto` 3, `dryoc` 1. ## Raw Results | Platform | Mode | Date/time | Parsed rows | Result | | --- | --- | --- | --- | --- | -| AMD Zen4 | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/amd-zen4/results.txt` | -| AMD Zen5 | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/amd-zen5/results.txt` | -| AWS Graviton3 | `ci` | `2026-07-04 15_26_12` | 2,367 | `benchmark_results/2026-07-04/linux/graviton3/results.txt` | -| AWS Graviton4 | `ci` | `2026-07-04 15_26_12` | 2,367 | `benchmark_results/2026-07-04/linux/graviton4/results.txt` | -| IBM Power10 | `ci` | `2026-07-04 15_26_12` | 2,107 | `benchmark_results/2026-07-04/linux/ibm-power10/results.txt` | -| IBM z16/s390x | `ci` | `2026-07-04 15_26_12` | 2,107 | `benchmark_results/2026-07-04/linux/ibm-s390x/results.txt` | -| Intel Ice Lake | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/intel-icl/results.txt` | -| Intel Sapphire Rapids | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/intel-spr/results.txt` | -| RISE RISC-V | `ci` | `2026-07-04 15_26_12` | 2,356 | `benchmark_results/2026-07-04/linux/rise-riscv/results.txt` | +| AMD Zen4 | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/amd-zen4/results.txt` | +| AMD Zen5 | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/amd-zen5/results.txt` | +| AWS Graviton3 | `ci` | `2026-08-18 21_03_07` | 2,308 | `benchmark_results/2026-08-18/linux/graviton3/results.txt` | +| AWS Graviton4 | `ci` | `2026-08-18 21_03_07` | 2,308 | `benchmark_results/2026-08-18/linux/graviton4/results.txt` | +| IBM Power10 | `ci` | `2026-08-18 21_03_07` | 2,055 | `benchmark_results/2026-08-18/linux/ibm-power10/results.txt` | +| IBM z16/s390x | `ci` | `2026-08-18 21_03_07` | 2,055 | `benchmark_results/2026-08-18/linux/ibm-s390x/results.txt` | +| Intel Ice Lake | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/intel-icl/results.txt` | +| Intel Sapphire Rapids | `ci` | `2026-08-18 21_03_07` | 2,304 | `benchmark_results/2026-08-18/linux/intel-spr/results.txt` | | macOS Apple Silicon | `local` | `2026-07-04 12_28_04` | 2,277 | `benchmark_results/2026-07-04/macos/aarch64/results.txt` | diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 00000000..be54ebfb --- /dev/null +++ b/clippy.toml @@ -0,0 +1,2 @@ +check-private-items = true +check-incompatible-msrv-in-tests = true diff --git a/deny.toml b/deny.toml index 8b889930..bfe406b2 100644 --- a/deny.toml +++ b/deny.toml @@ -1,30 +1,27 @@ [graph] -# Target coverage for rscrypto - canonical list for dependency analysis -# See .cargo/config.toml for per-target rustflags +# Dependency-analysis projection of .config/target-matrix.json. targets = [ "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", "aarch64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-musl", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "x86_64-unknown-linux-musl", - "aarch64-pc-windows-msvc", "aarch64-unknown-none", - "loongarch64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "riscv32imac-unknown-none-elf", "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-none-elf", "s390x-unknown-linux-gnu", "thumbv6m-none-eabi", + "wasm32-unknown-unknown", + "wasm32-wasip1", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", "x86_64-unknown-none", ] [advisories] -yanked = "warn" +yanked = "deny" # RustCrypto `rsa` is a dev-only differential oracle and benchmark baseline. # rscrypto never exposes it in production paths; production RSA verification # uses `src/auth/rsa.rs`. Keep this scoped to the known Marvin advisory until diff --git a/docs/benchmarking.md b/docs/benchmarking.md index f3235c0e..234e2bb3 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -7,11 +7,12 @@ Benchmark numbers are only meaningful with their platform, commit, feature set, and comparison shape. Treat every headline number as a pointer to the raw results in [`benchmark_results/`](../benchmark_results/). -The published 2026-07-04 aggregate is archival, not an equivalent-work -performance claim. Its RustCrypto HMAC-SHA-256 rows include key setup inside -the timed loop while the rscrypto, `ring`, and AWS-LC rows reuse keyed state. -The current benchmark source corrects that mismatch; publish a new aggregate -only after regenerating the complete artifact. +The published aggregate is the 2026-08-18 eight-runner Linux CI pass at commit +`7eb44e9`. The earlier RustCrypto HMAC-SHA-256 mismatch—key setup inside the +timed loop while the rscrypto, `ring`, and AWS-LC rows reused keyed state—is +corrected in the benchmark source and in this artifact, so the aggregate is an +equivalent-work claim. The RISE RISC-V runner did not execute in that run, so +row counts are not comparable to the nine-runner 2026-07-04 snapshot. ## Read the numbers @@ -26,8 +27,10 @@ Values above `1.00x` mean `rscrypto` was faster for that row. Values below The published W/T/L summaries classify ratios above `1.05x` as wins, ratios from `0.95x` through `1.05x` as ties, and ratios below `0.95x` as losses. Use -individual equivalent-work rows—not the archival aggregate—when a primitive or -message size matters to a deployment. +individual rows—not the crate-wide aggregate—when a primitive or message size +matters to a deployment. A single regressed runner can move a whole category +aggregate: in the 2026-08-18 pass the s390x ECDSA regression pulls the ECDSA +geomean from above parity to `0.87x`. ## Published sources diff --git a/docs/features.md b/docs/features.md index 565c2d36..e7b70b36 100644 --- a/docs/features.md +++ b/docs/features.md @@ -14,28 +14,28 @@ algorithm feature explicitly. ```toml # One algorithm, no_std. -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } # RSA public-key import and verification, no_std + alloc. -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa"] } # RSA key generation, signing, encryption, and private-operation blinding. -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa", "getrandom"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa", "getrandom"] } # ECDSA P-256/SHA-256 and P-384/SHA-384 signing and verification. -rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa"] } # FIPS 203 ML-KEM-512/768/1024 KEM APIs with caller-supplied randomness. -rscrypto = { version = "0.7.8", default-features = false, features = ["ml-kem"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ml-kem"] } # Everything. -rscrypto = { version = "0.7.8", features = ["full", "getrandom"] } +rscrypto = { version = "0.8.1", features = ["full", "getrandom"] } # Everything, with parallel BLAKE3 / Argon2 lanes via Rayon. -rscrypto = { version = "0.7.8", features = ["full", "parallel", "getrandom"] } +rscrypto = { version = "0.8.1", features = ["full", "parallel", "getrandom"] } # Audit-constrained: makes runtime dispatch ignore host SIMD/ASM capabilities. -rscrypto = { version = "0.7.8", features = ["full", "portable-only"] } +rscrypto = { version = "0.8.1", features = ["full", "portable-only"] } ``` ## Complete feature index diff --git a/docs/migration/README.md b/docs/migration/README.md index 01f2ecb3..e5ce58e8 100644 --- a/docs/migration/README.md +++ b/docs/migration/README.md @@ -29,12 +29,12 @@ For projects upgrading rscrypto itself, start with | From | To | Status | |---|---|---| -| [`blake3`](blake3.md) | `Blake3` | Verified against `blake3 1.8.5` | +| [`blake3`](blake3.md) | `Blake3` | Verified against `blake3 1.8.6` | | [`sha2`](RustCrypto/sha2.md) (RustCrypto) | `Sha224`, `Sha256`, `Sha384`, `Sha512`, `Sha512_256` | Verified against `sha2 0.11.0` | | [`sha3`](RustCrypto/sha3.md) (RustCrypto) | `Sha3_224`, `Sha3_256`, `Sha3_384`, `Sha3_512`, `Shake128`, `Shake256`, `Cshake128`, `Cshake256` | Verified against `sha3 0.12.0` | -| [`blake2`](RustCrypto/blake2.md) (RustCrypto) | `Blake2b256`, `Blake2b512`, `Blake2s128`, `Blake2s256` | Verified against `blake2 0.11.0-rc.6` | +| [`blake2`](RustCrypto/blake2.md) (RustCrypto) | `Blake2b256`, `Blake2b512`, `Blake2s128`, `Blake2s256` | Verified against `blake2 0.10.6` | | [`ascon-hash`](RustCrypto/ascon-hash.md) (RustCrypto) | `AsconHash256`, `AsconXof`, `AsconCxof128` | Verified against `ascon-hash 0.4.0` | -| [`xxhash-rust`](xxhash-rust.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | Verified against `xxhash-rust 0.8.16` | +| [`xxhash-rust`](xxhash-rust.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | Verified against `xxhash-rust 0.8.18` | | [`twox-hash`](twox-hash.md) | `Xxh3`, `Xxh3_128`, `Xxh3Hasher`, `Xxh3_128Hasher`, `Xxh3BuildHasher` | API migration guidance; XXH3 output covered by `xxhash-rust` oracle tests | | [`rapidhash`](rapidhash.md) | `RapidHash64`, `RapidStreamHasher`, `RapidHasher`, `RapidSeededState`, `RapidRandomState` | Verified against `rapidhash 4.5.1` | @@ -53,25 +53,25 @@ For projects upgrading rscrypto itself, start with | From | To | Status | |---|---|---| | [`aes-gcm`](RustCrypto/aes-gcm.md) (RustCrypto) | `Aes128Gcm`, `Aes256Gcm` | Verified against `aes-gcm 0.11.0` | -| [`aes-gcm-siv`](RustCrypto/aes-gcm-siv.md) (RustCrypto) | `Aes128GcmSiv`, `Aes256GcmSiv` | Verified against `aes-gcm-siv 0.11.1` | +| [`aes-gcm-siv`](RustCrypto/aes-gcm-siv.md) (RustCrypto) | `Aes128GcmSiv`, `Aes256GcmSiv` | Verified against `aes-gcm-siv 0.12.0` | | [`chacha20poly1305`](RustCrypto/chacha20poly1305.md) (RustCrypto) | `ChaCha20Poly1305`, `XChaCha20Poly1305` | Verified against `chacha20poly1305 0.11.0` | | [`ascon-aead`](RustCrypto/ascon-aead.md) (RustCrypto) | `AsconAead128` | Verified against `ascon-aead 0.6.0` | -| [`aegis`](aegis.md) | `Aegis256` | Verified against `aegis 0.9.12` | +| [`aegis`](aegis.md) | `Aegis256` | Verified against `aegis 0.9.15` | ## Signatures and key exchange | From | To | Status | |---|---|---| -| [`p256`](RustCrypto/p256.md) / [`p384`](RustCrypto/p384.md) (RustCrypto) | `EcdsaP256SecretKey`, `EcdsaP384SecretKey`, `EcdsaP256PublicKey`, `EcdsaP384PublicKey`, raw/DER signatures | Signing and verification tested against RustCrypto `p256 0.14.0` / `p384 0.13.1` | -| [`ed25519-dalek`](RustCrypto/ed25519-dalek.md) | `Ed25519SecretKey`, `Ed25519PublicKey`, `Ed25519Signature`, `Ed25519Keypair` | Verified against `ed25519-dalek 2.2.0` | +| [`p256`](RustCrypto/p256.md) / [`p384`](RustCrypto/p384.md) (RustCrypto) | `EcdsaP256SecretKey`, `EcdsaP384SecretKey`, `EcdsaP256PublicKey`, `EcdsaP384PublicKey`, raw/DER signatures | Signing and verification tested against RustCrypto `p256 0.14.0` / `p384 0.14.0` | +| [`ed25519-dalek`](RustCrypto/ed25519-dalek.md) | `Ed25519SecretKey`, `Ed25519PublicKey`, `Ed25519Signature`, `Ed25519Keypair` | Verified against `ed25519-dalek 3.0.0` | | [`rsa`](RustCrypto/rsa.md) (RustCrypto) | `RsaPublicKey`, `RsaPrivateKey`, RSA-PSS, RSASSA-PKCS1-v1_5, OAEP | Partial; verified through CAVP, Wycheproof, and RustCrypto/ring/OpenSSL oracles | -| [`x25519-dalek`](RustCrypto/x25519-dalek.md) | `X25519SecretKey`, `X25519PublicKey`, `X25519SharedSecret` | Verified against `x25519-dalek 2.0.1` | +| [`x25519-dalek`](RustCrypto/x25519-dalek.md) | `X25519SecretKey`, `X25519PublicKey`, `X25519SharedSecret` | Verified against `x25519-dalek 3.0.0` | ## Password hashing | From | To | Status | |---|---|---| -| [`argon2`](RustCrypto/argon2.md) (RustCrypto) | Raw `Argon2{d,i,id}` KDFs; bounded `Argon2idPassword` records | Verified against `argon2 0.6.0-rc.8` | +| [`argon2`](RustCrypto/argon2.md) (RustCrypto) | Raw `Argon2{d,i,id}` KDFs; bounded `Argon2idPassword` records | Verified against `argon2 0.5.3` | | [`scrypt`](RustCrypto/scrypt.md) (RustCrypto) | Raw `Scrypt` KDF; bounded `ScryptPassword` records | Verified against `scrypt 0.12.0` | ## Stack migrations diff --git a/docs/migration/RustCrypto/aes-gcm-siv.md b/docs/migration/RustCrypto/aes-gcm-siv.md index b1d666aa..bff269c5 100644 --- a/docs/migration/RustCrypto/aes-gcm-siv.md +++ b/docs/migration/RustCrypto/aes-gcm-siv.md @@ -4,14 +4,14 @@ > `Payload { msg, aad }` with rscrypto's named types and a caller-buffer API. > The RFC 8452 construction and combined ciphertext-and-tag bytes are unchanged. -Verified against `aes-gcm-siv = "0.11.1"` and the `rscrypto` 0.7.8 line. +Verified against `aes-gcm-siv = "0.12.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/aes128gcmsiv_oracle.rs`, `tests/aes256gcmsiv_oracle.rs`, and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`aes-gcm-siv` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`aes-gcm-siv` 0.12.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `aes-gcm-siv = "0.11"` | `rscrypto = { version = "0.7.8", features = ["aes-gcm-siv"] }` | +| Cargo dep | `aes-gcm-siv = "0.12"` | `rscrypto = { version = "0.8.1", features = ["aes-gcm-siv"] }` | | Import | `use aes_gcm_siv::{Aes256GcmSiv, Key, Nonce, KeyInit, aead::{Aead, Payload}};` | `use rscrypto::{Aead, Aes256GcmSiv, Aes256GcmSivKey, aead::{Nonce96, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -20,13 +20,13 @@ Evidence: `tests/aes128gcmsiv_oracle.rs`, `tests/aes256gcmsiv_oracle.rs`, and `t ```toml # Before [dependencies] -aes-gcm-siv = "0.11" +aes-gcm-siv = "0.12" ``` ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["aes-gcm-siv"] } +rscrypto = { version = "0.8.1", features = ["aes-gcm-siv"] } ``` ## Algorithm map @@ -50,10 +50,10 @@ The migration recipe below uses `Aes256GcmSiv` throughout; substitute use aes_gcm_siv::{Aes256GcmSiv, Key, Nonce, KeyInit}; use aes_gcm_siv::aead::{Aead, Payload}; -let key = Key::::from_slice(&[0u8; 32]); -let cipher = Aes256GcmSiv::new(key); -let nonce = Nonce::from_slice(&[0u8; 12]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::::from([0u8; 32]); +let cipher = Aes256GcmSiv::new(&key); +let nonce = Nonce::from([0u8; 12]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust diff --git a/docs/migration/RustCrypto/aes-gcm.md b/docs/migration/RustCrypto/aes-gcm.md index f0698bae..f21f7304 100644 --- a/docs/migration/RustCrypto/aes-gcm.md +++ b/docs/migration/RustCrypto/aes-gcm.md @@ -4,14 +4,14 @@ Replace the `Aes256Gcm` / `Key` / `Nonce` / `Payload { msg, aad }` builder with rscrypto's named types and a caller-buffer API. The combined ciphertext-and-tag bytes remain interoperable. -Verified against `aes-gcm = "0.11.0"` and the `rscrypto` 0.7.8 line. +Verified against `aes-gcm = "0.11.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/aes128gcm_oracle.rs`, `tests/aes256gcm_oracle.rs`, and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`aes-gcm` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`aes-gcm` 0.11.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `aes-gcm = "0.11"` | `rscrypto = { version = "0.7.8", features = ["aes-gcm"] }` | +| Cargo dep | `aes-gcm = "0.11"` | `rscrypto = { version = "0.8.1", features = ["aes-gcm"] }` | | Import | `use aes_gcm::{Aes256Gcm, Key, Nonce, KeyInit, aead::{Aead, Payload}};` | `use rscrypto::{Aead, Aes256Gcm, Aes256GcmKey, aead::{Nonce96, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` (returns `Vec`) | `cipher.encrypt(&nonce, aad, msg, &mut out)?` (writes into caller buffer) | @@ -26,7 +26,7 @@ aes-gcm = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["aes-gcm"] } +rscrypto = { version = "0.8.1", features = ["aes-gcm"] } ``` ## Algorithm map @@ -50,10 +50,10 @@ The migration recipe below uses `Aes256Gcm` throughout; substitute use aes_gcm::{Aes256Gcm, Key, Nonce, KeyInit}; use aes_gcm::aead::{Aead, Payload}; -let key = Key::::from_slice(&[0u8; 32]); -let cipher = Aes256Gcm::new(key); -let nonce = Nonce::from_slice(&[0u8; 12]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::::from([0u8; 32]); +let cipher = Aes256Gcm::new(&key); +let nonce = Nonce::from([0u8; 12]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); // ct: Vec, last 16 bytes are the tag. ``` diff --git a/docs/migration/RustCrypto/argon2.md b/docs/migration/RustCrypto/argon2.md index 911ef625..b4f0883b 100644 --- a/docs/migration/RustCrypto/argon2.md +++ b/docs/migration/RustCrypto/argon2.md @@ -5,16 +5,16 @@ rscrypto separates two jobs that should not share an API: - `Argon2d`, `Argon2i`, and `Argon2id` are deterministic raw KDFs. - `Argon2idPassword` generates canonical password records and verifies hostile PHC input under finite resource limits. -The raw implementations are checked against RFC 9106 vectors and the RustCrypto `argon2 0.6.0-rc.8` oracle in `tests/argon2_vectors.rs`, `tests/argon2_differential.rs`, `tests/argon2_kernels.rs`, and `tests/argon2_parallel.rs`. +The raw implementations are checked against RFC 9106 vectors and the RustCrypto `argon2 0.5.3` oracle in `tests/argon2_vectors.rs`, `tests/argon2_differential.rs`, `tests/argon2_kernels.rs`, and `tests/argon2_parallel.rs`. ## Cargo features ```toml # Raw Argon2 KDF -rscrypto = { version = "0.7.8", default-features = false, features = ["argon2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["argon2"] } # Password-record generation and verification with caller-owned entropy -rscrypto = { version = "0.7.8", default-features = false, features = [ +rscrypto = { version = "0.8.1", default-features = false, features = [ "argon2", "phc-strings", ] } diff --git a/docs/migration/RustCrypto/ascon-aead.md b/docs/migration/RustCrypto/ascon-aead.md index d7cb6374..2c8436e4 100644 --- a/docs/migration/RustCrypto/ascon-aead.md +++ b/docs/migration/RustCrypto/ascon-aead.md @@ -4,14 +4,14 @@ > `Payload { msg, aad }` with rscrypto's named types and a caller-buffer API. > NIST SP 800-232 specifies a 16-byte key, nonce, and tag. -Verified against `ascon-aead = "0.6.0"` and the `rscrypto` 0.7.8 line. +Verified against `ascon-aead = "0.6.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/ascon_aead_oracle.rs`. ## TL;DR -| | Before (`ascon-aead` 0.6.x) | After (`rscrypto` 0.7.8) | +| | Before (`ascon-aead` 0.6.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `ascon-aead = "0.6"` | `rscrypto = { version = "0.7.8", features = ["ascon-aead"] }` | +| Cargo dep | `ascon-aead = "0.6"` | `rscrypto = { version = "0.8.1", features = ["ascon-aead"] }` | | Import | `use ascon_aead::{AsconAead128, Key, Nonce, aead::{Aead, KeyInit, Payload}};` | `use rscrypto::{Aead, AsconAead128, AsconAead128Key, aead::{Nonce128, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -26,7 +26,7 @@ ascon-aead = "0.6" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["ascon-aead"] } +rscrypto = { version = "0.8.1", features = ["ascon-aead"] } ``` ## Algorithm map @@ -46,10 +46,10 @@ rscrypto = { version = "0.7.8", features = ["ascon-aead"] } use ascon_aead::{AsconAead128, Key, Nonce}; use ascon_aead::aead::{Aead, KeyInit, Payload}; -let key = Key::::from_slice(&[0u8; 16]); -let cipher = AsconAead128::new(key); -let nonce = Nonce::::from_slice(&[0u8; 16]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::::from([0u8; 16]); +let cipher = AsconAead128::new(&key); +let nonce = Nonce::::from([0u8; 16]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust diff --git a/docs/migration/RustCrypto/ascon-hash.md b/docs/migration/RustCrypto/ascon-hash.md index 51cada0a..538925b3 100644 --- a/docs/migration/RustCrypto/ascon-hash.md +++ b/docs/migration/RustCrypto/ascon-hash.md @@ -4,14 +4,14 @@ > the SP 800-232 Ascon-Hash256 parameter set; the output bytes, `update`, and > `finalize` flow are unchanged. -Verified against `ascon-hash = "0.4.0"` and the `rscrypto` 0.7.8 line. +Verified against `ascon-hash = "0.4.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/ascon_official_vectors.rs`, `tests/ascon_hash_oracle.rs`, `tests/ascon_cxof_vectors.rs`, and `tests/ascon_differential.rs`. ## TL;DR -| | Before (`ascon-hash` 0.4.x) | After (`rscrypto` 0.7.8) | +| | Before (`ascon-hash` 0.4.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `ascon-hash = "0.4"` | `rscrypto = { version = "0.7.8", features = ["ascon-hash"] }` | +| Cargo dep | `ascon-hash = "0.4"` | `rscrypto = { version = "0.8.1", features = ["ascon-hash"] }` | | Import | `use ascon_hash::{AsconHash256, digest::Digest};` | `use rscrypto::{AsconHash256, Digest};` | | Call | `AsconHash256::digest(data)` | `AsconHash256::digest(data)` | @@ -26,7 +26,7 @@ ascon-hash = "0.4" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["ascon-hash"] } +rscrypto = { version = "0.8.1", features = ["ascon-hash"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/blake2.md b/docs/migration/RustCrypto/blake2.md index 740ccb12..907b5a5f 100644 --- a/docs/migration/RustCrypto/blake2.md +++ b/docs/migration/RustCrypto/blake2.md @@ -2,16 +2,16 @@ > Same algorithms (RFC 7693), with named convenience types replacing `Blake2b`-style generic instantiations and `Blake2b256::keyed_digest(key, data)` replacing the separate `Blake2bMac` MAC type. -Verified against `blake2 = "0.11.0-rc.6"` and the `rscrypto` 0.7.8 line. +Verified against `blake2 = "0.10.6"` and the `rscrypto` 0.8.1 line. Evidence: `tests/blake2_official_vectors.rs` and `tests/blake2_differential.rs`. Code samples use the 0.10-style names where they remain the clearest migration shape for existing projects. ## TL;DR -| | Before (`blake2` 0.10.x) | After (`rscrypto` 0.7.8) | +| | Before (`blake2` 0.10.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `blake2 = "0.10"` | `rscrypto = { version = "0.7.8", features = ["blake2b", "blake2s"] }` | +| Cargo dep | `blake2 = "0.10"` | `rscrypto = { version = "0.8.1", features = ["blake2b", "blake2s"] }` | | Import | `use blake2::{Blake2b512, Digest};` | `use rscrypto::{Blake2b512, Digest};` | | Call | `Blake2b512::digest(data)` | `Blake2b512::digest(data)` | @@ -28,7 +28,7 @@ blake2 = "0.10" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["blake2b", "blake2s"] } +rscrypto = { version = "0.8.1", features = ["blake2b", "blake2s"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/chacha20poly1305.md b/docs/migration/RustCrypto/chacha20poly1305.md index 1fb07d1c..e90e6462 100644 --- a/docs/migration/RustCrypto/chacha20poly1305.md +++ b/docs/migration/RustCrypto/chacha20poly1305.md @@ -5,14 +5,14 @@ > ChaCha20-Poly1305 and XChaCha20-Poly1305 operations preserve ciphertext and > tag bytes. -Verified against `chacha20poly1305 = "0.11.0"` and the `rscrypto` 0.7.8 line. +Verified against `chacha20poly1305 = "0.11.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/chacha20poly1305.rs`, `tests/xchacha20poly1305.rs`, and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`chacha20poly1305` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`chacha20poly1305` 0.11.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `chacha20poly1305 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["chacha20poly1305", "xchacha20poly1305"] }` | +| Cargo dep | `chacha20poly1305 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["chacha20poly1305", "xchacha20poly1305"] }` | | Import | `use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, KeyInit, aead::{Aead, Payload}};` | `use rscrypto::{Aead, ChaCha20Poly1305, ChaCha20Poly1305Key, aead::{Nonce96, expert::AeadWithNonce}};` | | Encrypt | `cipher.encrypt(nonce, Payload { msg, aad })?` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -29,7 +29,7 @@ chacha20poly1305 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["chacha20poly1305", "xchacha20poly1305"] } +rscrypto = { version = "0.8.1", features = ["chacha20poly1305", "xchacha20poly1305"] } ``` ## Algorithm map @@ -49,10 +49,10 @@ rscrypto = { version = "0.7.8", features = ["chacha20poly1305", "xchacha20poly13 use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, KeyInit}; use chacha20poly1305::aead::{Aead, Payload}; -let key = Key::from_slice(&[0u8; 32]); // Key is non-generic -let cipher = ChaCha20Poly1305::new(key); -let nonce = Nonce::from_slice(&[0u8; 12]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::from([0u8; 32]); // Key is non-generic +let cipher = ChaCha20Poly1305::new(&key); +let nonce = Nonce::from([0u8; 12]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust @@ -76,10 +76,10 @@ cipher.encrypt(&nonce, aad, plaintext, &mut ct)?; use chacha20poly1305::{XChaCha20Poly1305, Key, XNonce, KeyInit}; use chacha20poly1305::aead::{Aead, Payload}; -let key = Key::from_slice(&[0u8; 32]); -let cipher = XChaCha20Poly1305::new(key); -let nonce = XNonce::from_slice(&[0u8; 24]); -let ct = cipher.encrypt(nonce, Payload { msg: plaintext, aad }).unwrap(); +let key = Key::from([0u8; 32]); +let cipher = XChaCha20Poly1305::new(&key); +let nonce = XNonce::from([0u8; 24]); +let ct = cipher.encrypt(&nonce, Payload { msg: plaintext, aad }).unwrap(); ``` ```rust diff --git a/docs/migration/RustCrypto/ed25519-dalek.md b/docs/migration/RustCrypto/ed25519-dalek.md index 14e28ac7..11e904e4 100644 --- a/docs/migration/RustCrypto/ed25519-dalek.md +++ b/docs/migration/RustCrypto/ed25519-dalek.md @@ -4,14 +4,14 @@ > `Ed25519SecretKey` / `Ed25519PublicKey` / `Ed25519Signature`. Signing > preserves RFC 8032 output bytes, and the rscrypto verifier is always strict. -Verified against `ed25519-dalek = "2.2.0"` and the `rscrypto` 0.7.8 line. +Verified against `ed25519-dalek = "3.0.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs`, and `tests/ed25519_wycheproof.rs`. ## TL;DR -| | Before (`ed25519-dalek` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`ed25519-dalek` 3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `ed25519-dalek = "2.2"` | `rscrypto = { version = "0.7.8", features = ["ed25519"] }` | +| Cargo dep | `ed25519-dalek = "3.0"` | `rscrypto = { version = "0.8.1", features = ["ed25519"] }` | | Import | `use ed25519_dalek::{SigningKey, Signer, Verifier};` | `use rscrypto::{Ed25519SecretKey, Ed25519PublicKey, Ed25519Signature};` | | Sign | `signing_key.sign(msg)` | `secret.sign(msg)` | | Verify | `verifying_key.verify_strict(msg, &sig)?` | `public_key.verify(msg, &sig)?` | @@ -21,13 +21,13 @@ Evidence: `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs`, and `te ```toml # Before [dependencies] -ed25519-dalek = "2.2" +ed25519-dalek = "3.0" ``` ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["ed25519"] } +rscrypto = { version = "0.8.1", features = ["ed25519"] } ``` The `ed25519` feature implies `sha2` (Ed25519 uses SHA-512 internally per RFC 8032). @@ -119,7 +119,7 @@ rscrypto's `verify` is *always* strict (rejects small-order public keys, non-can ### Keypair (combined secret + public) ```rust -// Before: no first-class Keypair type in dalek 2.x; use SigningKey directly. +// Before: no first-class Keypair type in dalek 3.x; use SigningKey directly. let signing_key = SigningKey::from_bytes(&seed); let pk = signing_key.verifying_key(); ``` diff --git a/docs/migration/RustCrypto/hkdf.md b/docs/migration/RustCrypto/hkdf.md index 7fe9290f..a54b6b52 100644 --- a/docs/migration/RustCrypto/hkdf.md +++ b/docs/migration/RustCrypto/hkdf.md @@ -2,14 +2,14 @@ > Replace `Hkdf::::new(Some(salt), ikm)` with `HkdfSha256::new(salt, ikm)`. The `Option<&[u8]>` salt becomes a plain `&[u8]` (empty slice == "no salt"); fused one-shot `HkdfSha256::derive_array::(...)` collapses extract+expand. -Verified against `hkdf = "0.13.0"` and the `rscrypto` 0.7.8 line. +Verified against `hkdf = "0.13.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/hkdf_sha256_vectors.rs`, `tests/hkdf_sha384_vectors.rs`, `tests/hkdf_sha512_vectors.rs`, the HKDF proptests, and `tests/hkdf_wycheproof.rs`. ## TL;DR -| | Before (`hkdf` 0.13.x) | After (`rscrypto` 0.7.8) | +| | Before (`hkdf` 0.13.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `hkdf = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["hkdf"] }` | +| Cargo dep | `hkdf = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["hkdf"] }` | | Import | `use hkdf::Hkdf; use sha2::Sha256;` | `use rscrypto::HkdfSha256;` | | Call | `Hkdf::::new(Some(salt), ikm).expand(info, &mut okm)?` | `HkdfSha256::new(salt, ikm).expand(info, &mut okm)?` | @@ -25,7 +25,7 @@ sha2 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["hkdf"] } +rscrypto = { version = "0.8.1", features = ["hkdf"] } ``` The `hkdf` feature implies `hmac` which implies `sha2`. diff --git a/docs/migration/RustCrypto/hmac.md b/docs/migration/RustCrypto/hmac.md index b037de9b..edc7745c 100644 --- a/docs/migration/RustCrypto/hmac.md +++ b/docs/migration/RustCrypto/hmac.md @@ -2,14 +2,14 @@ > Replace `Hmac::` / `Hmac::` (generic over digest) with named rscrypto types such as `HmacSha256` and `HmacSha3_256`. Key construction is infallible, `finalize()` borrows, and one-shot helpers return typed tags with sealed comparison decisions. -Verified against `hmac = "0.13.0"` and the `rscrypto` 0.7.8 line. +Verified against `hmac = "0.13.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/hmac_sha256_vectors.rs`, `tests/hmac_sha2_family_vectors.rs`, `tests/hmac_sha3_vectors.rs`, the HMAC proptests, and `tests/hmac_wycheproof.rs`. ## TL;DR -| | Before (`hmac` 0.13.x) | After (`rscrypto` 0.7.8) | +| | Before (`hmac` 0.13.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `hmac = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["hmac"] }` | +| Cargo dep | `hmac = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["hmac"] }` | | Import | `use hmac::{Hmac, Mac, KeyInit}; use sha2::Sha256;` | `use rscrypto::{HmacSha256, Mac};` | | Call | `Hmac::::new_from_slice(key).unwrap().chain_update(data).finalize().into_bytes()` | `HmacSha256::mac(key, data)` | @@ -25,7 +25,7 @@ sha2 = "0.11" # required as the generic parameter ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["hmac"] } +rscrypto = { version = "0.8.1", features = ["hmac"] } ``` The `hmac` feature implies `sha2`: no second dep to manage for SHA-2 HMAC. Use `features = ["hmac-sha3"]` for HMAC-SHA3, or `features = ["macs"]` when you want both SHA-2 and SHA-3 HMAC families. diff --git a/docs/migration/RustCrypto/p256.md b/docs/migration/RustCrypto/p256.md index affa3d7b..74ffe26c 100644 --- a/docs/migration/RustCrypto/p256.md +++ b/docs/migration/RustCrypto/p256.md @@ -13,9 +13,9 @@ Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and ## TL;DR -| | Before (`p256` 0.14) | After (`rscrypto` 0.7.8) | +| | Before (`p256` 0.14) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `p256 = { version = "0.14", features = ["ecdsa"] }` | `rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p256"] }` | +| Cargo dep | `p256 = { version = "0.14", features = ["ecdsa"] }` | `rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p256"] }` | | Import | `use p256::ecdsa::{SigningKey, Signature};` | `use rscrypto::{EcdsaP256SecretKey, EcdsaP256PublicKey, EcdsaP256Signature};` | | Sign | `signing_key.sign(msg)` | `secret.try_sign(msg)?` | | Verify | `verifying_key.verify(msg, &sig)?` | `public.verify(msg, &sig)?` | @@ -27,7 +27,7 @@ Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and p256 = { version = "0.14", features = ["ecdsa"] } # After -rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p256"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p256"] } ``` The `ecdsa-p256` feature implies `hmac`, which is used for deterministic nonce diff --git a/docs/migration/RustCrypto/p384.md b/docs/migration/RustCrypto/p384.md index 3032e5c0..4f1328b8 100644 --- a/docs/migration/RustCrypto/p384.md +++ b/docs/migration/RustCrypto/p384.md @@ -6,16 +6,16 @@ rscrypto supports the fixed P-384/SHA-384 ECDSA profile, raw `r || s` signatures, DER signature import, SEC1/SPKI public-key import, deterministic signing, and caller-blinded signing. -Verified against RustCrypto `p384 0.13.1`. +Verified against RustCrypto `p384 0.14.0`. Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and `fuzz/target_impls/auth_ecdsa_verify.rs` / `fuzz/target_impls/auth_ecdsa_sign.rs`. ## TL;DR -| | Before (`p384` 0.13) | After (`rscrypto` 0.7.8) | +| | Before (`p384` 0.14) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `p384 = { version = "0.13", features = ["ecdsa"] }` | `rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p384"] }` | +| Cargo dep | `p384 = { version = "0.14", features = ["ecdsa"] }` | `rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p384"] }` | | Import | `use p384::ecdsa::{SigningKey, Signature};` | `use rscrypto::{EcdsaP384SecretKey, EcdsaP384PublicKey, EcdsaP384Signature};` | | Sign | `signing_key.sign(msg)` | `secret.try_sign(msg)?` | | Verify | `verifying_key.verify(msg, &sig)?` | `public.verify(msg, &sig)?` | @@ -24,10 +24,10 @@ Evidence: `tests/ecdsa_oracle.rs`, `src/auth/ecdsa.rs` unit tests, and ```toml # Before -p384 = { version = "0.13", features = ["ecdsa"] } +p384 = { version = "0.14", features = ["ecdsa"] } # After -rscrypto = { version = "0.7.8", default-features = false, features = ["ecdsa-p384"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa-p384"] } ``` The `ecdsa-p384` feature implies `hmac`, which is used for deterministic nonce diff --git a/docs/migration/RustCrypto/pbkdf2.md b/docs/migration/RustCrypto/pbkdf2.md index e7433ecb..4ac1a8ed 100644 --- a/docs/migration/RustCrypto/pbkdf2.md +++ b/docs/migration/RustCrypto/pbkdf2.md @@ -2,14 +2,14 @@ > Replace the free function `pbkdf2_hmac::(password, salt, iters, &mut out)` with `Pbkdf2Sha256::derive_key_array::(password, salt, iters)?`. The password helpers enforce the current PBKDF2 iteration and salt floors by default, while `*_primitive` APIs remain available for RFC vectors and legacy compatibility. -Verified against `pbkdf2 = "0.13.0"` and the `rscrypto` 0.7.8 line. +Verified against `pbkdf2 = "0.13.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/pbkdf2_kat_vectors.rs`, `tests/pbkdf2_differential.rs`, and `tests/pbkdf2_wycheproof.rs`. ## TL;DR -| | Before (`pbkdf2` 0.13.x) | After (`rscrypto` 0.7.8) | +| | Before (`pbkdf2` 0.13.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `pbkdf2 = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["pbkdf2"] }` | +| Cargo dep | `pbkdf2 = "0.13"` + `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["pbkdf2"] }` | | Import | `use pbkdf2::pbkdf2_hmac; use sha2::Sha256;` | `use rscrypto::Pbkdf2Sha256;` | | Call | `pbkdf2_hmac::(pw, salt, iters, &mut okm)` | `Pbkdf2Sha256::derive_key(pw, salt, iters, &mut okm)?` | @@ -25,7 +25,7 @@ sha2 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["pbkdf2"] } +rscrypto = { version = "0.8.1", features = ["pbkdf2"] } ``` The `pbkdf2` feature implies `hmac` which implies `sha2`. diff --git a/docs/migration/RustCrypto/rsa.md b/docs/migration/RustCrypto/rsa.md index f565ef57..7bcf1867 100644 --- a/docs/migration/RustCrypto/rsa.md +++ b/docs/migration/RustCrypto/rsa.md @@ -17,10 +17,10 @@ interoperability tests used by the stack migration guides. rsa = { version = "0.9", features = ["sha2"] } # After -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa"] } # After, when generating keys or using randomized private/encryption APIs -rscrypto = { version = "0.7.8", default-features = false, features = ["rsa", "getrandom"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["rsa", "getrandom"] } ``` ## Map diff --git a/docs/migration/RustCrypto/scrypt.md b/docs/migration/RustCrypto/scrypt.md index ff188461..46668b11 100644 --- a/docs/migration/RustCrypto/scrypt.md +++ b/docs/migration/RustCrypto/scrypt.md @@ -11,10 +11,10 @@ The implementation is checked against RFC 7914 vectors and the RustCrypto `scryp ```toml # Raw scrypt KDF -rscrypto = { version = "0.7.8", default-features = false, features = ["scrypt"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["scrypt"] } # Password-record generation and verification with caller-owned entropy -rscrypto = { version = "0.7.8", default-features = false, features = [ +rscrypto = { version = "0.8.1", default-features = false, features = [ "scrypt", "phc-strings", ] } diff --git a/docs/migration/RustCrypto/sha2.md b/docs/migration/RustCrypto/sha2.md index 60f62560..116d77ff 100644 --- a/docs/migration/RustCrypto/sha2.md +++ b/docs/migration/RustCrypto/sha2.md @@ -2,14 +2,14 @@ > Same algorithms (FIPS 180-4), same `Sha224` / `Sha256` / `Sha384` / `Sha512` / `Sha512_256` names, same `new()` / `update()` / `finalize()` shape. The output type changes from `Output` (a `GenericArray`) to `[u8; N]`, and `finalize()` now borrows instead of consuming. -Verified against `sha2 = "0.11.0"` and the `rscrypto` 0.7.8 line. +Verified against `sha2 = "0.11.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/sha2_official_vectors.rs`, `tests/sha256_differential.rs`, and `tests/sha512_differential.rs`. ## TL;DR -| | Before (`sha2` 0.11.x) | After (`rscrypto` 0.7.8) | +| | Before (`sha2` 0.11.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `sha2 = "0.11"` | `rscrypto = { version = "0.7.8", features = ["sha2"] }` | +| Cargo dep | `sha2 = "0.11"` | `rscrypto = { version = "0.8.1", features = ["sha2"] }` | | Import | `use sha2::{Sha256, Digest};` | `use rscrypto::Sha256;` | | Call | `Sha256::digest(data)` | `Sha256::digest(data)` | @@ -24,7 +24,7 @@ sha2 = "0.11" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["sha2"] } +rscrypto = { version = "0.8.1", features = ["sha2"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/sha3.md b/docs/migration/RustCrypto/sha3.md index b2b59e0a..e27ce7f1 100644 --- a/docs/migration/RustCrypto/sha3.md +++ b/docs/migration/RustCrypto/sha3.md @@ -2,14 +2,14 @@ > Same algorithms (FIPS 202: SHA3-224/256/384/512, SHAKE128/256; SP 800-185: cSHAKE128/256), `[u8; N]` outputs replace `Output`, and the SHAKE/cSHAKE reader chain uses `squeeze(&mut out)`. -Verified against `sha3 = "0.12.0"` and the `rscrypto` 0.7.8 line. +Verified against `sha3 = "0.12.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/sha3_official_vectors.rs`, `tests/sha3_differential.rs`, `tests/shake128_differential.rs`, `tests/shake256_differential.rs`, `tests/cshake256_nist_vectors.rs`, and `tests/cshake256_differential.rs`. ## TL;DR -| | Before (`sha3` 0.12.x) | After (`rscrypto` 0.7.8) | +| | Before (`sha3` 0.12.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `sha3 = "0.12"` | `rscrypto = { version = "0.7.8", features = ["sha3"] }` | +| Cargo dep | `sha3 = "0.12"` | `rscrypto = { version = "0.8.1", features = ["sha3"] }` | | Import | `use sha3::{Sha3_256, Digest};` | `use rscrypto::{Sha3_256, Digest};` | | Call | `Sha3_256::digest(data)` | `Sha3_256::digest(data)` | @@ -24,7 +24,7 @@ sha3 = "0.12" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["sha3"] } +rscrypto = { version = "0.8.1", features = ["sha3"] } ``` ## Algorithm map diff --git a/docs/migration/RustCrypto/x25519-dalek.md b/docs/migration/RustCrypto/x25519-dalek.md index b2e21aff..5f8590b0 100644 --- a/docs/migration/RustCrypto/x25519-dalek.md +++ b/docs/migration/RustCrypto/x25519-dalek.md @@ -5,14 +5,14 @@ > `X25519SharedSecret`. Valid RFC 7748 inputs produce the same shared-secret > bytes; rscrypto returns an explicit error for an all-zero result. -Verified against `x25519-dalek = "2.0.1"` and the `rscrypto` 0.7.8 line. +Verified against `x25519-dalek = "3.0.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs`, and `tests/x25519_wycheproof.rs`. ## TL;DR -| | Before (`x25519-dalek` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`x25519-dalek` 3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `x25519-dalek = { version = "2.0", features = ["static_secrets"] }` | `rscrypto = { version = "0.7.8", features = ["x25519"] }` | +| Cargo dep | `x25519-dalek = { version = "3.0", features = ["static_secrets"] }` | `rscrypto = { version = "0.8.1", features = ["x25519"] }` | | Import | `use x25519_dalek::{StaticSecret, PublicKey};` | `use rscrypto::{X25519SecretKey, X25519PublicKey};` | | DH | `secret.diffie_hellman(&peer_pub)` (returns `SharedSecret`) | `secret.diffie_hellman(&peer_pub)?` (returns `Result`) | @@ -21,13 +21,13 @@ Evidence: `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs`, and `tests/x25519 ```toml # Before [dependencies] -x25519-dalek = { version = "2.0", features = ["static_secrets"] } +x25519-dalek = { version = "3.0", features = ["static_secrets"] } ``` ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["x25519"] } +rscrypto = { version = "0.8.1", features = ["x25519"] } ``` The `x25519` feature has no transitive dependencies: X25519 needs nothing beyond Curve25519 arithmetic. diff --git a/docs/migration/aegis.md b/docs/migration/aegis.md index 83796170..fd8f771c 100644 --- a/docs/migration/aegis.md +++ b/docs/migration/aegis.md @@ -4,14 +4,14 @@ > rscrypto's caller-buffer `encrypt(&nonce, aad, msg, &mut out)`. The mapped > AEGIS-256 operation preserves ciphertext and tag bytes. -Verified against `aegis = "0.9.12"` and the `rscrypto` 0.7.8 line. +Verified against `aegis = "0.9.15"` and the `rscrypto` 0.8.1 line. Evidence: `tests/aegis256_oracle.rs` and `tests/aead_wycheproof.rs`. ## TL;DR -| | Before (`aegis` 0.9.x) | After (`rscrypto` 0.7.8) | +| | Before (`aegis` 0.9.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `aegis = "0.9"` | `rscrypto = { version = "0.7.8", features = ["aegis256"] }` | +| Cargo dep | `aegis = "0.9"` | `rscrypto = { version = "0.8.1", features = ["aegis256"] }` | | Import | `use aegis::aegis256::Aegis256;` | `use rscrypto::{Aead, Aegis256, Aegis256Key, aead::{Nonce256, expert::AeadWithNonce}};` | | Encrypt | `Aegis256::<16>::new(&key, &nonce).encrypt(msg, aad) -> (Vec, [u8; 16])` | `cipher.encrypt(&nonce, aad, msg, &mut out)?` | @@ -26,7 +26,7 @@ aegis = "0.9" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["aegis256"] } +rscrypto = { version = "0.8.1", features = ["aegis256"] } ``` ## Algorithm map diff --git a/docs/migration/api-boundaries.md b/docs/migration/api-boundaries.md index 5ceb929c..74e02204 100644 --- a/docs/migration/api-boundaries.md +++ b/docs/migration/api-boundaries.md @@ -40,6 +40,28 @@ protocol defines a shorter value. Panicking `random()` constructors were removed. Use `try_random()` or the type-specific `try_generate()` method and propagate entropy failures. +## RSA caller-supplied blinding + +Caller-supplied RSA blinding methods now take one borrowed `RsaBlindingPair` +instead of adjacent factor and inverse slices: + +```rust +use rscrypto::{RsaBlindingPair, RsaPkcs1v15Profile}; + +let blinding = RsaBlindingPair::new(blinding_factor, blinding_factor_inverse); +private_key.sign_pkcs1v15_with_blinding_factor( + RsaPkcs1v15Profile::Sha256, + message, + blinding, + signature, +)?; +# Ok::<(), rscrypto::RsaPrivateOpError>(()) +``` + +The pair borrows both inputs without allocation or copying their bytes. Every +accepting operation still validates their width, range, and inverse relation +against its RSA key. Error variants and output clearing are unchanged. + ## AEAD nonces Normal `Aead` sealing generates a fresh OS nonce. AES-GCM also supports the diff --git a/docs/migration/aws-lc-rs.md b/docs/migration/aws-lc-rs.md index a8fab39c..261b7f99 100644 --- a/docs/migration/aws-lc-rs.md +++ b/docs/migration/aws-lc-rs.md @@ -12,7 +12,7 @@ The direct examples below are covered by `tests/migration_aws_lc_rs.rs`. aws-lc-rs = "1.17" # After: enable only the primitives you use -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "x25519", "rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "x25519", "rsa"] } ``` ## Map diff --git a/docs/migration/blake3.md b/docs/migration/blake3.md index 86b7e509..8e70bd1b 100644 --- a/docs/migration/blake3.md +++ b/docs/migration/blake3.md @@ -4,14 +4,14 @@ > Unkeyed hashes return `[u8; 32]`; keyed hashes return `Blake3KeyedHash` so > authenticator comparison requires an explicit sealed decision. -Verified against `blake3 = "1.8.5"` and the `rscrypto` 0.7.8 line. +Verified against `blake3 = "1.8.6"` and the `rscrypto` 0.8.1 line. Evidence: `tests/blake3_official_vectors.rs` and `tests/blake3_differential.rs`. ## TL;DR -| | Before (`blake3` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`blake3` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `blake3 = "1.8"` | `rscrypto = { version = "0.7.8", features = ["blake3"] }` | +| Cargo dep | `blake3 = "1.8"` | `rscrypto = { version = "0.8.1", features = ["blake3"] }` | | Import | `use blake3::Hasher;` | `use rscrypto::{Blake3, prelude::*};` | | Call | `blake3::hash(data).as_bytes()` | `&Blake3::digest(data)` | @@ -26,7 +26,7 @@ blake3 = "1.8" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["blake3"] } +rscrypto = { version = "0.8.1", features = ["blake3"] } ``` ## API patterns diff --git a/docs/migration/crc-fast.md b/docs/migration/crc-fast.md index dc0bf553..d2b791f8 100644 --- a/docs/migration/crc-fast.md +++ b/docs/migration/crc-fast.md @@ -4,15 +4,15 @@ Replace `crc-fast`'s enum-driven `checksum(CrcAlgorithm::*, data)` calls with rscrypto's named CRC types. The mapped algorithms keep the same CRC parameters, but rscrypto returns each algorithm's natural integer width instead of `u64`. -Verified against `crc-fast = "1.10.0"` and the `rscrypto` 0.7.8 line. +Verified against `crc-fast = "1.10.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/crc16_properties.rs`, `tests/crc32_properties.rs`, and `tests/crc64_properties.rs`. ## TL;DR -| | Before (`crc-fast` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc-fast` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc-fast = "1.10"` | `rscrypto = { version = "0.7.8", features = ["crc32", "crc64"] }` | +| Cargo dep | `crc-fast = "1.10"` | `rscrypto = { version = "0.8.1", features = ["crc32", "crc64"] }` | | Import | `use crc_fast::{checksum, CrcAlgorithm};` | `use rscrypto::checksum::{Checksum, Crc32};` | | Call | `checksum(CrcAlgorithm::Crc32IsoHdlc, data) as u32` | `Crc32::checksum(data)` | @@ -27,7 +27,7 @@ crc-fast = "1.10" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc32", "crc64"] } +rscrypto = { version = "0.8.1", features = ["crc32", "crc64"] } ``` Add `crc16` or `crc24` only when you migrate a mapped variant from those diff --git a/docs/migration/crc.md b/docs/migration/crc.md index 91b06595..89102218 100644 --- a/docs/migration/crc.md +++ b/docs/migration/crc.md @@ -4,14 +4,14 @@ Replace supported `Crc::::new(&CRC_*)` configurations with rscrypto's named CRC types. The mapped algorithms retain their output parameters and have a portable fallback plus selected target-specific backends. -Verified against `crc = "3.4.0"` and the `rscrypto` 0.7.8 line. +Verified against `crc = "3.4.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/crc16_properties.rs`, `tests/crc24_properties.rs`, `tests/crc32_properties.rs`, and `tests/crc64_properties.rs`. ## TL;DR -| | Before (`crc` 3.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc` 3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc = "3.4"` | `rscrypto = { version = "0.7.8", features = ["checksums"] }` | +| Cargo dep | `crc = "3.4"` | `rscrypto = { version = "0.8.1", features = ["checksums"] }` | | Import | `use crc::{Crc, CRC_32_ISO_HDLC};` | `use rscrypto::checksum::{Checksum, Crc32};` | | Call | `Crc::::new(&CRC_32_ISO_HDLC).checksum(data)` | `Crc32::checksum(data)` | @@ -26,7 +26,7 @@ crc = "3.4" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["checksums"] } +rscrypto = { version = "0.8.1", features = ["checksums"] } ``` `features = ["checksums"]` enables every CRC family. To trim the build, pick only what you use: `crc16`, `crc24`, `crc32` (covers IEEE + Castagnoli), `crc64` (covers XZ + NVME). diff --git a/docs/migration/crc32c.md b/docs/migration/crc32c.md index 6c65db57..8f5d3fc3 100644 --- a/docs/migration/crc32c.md +++ b/docs/migration/crc32c.md @@ -9,9 +9,9 @@ Output is covered by the CRC-32C oracle/property tests in `tests/crc32_propertie ## TL;DR -| | Before (`crc32c` 0.6.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc32c` 0.6.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc32c = "0.6"` | `rscrypto = { version = "0.7.8", features = ["crc32"] }` | +| Cargo dep | `crc32c = "0.6"` | `rscrypto = { version = "0.8.1", features = ["crc32"] }` | | Import | `use crc32c::{crc32c, crc32c_append, crc32c_combine};` | `use rscrypto::checksum::{Checksum, ChecksumCombine, Crc32C};` | | Call | `crc32c(data)` | `Crc32C::checksum(data)` | @@ -26,7 +26,7 @@ crc32c = "0.6" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc32"] } +rscrypto = { version = "0.8.1", features = ["crc32"] } ``` The `crc32` feature exposes both `Crc32` (IEEE) and `Crc32C` (Castagnoli). diff --git a/docs/migration/crc32fast.md b/docs/migration/crc32fast.md index 1aa5b225..e5768a69 100644 --- a/docs/migration/crc32fast.md +++ b/docs/migration/crc32fast.md @@ -6,9 +6,9 @@ Output is covered by the CRC-32 oracle/property tests in `tests/crc32_properties ## TL;DR -| | Before (`crc32fast` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc32fast` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc32fast = "1.5"` | `rscrypto = { version = "0.7.8", features = ["crc32"] }` | +| Cargo dep | `crc32fast = "1.5"` | `rscrypto = { version = "0.8.1", features = ["crc32"] }` | | Import | `use crc32fast::Hasher;` | `use rscrypto::checksum::{Checksum, Crc32};` | | Call | `crc32fast::hash(data)` | `Crc32::checksum(data)` | @@ -23,7 +23,7 @@ crc32fast = "1.5" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc32"] } +rscrypto = { version = "0.8.1", features = ["crc32"] } ``` `features = ["crc32"]` exposes both CRC-32/IEEE (`Crc32`) and diff --git a/docs/migration/crc64fast.md b/docs/migration/crc64fast.md index 67dedab3..1b3e3622 100644 --- a/docs/migration/crc64fast.md +++ b/docs/migration/crc64fast.md @@ -2,14 +2,14 @@ > Same CRC-64/XZ output with method renames from `Digest` / `write` / `sum64` to `Crc64` / `update` / `finalize`. The aside below also covers `crc64fast-nvme` → `Crc64Nvme`. -Verified against `crc64fast = "1.1.0"` for `Crc64`; `Crc64Nvme` oracle coverage uses `crc-fast = "1.10.0"` and the `rscrypto` 0.7.8 line. +Verified against `crc64fast = "1.1.0"` for `Crc64`; `Crc64Nvme` oracle coverage uses `crc-fast = "1.10.0"` and the `rscrypto` 0.8.1 line. Evidence: `tests/crc64_properties.rs` compares one-shot, streaming, and combine output against those oracle crates. ## TL;DR -| | Before (`crc64fast` 1.x) | After (`rscrypto` 0.7.8) | +| | Before (`crc64fast` 1.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `crc64fast = "1.1"` | `rscrypto = { version = "0.7.8", features = ["crc64"] }` | +| Cargo dep | `crc64fast = "1.1"` | `rscrypto = { version = "0.8.1", features = ["crc64"] }` | | Import | `use crc64fast::Digest;` | `use rscrypto::checksum::{Checksum, Crc64};` | | Call | `Digest::new(); .write(data); .sum64()` | `Crc64::new(); .update(data); .finalize()` | @@ -24,7 +24,7 @@ crc64fast = "1.1" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["crc64"] } +rscrypto = { version = "0.8.1", features = ["crc64"] } ``` `features = ["crc64"]` enables both `Crc64` (XZ / ECMA-182) and `Crc64Nvme`. @@ -120,7 +120,7 @@ Drop both `crc64fast` and `crc64fast-nvme` from Cargo.toml; `features = ["crc64" also matter. Do not substitute CRC-64/ISO. - **No reset, no resume in `crc64fast`.** Build a fresh `Digest` per checksum. rscrypto adds `.reset()` and `Crc64::resume(prev)` on top of the same shape. - **`no_std`.** `crc64fast` requires `std` for SIMD detection. rscrypto's `Crc64` is `no_std`-capable; runtime detection is gated on the `std` feature, with compile-time `target_feature` selection in `no_std` builds and a portable fallback always present. -- **Hardware coverage.** `crc64fast` ships x86_64 (PCLMUL) and aarch64 (PMULL) backends. rscrypto adds VPCLMULQDQ (large buffers on x86_64), SVE2 PMULL (aarch64), VPMSUMD (Power), VGFM (s390x), and Zbc/Zvbc (RISC-V). +- **Hardware coverage.** `crc64fast` ships x86_64 (PCLMUL) and aarch64 (PMULL) backends. rscrypto adds VPCLMULQDQ (large buffers on x86_64), SVE2 PMULL (aarch64), VPMSUMD (Power), and VGFM (s390x). RISC-V uses the portable slice-by-16 implementation. - **Force a backend.** `RSCRYPTO_CRC64_FORCE=portable` selects the portable CRC-64 runtime backend in `std` builds. The `portable-only` feature makes runtime capability detection ignore host acceleration; see diff --git a/docs/migration/dryoc.md b/docs/migration/dryoc.md index a134aad4..031710dd 100644 --- a/docs/migration/dryoc.md +++ b/docs/migration/dryoc.md @@ -14,7 +14,7 @@ The direct BLAKE2b, Ed25519, and X25519 examples below are covered by dryoc = "0.8" # After: choose only the primitive features you use -rscrypto = { version = "0.7.8", default-features = false, features = ["blake2b", "ed25519", "x25519", "argon2"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["blake2b", "ed25519", "x25519", "argon2"] } ``` ## Map diff --git a/docs/migration/openssl.md b/docs/migration/openssl.md index efbe7671..3ca47e4e 100644 --- a/docs/migration/openssl.md +++ b/docs/migration/openssl.md @@ -15,7 +15,7 @@ RSA test lane. openssl = "0.10" # After: choose only the primitive features you use -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ed25519", "x25519", "rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ed25519", "x25519", "rsa"] } ``` ## Map diff --git a/docs/migration/rapidhash.md b/docs/migration/rapidhash.md index 4c346ea2..2172298e 100644 --- a/docs/migration/rapidhash.md +++ b/docs/migration/rapidhash.md @@ -8,7 +8,7 @@ inherent methods can run at compile time. ```toml [dependencies] -rscrypto = { version = "0.7.8", features = ["rapidhash"] } +rscrypto = { version = "0.8.1", features = ["rapidhash"] } ``` ```rust diff --git a/docs/migration/ring.md b/docs/migration/ring.md index d6664eec..2600a697 100644 --- a/docs/migration/ring.md +++ b/docs/migration/ring.md @@ -13,7 +13,7 @@ The direct examples below are covered by `tests/migration_ring.rs`. ring = "0.17" # After: choose only the primitive features you use -rscrypto = { version = "0.7.8", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "rsa"] } +rscrypto = { version = "0.8.1", default-features = false, features = ["sha2", "hmac", "hkdf", "pbkdf2", "aes-gcm", "chacha20poly1305", "ecdsa", "ed25519", "rsa"] } ``` ## Map diff --git a/docs/migration/sha3-kmac.md b/docs/migration/sha3-kmac.md index 47ddf764..97a13aa2 100644 --- a/docs/migration/sha3-kmac.md +++ b/docs/migration/sha3-kmac.md @@ -13,9 +13,9 @@ Evidence: `tests/kmac128_nist_vectors.rs`, `tests/kmac128_differential.rs`, ## TL;DR -| | Before (`sha3-kmac` 0.3.x) | After (`rscrypto` 0.7.8) | +| | Before (`sha3-kmac` 0.3.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `sha3-kmac = "0.3"` | `rscrypto = { version = "0.7.8", features = ["kmac"] }` | +| Cargo dep | `sha3-kmac = "0.3"` | `rscrypto = { version = "0.8.1", features = ["kmac"] }` | | Import | `use sha3_kmac::Kmac256;` | `use rscrypto::Kmac256;` | | Call | `let mut k = Kmac256::new(key, custom)?; k.update(data); k.finalize_into(&mut tag);` | `Kmac256::mac_into(key, custom, data, &mut tag);` | @@ -30,7 +30,7 @@ sha3-kmac = "0.3" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["kmac"] } +rscrypto = { version = "0.8.1", features = ["kmac"] } ``` The `kmac` feature implies `sha3`. diff --git a/docs/migration/tiny-keccak.md b/docs/migration/tiny-keccak.md index e22ab527..4f96756a 100644 --- a/docs/migration/tiny-keccak.md +++ b/docs/migration/tiny-keccak.md @@ -5,14 +5,14 @@ Replace fixed-output `tiny_keccak::Kmac::v128` / `Kmac::v256` and `Kmac256` and `rscrypto::Cshake128` / `Cshake256`. KMAC construction is infallible and adds verification helpers. `KmacXof` is not mapped. -Verified against `tiny-keccak = "2.0.2"` (with `kmac` and `cshake` features) and the `rscrypto` 0.7.8 line. +Verified against `tiny-keccak = "2.0.2"` (with `kmac` and `cshake` features) and the `rscrypto` 0.8.1 line. Evidence: `tests/kmac128_differential.rs`, `tests/kmac256_differential.rs`, `tests/cshake256_differential.rs`, `tests/cshake256_nist_vectors.rs`, and `tests/kmac_wycheproof.rs`. ## TL;DR -| | Before (`tiny-keccak` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`tiny-keccak` 2.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `tiny-keccak = { version = "2.0", features = ["kmac", "cshake"] }` | `rscrypto = { version = "0.7.8", features = ["kmac"] }` | +| Cargo dep | `tiny-keccak = { version = "2.0", features = ["kmac", "cshake"] }` | `rscrypto = { version = "0.8.1", features = ["kmac"] }` | | KMAC import | `use tiny_keccak::{Hasher, Kmac};` | `use rscrypto::{Kmac128, Kmac256};` | | KMAC call | `let mut k = Kmac::v256(key, custom); k.update(data); k.finalize(&mut tag);` | `Kmac256::mac_into(key, custom, data, &mut tag);` | | cSHAKE import | `use tiny_keccak::{Hasher, CShake};` | `use rscrypto::{Cshake128, Cshake256, Xof};` | @@ -29,7 +29,7 @@ tiny-keccak = { version = "2.0", features = ["kmac", "cshake"] } ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["kmac"] } +rscrypto = { version = "0.8.1", features = ["kmac"] } ``` The `kmac` feature implies `sha3` (which provides the underlying `Cshake128` / `Cshake256` sponges for both KMAC variants and the standalone cSHAKE primitives). diff --git a/docs/migration/twox-hash.md b/docs/migration/twox-hash.md index 686a0d55..32fb95f4 100644 --- a/docs/migration/twox-hash.md +++ b/docs/migration/twox-hash.md @@ -7,9 +7,9 @@ The `twox-hash` mapping is API migration guidance for its 2.x XXH3 surface. ## TL;DR -| | Before (`twox-hash` 2.x) | After (`rscrypto` 0.7.8) | +| | Before (`twox-hash` 2.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `twox-hash = "2.1"` | `rscrypto = { version = "0.7.8", features = ["xxh3"] }` | +| Cargo dep | `twox-hash = "2.1"` | `rscrypto = { version = "0.8.1", features = ["xxh3"] }` | | Import | `use twox_hash::XxHash3_64;` | `use rscrypto::{FastHash, Xxh3};` | | Call | `XxHash3_64::oneshot(data)` | `Xxh3::hash(data)` | @@ -24,7 +24,7 @@ twox-hash = "2.1" ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["xxh3"] } +rscrypto = { version = "0.8.1", features = ["xxh3"] } ``` ## Algorithm map diff --git a/docs/migration/xxhash-rust.md b/docs/migration/xxhash-rust.md index 90097fe4..20e056ac 100644 --- a/docs/migration/xxhash-rust.md +++ b/docs/migration/xxhash-rust.md @@ -2,14 +2,14 @@ > Replace `xxhash_rust::xxh3::xxh3_64` with `rscrypto::Xxh3::hash` (and `xxh3_128` with `Xxh3_128::hash`). One-shot bit-equivalent. Streaming routes through `Xxh3Hasher` (a `core::hash::Hasher`). -Verified against `xxhash-rust = "0.8.16"` (with `xxh3` feature) and the `rscrypto` 0.7.8 line. +Verified against `xxhash-rust = "0.8.18"` (with `xxh3` feature) and the `rscrypto` 0.8.1 line. Evidence: `tests/xxh3_differential.rs`. ## TL;DR -| | Before (`xxhash-rust` 0.8.x) | After (`rscrypto` 0.7.8) | +| | Before (`xxhash-rust` 0.8.x) | After (`rscrypto` 0.8.1) | |---|---|---| -| Cargo dep | `xxhash-rust = { version = "0.8", features = ["xxh3"] }` | `rscrypto = { version = "0.7.8", features = ["xxh3"] }` | +| Cargo dep | `xxhash-rust = { version = "0.8", features = ["xxh3"] }` | `rscrypto = { version = "0.8.1", features = ["xxh3"] }` | | Import | `use xxhash_rust::xxh3::xxh3_64;` | `use rscrypto::{FastHash, Xxh3};` | | Call | `xxh3_64(data)` | `Xxh3::hash(data)` | @@ -24,7 +24,7 @@ xxhash-rust = { version = "0.8", features = ["xxh3"] } ```toml # After [dependencies] -rscrypto = { version = "0.7.8", features = ["xxh3"] } +rscrypto = { version = "0.8.1", features = ["xxh3"] } ``` ## Algorithm map diff --git a/docs/platforms.md b/docs/platforms.md index d672ad9a..fa140ec0 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -66,8 +66,10 @@ Build coverage does not establish a constant-time claim; use ## Per-platform benchmark evidence -The historical 2026-07-04 per-platform results live in -[`benchmark_results/OVERVIEW.md`](../benchmark_results/OVERVIEW.md#coverage-matrix). -Its aggregate includes the equivalent-work limitation documented in -[`benchmarking.md`](benchmarking.md). Benchmark the deployment workload on its -target CPU before choosing a performance-sensitive backend or feature set. +The 2026-08-18 per-platform results live in +[`benchmark_results/OVERVIEW.md`](../benchmark_results/OVERVIEW.md#coverage-matrix), +covering eight Linux CI runners; the RISE RISC-V runner did not execute in that +run. Per-platform results diverge: the same commit measures `2.77x` on IBM +z16/s390x and `1.36x` on AWS Graviton3, and s390x carries a large ECDSA +regression in this snapshot. Benchmark the deployment workload on its target CPU +before choosing a performance-sensitive backend or feature set. diff --git a/docs/secret-ownership.md b/docs/secret-ownership.md index 54eaf624..a9b69c34 100644 --- a/docs/secret-ownership.md +++ b/docs/secret-ownership.md @@ -43,7 +43,8 @@ capability for permanent retention. | ML-KEM prepared decapsulation keys | Explicit duplicate; no `Clone` or `Copy` | Masked | `SecretBytes` export; no Serde | Inline | Reuse of validated private arithmetic without making implicit copies | | `RsaPrivateKey` | Neither | Public key plus a redacted private-components field | PKCS#1/PKCS#8 DER into `SecretVec`; no Serde | Heap-backed big integers and Montgomery state | Standards-compatible private-key storage/export while keeping the returned allocation typed as secret | | `RsaPrivateScratch` | Neither | Public sizing metadata only | None | Reusable heap buffers and limb vectors | Amortizes private-operation allocation while keeping intermediate ownership with the caller | -| `RsaPrivateKeyParts<'a>` | `Clone + Copy` | No `Debug` | Borrowed import fields; no Serde | Borrowed | Pass-by-value import description; copying duplicates references, not private bytes | +| `RsaPrivateKeyParts<'a>` | `Clone + Copy` | Modulus size and public exponent only; private fields masked | Borrowed import fields; no Serde | Borrowed | Pass-by-value import description; copying duplicates references, not private bytes | +| `RsaBlindingPair<'a>` | `Clone + Copy` | Factor and inverse masked | Borrowed factor and inverse; no Serde | Borrowed | Couples the two caller-blinding inputs; private operations validate them against the selected key | | `RsaSignatureSigner<'a>` | `Clone + Copy` | Signature profile only | None | Borrowed | Reusable profile-bound signing handle; copying duplicates a private-key reference, not the key | | HMAC-SHA-2 and HMAC-SHA-3 states | Neither | Masked | None | Inline | Reuse through `reset`; independent streamed owners require explicit keyed construction rather than an implicit state copy | | HKDF-SHA-2 states | Neither | Masked | None | Inline | Repeated expansion borrows one extracted PRK owner, which can also be shared by reference | diff --git a/docs/test-vector-coverage.md b/docs/test-vector-coverage.md index f905818a..5882e3cf 100644 --- a/docs/test-vector-coverage.md +++ b/docs/test-vector-coverage.md @@ -65,7 +65,7 @@ the concrete inputs and outputs of each cryptography API. | XChaCha20-Poly1305 | `tests/xchacha20poly1305.rs` | `tests/aead_wycheproof.rs` covers Wycheproof 192-bit nonce open failure; unit/integration tests cover wrong nonce/tag/AAD | Current suite maps directly after nonce-size filtering | | AEGIS-256 | `tests/aegis256_oracle.rs` | `tests/aead_wycheproof.rs` covers Wycheproof AEGIS-256 open failure; unit/integration tests cover wrong nonce/tag/AAD | Current suite maps directly | | Ascon-AEAD128 | Final SP 800-232 reference corpus in `tests/ascon_final_kats.rs` and `testdata/ascon/asconaead128.txt`; independent coverage in `tests/ascon_aead_oracle.rs` | Unit/integration tests cover wrong nonce/tag/AAD and oracle decrypt failure | Current Wycheproof `ASCON128` vectors do not match this crate's NIST Ascon-AEAD128 variant, so they are not vendored | -| ECDSA P-256/P-384 signing and verification | `tests/ecdsa_oracle.rs`; `tests/ecdsa_wycheproof.rs`; `src/auth/ecdsa.rs` unit tests; RustCrypto `p256 0.14.0` / `p384 0.13.1` oracles; `fuzz/target_impls/auth_ecdsa_verify.rs`; `fuzz/target_impls/auth_ecdsa_sign.rs` | Current Wycheproof P-256/SHA-256 and P-384/SHA-384 DER suites cover valid and invalid verification cases; unit, oracle, and fuzz tests cover SEC1/SPKI/DER/raw parser boundaries, zero/out-of-range scalars and coordinates, tampered signatures, wrong messages, deterministic and blinded signing, low-S normalization, and public-key derivation | CT evidence covers blinded signing. Public verification remains public-input work unless promoted by the CT manifest. Variable-length P1363 parser cases do not map to the fixed-size raw signature APIs. | +| ECDSA P-256/P-384 signing and verification | `tests/ecdsa_oracle.rs`; `tests/ecdsa_wycheproof.rs`; `src/auth/ecdsa.rs` unit tests; RustCrypto `p256 0.14.0` / `p384 0.14.0` oracles; `fuzz/target_impls/auth_ecdsa_verify.rs`; `fuzz/target_impls/auth_ecdsa_sign.rs` | Current Wycheproof P-256/SHA-256 and P-384/SHA-384 DER suites cover valid and invalid verification cases; unit, oracle, and fuzz tests cover SEC1/SPKI/DER/raw parser boundaries, zero/out-of-range scalars and coordinates, tampered signatures, wrong messages, deterministic and blinded signing, low-S normalization, and public-key derivation | CT evidence covers blinded signing. Public verification remains public-input work unless promoted by the CT manifest. Variable-length P1363 parser cases do not map to the fixed-size raw signature APIs. | | Ed25519 | `tests/ed25519_rfc8032_vectors.rs`, `tests/ed25519_oracle.rs` | `tests/ed25519_wycheproof.rs` covers Wycheproof valid/invalid signatures and invalid public/signature encodings; unit tests cover small-order and non-canonical signatures | Current suite maps directly | | X25519 | `tests/x25519_vectors.rs`, `tests/x25519_oracle.rs` | `tests/x25519_wycheproof.rs` covers Wycheproof valid/acceptable XDH vectors and rejects all-zero shared secrets; RFC low-order and non-canonical public cases remain in `tests/x25519_vectors.rs` | ASN/JWK/PEM suites do not apply to byte-array API | | ML-KEM-512/768/1024 | `tests/mlkem_acvp.rs` covers NIST ACVP FIPS 203 keyGen, encapsulation, decapsulation, decapsulationKeyCheck, and encapsulationKeyCheck vectors for all parameter sets; `tests/mlkem_properties.rs` differentials arbitrary seeds against the `fips203` crate; `tests/mlkem_types.rs` checks FIPS 203 sizes, randomness, security categories, byte wrappers, secret redaction, public-value equality, and sealed secret-owner comparison | `tests/mlkem_ops.rs` covers non-canonical public-key rejection before randomness, prepared-key parity, prepared-key invalid material, wrong-length parsers, decapsulation-key hash mismatch, and modified-ciphertext implicit rejection; `fuzz/target_impls/auth_mlkem512.rs`, `auth_mlkem768.rs`, and `auth_mlkem1024.rs` cover round trips, parser inputs, and modified ciphertexts | No vendored Wycheproof ML-KEM suite is currently mapped; official ACVP vectors plus all-profile FIPS 203 differential/property coverage are the primary oracle set | diff --git a/docs/types.md b/docs/types.md index c5d913fa..5480ba5e 100644 --- a/docs/types.md +++ b/docs/types.md @@ -144,7 +144,7 @@ Features: `signatures` / `key-exchange` or `ecdsa` / `ed25519` / `rsa` / `x25519 | `EcdsaP256Keypair` / `EcdsaP384Keypair` | secret + public | FIPS 186-5 / SEC 1 | | `Ed25519SecretKey` / `Ed25519PublicKey` / `Ed25519Signature` | 32/32/64B | RFC 8032 | | `Ed25519Keypair` | -- | RFC 8032 | -| `RsaPublicKey`, `RsaPrivateKey`, `RsaPrivateKeyParts`, `RsaX509PublicKey`, `RsaPublicScratch`, `RsaPrivateScratch` | variable | RFC 8017 / RFC 4055 | +| `RsaPublicKey`, `RsaPrivateKey`, `RsaPrivateKeyParts`, `RsaBlindingPair`, `RsaX509PublicKey`, `RsaPublicScratch`, `RsaPrivateScratch` | variable | RFC 8017 / RFC 4055 | | `RsaSignatureSigner`, `RsaSignatureVerifier` | profile-bound wrappers | RFC 8017 / RFC 4055 | | `RsaJwtAlgorithm`, `RsaJwtVerifier` | verifier-owned JWT/JWS policy | RFC 7515 / RFC 8725 | | `RsaSignatureProfile`, `RsaPssProfile`, `RsaPkcs1v15Profile`, `RsaOaepProfile`, `RsaPublicKeyPolicy`, `RsaKeyGenerationContract` | -- | RFC 8017 / RFC 4055 / FIPS 186-5 / protocol-specific profiles | diff --git a/examples/aead_seal_open.rs b/examples/aead_seal_open.rs index 4484efee..249756d0 100644 --- a/examples/aead_seal_open.rs +++ b/examples/aead_seal_open.rs @@ -11,7 +11,9 @@ fn main() -> Result<(), Box> { let (nonce, sealed) = cipher.seal_random_to_vec(aad, plaintext)?; let opened = cipher.decrypt_to_vec(&nonce, aad, &sealed)?; - assert_eq!(opened, plaintext); + if opened != plaintext { + return Err(std::io::Error::other("AEAD round-trip changed the plaintext").into()); + } println!("ChaCha20-Poly1305 sealed {} bytes", plaintext.len()); Ok(()) } diff --git a/examples/basic.rs b/examples/basic.rs index b2ac356d..03c632cf 100644 --- a/examples/basic.rs +++ b/examples/basic.rs @@ -3,7 +3,7 @@ //! //! Run with: `cargo run --example basic --features full,getrandom` -use std::io::{Cursor, Read, Write}; +use std::io::{Read, Write}; use rscrypto::{ Aead, Blake3, ChaCha20Poly1305, ChaCha20Poly1305Key, Checksum, Crc32C, Digest, Ed25519Keypair, Ed25519SecretKey, @@ -86,17 +86,19 @@ fn auth_api() -> Result<(), Box> { let mut mac = HmacSha256::new(key); mac.update(b"hello "); mac.update(b"world"); - assert!(mac.verify(&tag).is_ok()); + mac.verify(&tag)?; mac.reset(); mac.update(data); - assert!(mac.verify(&tag).is_ok()); + mac.verify(&tag)?; let hkdf = HkdfSha256::new(b"salt", b"input key material"); let mut okm = [0u8; 42]; hkdf.expand(b"context", &mut okm)?; let oneshot = HkdfSha256::derive_array::<42>(b"salt", b"input key material", b"context")?; - assert_eq!(okm, oneshot); + if okm != oneshot { + return Err(std::io::Error::other("HKDF streaming and one-shot outputs differ").into()); + } println!("HMAC-SHA256 tag size = {} bytes", tag.as_slice().len()); println!("HKDF-SHA256 output = {} bytes\n", okm.len()); @@ -116,7 +118,9 @@ fn aead_api() -> Result<(), Box> { let mut opened = [0u8; 5]; aead.decrypt(&nonce, b"", &sealed, &mut opened)?; - assert_eq!(&opened, b"hello"); + if &opened != b"hello" { + return Err(std::io::Error::other("AEAD round-trip changed the plaintext").into()); + } println!("ChaCha20-Poly1305 round-trip succeeded"); println!(" nonce = {nonce}\n"); @@ -135,12 +139,16 @@ fn hex_api() -> Result<(), Box> { println!("Debug: {nonce:?}"); let parsed: Nonce96 = "abababababababababababab".parse()?; - assert_eq!(parsed, nonce); + if parsed != nonce { + return Err(std::io::Error::other("nonce hex round-trip changed the value").into()); + } println!("FromStr: round-trip succeeded"); let key = ChaCha20Poly1305Key::from_bytes([0x42; 32]); let key_debug = format!("{key:?}"); - assert_eq!(key_debug, "ChaCha20Poly1305Key(****)"); + if key_debug != "ChaCha20Poly1305Key(****)" { + return Err(std::io::Error::other("secret-key Debug output exposed an unexpected representation").into()); + } println!("\nSecret Debug: {key_debug}"); let ed_sk = Ed25519SecretKey::from_bytes([7u8; 32]); @@ -215,23 +223,35 @@ fn io_api() -> std::io::Result<()> { let data = b"stream me through adapters"; - let mut reader = Sha256::reader(Cursor::new(data.to_vec())); + let mut reader = Sha256::reader(data.as_slice()); let mut copied = Vec::new(); reader.read_to_end(&mut copied)?; - assert_eq!(copied, data); - assert_eq!(reader.digest(), Sha256::digest(data)); + if copied != data { + return Err(std::io::Error::other("digest reader changed the copied data")); + } + if reader.digest() != Sha256::digest(data) { + return Err(std::io::Error::other("digest reader produced the wrong digest")); + } let mut checksum_writer = Crc32C::writer(Vec::new()); checksum_writer.write_all(data)?; let (written, crc) = checksum_writer.into_parts(); - assert_eq!(written, data); - assert_eq!(crc, Crc32C::checksum(data)); + if written != data { + return Err(std::io::Error::other("checksum writer changed the written data")); + } + if crc != Crc32C::checksum(data) { + return Err(std::io::Error::other("checksum writer produced the wrong checksum")); + } let mut digest_writer = Blake3::writer(Vec::new()); digest_writer.write_all(data)?; let (written, digest) = digest_writer.into_parts(); - assert_eq!(written, data); - assert_eq!(digest, Blake3::digest(data)); + if written != data { + return Err(std::io::Error::other("digest writer changed the written data")); + } + if digest != Blake3::digest(data) { + return Err(std::io::Error::other("digest writer produced the wrong digest")); + } println!("reader digest matches Sha256::digest()"); println!("writer checksum matches Crc32C::checksum()"); diff --git a/examples/mlkem_encapsulation.rs b/examples/mlkem_encapsulation.rs index 83721d4d..577a650c 100644 --- a/examples/mlkem_encapsulation.rs +++ b/examples/mlkem_encapsulation.rs @@ -7,7 +7,9 @@ fn main() -> Result<(), Box> { let (ciphertext, shared_secret) = MlKem768::try_encapsulate(&encapsulation_key)?; let decapsulated = MlKem768::decapsulate(&decapsulation_key, &ciphertext)?; - assert!(shared_secret.ct_eq(&decapsulated).declassify()); + if !shared_secret.ct_eq(&decapsulated).declassify() { + return Err(std::io::Error::other("ML-KEM encapsulation and decapsulation secrets differ").into()); + } println!( "ML-KEM-768 encapsulated {} shared-secret bytes", shared_secret.as_bytes().len() diff --git a/examples/parallel.rs b/examples/parallel.rs index 8f924712..f66d8aaa 100644 --- a/examples/parallel.rs +++ b/examples/parallel.rs @@ -63,7 +63,7 @@ fn parallel_chunks() { println!("Parallel chunk processing\n"); // Simulate large data (in practice, this could be a memory-mapped file) - let data: Vec = (0..1_000_000).map(|i| (i % 256) as u8).collect(); + let data: Vec = (0usize..1_000_000).map(|i| i.to_le_bytes()[0]).collect(); let chunk_size = 250_000; // 4 chunks of 250KB each @@ -74,12 +74,9 @@ fn parallel_chunks() { // Parallel: compute each chunk's CRC, then combine let chunks: Vec<_> = data.chunks(chunk_size).collect(); let chunk_crcs: Vec<_> = thread::scope(|scope| { - let handles: Vec<_> = chunks + chunks .iter() .map(|&chunk| scope.spawn(move || Crc64::checksum(chunk))) - .collect(); - handles - .into_iter() .map(|handle| handle.join().expect("thread panicked")) .collect() }); @@ -100,7 +97,12 @@ fn threaded_example() { println!("Threaded processing\n"); // Generate test data - let data: Vec = (0..4_000_000).map(|i| ((i * 17) % 256) as u8).collect(); + let data: Vec = (0usize..4_000_000) + .map(|i| { + let value = i.strict_mul(17).strict_rem(256); + u8::try_from(value).expect("threaded fixture byte must fit in u8") + }) + .collect(); let num_threads = 4; let chunk_size = data.len() / num_threads; @@ -114,12 +116,9 @@ fn threaded_example() { // Spawn scoped threads to compute each chunk's CRC without copying chunk data. let mut results: Vec<(usize, u64, usize)> = thread::scope(|scope| { - let handles: Vec<_> = chunks + chunks .iter() .map(|&(idx, chunk)| scope.spawn(move || (idx, Crc64::checksum(chunk), chunk.len()))) - .collect(); - handles - .into_iter() .map(|handle| handle.join().expect("thread panicked")) .collect() }); diff --git a/examples/password_hashing.rs b/examples/password_hashing.rs index 071cb91d..a93ad8cd 100644 --- a/examples/password_hashing.rs +++ b/examples/password_hashing.rs @@ -13,13 +13,17 @@ fn main() -> Result<(), Box> { let argon2 = Argon2idPassword::default(); let argon2_phc = argon2.hash_password(password)?; - assert!(argon2.verify_password(password, &argon2_phc).is_ok()); - assert!(argon2.verify_password(b"wrong password", &argon2_phc).is_err()); + argon2.verify_password(password, &argon2_phc)?; + if argon2.verify_password(b"wrong password", &argon2_phc).is_ok() { + return Err(std::io::Error::other("Argon2id accepted the wrong password").into()); + } let scrypt = ScryptPassword::default(); let scrypt_phc = scrypt.hash_password(password)?; - assert!(scrypt.verify_password(password, &scrypt_phc).is_ok()); - assert!(scrypt.verify_password(b"wrong password", &scrypt_phc).is_err()); + scrypt.verify_password(password, &scrypt_phc)?; + if scrypt.verify_password(b"wrong password", &scrypt_phc).is_ok() { + return Err(std::io::Error::other("scrypt accepted the wrong password").into()); + } println!("{argon2_phc}"); println!("{scrypt_phc}"); diff --git a/fuzz-packages/aead-aegis256/Cargo.lock b/fuzz-packages/aead-aegis256/Cargo.lock index 3a480232..309d2ad4 100644 --- a/fuzz-packages/aead-aegis256/Cargo.lock +++ b/fuzz-packages/aead-aegis256/Cargo.lock @@ -20,9 +20,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -38,9 +38,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -87,7 +87,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aegis256" diff --git a/fuzz-packages/aead-aes128gcm/Cargo.lock b/fuzz-packages/aead-aes128gcm/Cargo.lock index 520c1bcb..eb550e78 100644 --- a/fuzz-packages/aead-aes128gcm/Cargo.lock +++ b/fuzz-packages/aead-aes128gcm/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -218,7 +218,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes128gcm" diff --git a/fuzz-packages/aead-aes128gcmsiv/Cargo.lock b/fuzz-packages/aead-aes128gcmsiv/Cargo.lock index 7e8bc8ba..d374eba4 100644 --- a/fuzz-packages/aead-aes128gcmsiv/Cargo.lock +++ b/fuzz-packages/aead-aes128gcmsiv/Cargo.lock @@ -4,30 +4,30 @@ version = 4 [[package]] name = "aead" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common", - "generic-array", + "inout", ] [[package]] name = "aes" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ "aead", "aes", @@ -35,7 +35,6 @@ dependencies = [ "ctr", "polyval", "subtle", - "zeroize", ] [[package]] @@ -44,11 +43,20 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -64,57 +72,68 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer", "crypto-common", "inout", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "ctr" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ "cipher", ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "ctutils" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] [[package]] -name = "generic-array" -version = "0.14.7" +name = "find-msvc-tools" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -127,13 +146,22 @@ dependencies = [ "r-efi", ] +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -162,21 +190,14 @@ dependencies = [ "cc", ] -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "polyval" -version = "0.6.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ - "cfg-if", + "cpubits", "cpufeatures", - "opaque-debug", "universal-hash", ] @@ -188,7 +209,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes128gcmsiv" @@ -227,22 +248,10 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "universal-hash" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" dependencies = [ "crypto-common", - "subtle", + "ctutils", ] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/fuzz-packages/aead-aes128gcmsiv/Cargo.toml b/fuzz-packages/aead-aes128gcmsiv/Cargo.toml index 2f86b491..e9b6ce9d 100644 --- a/fuzz-packages/aead-aes128gcmsiv/Cargo.toml +++ b/fuzz-packages/aead-aes128gcmsiv/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "aes-gcm-siv"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support", features = ["aead"] } -aes-gcm-siv = { version = "0.11", default-features = false, features = ["alloc", "aes"] } +aes-gcm-siv = { version = "0.12", default-features = false, features = ["alloc", "aes"] } [workspace] members = ["."] diff --git a/fuzz-packages/aead-aes256gcm/Cargo.lock b/fuzz-packages/aead-aes256gcm/Cargo.lock index 5ef7f9e0..a94e9b6d 100644 --- a/fuzz-packages/aead-aes256gcm/Cargo.lock +++ b/fuzz-packages/aead-aes256gcm/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -218,7 +218,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes256gcm" diff --git a/fuzz-packages/aead-aes256gcmsiv/Cargo.lock b/fuzz-packages/aead-aes256gcmsiv/Cargo.lock index dfa5d6a3..0a87b8b3 100644 --- a/fuzz-packages/aead-aes256gcmsiv/Cargo.lock +++ b/fuzz-packages/aead-aes256gcmsiv/Cargo.lock @@ -4,30 +4,30 @@ version = 4 [[package]] name = "aead" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common", - "generic-array", + "inout", ] [[package]] name = "aes" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ "aead", "aes", @@ -35,7 +35,6 @@ dependencies = [ "ctr", "polyval", "subtle", - "zeroize", ] [[package]] @@ -44,11 +43,20 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -64,57 +72,68 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer", "crypto-common", "inout", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "ctr" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ "cipher", ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "ctutils" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] [[package]] -name = "generic-array" -version = "0.14.7" +name = "find-msvc-tools" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -127,13 +146,22 @@ dependencies = [ "r-efi", ] +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -162,21 +190,14 @@ dependencies = [ "cc", ] -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "polyval" -version = "0.6.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ - "cfg-if", + "cpubits", "cpufeatures", - "opaque-debug", "universal-hash", ] @@ -188,7 +209,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-aes256gcmsiv" @@ -227,22 +248,10 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "universal-hash" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" dependencies = [ "crypto-common", - "subtle", + "ctutils", ] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/fuzz-packages/aead-aes256gcmsiv/Cargo.toml b/fuzz-packages/aead-aes256gcmsiv/Cargo.toml index 7c2a09e6..aa480bc9 100644 --- a/fuzz-packages/aead-aes256gcmsiv/Cargo.toml +++ b/fuzz-packages/aead-aes256gcmsiv/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "aes-gcm-siv"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support", features = ["aead"] } -aes-gcm-siv = { version = "0.11", default-features = false, features = ["alloc", "aes"] } +aes-gcm-siv = { version = "0.12", default-features = false, features = ["alloc", "aes"] } [workspace] members = ["."] diff --git a/fuzz-packages/aead-ascon128/Cargo.lock b/fuzz-packages/aead-ascon128/Cargo.lock index 15bc01d0..575ddea4 100644 --- a/fuzz-packages/aead-ascon128/Cargo.lock +++ b/fuzz-packages/aead-ascon128/Cargo.lock @@ -38,9 +38,9 @@ checksum = "c90c8493a0b3bf8678a82f096821a2c527dd6344bbcef6265bfcc09745f44e24" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -65,9 +65,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -132,7 +132,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-ascon128" diff --git a/fuzz-packages/aead-chacha20poly1305/Cargo.lock b/fuzz-packages/aead-chacha20poly1305/Cargo.lock index cd2ff2a1..fbfc0c17 100644 --- a/fuzz-packages/aead-chacha20poly1305/Cargo.lock +++ b/fuzz-packages/aead-chacha20poly1305/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -191,7 +191,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-chacha20poly1305" diff --git a/fuzz-packages/aead-nonce-counter/Cargo.lock b/fuzz-packages/aead-nonce-counter/Cargo.lock index 5b5f25ce..e05e16db 100644 --- a/fuzz-packages/aead-nonce-counter/Cargo.lock +++ b/fuzz-packages/aead-nonce-counter/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-nonce-counter" diff --git a/fuzz-packages/aead-xchacha20poly1305/Cargo.lock b/fuzz-packages/aead-xchacha20poly1305/Cargo.lock index 3b19f3f1..33393cc2 100644 --- a/fuzz-packages/aead-xchacha20poly1305/Cargo.lock +++ b/fuzz-packages/aead-xchacha20poly1305/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -191,7 +191,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-aead-xchacha20poly1305" diff --git a/fuzz-packages/auth-argon2/Cargo.lock b/fuzz-packages/auth-argon2/Cargo.lock index 8fb4467c..778af033 100644 --- a/fuzz-packages/auth-argon2/Cargo.lock +++ b/fuzz-packages/auth-argon2/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "argon2" -version = "0.6.0-rc.8" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", @@ -28,27 +28,27 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ "digest", ] [[package]] name = "block-buffer" -version = "0.12.1" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "hybrid-array", + "generic-array", ] [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -62,55 +62,51 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "ctutils" -version = "0.4.2" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "cmov", + "generic-array", + "typenum", ] [[package]] name = "digest" -version = "0.11.3" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "ctutils", + "subtle", ] [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] [[package]] name = "getrandom" @@ -123,15 +119,6 @@ dependencies = [ "r-efi", ] -[[package]] -name = "hybrid-array" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" -dependencies = [ - "typenum", -] - [[package]] name = "jobserver" version = "0.1.35" @@ -160,21 +147,13 @@ dependencies = [ [[package]] name = "password-hash" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" -dependencies = [ - "phc", -] - -[[package]] -name = "phc" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "ctutils", + "rand_core", + "subtle", ] [[package]] @@ -183,9 +162,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-argon2" @@ -210,8 +195,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/fuzz-packages/auth-argon2/Cargo.toml b/fuzz-packages/auth-argon2/Cargo.toml index 979d119e..5c417308 100644 --- a/fuzz-packages/auth-argon2/Cargo.toml +++ b/fuzz-packages/auth-argon2/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "argon2"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -argon2 = { version = "0.6.0-rc.8", default-features = false, features = ["alloc"] } +argon2 = { version = "0.5.3", default-features = false, features = ["alloc"] } [workspace] members = ["."] diff --git a/fuzz-packages/auth-ed25519/Cargo.lock b/fuzz-packages/auth-ed25519/Cargo.lock index 013bb2aa..12386b92 100644 --- a/fuzz-packages/auth-ed25519/Cargo.lock +++ b/fuzz-packages/auth-ed25519/Cargo.lock @@ -8,26 +8,20 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -41,36 +35,29 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", "cpufeatures", @@ -90,24 +77,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "zeroize", + "syn", ] [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", "crypto-common", @@ -115,23 +92,21 @@ dependencies = [ [[package]] name = "ed25519" -version = "2.2.3" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8", "signature", ] [[package]] name = "ed25519-dalek" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ "curve25519-dalek", "ed25519", - "serde", "sha2", "subtle", "zeroize", @@ -139,46 +114,34 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "generic-array" -version = "0.14.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" -version = "0.2.17" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "wasi", + "r-efi", ] [[package]] -name = "getrandom" -version = "0.4.3" +name = "hybrid-array" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ - "cfg-if", - "libc", - "r-efi", + "typenum", ] [[package]] @@ -187,7 +150,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.4.3", + "getrandom", "libc", ] @@ -207,16 +170,6 @@ dependencies = [ "cc", ] -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -241,18 +194,9 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-ed25519" @@ -286,40 +230,11 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", @@ -334,22 +249,9 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signature" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "rand_core", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" [[package]] name = "subtle" @@ -368,17 +270,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "typenum" version = "1.20.1" @@ -391,18 +282,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "zeroize" version = "1.9.0" diff --git a/fuzz-packages/auth-ed25519/Cargo.toml b/fuzz-packages/auth-ed25519/Cargo.toml index a56bbf2c..3dcfbad7 100644 --- a/fuzz-packages/auth-ed25519/Cargo.toml +++ b/fuzz-packages/auth-ed25519/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "ed25519"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -ed25519-dalek = "2.2" +ed25519-dalek = "3.0" [workspace] members = ["."] diff --git a/fuzz-packages/auth-hkdf/Cargo.lock b/fuzz-packages/auth-hkdf/Cargo.lock index c4719c32..88526fb3 100644 --- a/fuzz-packages/auth-hkdf/Cargo.lock +++ b/fuzz-packages/auth-hkdf/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -164,7 +164,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-hkdf" diff --git a/fuzz-packages/auth-hmac/Cargo.lock b/fuzz-packages/auth-hmac/Cargo.lock index e14dfe58..f9453d1f 100644 --- a/fuzz-packages/auth-hmac/Cargo.lock +++ b/fuzz-packages/auth-hmac/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -155,7 +155,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-hmac" diff --git a/fuzz-packages/auth-kmac256/Cargo.lock b/fuzz-packages/auth-kmac256/Cargo.lock index ed5bce65..3ecd0c71 100644 --- a/fuzz-packages/auth-kmac256/Cargo.lock +++ b/fuzz-packages/auth-kmac256/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -34,9 +34,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -83,7 +83,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-kmac256" diff --git a/fuzz-packages/auth-pbkdf2/Cargo.lock b/fuzz-packages/auth-pbkdf2/Cargo.lock index a661e37c..ea2872ba 100644 --- a/fuzz-packages/auth-pbkdf2/Cargo.lock +++ b/fuzz-packages/auth-pbkdf2/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -165,7 +165,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-pbkdf2" diff --git a/fuzz-packages/auth-phc/Cargo.lock b/fuzz-packages/auth-phc/Cargo.lock index 0aac3f41..39a50a0e 100644 --- a/fuzz-packages/auth-phc/Cargo.lock +++ b/fuzz-packages/auth-phc/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-phc" diff --git a/fuzz-packages/auth-scrypt/Cargo.lock b/fuzz-packages/auth-scrypt/Cargo.lock index 5a7deb35..f889a2a5 100644 --- a/fuzz-packages/auth-scrypt/Cargo.lock +++ b/fuzz-packages/auth-scrypt/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -178,7 +178,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-scrypt" diff --git a/fuzz-packages/auth-x25519/Cargo.lock b/fuzz-packages/auth-x25519/Cargo.lock index 4267b661..c8a42896 100644 --- a/fuzz-packages/auth-x25519/Cargo.lock +++ b/fuzz-packages/auth-x25519/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,18 +28,18 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", "cpufeatures", @@ -58,20 +58,20 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "fiat-crypto" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -136,13 +136,13 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand_core" -version = "0.6.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-auth-x25519" @@ -176,36 +176,6 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "shlex" version = "2.0.1" @@ -229,17 +199,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "unicode-ident" version = "1.0.24" @@ -248,13 +207,12 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "x25519-dalek" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ "curve25519-dalek", "rand_core", - "serde", "zeroize", ] @@ -263,17 +221,3 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] diff --git a/fuzz-packages/auth-x25519/Cargo.toml b/fuzz-packages/auth-x25519/Cargo.toml index eb7ec9b6..904467b1 100644 --- a/fuzz-packages/auth-x25519/Cargo.toml +++ b/fuzz-packages/auth-x25519/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "x25519"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } [workspace] members = ["."] diff --git a/fuzz-packages/checksums-crc16/Cargo.lock b/fuzz-packages/checksums-crc16/Cargo.lock index 1c1a2e88..6c59c848 100644 --- a/fuzz-packages/checksums-crc16/Cargo.lock +++ b/fuzz-packages/checksums-crc16/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc16" diff --git a/fuzz-packages/checksums-crc24/Cargo.lock b/fuzz-packages/checksums-crc24/Cargo.lock index b8a0bfa6..043526cd 100644 --- a/fuzz-packages/checksums-crc24/Cargo.lock +++ b/fuzz-packages/checksums-crc24/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc24" diff --git a/fuzz-packages/checksums-crc32/Cargo.lock b/fuzz-packages/checksums-crc32/Cargo.lock index 76ed2839..eb4945bb 100644 --- a/fuzz-packages/checksums-crc32/Cargo.lock +++ b/fuzz-packages/checksums-crc32/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc32" diff --git a/fuzz-packages/checksums-crc64/Cargo.lock b/fuzz-packages/checksums-crc64/Cargo.lock index c910382d..d9b51307 100644 --- a/fuzz-packages/checksums-crc64/Cargo.lock +++ b/fuzz-packages/checksums-crc64/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -43,9 +43,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -92,7 +92,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-checksums-crc64" diff --git a/fuzz-packages/fast-rapidhash/Cargo.lock b/fuzz-packages/fast-rapidhash/Cargo.lock index 0f99f2e3..6026a714 100644 --- a/fuzz-packages/fast-rapidhash/Cargo.lock +++ b/fuzz-packages/fast-rapidhash/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -86,7 +86,7 @@ dependencies = [ [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-fast-rapidhash" diff --git a/fuzz-packages/fast-xxh3/Cargo.lock b/fuzz-packages/fast-xxh3/Cargo.lock index 207a69f6..640364b1 100644 --- a/fuzz-packages/fast-xxh3/Cargo.lock +++ b/fuzz-packages/fast-xxh3/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-fast-xxh3" diff --git a/fuzz-packages/hash-ascon/Cargo.lock b/fuzz-packages/hash-ascon/Cargo.lock index 946b3949..c849763e 100644 --- a/fuzz-packages/hash-ascon/Cargo.lock +++ b/fuzz-packages/hash-ascon/Cargo.lock @@ -35,9 +35,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -130,7 +130,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-ascon" diff --git a/fuzz-packages/hash-blake2/Cargo.lock b/fuzz-packages/hash-blake2/Cargo.lock index d993c23c..c2567352 100644 --- a/fuzz-packages/hash-blake2/Cargo.lock +++ b/fuzz-packages/hash-blake2/Cargo.lock @@ -10,11 +10,20 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", ] [[package]] @@ -28,9 +37,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -50,6 +59,16 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -68,22 +87,43 @@ dependencies = [ "cmov", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.12.1", + "crypto-common 0.2.2", "ctutils", ] [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] [[package]] name = "getrandom" @@ -102,7 +142,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -148,14 +188,14 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-blake2" version = "0.0.0" dependencies = [ "blake2", - "digest", + "digest 0.11.3", "hmac", "libfuzzer-sys", "rscrypto", @@ -175,8 +215,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/fuzz-packages/hash-blake2/Cargo.toml b/fuzz-packages/hash-blake2/Cargo.toml index e9db192f..b9424e7b 100644 --- a/fuzz-packages/hash-blake2/Cargo.toml +++ b/fuzz-packages/hash-blake2/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rscrypto = { path = "../..", features = ["std", "blake2b", "blake2s"] } rscrypto_fuzz = { package = "rscrypto-fuzz-support", path = "../../fuzz/support" } -blake2 = "0.11.0-rc.6" +blake2 = "0.10.6" digest = "0.11" hmac = "0.13" diff --git a/fuzz-packages/hash-blake3/Cargo.lock b/fuzz-packages/hash-blake3/Cargo.lock index 61858c38..fd940359 100644 --- a/fuzz-packages/hash-blake3/Cargo.lock +++ b/fuzz-packages/hash-blake3/Cargo.lock @@ -22,9 +22,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -69,9 +69,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -118,7 +118,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-blake3" diff --git a/fuzz-packages/hash-sha2/Cargo.lock b/fuzz-packages/hash-sha2/Cargo.lock index b66776e7..3a7a8f3e 100644 --- a/fuzz-packages/hash-sha2/Cargo.lock +++ b/fuzz-packages/hash-sha2/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -72,9 +72,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -130,7 +130,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-sha2" diff --git a/fuzz-packages/hash-sha3/Cargo.lock b/fuzz-packages/hash-sha3/Cargo.lock index 23941a0e..bdfb4385 100644 --- a/fuzz-packages/hash-sha3/Cargo.lock +++ b/fuzz-packages/hash-sha3/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" dependencies = [ "cfg-if", "cpufeatures", @@ -157,7 +157,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-hash-sha3" diff --git a/fuzz-packages/surface-hex-parse/Cargo.lock b/fuzz-packages/surface-hex-parse/Cargo.lock index 722ecdf9..a6214cf4 100644 --- a/fuzz-packages/surface-hex-parse/Cargo.lock +++ b/fuzz-packages/surface-hex-parse/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-support" diff --git a/fuzz-packages/traits-io/Cargo.lock b/fuzz-packages/traits-io/Cargo.lock index 0a4d28bb..27ef473c 100644 --- a/fuzz-packages/traits-io/Cargo.lock +++ b/fuzz-packages/traits-io/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -28,9 +28,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "getrandom" @@ -77,7 +77,7 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz-support" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 73ccdeee..b438ed0e 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2,16 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common 0.1.7", - "generic-array", -] - [[package]] name = "aead" version = "0.6.1" @@ -19,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -32,24 +22,13 @@ dependencies = [ "softaes", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", -] - [[package]] name = "aes" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ - "cipher 0.5.2", + "cipher", "cpubits", "cpufeatures 0.3.0", ] @@ -60,27 +39,26 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ - "aead 0.6.1", - "aes 0.9.2", - "cipher 0.5.2", - "ctr 0.10.1", + "aead", + "aes", + "cipher", + "ctr", "ghash", "subtle", ] [[package]] name = "aes-gcm-siv" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "1f437e5b075722bda3f54039e95d60d1c142f140e6c45b718e4f9fca3e5a1514" dependencies = [ - "aead 0.5.2", - "aes 0.8.4", - "cipher 0.4.4", - "ctr 0.9.2", - "polyval 0.6.2", + "aead", + "aes", + "cipher", + "ctr", + "polyval", "subtle", - "zeroize", ] [[package]] @@ -91,13 +69,13 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "argon2" -version = "0.6.0-rc.8" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures 0.3.0", + "cpufeatures 0.2.17", "password-hash", ] @@ -119,9 +97,9 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cba922132fed390c20be89a16fa713999c46b2ce84ac4b0ecee0f3586a39d88" dependencies = [ - "aead 0.6.1", + "aead", "ascon-core", - "inout 0.2.2", + "inout", "subtle", ] @@ -147,12 +125,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base16ct" version = "1.0.0" @@ -167,18 +139,18 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "blake2" -version = "0.11.0-rc.6" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.11.3", + "digest 0.10.7", ] [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -208,9 +180,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -231,7 +203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", "cpufeatures 0.3.0", ] @@ -241,22 +213,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" dependencies = [ - "aead 0.6.1", + "aead", "chacha20", - "cipher 0.5.2", + "cipher", "poly1305", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.2" @@ -265,7 +227,7 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -274,12 +236,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-oid" version = "0.10.2" @@ -337,18 +293,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.7.5" @@ -357,7 +301,7 @@ checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" dependencies = [ "cpubits", "ctutils", - "getrandom 0.4.3", + "getrandom", "hybrid-array", "num-traits", "rand_core 0.10.1", @@ -381,7 +325,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "getrandom 0.4.3", + "getrandom", "hybrid-array", "rand_core 0.10.1", ] @@ -397,22 +341,13 @@ dependencies = [ "sponge-cursor", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher 0.4.4", -] - [[package]] name = "ctr" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher 0.5.2", + "cipher", ] [[package]] @@ -427,14 +362,14 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", "curve25519-dalek-derive", - "digest 0.10.7", + "digest 0.11.3", "fiat-crypto", "rustc_version", "subtle", @@ -452,23 +387,13 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "zeroize", -] - [[package]] name = "der" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ - "const-oid 0.10.2", + "const-oid", "zeroize", ] @@ -479,7 +404,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -491,79 +415,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid 0.10.2", + "const-oid", "crypto-common 0.2.2", "ctutils", ] -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der 0.7.10", - "digest 0.10.7", - "elliptic-curve 0.13.8", - "rfc6979 0.4.0", - "signature 2.2.0", - "spki 0.7.3", -] - [[package]] name = "ecdsa" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der 0.8.1", + "der", "digest 0.11.3", - "elliptic-curve 0.14.1", - "rfc6979 0.6.0", - "signature 3.0.0", - "spki 0.8.0", + "elliptic-curve", + "rfc6979", + "signature", + "spki", "zeroize", ] [[package]] name = "ed25519" -version = "2.2.3" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8 0.10.2", - "signature 2.2.0", + "signature", ] [[package]] name = "ed25519-dalek" -version = "2.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" dependencies = [ "curve25519-dalek", "ed25519", - "serde", - "sha2 0.10.9", - "subtle", - "zeroize", -] - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct 0.2.0", - "crypto-bigint 0.5.5", - "digest 0.10.7", - "ff 0.13.1", - "generic-array", - "group 0.13.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", - "sec1 0.7.3", + "sha2", "subtle", "zeroize", ] @@ -574,30 +463,20 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ - "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "base16ct", + "crypto-bigint", "crypto-common 0.2.2", "digest 0.11.3", - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", - "pkcs8 0.11.0", + "pkcs8", "rand_core 0.10.1", - "sec1 0.8.1", + "sec1", "subtle", "zeroize", ] -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "ff" version = "0.14.0" @@ -610,15 +489,15 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.9" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "generic-array" @@ -628,18 +507,6 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", - "zeroize", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", ] [[package]] @@ -660,18 +527,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "polyval 0.7.3", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff 0.13.1", - "rand_core 0.6.4", - "subtle", + "polyval", ] [[package]] @@ -680,7 +536,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", ] @@ -691,16 +547,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac 0.13.0", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", + "hmac", ] [[package]] @@ -723,15 +570,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "inout" version = "0.2.2" @@ -747,15 +585,15 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.4.3", + "getrandom", "libc", ] [[package]] name = "keccak" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -792,44 +630,42 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "p256" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ - "ecdsa 0.17.0", - "elliptic-curve 0.14.1", + "ecdsa", + "elliptic-curve", "primefield", - "primeorder 0.14.0", - "sha2 0.11.0", + "primeorder", + "sha2", ] [[package]] name = "p384" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +checksum = "d17b851e6b3e378ab4ecb07fa2ed23f4d15f075735f8fec9fa1e7bdce5f8301f" dependencies = [ - "ecdsa 0.16.9", - "elliptic-curve 0.13.8", - "primeorder 0.13.6", - "sha2 0.10.9", + "ecdsa", + "elliptic-curve", + "fiat-crypto", + "primefield", + "primeorder", + "sha2", ] [[package]] name = "password-hash" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ - "phc", + "base64ct", + "rand_core 0.6.4", + "subtle", ] [[package]] @@ -839,27 +675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest 0.11.3", - "hmac 0.13.0", -] - -[[package]] -name = "phc" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" -dependencies = [ - "base64ct", - "ctutils", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der 0.7.10", - "spki 0.7.3", + "hmac", ] [[package]] @@ -868,8 +684,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.1", - "spki 0.8.0", + "der", + "spki", ] [[package]] @@ -879,19 +695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" dependencies = [ "cpufeatures 0.3.0", - "universal-hash 0.6.1", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", - "universal-hash 0.5.1", + "universal-hash", ] [[package]] @@ -902,7 +706,7 @@ checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", "cpufeatures 0.3.0", - "universal-hash 0.6.1", + "universal-hash", ] [[package]] @@ -911,30 +715,21 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint", "crypto-common 0.2.2", - "ff 0.14.0", + "ff", "rand_core 0.10.1", "subtle", "zeroize", ] -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve 0.13.8", -] - [[package]] name = "primeorder" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ - "elliptic-curve 0.14.1", + "elliptic-curve", "once_cell", "primefield", "serdect", @@ -970,9 +765,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] [[package]] name = "rand_core" @@ -989,29 +781,19 @@ dependencies = [ "rustversion", ] -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac 0.12.1", - "subtle", -] - [[package]] name = "rfc6979" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "crypto-bigint 0.7.5", - "hmac 0.13.0", + "crypto-bigint", + "hmac", ] [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-fuzz" @@ -1031,7 +813,7 @@ dependencies = [ "digest 0.11.3", "ed25519-dalek", "hkdf", - "hmac 0.13.0", + "hmac", "libfuzzer-sys", "p256", "p384", @@ -1040,7 +822,7 @@ dependencies = [ "rscrypto", "rscrypto-fuzz-support", "scrypt", - "sha2 0.11.0", + "sha2", "sha3", "tiny-keccak", "x25519-dalek", @@ -1076,7 +858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" dependencies = [ "cfg-if", - "cipher 0.5.2", + "cipher", ] [[package]] @@ -1088,21 +870,7 @@ dependencies = [ "cfg-if", "pbkdf2", "salsa20", - "sha2 0.11.0", -] - -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", - "subtle", - "zeroize", + "sha2", ] [[package]] @@ -1111,9 +879,9 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 1.0.0", + "base16ct", "ctutils", - "der 0.8.1", + "der", "hybrid-array", "subtle", "zeroize", @@ -1132,7 +900,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", - "serde_derive", ] [[package]] @@ -1161,21 +928,10 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" dependencies = [ - "base16ct 1.0.0", + "base16ct", "serde", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -1204,16 +960,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "3.0.0" @@ -1230,16 +976,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der 0.7.10", -] - [[package]] name = "spki" version = "0.8.0" @@ -1247,7 +983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.1", + "der", ] [[package]] @@ -1305,16 +1041,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common 0.1.7", - "subtle", -] - [[package]] name = "universal-hash" version = "0.6.1" @@ -1331,32 +1057,25 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wnaf" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1" dependencies = [ - "ff 0.14.0", - "group 0.14.0", + "ff", + "group", "hybrid-array", ] [[package]] name = "x25519-dalek" -version = "2.0.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ "curve25519-dalek", - "rand_core 0.6.4", - "serde", + "rand_core 0.10.1", "zeroize", ] @@ -1371,17 +1090,3 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index ce331f94..a60b0fb3 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -8,7 +8,7 @@ edition = "2024" cargo-fuzz = true [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = [ +unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(fuzzing)', 'cfg(rscrypto_internal_fuzzing)', ] } @@ -28,24 +28,24 @@ rscrypto = { path = "..", features = ["std", "checksums", "hashes", "auth", "aea # AEAD ascon-aead = { version = "0.6", default-features = false, features = ["alloc"] } aes-gcm = { version = "0.11", default-features = false, features = ["alloc", "aes"] } -aes-gcm-siv = { version = "0.11", default-features = false, features = ["alloc", "aes"] } +aes-gcm-siv = { version = "0.12", default-features = false, features = ["alloc", "aes"] } aegis = "0.9" chacha20poly1305 = { version = "0.11", default-features = false, features = ["alloc"] } # Auth -argon2 = { version = "0.6.0-rc.8", default-features = false, features = ["alloc"] } -ed25519-dalek = "2.2" +argon2 = { version = "0.5.3", default-features = false, features = ["alloc"] } +ed25519-dalek = "3.0" hmac = "0.13" hkdf = "0.13" p256 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } -p384 = { version = "0.13.1", default-features = false, features = ["ecdsa", "pkcs8", "std"] } +p384 = { version = "0.14.0", default-features = false, features = ["ecdsa", "pkcs8", "std"] } pbkdf2 = "0.13" scrypt = { version = "0.12", default-features = false } -x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +x25519-dalek = { version = "3.0.0", features = ["static_secrets"] } # Hashes ascon-hash = "0.4" -blake2 = "0.11.0-rc.6" +blake2 = "0.10.6" blake3 = "1.8" cshake = { version = "0.2.1", default-features = false } digest = "0.11" diff --git a/fuzz/fuzz_targets/aead_aegis256.rs b/fuzz/fuzz_targets/aead_aegis256.rs index 44802516..09aaf1af 100644 --- a/fuzz/fuzz_targets/aead_aegis256.rs +++ b/fuzz/fuzz_targets/aead_aegis256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes128gcm.rs b/fuzz/fuzz_targets/aead_aes128gcm.rs index 30d3a805..dcbcb90f 100644 --- a/fuzz/fuzz_targets/aead_aes128gcm.rs +++ b/fuzz/fuzz_targets/aead_aes128gcm.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes128gcmsiv.rs b/fuzz/fuzz_targets/aead_aes128gcmsiv.rs index 3276daf4..ea606ddd 100644 --- a/fuzz/fuzz_targets/aead_aes128gcmsiv.rs +++ b/fuzz/fuzz_targets/aead_aes128gcmsiv.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes256gcm.rs b/fuzz/fuzz_targets/aead_aes256gcm.rs index 691613d5..833a768c 100644 --- a/fuzz/fuzz_targets/aead_aes256gcm.rs +++ b/fuzz/fuzz_targets/aead_aes256gcm.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_aes256gcmsiv.rs b/fuzz/fuzz_targets/aead_aes256gcmsiv.rs index e182e649..f5aefce3 100644 --- a/fuzz/fuzz_targets/aead_aes256gcmsiv.rs +++ b/fuzz/fuzz_targets/aead_aes256gcmsiv.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_ascon128.rs b/fuzz/fuzz_targets/aead_ascon128.rs index e60350d4..7cb5625b 100644 --- a/fuzz/fuzz_targets/aead_ascon128.rs +++ b/fuzz/fuzz_targets/aead_ascon128.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_chacha20poly1305.rs b/fuzz/fuzz_targets/aead_chacha20poly1305.rs index 86673314..b18a79c9 100644 --- a/fuzz/fuzz_targets/aead_chacha20poly1305.rs +++ b/fuzz/fuzz_targets/aead_chacha20poly1305.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_nonce_counter.rs b/fuzz/fuzz_targets/aead_nonce_counter.rs index 35d353da..97ef6b9b 100644 --- a/fuzz/fuzz_targets/aead_nonce_counter.rs +++ b/fuzz/fuzz_targets/aead_nonce_counter.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/aead_xchacha20poly1305.rs b/fuzz/fuzz_targets/aead_xchacha20poly1305.rs index 489e289e..2d0d6078 100644 --- a/fuzz/fuzz_targets/aead_xchacha20poly1305.rs +++ b/fuzz/fuzz_targets/aead_xchacha20poly1305.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_argon2d.rs b/fuzz/fuzz_targets/auth_argon2d.rs index a45efb2f..3d97698d 100644 --- a/fuzz/fuzz_targets/auth_argon2d.rs +++ b/fuzz/fuzz_targets/auth_argon2d.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_argon2i.rs b/fuzz/fuzz_targets/auth_argon2i.rs index 92a97153..06414a6f 100644 --- a/fuzz/fuzz_targets/auth_argon2i.rs +++ b/fuzz/fuzz_targets/auth_argon2i.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_argon2id.rs b/fuzz/fuzz_targets/auth_argon2id.rs index 7d86ce8d..82aaffa8 100644 --- a/fuzz/fuzz_targets/auth_argon2id.rs +++ b/fuzz/fuzz_targets/auth_argon2id.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_ed25519.rs b/fuzz/fuzz_targets/auth_ed25519.rs index 1447944c..bdc6c808 100644 --- a/fuzz/fuzz_targets/auth_ed25519.rs +++ b/fuzz/fuzz_targets/auth_ed25519.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_ed25519_verify.rs b/fuzz/fuzz_targets/auth_ed25519_verify.rs index 1feb01d4..a61f9316 100644 --- a/fuzz/fuzz_targets/auth_ed25519_verify.rs +++ b/fuzz/fuzz_targets/auth_ed25519_verify.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hkdf_sha256.rs b/fuzz/fuzz_targets/auth_hkdf_sha256.rs index 7a356e3b..9beee577 100644 --- a/fuzz/fuzz_targets/auth_hkdf_sha256.rs +++ b/fuzz/fuzz_targets/auth_hkdf_sha256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hkdf_sha384.rs b/fuzz/fuzz_targets/auth_hkdf_sha384.rs index 41edb832..d48279d1 100644 --- a/fuzz/fuzz_targets/auth_hkdf_sha384.rs +++ b/fuzz/fuzz_targets/auth_hkdf_sha384.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hmac_sha256.rs b/fuzz/fuzz_targets/auth_hmac_sha256.rs index 4249bbbf..a02999eb 100644 --- a/fuzz/fuzz_targets/auth_hmac_sha256.rs +++ b/fuzz/fuzz_targets/auth_hmac_sha256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hmac_sha384.rs b/fuzz/fuzz_targets/auth_hmac_sha384.rs index 3c867e00..e372fcb9 100644 --- a/fuzz/fuzz_targets/auth_hmac_sha384.rs +++ b/fuzz/fuzz_targets/auth_hmac_sha384.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_hmac_sha512.rs b/fuzz/fuzz_targets/auth_hmac_sha512.rs index 0205f50e..441a4a53 100644 --- a/fuzz/fuzz_targets/auth_hmac_sha512.rs +++ b/fuzz/fuzz_targets/auth_hmac_sha512.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_kmac256.rs b/fuzz/fuzz_targets/auth_kmac256.rs index c87e8e57..c33d9039 100644 --- a/fuzz/fuzz_targets/auth_kmac256.rs +++ b/fuzz/fuzz_targets/auth_kmac256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_pbkdf2.rs b/fuzz/fuzz_targets/auth_pbkdf2.rs index ff7a4668..c70fea3d 100644 --- a/fuzz/fuzz_targets/auth_pbkdf2.rs +++ b/fuzz/fuzz_targets/auth_pbkdf2.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_phc.rs b/fuzz/fuzz_targets/auth_phc.rs index a77ff641..126149a9 100644 --- a/fuzz/fuzz_targets/auth_phc.rs +++ b/fuzz/fuzz_targets/auth_phc.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_scrypt.rs b/fuzz/fuzz_targets/auth_scrypt.rs index 177e0bf6..be949324 100644 --- a/fuzz/fuzz_targets/auth_scrypt.rs +++ b/fuzz/fuzz_targets/auth_scrypt.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/auth_x25519.rs b/fuzz/fuzz_targets/auth_x25519.rs index 04ba47b1..36703851 100644 --- a/fuzz/fuzz_targets/auth_x25519.rs +++ b/fuzz/fuzz_targets/auth_x25519.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/checksum_crc.rs b/fuzz/fuzz_targets/checksum_crc.rs index 8ce3e7d4..092575b6 100644 --- a/fuzz/fuzz_targets/checksum_crc.rs +++ b/fuzz/fuzz_targets/checksum_crc.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/fast_rapidhash.rs b/fuzz/fuzz_targets/fast_rapidhash.rs index 6826985a..212c8422 100644 --- a/fuzz/fuzz_targets/fast_rapidhash.rs +++ b/fuzz/fuzz_targets/fast_rapidhash.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/fast_xxh3.rs b/fuzz/fuzz_targets/fast_xxh3.rs index e1967fa2..dbd78f4d 100644 --- a/fuzz/fuzz_targets/fast_xxh3.rs +++ b/fuzz/fuzz_targets/fast_xxh3.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_ascon.rs b/fuzz/fuzz_targets/hash_ascon.rs index 6a652898..bd00a12d 100644 --- a/fuzz/fuzz_targets/hash_ascon.rs +++ b/fuzz/fuzz_targets/hash_ascon.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_ascon_cxof.rs b/fuzz/fuzz_targets/hash_ascon_cxof.rs index bb51e4f9..3d93175d 100644 --- a/fuzz/fuzz_targets/hash_ascon_cxof.rs +++ b/fuzz/fuzz_targets/hash_ascon_cxof.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake2b.rs b/fuzz/fuzz_targets/hash_blake2b.rs index d9416c39..288c2abb 100644 --- a/fuzz/fuzz_targets/hash_blake2b.rs +++ b/fuzz/fuzz_targets/hash_blake2b.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake2s.rs b/fuzz/fuzz_targets/hash_blake2s.rs index 5cee46ce..e6b431df 100644 --- a/fuzz/fuzz_targets/hash_blake2s.rs +++ b/fuzz/fuzz_targets/hash_blake2s.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake3.rs b/fuzz/fuzz_targets/hash_blake3.rs index ad8d8210..b2835b66 100644 --- a/fuzz/fuzz_targets/hash_blake3.rs +++ b/fuzz/fuzz_targets/hash_blake3.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake3_derive.rs b/fuzz/fuzz_targets/hash_blake3_derive.rs index 0d92de67..609f2c3a 100644 --- a/fuzz/fuzz_targets/hash_blake3_derive.rs +++ b/fuzz/fuzz_targets/hash_blake3_derive.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_blake3_keyed.rs b/fuzz/fuzz_targets/hash_blake3_keyed.rs index adbde5eb..1daf7c28 100644 --- a/fuzz/fuzz_targets/hash_blake3_keyed.rs +++ b/fuzz/fuzz_targets/hash_blake3_keyed.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_cshake256.rs b/fuzz/fuzz_targets/hash_cshake256.rs index 5ebd4601..52c2bc46 100644 --- a/fuzz/fuzz_targets/hash_cshake256.rs +++ b/fuzz/fuzz_targets/hash_cshake256.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_sha2.rs b/fuzz/fuzz_targets/hash_sha2.rs index f7397733..306888bc 100644 --- a/fuzz/fuzz_targets/hash_sha2.rs +++ b/fuzz/fuzz_targets/hash_sha2.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hash_sha3.rs b/fuzz/fuzz_targets/hash_sha3.rs index 7d72a358..49b9f46d 100644 --- a/fuzz/fuzz_targets/hash_sha3.rs +++ b/fuzz/fuzz_targets/hash_sha3.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/hex_parse.rs b/fuzz/fuzz_targets/hex_parse.rs index 45ee5985..8a947d3b 100644 --- a/fuzz/fuzz_targets/hex_parse.rs +++ b/fuzz/fuzz_targets/hex_parse.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/fuzz_targets/traits_io.rs b/fuzz/fuzz_targets/traits_io.rs index e741afd4..fe50fb54 100644 --- a/fuzz/fuzz_targets/traits_io.rs +++ b/fuzz/fuzz_targets/traits_io.rs @@ -4,5 +4,5 @@ mod target_impl; libfuzzer_sys::fuzz_target!(|data: &[u8]| { - target_impl::run(data); + target_impl::run(data); }); diff --git a/fuzz/support/src/lib.rs b/fuzz/support/src/lib.rs index ab1ff4f4..a2e0c9a5 100644 --- a/fuzz/support/src/lib.rs +++ b/fuzz/support/src/lib.rs @@ -51,12 +51,12 @@ where ); let mut files = fs::read_dir(corpus_dir) - .unwrap_or_else(|err| panic!("corpus replay: failed to read {}: {err}", corpus_dir.display())) - .map(|entry| entry.unwrap_or_else(|err| panic!("corpus replay: failed to read directory entry: {err}"))) + .expect("corpus replay directory must be readable") + .map(|entry| entry.expect("corpus replay directory entries must be readable")) .filter_map(|entry| { let file_type = entry .file_type() - .unwrap_or_else(|err| panic!("corpus replay: failed to inspect {}: {err}", entry.path().display())); + .expect("corpus replay directory entry metadata must be readable"); if file_type.is_file() || file_type.is_symlink() { Some(entry.path()) } else { @@ -74,7 +74,7 @@ where let mut count = 0usize; for path in files { - let data = fs::read(&path).unwrap_or_else(|err| panic!("corpus replay: failed to read {}: {err}", path.display())); + let data = fs::read(&path).expect("corpus replay input must be readable"); run(&data); count = count.checked_add(1).expect("corpus replay file count overflow"); } @@ -154,7 +154,7 @@ pub fn pad_salt_to(material: &[u8], filler: u8) -> [u8; N] { out[..N / 2].fill(filler); } else { for (i, slot) in out.iter_mut().enumerate() { - *slot = material[i % material.len()]; + *slot = material[i.rem_euclid(material.len())]; } } out @@ -258,7 +258,7 @@ pub fn assert_aead_forgery(cipher: &A, nonce: &A::Nonce, aad: &[u8], pl match target { 0 if !ct.is_empty() => { let mut forged = ct.clone(); - let idx = seed as usize % forged.len(); + let idx = usize::from(seed).rem_euclid(forged.len()); forged[idx] ^= 1u8 << (seed as u32 & 7); assert!( cipher.decrypt_in_place(nonce, aad, &mut forged, &tag).is_err(), @@ -267,7 +267,7 @@ pub fn assert_aead_forgery(cipher: &A, nonce: &A::Nonce, aad: &[u8], pl } 2 if !aad.is_empty() => { let mut forged_aad = aad.to_vec(); - let idx = seed as usize % forged_aad.len(); + let idx = usize::from(seed).rem_euclid(forged_aad.len()); forged_aad[idx] ^= 1u8 << (seed as u32 & 7); let mut ct_copy = ct.clone(); assert!( @@ -278,9 +278,9 @@ pub fn assert_aead_forgery(cipher: &A, nonce: &A::Nonce, aad: &[u8], pl _ => { let tag_ref = tag.as_ref(); let mut tag_bytes = tag_ref.to_vec(); - let idx = seed as usize % tag_bytes.len(); + let idx = usize::from(seed).rem_euclid(tag_bytes.len()); tag_bytes[idx] ^= 1u8 << (seed as u32 & 7); - let forged_tag = A::tag_from_slice(&tag_bytes).unwrap(); + let forged_tag = A::tag_from_slice(&tag_bytes).expect("forged tag preserves the algorithm tag length"); let mut ct_copy = ct.clone(); assert!( cipher.decrypt_in_place(nonce, aad, &mut ct_copy, &forged_tag).is_err(), diff --git a/fuzz/target_impls/aead_aegis256.rs b/fuzz/target_impls/aead_aegis256.rs index af12a9c0..4fc1f8c6 100644 --- a/fuzz/target_impls/aead_aegis256.rs +++ b/fuzz/target_impls/aead_aegis256.rs @@ -1,10 +1,10 @@ use rscrypto::{ - Aegis256, Aegis256Key, - aead::{Nonce256, expert::AeadWithNonce}, + Aegis256, Aegis256Key, + aead::{Nonce256, expert::AeadWithNonce}, }; use rscrypto_fuzz::{FuzzInput, assert_aead_forgery, assert_aead_roundtrip, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 32] = some_or_return!(input.bytes()); @@ -26,9 +26,14 @@ pub fn run(data: &[u8]) { let tag = cipher .encrypt_in_place(&nonce, aad, &mut ct) .expect("differential: rscrypto encrypt must succeed"); - let tag_arr: [u8; 16] = tag.as_ref().try_into().unwrap(); + let tag_arr: [u8; 16] = tag + .as_ref() + .try_into() + .expect("AEGIS-256 produces a 16-byte authentication tag"); let oracle = OracleAegis::<16>::new(&key_bytes, &nonce_bytes); - let pt = oracle.decrypt(&ct, &tag_arr, aad).unwrap(); + let pt = oracle + .decrypt(&ct, &tag_arr, aad) + .expect("oracle must accept the equivalent rscrypto ciphertext"); assert_eq!(pt, plaintext, "oracle failed to decrypt our ciphertext"); // oracle encrypt → rscrypto decrypt @@ -36,7 +41,12 @@ pub fn run(data: &[u8]) { let (oct, otag) = oracle_enc.encrypt(plaintext, aad); let mut buf = oct; cipher - .decrypt_in_place(&nonce, aad, &mut buf, &Aegis256::tag_from_slice(&otag).unwrap()) + .decrypt_in_place( + &nonce, + aad, + &mut buf, + &Aegis256::tag_from_slice(&otag).expect("oracle produces a 16-byte authentication tag"), + ) .expect("we failed to decrypt oracle ciphertext"); assert_eq!(buf, plaintext, "decrypt mismatch on oracle ciphertext"); } diff --git a/fuzz/target_impls/aead_aes128gcm.rs b/fuzz/target_impls/aead_aes128gcm.rs index b810880f..2a7f88b9 100644 --- a/fuzz/target_impls/aead_aes128gcm.rs +++ b/fuzz/target_impls/aead_aes128gcm.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 16] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm crate. use aes_gcm::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm::Aes128Gcm::new_from_slice(&key_bytes).unwrap(); + let oracle = aes_gcm::Aes128Gcm::new_from_slice(&key_bytes).expect("AES-128-GCM accepts a 16-byte key"); let on = aes_gcm::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_aes128gcmsiv.rs b/fuzz/target_impls/aead_aes128gcmsiv.rs index 036ec8cc..3d0055bc 100644 --- a/fuzz/target_impls/aead_aes128gcmsiv.rs +++ b/fuzz/target_impls/aead_aes128gcmsiv.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 16] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm-siv crate. use aes_gcm_siv::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm_siv::Aes128GcmSiv::new_from_slice(&key_bytes).unwrap(); - let on = aes_gcm_siv::Nonce::from_slice(&nonce_bytes); + let oracle = aes_gcm_siv::Aes128GcmSiv::new_from_slice(&key_bytes).expect("AES-128-GCM-SIV accepts a 16-byte key"); + let on = aes_gcm_siv::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_aes256gcm.rs b/fuzz/target_impls/aead_aes256gcm.rs index 73e0d142..40a49538 100644 --- a/fuzz/target_impls/aead_aes256gcm.rs +++ b/fuzz/target_impls/aead_aes256gcm.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm crate. use aes_gcm::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm::Aes256Gcm::new_from_slice(&key_bytes).unwrap(); + let oracle = aes_gcm::Aes256Gcm::new_from_slice(&key_bytes).expect("AES-256-GCM accepts a 32-byte key"); let on = aes_gcm::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_aes256gcmsiv.rs b/fuzz/target_impls/aead_aes256gcmsiv.rs index 5c171acd..34ce2186 100644 --- a/fuzz/target_impls/aead_aes256gcmsiv.rs +++ b/fuzz/target_impls/aead_aes256gcmsiv.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ aes-gcm-siv crate. use aes_gcm_siv::aead::{Aead as _, KeyInit, Payload}; - let oracle = aes_gcm_siv::Aes256GcmSiv::new_from_slice(&key_bytes).unwrap(); - let on = aes_gcm_siv::Nonce::from_slice(&nonce_bytes); + let oracle = aes_gcm_siv::Aes256GcmSiv::new_from_slice(&key_bytes).expect("AES-256-GCM-SIV accepts a 32-byte key"); + let on = aes_gcm_siv::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_ascon128.rs b/fuzz/target_impls/aead_ascon128.rs index d2fd430e..fa6dc6df 100644 --- a/fuzz/target_impls/aead_ascon128.rs +++ b/fuzz/target_impls/aead_ascon128.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 16] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 16] = some_or_return!(input.bytes()); @@ -18,14 +18,22 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ ascon-aead crate. use ascon_aead::aead::{Aead as _, KeyInit, Payload, array::Array}; - let oracle = ascon_aead::AsconAead128::new_from_slice(&key_bytes).unwrap(); + let oracle = ascon_aead::AsconAead128::new_from_slice(&key_bytes).expect("Ascon-AEAD128 accepts a 16-byte key"); let on = Array(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_chacha20poly1305.rs b/fuzz/target_impls/aead_chacha20poly1305.rs index 296d041c..039c0ab9 100644 --- a/fuzz/target_impls/aead_chacha20poly1305.rs +++ b/fuzz/target_impls/aead_chacha20poly1305.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); @@ -18,14 +18,23 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ chacha20poly1305 crate. use chacha20poly1305::aead::{Aead as _, KeyInit, Payload}; - let oracle = chacha20poly1305::ChaCha20Poly1305::new_from_slice(&key_bytes).unwrap(); + let oracle = + chacha20poly1305::ChaCha20Poly1305::new_from_slice(&key_bytes).expect("ChaCha20-Poly1305 accepts a 32-byte key"); let on = chacha20poly1305::Nonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/aead_nonce_counter.rs b/fuzz/target_impls/aead_nonce_counter.rs index cd681fa2..b764fc48 100644 --- a/fuzz/target_impls/aead_nonce_counter.rs +++ b/fuzz/target_impls/aead_nonce_counter.rs @@ -1,6 +1,6 @@ use rscrypto::{ - Aes256Gcm, Aes256GcmKey, - aead::{NonceCounter, expert::AeadWithNonce}, + Aes256Gcm, Aes256GcmKey, + aead::{NonceCounter, expert::AeadWithNonce}, }; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; @@ -87,7 +87,7 @@ fn assert_resume_equivalence(prefix: [u8; 4], advance: u32) { ); } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let prefix: [u8; 4] = some_or_return!(input.bytes()); @@ -101,12 +101,12 @@ pub fn run(data: &[u8]) { // are pinned by `aes_gcm_nonce_counter_exhausts_cleanly` and // `aes_gcm_nonce_counter_with_counter_rejects_max` in src/aead/nonce_counter.rs. let initial = u64::from_le_bytes(initial_bytes) - % NonceCounter::::MAX_MESSAGES.strict_sub(u64::from(MAX_NONCES_PER_ITER)); - let burst = (u32::from(burst_byte) % MAX_NONCES_PER_ITER).strict_add(1); + .rem_euclid(NonceCounter::::MAX_MESSAGES.strict_sub(u64::from(MAX_NONCES_PER_ITER))); + let burst = u32::from(burst_byte).rem_euclid(MAX_NONCES_PER_ITER).strict_add(1); // Resume-equivalence advance bounded so the fresh-counter loop runs // ≤ MAX_NONCES_PER_ITER times — exercised every iteration rather than // gated on a 10⁻¹² fuzzer probability. - let resume_advance = u32::from(burst_byte) % MAX_NONCES_PER_ITER; + let resume_advance = u32::from(burst_byte).rem_euclid(MAX_NONCES_PER_ITER); let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes(key_bytes)); let (aad, plaintext) = split_at_ratio(rest, aad_split); diff --git a/fuzz/target_impls/aead_xchacha20poly1305.rs b/fuzz/target_impls/aead_xchacha20poly1305.rs index b0341567..5f08f3b9 100644 --- a/fuzz/target_impls/aead_xchacha20poly1305.rs +++ b/fuzz/target_impls/aead_xchacha20poly1305.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_aead_against_oracle, assert_aead_forgery, assert_aead_roundtrip, some_or_return, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let nonce_bytes: [u8; 24] = some_or_return!(input.bytes()); @@ -18,14 +18,23 @@ pub fn run(data: &[u8]) { // Differential: rscrypto ↔ chacha20poly1305 crate (XChaCha variant). use chacha20poly1305::aead::{Aead as _, KeyInit, Payload}; - let oracle = chacha20poly1305::XChaCha20Poly1305::new_from_slice(&key_bytes).unwrap(); + let oracle = + chacha20poly1305::XChaCha20Poly1305::new_from_slice(&key_bytes).expect("XChaCha20-Poly1305 accepts a 32-byte key"); let on = chacha20poly1305::XNonce::from(nonce_bytes); assert_aead_against_oracle( &cipher, &nonce, aad, plaintext, - |pt, aad| oracle.encrypt(&on, Payload { msg: pt, aad }).unwrap(), - |ct, aad| oracle.decrypt(&on, Payload { msg: ct, aad }).unwrap(), + |pt, aad| { + oracle + .encrypt(&on, Payload { msg: pt, aad }) + .expect("oracle encryption accepts the fuzz input") + }, + |ct, aad| { + oracle + .decrypt(&on, Payload { msg: ct, aad }) + .expect("oracle must accept the equivalent rscrypto ciphertext") + }, ); } diff --git a/fuzz/target_impls/auth_argon2d.rs b/fuzz/target_impls/auth_argon2d.rs index 0b1c083d..2c10d2b9 100644 --- a/fuzz/target_impls/auth_argon2d.rs +++ b/fuzz/target_impls/auth_argon2d.rs @@ -1,7 +1,7 @@ use rscrypto::{Argon2Params, Argon2d}; use rscrypto_fuzz::{FuzzInput, pad_salt_to, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -13,8 +13,7 @@ pub fn run(data: &[u8]) { let t = 1u32.strict_add(u32::from(t_byte) % 4); let out_len = 4u32.strict_add(u32::from(out_len_byte) % 29); - let params = Argon2Params::new(m_kib, t, 1) - .expect("params must be valid for fuzzer ranges"); + let params = Argon2Params::new(m_kib, t, 1).expect("params must be valid for fuzzer ranges"); // See `auth_argon2id.rs` for cost-parameter rationale. let (password, salt_material) = split_at_ratio(rest, pw_salt_split); diff --git a/fuzz/target_impls/auth_argon2i.rs b/fuzz/target_impls/auth_argon2i.rs index 799f2adf..0430dce9 100644 --- a/fuzz/target_impls/auth_argon2i.rs +++ b/fuzz/target_impls/auth_argon2i.rs @@ -1,7 +1,7 @@ use rscrypto::{Argon2Params, Argon2i}; use rscrypto_fuzz::{FuzzInput, pad_salt_to, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -13,8 +13,7 @@ pub fn run(data: &[u8]) { let t = 1u32.strict_add(u32::from(t_byte) % 4); let out_len = 4u32.strict_add(u32::from(out_len_byte) % 29); - let params = Argon2Params::new(m_kib, t, 1) - .expect("params must be valid for fuzzer ranges"); + let params = Argon2Params::new(m_kib, t, 1).expect("params must be valid for fuzzer ranges"); // See `auth_argon2id.rs` for cost-parameter rationale. let (password, salt_material) = split_at_ratio(rest, pw_salt_split); diff --git a/fuzz/target_impls/auth_argon2id.rs b/fuzz/target_impls/auth_argon2id.rs index 57fef803..7e0bc4a8 100644 --- a/fuzz/target_impls/auth_argon2id.rs +++ b/fuzz/target_impls/auth_argon2id.rs @@ -1,7 +1,7 @@ use rscrypto::{Argon2Params, Argon2id}; use rscrypto_fuzz::{FuzzInput, pad_salt_to, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -16,8 +16,7 @@ pub fn run(data: &[u8]) { let t = 1u32.strict_add(u32::from(t_byte) % 4); let out_len = 4u32.strict_add(u32::from(out_len_byte) % 29); // 4..=32 per RFC 9106 §3.1 - let params = Argon2Params::new(m_kib, t, 1) - .expect("params must be valid for fuzzer ranges"); + let params = Argon2Params::new(m_kib, t, 1).expect("params must be valid for fuzzer ranges"); let (password, salt_material) = split_at_ratio(rest, pw_salt_split); let salt_buf = pad_salt_to::<16>(salt_material, pw_salt_split); diff --git a/fuzz/target_impls/auth_ecdsa_sign.rs b/fuzz/target_impls/auth_ecdsa_sign.rs index fb1dd2e7..f796f4bc 100644 --- a/fuzz/target_impls/auth_ecdsa_sign.rs +++ b/fuzz/target_impls/auth_ecdsa_sign.rs @@ -7,7 +7,7 @@ use p384::ecdsa::{ use rscrypto::{EcdsaP256Keypair, EcdsaP256SecretKey, EcdsaP384Keypair, EcdsaP384SecretKey}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let selector = some_or_return!(input.byte()); @@ -25,15 +25,21 @@ fn run_p256(input: &mut FuzzInput<'_>) { let message = input.rest(); let public = keypair.public_key(); let signature = some_or_return!(keypair.try_sign(message).ok()); - let oracle_public = P256OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-256 public key"); + let oracle_public = + P256OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-256 public key"); let oracle_signature = P256OracleSignature::from_slice(signature.as_bytes()).expect("derived P-256 signature"); - assert!(public.verify(message, &signature).is_ok()); - assert!(P256Verifier::verify(&oracle_public, message, &oracle_signature).is_ok()); + public + .verify(message, &signature) + .expect("P-256 public key must verify its own signature"); + P256Verifier::verify(&oracle_public, message, &oracle_signature) + .expect("P-256 oracle must verify the equivalent signature"); let mut tampered = message.to_vec(); tampered.push(0x80); - assert!(public.verify(&tampered, &signature).is_err()); + let _verification_error = public + .verify(&tampered, &signature) + .expect_err("P-256 verification must reject a tampered message"); } fn run_p384(input: &mut FuzzInput<'_>) { @@ -43,13 +49,19 @@ fn run_p384(input: &mut FuzzInput<'_>) { let message = input.rest(); let public = keypair.public_key(); let signature = some_or_return!(keypair.try_sign(message).ok()); - let oracle_public = P384OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-384 public key"); + let oracle_public = + P384OracleVerifyingKey::from_sec1_bytes(&public.to_sec1_bytes()).expect("derived P-384 public key"); let oracle_signature = P384OracleSignature::from_slice(signature.as_bytes()).expect("derived P-384 signature"); - assert!(public.verify(message, &signature).is_ok()); - assert!(P384Verifier::verify(&oracle_public, message, &oracle_signature).is_ok()); + public + .verify(message, &signature) + .expect("P-384 public key must verify its own signature"); + P384Verifier::verify(&oracle_public, message, &oracle_signature) + .expect("P-384 oracle must verify the equivalent signature"); let mut tampered = message.to_vec(); tampered.push(0x80); - assert!(public.verify(&tampered, &signature).is_err()); + let _verification_error = public + .verify(&tampered, &signature) + .expect_err("P-384 verification must reject a tampered message"); } diff --git a/fuzz/target_impls/auth_ecdsa_verify.rs b/fuzz/target_impls/auth_ecdsa_verify.rs index 2c04d88f..b38245c9 100644 --- a/fuzz/target_impls/auth_ecdsa_verify.rs +++ b/fuzz/target_impls/auth_ecdsa_verify.rs @@ -13,7 +13,7 @@ fn array_from_slice(bytes: &[u8]) -> [u8; N] { bytes.try_into().expect("oracle ECDSA signature length must match") } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let selector = some_or_return!(input.byte()); let derive_public = selector & 2 != 0; diff --git a/fuzz/target_impls/auth_ed25519.rs b/fuzz/target_impls/auth_ed25519.rs index fca0dbb8..fb1dae1c 100644 --- a/fuzz/target_impls/auth_ed25519.rs +++ b/fuzz/target_impls/auth_ed25519.rs @@ -1,7 +1,7 @@ use rscrypto::{Ed25519SecretKey, Ed25519Signature}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_bytes: [u8; 32] = some_or_return!(input.bytes()); let message = input.rest(); diff --git a/fuzz/target_impls/auth_ed25519_verify.rs b/fuzz/target_impls/auth_ed25519_verify.rs index 5f5fc732..a1f63c87 100644 --- a/fuzz/target_impls/auth_ed25519_verify.rs +++ b/fuzz/target_impls/auth_ed25519_verify.rs @@ -1,7 +1,7 @@ use rscrypto::{Ed25519PublicKey, Ed25519Signature}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let public_bytes: [u8; 32] = some_or_return!(input.bytes()); let signature_bytes: [u8; 64] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/auth_hkdf_sha256.rs b/fuzz/target_impls/auth_hkdf_sha256.rs index 8091ec39..bcd3b119 100644 --- a/fuzz/target_impls/auth_hkdf_sha256.rs +++ b/fuzz/target_impls/auth_hkdf_sha256.rs @@ -1,7 +1,7 @@ use rscrypto::HkdfSha256; use rscrypto_fuzz::{FuzzInput, assert_hkdf_against_oracle, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let salt_split: u8 = some_or_return!(input.byte()); let ikm_split: u8 = some_or_return!(input.byte()); @@ -14,7 +14,7 @@ pub fn run(data: &[u8]) { // Exercise both valid and over-length expand requests without allocating // unbounded buffers. - let out_len = usize::from(u16::from_le_bytes(out_len_bytes)) % HkdfSha256::MAX_OUTPUT_SIZE.strict_add(33); + let out_len = usize::from(u16::from_le_bytes(out_len_bytes)).rem_euclid(HkdfSha256::MAX_OUTPUT_SIZE.strict_add(33)); let hk = HkdfSha256::new(salt, ikm); let mut okm = vec![0u8; out_len]; diff --git a/fuzz/target_impls/auth_hkdf_sha384.rs b/fuzz/target_impls/auth_hkdf_sha384.rs index 8ba07a37..45c6be9d 100644 --- a/fuzz/target_impls/auth_hkdf_sha384.rs +++ b/fuzz/target_impls/auth_hkdf_sha384.rs @@ -1,7 +1,7 @@ use rscrypto::HkdfSha384; use rscrypto_fuzz::{FuzzInput, assert_hkdf_against_oracle, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let salt_split: u8 = some_or_return!(input.byte()); let ikm_split: u8 = some_or_return!(input.byte()); @@ -11,7 +11,7 @@ pub fn run(data: &[u8]) { let (salt, remainder) = split_at_ratio(rest, salt_split); let (ikm, info) = split_at_ratio(remainder, ikm_split); - let out_len = usize::from(u16::from_le_bytes(out_len_bytes)) % HkdfSha384::MAX_OUTPUT_SIZE.strict_add(33); + let out_len = usize::from(u16::from_le_bytes(out_len_bytes)).rem_euclid(HkdfSha384::MAX_OUTPUT_SIZE.strict_add(33)); let hk = HkdfSha384::new(salt, ikm); let mut okm = vec![0u8; out_len]; diff --git a/fuzz/target_impls/auth_hmac_sha256.rs b/fuzz/target_impls/auth_hmac_sha256.rs index 14983aa3..446ddc3a 100644 --- a/fuzz/target_impls/auth_hmac_sha256.rs +++ b/fuzz/target_impls/auth_hmac_sha256.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_mac_against_oracle, assert_mac_reset, assert_mac_streaming, some_or_return, split_at_ratio, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -20,7 +20,7 @@ pub fn run(data: &[u8]) { assert_mac_against_oracle::(key, message, &tag, |key, msg| { use hmac::{Hmac, KeyInit, Mac as _}; - let mut oracle = as KeyInit>::new_from_slice(key).unwrap(); + let mut oracle = as KeyInit>::new_from_slice(key).expect("HMAC accepts keys of every length"); oracle.update(msg); oracle.finalize().into_bytes().to_vec() }); diff --git a/fuzz/target_impls/auth_hmac_sha384.rs b/fuzz/target_impls/auth_hmac_sha384.rs index 73aa94cf..3b221513 100644 --- a/fuzz/target_impls/auth_hmac_sha384.rs +++ b/fuzz/target_impls/auth_hmac_sha384.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_mac_against_oracle, assert_mac_reset, assert_mac_streaming, some_or_return, split_at_ratio, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -19,7 +19,7 @@ pub fn run(data: &[u8]) { assert_mac_against_oracle::(key, message, &tag, |key, msg| { use hmac::{Hmac, KeyInit, Mac as _}; - let mut oracle = as KeyInit>::new_from_slice(key).unwrap(); + let mut oracle = as KeyInit>::new_from_slice(key).expect("HMAC accepts keys of every length"); oracle.update(msg); oracle.finalize().into_bytes().to_vec() }); diff --git a/fuzz/target_impls/auth_hmac_sha512.rs b/fuzz/target_impls/auth_hmac_sha512.rs index 9db49b07..5c2b44a4 100644 --- a/fuzz/target_impls/auth_hmac_sha512.rs +++ b/fuzz/target_impls/auth_hmac_sha512.rs @@ -3,7 +3,7 @@ use rscrypto_fuzz::{ FuzzInput, assert_mac_against_oracle, assert_mac_reset, assert_mac_streaming, some_or_return, split_at_ratio, }; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -19,7 +19,7 @@ pub fn run(data: &[u8]) { assert_mac_against_oracle::(key, message, &tag, |key, msg| { use hmac::{Hmac, KeyInit, Mac as _}; - let mut oracle = as KeyInit>::new_from_slice(key).unwrap(); + let mut oracle = as KeyInit>::new_from_slice(key).expect("HMAC accepts keys of every length"); oracle.update(msg); oracle.finalize().into_bytes().to_vec() }); diff --git a/fuzz/target_impls/auth_kmac256.rs b/fuzz/target_impls/auth_kmac256.rs index 31854463..c4e7c75a 100644 --- a/fuzz/target_impls/auth_kmac256.rs +++ b/fuzz/target_impls/auth_kmac256.rs @@ -3,7 +3,11 @@ use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; fn encoded_string_len(len: usize) -> usize { let bits = len.strict_mul(8); - let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); + let width_bits = usize::BITS.strict_sub(bits.leading_zeros()); + let width = usize::try_from(width_bits) + .expect("the bit width fits usize") + .div_ceil(8) + .max(1); 1usize.strict_add(width).strict_add(len) } @@ -15,7 +19,7 @@ fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { encoded_len.is_multiple_of(rate) } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_split: u8 = some_or_return!(input.byte()); @@ -24,7 +28,7 @@ pub fn run(data: &[u8]) { let (key, remainder) = split_at_ratio(rest, key_split); let (custom, message) = split_at_ratio(remainder, split); - let out_len = (out_len_byte as usize % 128).strict_add(1); + let out_len = usize::from(out_len_byte).rem_euclid(128).strict_add(1); // Property: streaming equivalence let mut expected = vec![0u8; out_len]; diff --git a/fuzz/target_impls/auth_mlkem1024.rs b/fuzz/target_impls/auth_mlkem1024.rs index 698c8138..fa6de78f 100644 --- a/fuzz/target_impls/auth_mlkem1024.rs +++ b/fuzz/target_impls/auth_mlkem1024.rs @@ -3,7 +3,7 @@ use rscrypto::{ }; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_random: [u8; MlKem1024::KEY_GENERATION_RANDOM_SIZE] = some_or_return!(input.bytes()); let encaps_random: [u8; MlKem1024::ENCAPSULATION_RANDOM_SIZE] = some_or_return!(input.bytes()); @@ -27,9 +27,9 @@ pub fn run(data: &[u8]) { ); let parse_material = input.rest(); - let _ = MlKem1024EncapsulationKey::try_from_slice(parse_material); - let _ = MlKem1024DecapsulationKey::try_from_slice(parse_material); - let _ = MlKem1024Ciphertext::try_from_slice(parse_material); + let _encapsulation_key_result = MlKem1024EncapsulationKey::try_from_slice(parse_material); + let _decapsulation_key_result = MlKem1024DecapsulationKey::try_from_slice(parse_material); + let _ciphertext_result = MlKem1024Ciphertext::try_from_slice(parse_material); let Some(byte_idx) = input.byte() else { return; @@ -39,7 +39,7 @@ pub fn run(data: &[u8]) { }; let mut modified = ciphertext.to_bytes(); - modified[byte_idx as usize % MlKem1024::CIPHERTEXT_SIZE] ^= 1u8 << (bit_idx & 7); + modified[usize::from(byte_idx).rem_euclid(MlKem1024::CIPHERTEXT_SIZE)] ^= 1u8.strict_shl(u32::from(bit_idx & 7)); let rejected = MlKem1024::decapsulate(&dk, &MlKem1024Ciphertext::from_bytes(modified)) .expect("ML-KEM implicit rejection returns a shared secret"); assert!( diff --git a/fuzz/target_impls/auth_mlkem512.rs b/fuzz/target_impls/auth_mlkem512.rs index 3722449a..23d7b56a 100644 --- a/fuzz/target_impls/auth_mlkem512.rs +++ b/fuzz/target_impls/auth_mlkem512.rs @@ -3,7 +3,7 @@ use rscrypto::{ }; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_random: [u8; MlKem512::KEY_GENERATION_RANDOM_SIZE] = some_or_return!(input.bytes()); let encaps_random: [u8; MlKem512::ENCAPSULATION_RANDOM_SIZE] = some_or_return!(input.bytes()); @@ -27,9 +27,9 @@ pub fn run(data: &[u8]) { ); let parse_material = input.rest(); - let _ = MlKem512EncapsulationKey::try_from_slice(parse_material); - let _ = MlKem512DecapsulationKey::try_from_slice(parse_material); - let _ = MlKem512Ciphertext::try_from_slice(parse_material); + let _encapsulation_key_result = MlKem512EncapsulationKey::try_from_slice(parse_material); + let _decapsulation_key_result = MlKem512DecapsulationKey::try_from_slice(parse_material); + let _ciphertext_result = MlKem512Ciphertext::try_from_slice(parse_material); let Some(byte_idx) = input.byte() else { return; @@ -39,7 +39,7 @@ pub fn run(data: &[u8]) { }; let mut modified = ciphertext.to_bytes(); - modified[byte_idx as usize % MlKem512::CIPHERTEXT_SIZE] ^= 1u8 << (bit_idx & 7); + modified[usize::from(byte_idx).rem_euclid(MlKem512::CIPHERTEXT_SIZE)] ^= 1u8.strict_shl(u32::from(bit_idx & 7)); let rejected = MlKem512::decapsulate(&dk, &MlKem512Ciphertext::from_bytes(modified)) .expect("ML-KEM implicit rejection returns a shared secret"); assert!( diff --git a/fuzz/target_impls/auth_mlkem768.rs b/fuzz/target_impls/auth_mlkem768.rs index e8810455..f5aabf80 100644 --- a/fuzz/target_impls/auth_mlkem768.rs +++ b/fuzz/target_impls/auth_mlkem768.rs @@ -3,7 +3,7 @@ use rscrypto::{ }; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key_random: [u8; MlKem768::KEY_GENERATION_RANDOM_SIZE] = some_or_return!(input.bytes()); let encaps_random: [u8; MlKem768::ENCAPSULATION_RANDOM_SIZE] = some_or_return!(input.bytes()); @@ -27,9 +27,9 @@ pub fn run(data: &[u8]) { ); let parse_material = input.rest(); - let _ = MlKem768EncapsulationKey::try_from_slice(parse_material); - let _ = MlKem768DecapsulationKey::try_from_slice(parse_material); - let _ = MlKem768Ciphertext::try_from_slice(parse_material); + let _encapsulation_key_result = MlKem768EncapsulationKey::try_from_slice(parse_material); + let _decapsulation_key_result = MlKem768DecapsulationKey::try_from_slice(parse_material); + let _ciphertext_result = MlKem768Ciphertext::try_from_slice(parse_material); let Some(byte_idx) = input.byte() else { return; @@ -39,7 +39,7 @@ pub fn run(data: &[u8]) { }; let mut modified = ciphertext.to_bytes(); - modified[byte_idx as usize % MlKem768::CIPHERTEXT_SIZE] ^= 1u8 << (bit_idx & 7); + modified[usize::from(byte_idx).rem_euclid(MlKem768::CIPHERTEXT_SIZE)] ^= 1u8.strict_shl(u32::from(bit_idx & 7)); let rejected = MlKem768::decapsulate(&dk, &MlKem768Ciphertext::from_bytes(modified)) .expect("ML-KEM implicit rejection returns a shared secret"); assert!( diff --git a/fuzz/target_impls/auth_pbkdf2.rs b/fuzz/target_impls/auth_pbkdf2.rs index dd6c921f..a02ed27c 100644 --- a/fuzz/target_impls/auth_pbkdf2.rs +++ b/fuzz/target_impls/auth_pbkdf2.rs @@ -1,7 +1,7 @@ use rscrypto::{Pbkdf2Sha256, Pbkdf2Sha512}; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -9,20 +9,25 @@ pub fn run(data: &[u8]) { let data = input.rest(); let (password, salt) = split_at_ratio(data, split); - let out_len = (out_len_byte as usize % 96).strict_add(1); - let iterations = (u32::from(u16::from_le_bytes(iterations_bytes)) % 64).strict_add(1); + let out_len = usize::from(out_len_byte).rem_euclid(96).strict_add(1); + let iterations = u32::from(u16::from_le_bytes(iterations_bytes)) + .rem_euclid(64) + .strict_add(1); let mut ours_256 = vec![0u8; out_len]; let mut ours_256_state = vec![0u8; out_len]; - Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut ours_256).unwrap(); + Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut ours_256) + .expect("the bounded PBKDF2-SHA-256 request is valid"); Pbkdf2Sha256::new(password) .derive(salt, iterations, &mut ours_256_state) - .unwrap(); + .expect("the bounded PBKDF2-SHA-256 request is valid"); assert_eq!(ours_256, ours_256_state, "pbkdf2-sha256 state reuse mismatch"); - assert!(Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &ours_256).is_ok()); + Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &ours_256) + .expect("PBKDF2-SHA-256 must verify its own output"); let mut wrong_256 = ours_256.clone(); wrong_256[0] ^= 1; - assert!(Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &wrong_256).is_err()); + let _verification_error = Pbkdf2Sha256::verify_password_primitive(password, salt, iterations, &wrong_256) + .expect_err("PBKDF2-SHA-256 must reject a corrupted output"); let mut oracle_256 = vec![0u8; out_len]; pbkdf2::pbkdf2_hmac::(password, salt, iterations, &mut oracle_256); @@ -30,15 +35,18 @@ pub fn run(data: &[u8]) { let mut ours_512 = vec![0u8; out_len]; let mut ours_512_state = vec![0u8; out_len]; - Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut ours_512).unwrap(); + Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut ours_512) + .expect("the bounded PBKDF2-SHA-512 request is valid"); Pbkdf2Sha512::new(password) .derive(salt, iterations, &mut ours_512_state) - .unwrap(); + .expect("the bounded PBKDF2-SHA-512 request is valid"); assert_eq!(ours_512, ours_512_state, "pbkdf2-sha512 state reuse mismatch"); - assert!(Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &ours_512).is_ok()); + Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &ours_512) + .expect("PBKDF2-SHA-512 must verify its own output"); let mut wrong_512 = ours_512.clone(); wrong_512[0] ^= 1; - assert!(Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &wrong_512).is_err()); + let _verification_error = Pbkdf2Sha512::verify_password_primitive(password, salt, iterations, &wrong_512) + .expect_err("PBKDF2-SHA-512 must reject a corrupted output"); let mut oracle_512 = vec![0u8; out_len]; pbkdf2::pbkdf2_hmac::(password, salt, iterations, &mut oracle_512); diff --git a/fuzz/target_impls/auth_phc.rs b/fuzz/target_impls/auth_phc.rs index 0bced237..c3da30cf 100644 --- a/fuzz/target_impls/auth_phc.rs +++ b/fuzz/target_impls/auth_phc.rs @@ -4,24 +4,18 @@ // canonical record remains cheap while malformed or over-budget inputs // exercise the parser and approval boundary at full throughput. -use rscrypto::{ - Argon2Params, Argon2idPassword, ScryptParams, ScryptPassword, -}; +use rscrypto::{Argon2Params, Argon2idPassword, ScryptParams, ScryptPassword}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let split = data.len() / 2; let (password, encoded_bytes) = data.split_at(split); let encoded = String::from_utf8_lossy(encoded_bytes); - let argon2 = Argon2idPassword::new( - Argon2Params::new(8, 1, 1).expect("fixed Argon2 fuzz profile is valid"), - ) - .expect("fixed Argon2 fuzz profile fits the target"); - let scrypt = ScryptPassword::new( - ScryptParams::new(1, 1, 1).expect("fixed scrypt fuzz profile is valid"), - ) - .expect("fixed scrypt fuzz profile fits the target"); + let argon2 = Argon2idPassword::new(Argon2Params::new(8, 1, 1).expect("fixed Argon2 fuzz profile is valid")) + .expect("fixed Argon2 fuzz profile fits the target"); + let scrypt = ScryptPassword::new(ScryptParams::new(1, 1, 1).expect("fixed scrypt fuzz profile is valid")) + .expect("fixed scrypt fuzz profile fits the target"); - let _ = argon2.verify_password(password, &encoded); - let _ = scrypt.verify_password(password, &encoded); + let _argon2_result = argon2.verify_password(password, &encoded); + let _scrypt_result = scrypt.verify_password(password, &encoded); } diff --git a/fuzz/target_impls/auth_rsa_import.rs b/fuzz/target_impls/auth_rsa_import.rs index b63c8ca6..6e71a2a5 100644 --- a/fuzz/target_impls/auth_rsa_import.rs +++ b/fuzz/target_impls/auth_rsa_import.rs @@ -1,10 +1,10 @@ #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -use rscrypto_fuzz::{FuzzInput, some_or_return}; -#[cfg(any(fuzzing, rscrypto_internal_fuzzing))] use rscrypto::auth::rsa::fuzz_rsa_import_der; +#[cfg(any(fuzzing, rscrypto_internal_fuzzing))] +use rscrypto_fuzz::{FuzzInput, some_or_return}; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let format = some_or_return!(input.byte()); let (der, expected) = if input.rest().first().copied() == Some(b'V') { @@ -15,12 +15,15 @@ pub fn run(data: &[u8]) { let accepted = fuzz_rsa_import_der(format, &der); if let Some(expected) = expected { - assert_eq!(accepted, expected, "generated RSA private-key import fixture expectation drifted"); + assert_eq!( + accepted, expected, + "generated RSA private-key import fixture expectation drifted" + ); } } #[cfg(not(any(fuzzing, rscrypto_internal_fuzzing)))] -pub fn run(_data: &[u8]) {} +pub(super) fn run(_data: &[u8]) {} #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn decoded_der(input: &[u8]) -> Vec { @@ -57,9 +60,21 @@ fn hex_value(byte: u8) -> Option { fn valid_private_key_der(format: u8, control: &[u8]) -> (Vec, Option) { let control = control.strip_suffix(b"\n").unwrap_or(control); let pkcs1 = match control.first().copied() { - Some(b'P') => pkcs1_private_key_der_with_crt(&[1], &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX)), - Some(b'Q') => pkcs1_private_key_der_with_crt(&hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), &[1], &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX)), - Some(b'C') => pkcs1_private_key_der_with_crt(&hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), &[1]), + Some(b'P') => pkcs1_private_key_der_with_crt( + &[1], + &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), + &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX), + ), + Some(b'Q') => pkcs1_private_key_der_with_crt( + &hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), + &[1], + &hex_to_vec(RSA_PRIVATE_COEFFICIENT_HEX), + ), + Some(b'C') => pkcs1_private_key_der_with_crt( + &hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), + &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), + &[1], + ), Some(b'N') => pkcs1_private_key_der_with_noncanonical_version(), _ => valid_pkcs1_private_key_der(), }; @@ -94,7 +109,7 @@ fn valid_private_key_der(format: u8, control: &[u8]) -> (Vec, Option) } #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -pub fn valid_pkcs1_private_key_der() -> Vec { +pub(super) fn valid_pkcs1_private_key_der() -> Vec { pkcs1_private_key_der_with_crt( &hex_to_vec(RSA_PRIVATE_EXPONENT_P_HEX), &hex_to_vec(RSA_PRIVATE_EXPONENT_Q_HEX), @@ -143,7 +158,11 @@ fn private_key_der_for_format(format: u8, pkcs1: &[u8]) -> Vec { #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn valid_pkcs8_private_key_der(pkcs1: &[u8]) -> Vec { - sequence(&[integer_unsigned(&[0]), algorithm_identifier(RSA_ENCRYPTION_OID, Some(&der_null())), tlv(0x04, pkcs1)]) + sequence(&[ + integer_unsigned(&[0]), + algorithm_identifier(RSA_ENCRYPTION_OID, Some(&der_null())), + tlv(0x04, pkcs1), + ]) } #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] diff --git a/fuzz/target_impls/auth_rsa_private_ops.rs b/fuzz/target_impls/auth_rsa_private_ops.rs index 706422a4..6b0168c0 100644 --- a/fuzz/target_impls/auth_rsa_private_ops.rs +++ b/fuzz/target_impls/auth_rsa_private_ops.rs @@ -1,13 +1,12 @@ #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] use rscrypto::{ - RsaEncryptionError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, - RsaSignatureProfile, + RsaBlindingPair, RsaEncryptionError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, + RsaPublicKeyPolicy, RsaSignatureProfile, }; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -#[allow(dead_code)] #[path = "auth_rsa_import.rs"] mod rsa_import_fixture; @@ -23,7 +22,7 @@ const PSS_PROFILES: [RsaPssProfile; 3] = [RsaPssProfile::Sha256, RsaPssProfile:: const OAEP_PROFILES: [RsaOaepProfile; 3] = [RsaOaepProfile::Sha256, RsaOaepProfile::Sha384, RsaOaepProfile::Sha512]; #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let selector = some_or_return!(input.byte()); @@ -37,12 +36,12 @@ pub fn run(data: &[u8]) { .expect("fuzz RSA private-key fixture must parse"); let (one, one_inverse) = factor_one(key.signature_len()); - match mode % 10 { + match mode.rem_euclid(10) { 0 => { let profile = PKCS1_PROFILES[usize::from(selector) % PKCS1_PROFILES.len()]; let mut signature = vec![0u8; key.signature_len()]; key - .sign_pkcs1v15_with_blinding_factor(profile, left, &one, &one_inverse, &mut signature) + .sign_pkcs1v15_with_blinding_factor(profile, left, RsaBlindingPair::new(&one, &one_inverse), &mut signature) .expect("fixture RSA-PKCS1-v1_5 signing must succeed"); key .public_key() @@ -54,7 +53,13 @@ pub fn run(data: &[u8]) { let salt = bounded_slice(right, profile.digest_len()); let mut signature = vec![0u8; key.signature_len()]; key - .sign_pss_with_salt_and_blinding_factor(profile, left, salt, &one, &one_inverse, &mut signature) + .sign_pss_with_salt_and_blinding_factor( + profile, + left, + salt, + RsaBlindingPair::new(&one, &one_inverse), + &mut signature, + ) .expect("fixture RSA-PSS signing must succeed"); key .public_key() @@ -73,7 +78,13 @@ pub fn run(data: &[u8]) { .expect("fixture RSA-OAEP encryption must succeed for bounded message"); let mut plaintext = vec![0u8; key.signature_len()]; let plaintext_len = key - .decrypt_oaep_with_blinding_factor(profile, label, &ciphertext, &one, &one_inverse, &mut plaintext) + .decrypt_oaep_with_blinding_factor( + profile, + label, + &ciphertext, + RsaBlindingPair::new(&one, &one_inverse), + &mut plaintext, + ) .expect("self-produced RSA-OAEP ciphertext must decrypt"); assert_eq!(&plaintext[..plaintext_len], message); } @@ -84,8 +95,7 @@ pub fn run(data: &[u8]) { .sign_pkcs1v15_with_blinding_factor( profile.pkcs1v15_profile().expect("profile is PKCS1-v1_5"), left, - &one, - &one_inverse, + RsaBlindingPair::new(&one, &one_inverse), &mut signature, ) .expect("typed fixture RSA-PKCS1-v1_5 signing must succeed"); @@ -98,30 +108,47 @@ pub fn run(data: &[u8]) { let profile = OAEP_PROFILES[usize::from(selector) % OAEP_PROFILES.len()]; let mut plaintext = vec![0u8; key.signature_len()]; let ciphertext = full_width_candidate(left, key.signature_len()); - let _ = key.decrypt_oaep_with_blinding_factor(profile, right, &ciphertext, &one, &one_inverse, &mut plaintext); + let _decryption_result = key.decrypt_oaep_with_blinding_factor( + profile, + right, + &ciphertext, + RsaBlindingPair::new(&one, &one_inverse), + &mut plaintext, + ); } 5 => { let profile = PKCS1_PROFILES[usize::from(selector) % PKCS1_PROFILES.len()]; let mut signature = vec![0u8; key.signature_len()]; let bad_factor = full_width_candidate(left, key.signature_len()); - let _ = key.sign_pkcs1v15_with_blinding_factor(profile, right, &bad_factor, &one_inverse, &mut signature); + let _signing_result = key.sign_pkcs1v15_with_blinding_factor( + profile, + right, + RsaBlindingPair::new(&bad_factor, &one_inverse), + &mut signature, + ); } 6 => { let profile = PSS_PROFILES[usize::from(selector) % PSS_PROFILES.len()]; let mut short_signature = vec![0u8; key.signature_len().saturating_sub(1)]; - assert!(key - .sign_pss_with_salt_and_blinding_factor(profile, left, right, &one, &one_inverse, &mut short_signature) - .is_err()); + let _signing_error = key + .sign_pss_with_salt_and_blinding_factor( + profile, + left, + right, + RsaBlindingPair::new(&one, &one_inverse), + &mut short_signature, + ) + .expect_err("RSA-PSS signing must reject a short output buffer"); } 7 => { let profile = OAEP_PROFILES[usize::from(selector) % OAEP_PROFILES.len()]; let mut ciphertext = vec![0u8; key.signature_len()]; - assert!(key + let _encryption_error = key .public_key() .encrypt_oaep_with_random_fill(profile, left, right, &mut ciphertext, |_| { Err(RsaEncryptionError::EntropyUnavailable) }) - .is_err()); + .expect_err("RSA-OAEP encryption must propagate entropy failure"); } 8 => { let message = bounded_slice(right, pkcs1v15_message_limit(&key)); @@ -133,21 +160,25 @@ pub fn run(data: &[u8]) { .expect("fixture RSAES-PKCS1-v1_5 encryption must succeed for bounded message"); let mut plaintext = vec![0u8; key.signature_len()]; let plaintext_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &one, &one_inverse, &mut plaintext) + .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, RsaBlindingPair::new(&one, &one_inverse), &mut plaintext) .expect("self-produced RSAES-PKCS1-v1_5 ciphertext must decrypt"); assert_eq!(&plaintext[..plaintext_len], message); } 9 => { let mut plaintext = vec![0u8; key.signature_len()]; let ciphertext = full_width_candidate(left, key.signature_len()); - let _ = key.decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &one, &one_inverse, &mut plaintext); + let _decryption_result = key.decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&one, &one_inverse), + &mut plaintext, + ); } - _ => unreachable!("mode modulo 10 is always in 0..10"), + _ => {} } } #[cfg(not(any(fuzzing, rscrypto_internal_fuzzing)))] -pub fn run(_data: &[u8]) {} +pub(super) fn run(_data: &[u8]) {} #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn factor_one(len: usize) -> (Vec, Vec) { @@ -191,7 +222,8 @@ fn fill_random_from(bytes: &[u8]) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncr #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn oaep_message_limit(key: &RsaPrivateKey, profile: RsaOaepProfile) -> usize { - key.signature_len() + key + .signature_len() .saturating_sub(profile.digest_len().saturating_mul(2)) .saturating_sub(2) } @@ -213,10 +245,7 @@ fn pkcs1v15_message_limit(key: &RsaPrivateKey) -> usize { #[cfg(any(fuzzing, rscrypto_internal_fuzzing))] fn pkcs1v15_seed(key: &RsaPrivateKey, message_len: usize, selector: u8, left: &[u8], right: &[u8]) -> Vec { - let len = key - .signature_len() - .saturating_sub(message_len) - .saturating_sub(3); + let len = key.signature_len().saturating_sub(message_len).saturating_sub(3); let mut seed = vec![selector.wrapping_add(1).max(1); len]; for (index, byte) in left.iter().chain(right.iter()).copied().enumerate() { let seed_len = seed.len(); diff --git a/fuzz/target_impls/auth_rsa_protocol.rs b/fuzz/target_impls/auth_rsa_protocol.rs index 4c51e398..152b95ff 100644 --- a/fuzz/target_impls/auth_rsa_protocol.rs +++ b/fuzz/target_impls/auth_rsa_protocol.rs @@ -28,8 +28,8 @@ const X509_PSS_DEFAULT_SHA1_ALGORITHM: &[u8] = &[ const X509_MALFORMED_INDEFINITE_SEQUENCE: &[u8] = &[0x30, 0x80, 0x00, 0x00]; const JWT_ALGS: [&str; 16] = [ - "PS256", "PS384", "PS512", "RS256", "RS384", "RS512", "none", "HS256", "ES256", "EdDSA", "ps256", "", "PS1", - "RS1", "PS256\0", "RS256 ", + "PS256", "PS384", "PS512", "RS256", "RS384", "RS512", "none", "HS256", "ES256", "EdDSA", "ps256", "", "PS1", "RS1", + "PS256\0", "RS256 ", ]; const COSE_ALGORITHMS: [i64; 16] = [ -37, @@ -50,8 +50,8 @@ const COSE_ALGORITHMS: [i64; 16] = [ i64::MAX - 1, ]; const TLS_SCHEMES: [u16; 19] = [ - 0x0804, 0x0805, 0x0806, 0x0809, 0x080a, 0x080b, 0x0401, 0x0501, 0x0601, 0x0101, 0x0201, 0x0420, - 0x0520, 0x0620, 0x0301, 0x0203, 0x0403, 0, 0xffff, + 0x0804, 0x0805, 0x0806, 0x0809, 0x080a, 0x080b, 0x0401, 0x0501, 0x0601, 0x0101, 0x0201, 0x0420, 0x0520, 0x0620, + 0x0301, 0x0203, 0x0403, 0, 0xffff, ]; const X509_ALGORITHMS: [&[u8]; 6] = [ X509_PSS_SHA256_ALGORITHM, @@ -69,7 +69,7 @@ fn signature_candidate(material: &[u8], len: usize) -> Vec { } for (index, byte) in out.iter_mut().enumerate() { - *byte = material[index % material.len()]; + *byte = material[index.rem_euclid(material.len())]; } out } @@ -80,10 +80,12 @@ fn selected_signature<'a>(material: &'a [u8], full_width: &'a [u8], selector: u8 #[inline] fn select(items: &[T], selector: u8) -> &T { - &items[(selector as usize) % items.len()] + items + .get(usize::from(selector).rem_euclid(items.len())) + .expect("RSA protocol selector tables are nonempty") } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let selector = some_or_return!(input.byte()); @@ -96,7 +98,7 @@ pub fn run(data: &[u8]) { let signature = selected_signature(signature_material, &full_width_signature, selector); let mut scratch = key.public_scratch(); - match mode % 14 { + match mode.rem_euclid(14) { 0 => { key .jwt_verifier(RsaJwtAlgorithm::Ps256) @@ -134,12 +136,15 @@ pub fn run(data: &[u8]) { .expect("valid TLS certificate rsa_pkcs1_sha256 fixture must verify"); } 5 => { - let _ = key - .jwt_verifier(RsaJwtAlgorithm::Ps256) - .verify_with_scratch(select(&JWT_ALGS, selector), message, signature, &mut scratch); + let _verification_result = key.jwt_verifier(RsaJwtAlgorithm::Ps256).verify_with_scratch( + select(&JWT_ALGS, selector), + message, + signature, + &mut scratch, + ); } 6 => { - let _ = key.verify_cose_algorithm_id_with_scratch( + let _verification_result = key.verify_cose_algorithm_id_with_scratch( *select(&COSE_ALGORITHMS, selector), message, signature, @@ -147,7 +152,7 @@ pub fn run(data: &[u8]) { ); } 7 => { - let _ = x509_key.verify_signature_from_x509_algorithm_der_with_scratch( + let _verification_result = x509_key.verify_signature_from_x509_algorithm_der_with_scratch( select(&X509_ALGORITHMS, selector), message, signature, @@ -155,7 +160,7 @@ pub fn run(data: &[u8]) { ); } 8 => { - let _ = + let _verification_result = x509_key.verify_signature_from_x509_algorithm_der_with_scratch(message, MESSAGE_PSS, signature, &mut scratch); } 9 => { @@ -164,7 +169,8 @@ pub fn run(data: &[u8]) { } else { u16::from_be_bytes([selector, split]) }; - let _ = x509_key.verify_tls13_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); + let _verification_result = + x509_key.verify_tls13_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); } 10 => { let scheme = if selector & 1 == 0 { @@ -172,18 +178,20 @@ pub fn run(data: &[u8]) { } else { u16::from_be_bytes([selector, split]) }; - let _ = x509_key.verify_tls_certificate_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); + let _verification_result = + x509_key.verify_tls_certificate_signature_scheme_with_scratch(scheme, message, signature, &mut scratch); } 11 => { - let _ = RsaSignatureProfile::from_x509_signature_algorithm_der(message); + let _profile_result = RsaSignatureProfile::from_x509_signature_algorithm_der(message); } 12 => { - let _ = RsaSignatureProfile::from_cose_algorithm_id(*select(&COSE_ALGORITHMS, selector)); - let _ = RsaSignatureProfile::from_tls13_signature_scheme(u16::from_be_bytes([selector, split])); - let _ = RsaSignatureProfile::from_tls_certificate_signature_scheme(u16::from_be_bytes([selector, split])); + let _cose_profile = RsaSignatureProfile::from_cose_algorithm_id(*select(&COSE_ALGORITHMS, selector)); + let _tls13_profile = RsaSignatureProfile::from_tls13_signature_scheme(u16::from_be_bytes([selector, split])); + let _certificate_profile = + RsaSignatureProfile::from_tls_certificate_signature_scheme(u16::from_be_bytes([selector, split])); } _ => { - let _ = RsaX509PublicKey::from_spki_der(message); + let _key_result = RsaX509PublicKey::from_spki_der(message); } } } diff --git a/fuzz/target_impls/auth_rsa_public_key.rs b/fuzz/target_impls/auth_rsa_public_key.rs index ddd6668f..d0d48e8c 100644 --- a/fuzz/target_impls/auth_rsa_public_key.rs +++ b/fuzz/target_impls/auth_rsa_public_key.rs @@ -1,7 +1,7 @@ use rscrypto::{RsaPublicKey, RsaPublicKeyPolicy}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let raw_der = input.rest(); @@ -13,7 +13,7 @@ pub fn run(data: &[u8]) { raw_der }; - let parsed = match mode % 4 { + let parsed = match mode.rem_euclid(4) { 0 => RsaPublicKey::from_pkcs1_der(der), 1 => RsaPublicKey::from_spki_der(der), 2 => RsaPublicKey::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::modern_verification()), @@ -51,7 +51,7 @@ fn generated_public_key_der(mode: u8, control: &[u8]) -> Option> { Some(der) } Some(b'L') => Some(tlv_with_leading_zero_long_len(&public_key_der_for_mode(mode, &pkcs1))), - Some(b'U') if mode % 2 == 1 => Some(spki_public_key_der_with_unused_bits(&pkcs1)), + Some(b'U') if mode.rem_euclid(2) == 1 => Some(spki_public_key_der_with_unused_bits(&pkcs1)), Some(b'N') if mode.is_multiple_of(2) => Some(pkcs1_public_key_der_with_noncanonical_exponent()), _ => None, } @@ -122,7 +122,7 @@ fn algorithm_identifier(algorithm_oid: &[u8], params: Option<&[u8]>) -> Vec fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -131,7 +131,8 @@ fn integer_unsigned(value: &[u8]) -> Vec { } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = 1usize.strict_add(der_len(value.len()).len()).strict_add(value.len()); + let mut out = Vec::with_capacity(encoded_len); out.push(tag); out.extend_from_slice(&der_len(value.len())); out.extend_from_slice(value); @@ -146,7 +147,8 @@ fn tlv_with_leading_zero_long_len(der: &[u8]) -> Vec { let mut out = Vec::with_capacity(der.len().strict_add(1)); out.push(tag); - out.push(0x80 | (len_len.strict_add(1) as u8)); + let encoded_len_len = u8::try_from(len_len.strict_add(1)).expect("DER length-of-length fits one byte"); + out.push(0x80 | encoded_len_len); out.push(0); out.extend_from_slice(&der[2..]); out @@ -154,14 +156,18 @@ fn tlv_with_leading_zero_long_len(der: &[u8]) -> Vec { fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short-form DER length is below 128")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long-form DER length is nonzero"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + let len_len = u8::try_from(len_bytes.len()).expect("usize DER length uses at most 16 bytes"); + out.push(0x80 | len_len); out.extend_from_slice(len_bytes); out } @@ -171,21 +177,21 @@ fn der_null() -> Vec { } fn hex_to_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); + let mut out = Vec::with_capacity(hex.len().div_euclid(2)); for chunk in hex.as_bytes().chunks_exact(2) { - let hi = hex_value(chunk[0]); - let lo = hex_value(chunk[1]); - out.push((hi << 4) | lo); + let hi = hex_value(chunk[0]).expect("RSA modulus fixture must contain hexadecimal digits"); + let lo = hex_value(chunk[1]).expect("RSA modulus fixture must contain hexadecimal digits"); + out.push(hi.strict_shl(4) | lo); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => 0, + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } diff --git a/fuzz/target_impls/auth_rsa_verify.rs b/fuzz/target_impls/auth_rsa_verify.rs index 0fb1bd70..d4dadeb2 100644 --- a/fuzz/target_impls/auth_rsa_verify.rs +++ b/fuzz/target_impls/auth_rsa_verify.rs @@ -14,12 +14,12 @@ fn signature_candidate(material: &[u8], len: usize) -> Vec { } for (index, byte) in out.iter_mut().enumerate() { - *byte = material[index % material.len()]; + *byte = material[index.rem_euclid(material.len())]; } out } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let mode = some_or_return!(input.byte()); let split = some_or_return!(input.byte()); @@ -28,7 +28,7 @@ pub fn run(data: &[u8]) { let key = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("fuzz RSA fixture must parse"); let mut scratch = key.public_scratch(); - match mode % 6 { + match mode.rem_euclid(6) { 0 => { key .verify_pss_with_scratch(RsaPssProfile::Sha256, MESSAGE_PSS, RSA3072_PSS_SHA256, &mut scratch) @@ -46,17 +46,20 @@ pub fn run(data: &[u8]) { } 2 => { let signature = signature_candidate(signature_material, key.modulus().len()); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha256, message, &signature, &mut scratch); + let _verification_result = key.verify_pss_with_scratch(RsaPssProfile::Sha256, message, &signature, &mut scratch); } 3 => { let signature = signature_candidate(signature_material, key.modulus().len()); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, message, &signature, &mut scratch); + let _verification_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, message, &signature, &mut scratch); } 4 => { - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha384, message, signature_material, &mut scratch); + let _verification_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha384, message, signature_material, &mut scratch); } _ => { - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, message, signature_material, &mut scratch); + let _verification_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, message, signature_material, &mut scratch); } } } diff --git a/fuzz/target_impls/auth_scrypt.rs b/fuzz/target_impls/auth_scrypt.rs index 487dbfc9..bfba115b 100644 --- a/fuzz/target_impls/auth_scrypt.rs +++ b/fuzz/target_impls/auth_scrypt.rs @@ -1,7 +1,7 @@ use rscrypto::{Scrypt, ScryptParams}; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let pw_salt_split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); @@ -16,8 +16,7 @@ pub fn run(data: &[u8]) { let r = 1u32.strict_add(u32::from(r_byte) % 4); let out_len = 1u32.strict_add(u32::from(out_len_byte) % 64); - let params = ScryptParams::new(log_n, r, 1) - .expect("params must be valid for fuzzer ranges"); + let params = ScryptParams::new(log_n, r, 1).expect("params must be valid for fuzzer ranges"); let (password, salt) = split_at_ratio(rest, pw_salt_split); diff --git a/fuzz/target_impls/auth_x25519.rs b/fuzz/target_impls/auth_x25519.rs index d25e0521..a0c4ae17 100644 --- a/fuzz/target_impls/auth_x25519.rs +++ b/fuzz/target_impls/auth_x25519.rs @@ -1,7 +1,7 @@ use rscrypto::{X25519PublicKey, X25519SecretKey, X25519SharedSecret}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let secret_bytes: [u8; 32] = some_or_return!(input.bytes()); let peer_bytes: [u8; 32] = some_or_return!(input.bytes()); @@ -25,7 +25,11 @@ pub fn run(data: &[u8]) { match (&ours_shared, &helper_shared) { (Ok(ours), Ok(helper)) => assert!(ours.ct_eq(helper).declassify(), "x25519 helper mismatch"), (Err(_), Err(_)) => {} - _ => panic!("x25519 helper result mismatch"), + _ => assert_eq!( + ours_shared.is_ok(), + helper_shared.is_ok(), + "x25519 helper result mismatch" + ), } if dalek_shared.iter().all(|&byte| byte == 0) { diff --git a/fuzz/target_impls/checksum_crc.rs b/fuzz/target_impls/checksum_crc.rs index 2e116839..6ddde7c2 100644 --- a/fuzz/target_impls/checksum_crc.rs +++ b/fuzz/target_impls/checksum_crc.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm, Crc24OpenPgp, Crc32, Crc32C, Crc64, Crc64Nvme}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc16.rs b/fuzz/target_impls/checksum_crc16.rs index c6457f1b..1890bd6f 100644 --- a/fuzz/target_impls/checksum_crc16.rs +++ b/fuzz/target_impls/checksum_crc16.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, Crc16Ccitt, Crc16Ibm}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc24.rs b/fuzz/target_impls/checksum_crc24.rs index 74161872..bccbca3b 100644 --- a/fuzz/target_impls/checksum_crc24.rs +++ b/fuzz/target_impls/checksum_crc24.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, Crc24OpenPgp}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc32.rs b/fuzz/target_impls/checksum_crc32.rs index e91a65e3..edeaf35d 100644 --- a/fuzz/target_impls/checksum_crc32.rs +++ b/fuzz/target_impls/checksum_crc32.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, Crc32, Crc32C}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/checksum_crc64.rs b/fuzz/target_impls/checksum_crc64.rs index 2bfe5a3c..69d7c223 100644 --- a/fuzz/target_impls/checksum_crc64.rs +++ b/fuzz/target_impls/checksum_crc64.rs @@ -1,7 +1,7 @@ use rscrypto::{Checksum, ChecksumCombine, Crc64, Crc64Nvme}; use rscrypto_fuzz::{FuzzInput, assert_checksum_chunked, assert_checksum_combine, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/fast_rapidhash.rs b/fuzz/target_impls/fast_rapidhash.rs index d3585184..c4e8bd3b 100644 --- a/fuzz/target_impls/fast_rapidhash.rs +++ b/fuzz/target_impls/fast_rapidhash.rs @@ -3,7 +3,7 @@ use core::hash::Hasher; use rscrypto::{RapidHash64, RapidStreamHasher}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let seed_bytes: [u8; 8] = some_or_return!(input.bytes()); let partitions: [u8; 8] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/fast_xxh3.rs b/fuzz/target_impls/fast_xxh3.rs index 3a2c0cdf..12399a55 100644 --- a/fuzz/target_impls/fast_xxh3.rs +++ b/fuzz/target_impls/fast_xxh3.rs @@ -3,7 +3,7 @@ use core::hash::Hasher; use rscrypto::{FastHash, Xxh3, Xxh3_128, Xxh3_128Hasher, Xxh3Hasher}; use rscrypto_fuzz::{FuzzInput, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let seed_bytes: [u8; 8] = some_or_return!(input.bytes()); let partitions: [u8; 8] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/hash_ascon.rs b/fuzz/target_impls/hash_ascon.rs index 3529b687..c7cbdf80 100644 --- a/fuzz/target_impls/hash_ascon.rs +++ b/fuzz/target_impls/hash_ascon.rs @@ -1,7 +1,7 @@ use rscrypto::{AsconHash256, AsconXof, Digest, Xof}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_ascon_cxof.rs b/fuzz/target_impls/hash_ascon_cxof.rs index 145e9a4e..a5bdbe88 100644 --- a/fuzz/target_impls/hash_ascon_cxof.rs +++ b/fuzz/target_impls/hash_ascon_cxof.rs @@ -18,7 +18,7 @@ fn extend_to_overlong(input: &[u8], control: u8) -> Vec { out } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let customization_split: u8 = some_or_return!(input.byte()); let message_split: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_blake2b.rs b/fuzz/target_impls/hash_blake2b.rs index 78266587..6e438e98 100644 --- a/fuzz/target_impls/hash_blake2b.rs +++ b/fuzz/target_impls/hash_blake2b.rs @@ -1,13 +1,18 @@ -use blake2::{Blake2b256 as OracleBlake2b256, Blake2b512 as OracleBlake2b512, Blake2bMac, Digest as _}; -use digest::typenum::{U32, U64}; -use hmac::{Mac as _, digest::KeyInit}; +use blake2::{ + Blake2b as OracleBlake2b, Blake2b512 as OracleBlake2b512, Blake2bMac, + digest::{ + Digest as _, Mac as _, + consts::{U32, U64}, + }, +}; use rscrypto::{Blake2b256, Blake2b512, Blake2bKey, Digest}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return, split_at_ratio}; type OracleBlake2bMac256 = Blake2bMac; type OracleBlake2bMac512 = Blake2bMac; +type OracleBlake2b256 = OracleBlake2b; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_ratio: u8 = some_or_return!(input.byte()); @@ -30,7 +35,7 @@ pub fn run(data: &[u8]) { let split_idx = split_at_ratio(data, key_ratio).0.len(); let key_len = split_idx.clamp(1, 64); let (key, msg) = data.split_at(key_len); - let typed_key = Blake2bKey::new(key).unwrap(); + let typed_key = Blake2bKey::new(key).expect("the bounded BLAKE2b key length is valid"); let (msg_a, msg_b) = split_at_ratio(msg, split); let mut ours_256_stream = Blake2b256::new_keyed(typed_key); @@ -38,7 +43,8 @@ pub fn run(data: &[u8]) { ours_256_stream.update(msg_b); let ours_256_keyed = ours_256_stream.finalize(); - let mut oracle_256_mac = OracleBlake2bMac256::new_from_slice(key).unwrap(); + let mut oracle_256_mac = + OracleBlake2bMac256::new_from_slice(key).expect("the bounded BLAKE2b oracle key length is valid"); oracle_256_mac.update(msg); let oracle_256_keyed = oracle_256_mac.finalize().into_bytes(); assert_eq!(&ours_256_keyed[..], &oracle_256_keyed[..], "blake2b256 keyed mismatch"); @@ -48,7 +54,8 @@ pub fn run(data: &[u8]) { ours_512_stream.update(msg_b); let ours_512_keyed = ours_512_stream.finalize(); - let mut oracle_512_mac = OracleBlake2bMac512::new_from_slice(key).unwrap(); + let mut oracle_512_mac = + OracleBlake2bMac512::new_from_slice(key).expect("the bounded BLAKE2b oracle key length is valid"); oracle_512_mac.update(msg); let oracle_512_keyed = oracle_512_mac.finalize().into_bytes(); assert_eq!(&ours_512_keyed[..], &oracle_512_keyed[..], "blake2b512 keyed mismatch"); diff --git a/fuzz/target_impls/hash_blake2s.rs b/fuzz/target_impls/hash_blake2s.rs index 8446d3c6..22215ae8 100644 --- a/fuzz/target_impls/hash_blake2s.rs +++ b/fuzz/target_impls/hash_blake2s.rs @@ -1,13 +1,18 @@ -use blake2::{Blake2s128 as OracleBlake2s128, Blake2s256 as OracleBlake2s256, Blake2sMac, Digest as _}; -use digest::typenum::{U16, U32}; -use hmac::{Mac as _, digest::KeyInit}; +use blake2::{ + Blake2s as OracleBlake2s, Blake2s256 as OracleBlake2s256, Blake2sMac, + digest::{ + Digest as _, Mac as _, + consts::{U16, U32}, + }, +}; use rscrypto::{Blake2s128, Blake2s256, Blake2sKey, Digest}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return, split_at_ratio}; type OracleBlake2sMac128 = Blake2sMac; type OracleBlake2sMac256 = Blake2sMac; +type OracleBlake2s128 = OracleBlake2s; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let key_ratio: u8 = some_or_return!(input.byte()); @@ -30,7 +35,7 @@ pub fn run(data: &[u8]) { let split_idx = split_at_ratio(data, key_ratio).0.len(); let key_len = split_idx.clamp(1, 32); let (key, msg) = data.split_at(key_len); - let typed_key = Blake2sKey::new(key).unwrap(); + let typed_key = Blake2sKey::new(key).expect("the bounded BLAKE2s key length is valid"); let (msg_a, msg_b) = split_at_ratio(msg, split); let mut ours_128_stream = Blake2s128::new_keyed(typed_key); @@ -38,7 +43,8 @@ pub fn run(data: &[u8]) { ours_128_stream.update(msg_b); let ours_128_keyed = ours_128_stream.finalize(); - let mut oracle_128_mac = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle_128_mac = + OracleBlake2sMac128::new_from_slice(key).expect("the bounded BLAKE2s oracle key length is valid"); oracle_128_mac.update(msg); let oracle_128_keyed = oracle_128_mac.finalize().into_bytes(); assert_eq!(&ours_128_keyed[..], &oracle_128_keyed[..], "blake2s128 keyed mismatch"); @@ -48,7 +54,8 @@ pub fn run(data: &[u8]) { ours_256_stream.update(msg_b); let ours_256_keyed = ours_256_stream.finalize(); - let mut oracle_256_mac = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle_256_mac = + OracleBlake2sMac256::new_from_slice(key).expect("the bounded BLAKE2s oracle key length is valid"); oracle_256_mac.update(msg); let oracle_256_keyed = oracle_256_mac.finalize().into_bytes(); assert_eq!(&ours_256_keyed[..], &oracle_256_keyed[..], "blake2s256 keyed mismatch"); diff --git a/fuzz/target_impls/hash_blake3.rs b/fuzz/target_impls/hash_blake3.rs index 9878e469..888c9caa 100644 --- a/fuzz/target_impls/hash_blake3.rs +++ b/fuzz/target_impls/hash_blake3.rs @@ -1,7 +1,7 @@ use rscrypto::{Blake3, Digest}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_blake3_derive.rs b/fuzz/target_impls/hash_blake3_derive.rs index 36aaf65b..cb1d318c 100644 --- a/fuzz/target_impls/hash_blake3_derive.rs +++ b/fuzz/target_impls/hash_blake3_derive.rs @@ -6,7 +6,7 @@ use rscrypto_fuzz::{FuzzInput, assert_xof_prefix, some_or_return}; // separator, not the interesting input. const CONTEXT: &str = "rscrypto fuzz 2026-04-12 derive_key test context"; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/hash_blake3_keyed.rs b/fuzz/target_impls/hash_blake3_keyed.rs index 6da3b9ee..dacca9bc 100644 --- a/fuzz/target_impls/hash_blake3_keyed.rs +++ b/fuzz/target_impls/hash_blake3_keyed.rs @@ -1,7 +1,7 @@ use rscrypto::{Blake3, Blake3KeyedHash, Digest}; use rscrypto_fuzz::{FuzzInput, assert_xof_prefix, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let key: [u8; 32] = some_or_return!(input.bytes()); let split: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_cshake256.rs b/fuzz/target_impls/hash_cshake256.rs index 5ebdc605..f03b421f 100644 --- a/fuzz/target_impls/hash_cshake256.rs +++ b/fuzz/target_impls/hash_cshake256.rs @@ -1,7 +1,7 @@ use rscrypto::{Cshake256, Xof}; use rscrypto_fuzz::{FuzzInput, some_or_return, split_at_ratio}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let name_split: u8 = some_or_return!(input.byte()); let custom_split: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hash_sha2.rs b/fuzz/target_impls/hash_sha2.rs index 2dfcc9e2..b0e3be79 100644 --- a/fuzz/target_impls/hash_sha2.rs +++ b/fuzz/target_impls/hash_sha2.rs @@ -1,7 +1,7 @@ use rscrypto::{Sha224, Sha256, Sha384, Sha512, Sha512_256}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let data = input.rest(); diff --git a/fuzz/target_impls/hash_sha3.rs b/fuzz/target_impls/hash_sha3.rs index 2082a539..78dd50a6 100644 --- a/fuzz/target_impls/hash_sha3.rs +++ b/fuzz/target_impls/hash_sha3.rs @@ -1,7 +1,7 @@ use rscrypto::{Digest, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256, Xof}; use rscrypto_fuzz::{FuzzInput, assert_digest_chunked, assert_digest_reset, some_or_return}; -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let split: u8 = some_or_return!(input.byte()); let out_len_byte: u8 = some_or_return!(input.byte()); diff --git a/fuzz/target_impls/hex_parse.rs b/fuzz/target_impls/hex_parse.rs index 44a9e120..4eb36930 100644 --- a/fuzz/target_impls/hex_parse.rs +++ b/fuzz/target_impls/hex_parse.rs @@ -23,7 +23,7 @@ fn ascii_candidate(bytes: &[u8], len: usize) -> String { .copied() .cycle() .take(len) - .map(|byte| char::from(32 + (byte % 95))) + .map(|byte| char::from(byte.rem_euclid(95).strict_add(32))) .collect() } @@ -33,10 +33,20 @@ where { let lower = value.to_string(); let upper = format!("{value:X}"); - let short = &lower[..lower.len().strict_sub(1)]; + let short = lower + .get(..lower.len().strict_sub(1)) + .expect("hex rendering has an ASCII boundary before its final byte"); - assert_eq!(lower.parse::().unwrap(), value, "public lower parse mismatch"); - assert_eq!(upper.parse::().unwrap(), value, "public upper parse mismatch"); + assert_eq!( + lower.parse::().expect("lowercase rendering must parse"), + value, + "public lower parse mismatch" + ); + assert_eq!( + upper.parse::().expect("uppercase rendering must parse"), + value, + "public upper parse mismatch" + ); assert_eq!( short.parse::(), Err(InvalidHexError::InvalidLength), @@ -44,8 +54,19 @@ where ); if let Ok(parsed) = candidate.parse::() { - assert_eq!(parsed.to_string().parse::().unwrap(), parsed); - assert_eq!(format!("{parsed:X}").parse::().unwrap(), parsed); + assert_eq!( + parsed + .to_string() + .parse::() + .expect("parsed value must round-trip through lowercase rendering"), + parsed + ); + assert_eq!( + format!("{parsed:X}") + .parse::() + .expect("parsed value must round-trip through uppercase rendering"), + parsed + ); } } @@ -58,14 +79,22 @@ fn exercise_sensitive_parse( ) where T: Debug + FromStr, { - let short = &lower[..lower.len().strict_sub(1)]; + let short = lower + .get(..lower.len().strict_sub(1)) + .expect("hex rendering has an ASCII boundary before its final byte"); assert!( - equivalent(&lower.parse::().unwrap(), &value), + equivalent( + &lower.parse::().expect("lowercase secret rendering must parse"), + &value + ), "sensitive lower parse mismatch" ); assert!( - equivalent(&upper.parse::().unwrap(), &value), + equivalent( + &upper.parse::().expect("uppercase secret rendering must parse"), + &value + ), "sensitive upper parse mismatch" ); assert!( @@ -79,7 +108,7 @@ fn exercise_sensitive_parse( } } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let nonce_bytes: [u8; 12] = some_or_return!(input.bytes()); let tag_bytes: [u8; 16] = some_or_return!(input.bytes()); diff --git a/fuzz/target_impls/traits_io.rs b/fuzz/target_impls/traits_io.rs index 786e6f7c..02400efa 100644 --- a/fuzz/target_impls/traits_io.rs +++ b/fuzz/target_impls/traits_io.rs @@ -44,7 +44,7 @@ fn sequential_chunks(data: &[u8], lens: [usize; 3]) -> [&[u8]; 3] { } fn reader_step(reader: &mut R, seen: &mut Vec, control: u8) -> io::Result<()> { - match control % 3 { + match control.rem_euclid(3) { 0 => { let mut buf = vec![0u8; scalar_len(control)]; let n = reader.read(&mut buf)?; @@ -68,7 +68,9 @@ fn reader_step(reader: &mut R, seen: &mut Vec, control: u8) -> io:: } _ => { let n = reader.read(&mut [])?; - assert_eq!(n, 0, "zero-length read must report zero"); + if n != 0 { + return Err(io::Error::other("zero-length read returned a nonzero length")); + } } } @@ -82,7 +84,7 @@ fn writer_step( accepted: &mut Vec, control: u8, ) -> io::Result<()> { - match control % 4 { + match control.rem_euclid(4) { 0 => { let request = scalar_len(control).min(data.len().strict_sub(*cursor)); let end = cursor.strict_add(request); @@ -103,7 +105,9 @@ fn writer_step( } 2 => { let n = writer.write(&[])?; - assert_eq!(n, 0, "zero-length write must report zero"); + if n != 0 { + return Err(io::Error::other("zero-length write returned a nonzero length")); + } } _ => writer.flush()?, } @@ -275,7 +279,7 @@ fn fuzz_digest_writer(data: &[u8], ops: &[u8], max_per_call: usize) { assert_eq!(digest, Blake3::digest(data), "digest writer final mismatch"); } -pub fn run(data: &[u8]) { +pub(super) fn run(data: &[u8]) { let mut input = FuzzInput::new(data); let reader_limit = usize::from(some_or_return!(input.byte()) % 32).strict_add(1); let writer_limit = usize::from(some_or_return!(input.byte()) % 32).strict_add(1); diff --git a/fuzz/tests/corpus_replay.rs b/fuzz/tests/corpus_replay.rs index ad1ff5dc..c9152356 100644 --- a/fuzz/tests/corpus_replay.rs +++ b/fuzz/tests/corpus_replay.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use rscrypto_fuzz::replay_corpus_dir; fn corpus_dir(target: &str) -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("corpus").join(target) + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("corpus").join(target) } #[path = "../target_impls/aead_aegis256.rs"] @@ -149,282 +149,346 @@ mod traits_io; #[test] fn replay_aead_aegis256_corpus() { - let replayed = replay_corpus_dir("aead_aegis256", corpus_dir("aead_aegis256"), aead_aegis256::run); - assert_ne!(replayed, 0, "aead_aegis256 corpus should not be empty"); + let replayed = replay_corpus_dir("aead_aegis256", corpus_dir("aead_aegis256"), aead_aegis256::run); + assert_ne!(replayed, 0, "aead_aegis256 corpus should not be empty"); } #[test] fn replay_aead_aes256gcm_corpus() { - let replayed = replay_corpus_dir("aead_aes256gcm", corpus_dir("aead_aes256gcm"), aead_aes256gcm::run); - assert_ne!(replayed, 0, "aead_aes256gcm corpus should not be empty"); + let replayed = replay_corpus_dir("aead_aes256gcm", corpus_dir("aead_aes256gcm"), aead_aes256gcm::run); + assert_ne!(replayed, 0, "aead_aes256gcm corpus should not be empty"); } #[test] fn replay_aead_aes256gcmsiv_corpus() { - let replayed = replay_corpus_dir("aead_aes256gcmsiv", corpus_dir("aead_aes256gcmsiv"), aead_aes256gcmsiv::run); - assert_ne!(replayed, 0, "aead_aes256gcmsiv corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_aes256gcmsiv", + corpus_dir("aead_aes256gcmsiv"), + aead_aes256gcmsiv::run, + ); + assert_ne!(replayed, 0, "aead_aes256gcmsiv corpus should not be empty"); } #[test] fn replay_aead_ascon128_corpus() { - let replayed = replay_corpus_dir("aead_ascon128", corpus_dir("aead_ascon128"), aead_ascon128::run); - assert_ne!(replayed, 0, "aead_ascon128 corpus should not be empty"); + let replayed = replay_corpus_dir("aead_ascon128", corpus_dir("aead_ascon128"), aead_ascon128::run); + assert_ne!(replayed, 0, "aead_ascon128 corpus should not be empty"); } #[test] fn replay_aead_chacha20poly1305_corpus() { - let replayed = replay_corpus_dir("aead_chacha20poly1305", corpus_dir("aead_chacha20poly1305"), aead_chacha20poly1305::run); - assert_ne!(replayed, 0, "aead_chacha20poly1305 corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_chacha20poly1305", + corpus_dir("aead_chacha20poly1305"), + aead_chacha20poly1305::run, + ); + assert_ne!(replayed, 0, "aead_chacha20poly1305 corpus should not be empty"); } #[test] fn replay_aead_nonce_counter_corpus() { - let replayed = replay_corpus_dir("aead_nonce_counter", corpus_dir("aead_nonce_counter"), aead_nonce_counter::run); - assert_ne!(replayed, 0, "aead_nonce_counter corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_nonce_counter", + corpus_dir("aead_nonce_counter"), + aead_nonce_counter::run, + ); + assert_ne!(replayed, 0, "aead_nonce_counter corpus should not be empty"); } #[test] fn replay_aead_xchacha20poly1305_corpus() { - let replayed = replay_corpus_dir("aead_xchacha20poly1305", corpus_dir("aead_xchacha20poly1305"), aead_xchacha20poly1305::run); - assert_ne!(replayed, 0, "aead_xchacha20poly1305 corpus should not be empty"); + let replayed = replay_corpus_dir( + "aead_xchacha20poly1305", + corpus_dir("aead_xchacha20poly1305"), + aead_xchacha20poly1305::run, + ); + assert_ne!(replayed, 0, "aead_xchacha20poly1305 corpus should not be empty"); } #[test] fn replay_auth_argon2d_corpus() { - let replayed = replay_corpus_dir("auth_argon2d", corpus_dir("auth_argon2d"), auth_argon2d::run); - assert_ne!(replayed, 0, "auth_argon2d corpus should not be empty"); + let replayed = replay_corpus_dir("auth_argon2d", corpus_dir("auth_argon2d"), auth_argon2d::run); + assert_ne!(replayed, 0, "auth_argon2d corpus should not be empty"); } #[test] fn replay_auth_argon2i_corpus() { - let replayed = replay_corpus_dir("auth_argon2i", corpus_dir("auth_argon2i"), auth_argon2i::run); - assert_ne!(replayed, 0, "auth_argon2i corpus should not be empty"); + let replayed = replay_corpus_dir("auth_argon2i", corpus_dir("auth_argon2i"), auth_argon2i::run); + assert_ne!(replayed, 0, "auth_argon2i corpus should not be empty"); } #[test] fn replay_auth_argon2id_corpus() { - let replayed = replay_corpus_dir("auth_argon2id", corpus_dir("auth_argon2id"), auth_argon2id::run); - assert_ne!(replayed, 0, "auth_argon2id corpus should not be empty"); + let replayed = replay_corpus_dir("auth_argon2id", corpus_dir("auth_argon2id"), auth_argon2id::run); + assert_ne!(replayed, 0, "auth_argon2id corpus should not be empty"); } #[test] fn replay_auth_ed25519_corpus() { - let replayed = replay_corpus_dir("auth_ed25519", corpus_dir("auth_ed25519"), auth_ed25519::run); - assert_ne!(replayed, 0, "auth_ed25519 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_ed25519", corpus_dir("auth_ed25519"), auth_ed25519::run); + assert_ne!(replayed, 0, "auth_ed25519 corpus should not be empty"); } #[test] fn replay_auth_ed25519_verify_corpus() { - let replayed = replay_corpus_dir("auth_ed25519_verify", corpus_dir("auth_ed25519_verify"), auth_ed25519_verify::run); - assert_ne!(replayed, 0, "auth_ed25519_verify corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_ed25519_verify", + corpus_dir("auth_ed25519_verify"), + auth_ed25519_verify::run, + ); + assert_ne!(replayed, 0, "auth_ed25519_verify corpus should not be empty"); } #[test] fn replay_auth_ecdsa_verify_corpus() { - let replayed = replay_corpus_dir("auth_ecdsa_verify", corpus_dir("auth_ecdsa_verify"), auth_ecdsa_verify::run); - assert_ne!(replayed, 0, "auth_ecdsa_verify corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_ecdsa_verify", + corpus_dir("auth_ecdsa_verify"), + auth_ecdsa_verify::run, + ); + assert_ne!(replayed, 0, "auth_ecdsa_verify corpus should not be empty"); } #[test] fn replay_auth_ecdsa_sign_corpus() { - let replayed = replay_corpus_dir("auth_ecdsa_sign", corpus_dir("auth_ecdsa_sign"), auth_ecdsa_sign::run); - assert_ne!(replayed, 0, "auth_ecdsa_sign corpus should not be empty"); + let replayed = replay_corpus_dir("auth_ecdsa_sign", corpus_dir("auth_ecdsa_sign"), auth_ecdsa_sign::run); + assert_ne!(replayed, 0, "auth_ecdsa_sign corpus should not be empty"); } #[test] fn replay_auth_hkdf_sha256_corpus() { - let replayed = replay_corpus_dir("auth_hkdf_sha256", corpus_dir("auth_hkdf_sha256"), auth_hkdf_sha256::run); - assert_ne!(replayed, 0, "auth_hkdf_sha256 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hkdf_sha256", + corpus_dir("auth_hkdf_sha256"), + auth_hkdf_sha256::run, + ); + assert_ne!(replayed, 0, "auth_hkdf_sha256 corpus should not be empty"); } #[test] fn replay_auth_hkdf_sha384_corpus() { - let replayed = replay_corpus_dir("auth_hkdf_sha384", corpus_dir("auth_hkdf_sha384"), auth_hkdf_sha384::run); - assert_ne!(replayed, 0, "auth_hkdf_sha384 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hkdf_sha384", + corpus_dir("auth_hkdf_sha384"), + auth_hkdf_sha384::run, + ); + assert_ne!(replayed, 0, "auth_hkdf_sha384 corpus should not be empty"); } #[test] fn replay_auth_hmac_sha256_corpus() { - let replayed = replay_corpus_dir("auth_hmac_sha256", corpus_dir("auth_hmac_sha256"), auth_hmac_sha256::run); - assert_ne!(replayed, 0, "auth_hmac_sha256 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hmac_sha256", + corpus_dir("auth_hmac_sha256"), + auth_hmac_sha256::run, + ); + assert_ne!(replayed, 0, "auth_hmac_sha256 corpus should not be empty"); } #[test] fn replay_auth_hmac_sha384_corpus() { - let replayed = replay_corpus_dir("auth_hmac_sha384", corpus_dir("auth_hmac_sha384"), auth_hmac_sha384::run); - assert_ne!(replayed, 0, "auth_hmac_sha384 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hmac_sha384", + corpus_dir("auth_hmac_sha384"), + auth_hmac_sha384::run, + ); + assert_ne!(replayed, 0, "auth_hmac_sha384 corpus should not be empty"); } #[test] fn replay_auth_hmac_sha512_corpus() { - let replayed = replay_corpus_dir("auth_hmac_sha512", corpus_dir("auth_hmac_sha512"), auth_hmac_sha512::run); - assert_ne!(replayed, 0, "auth_hmac_sha512 corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_hmac_sha512", + corpus_dir("auth_hmac_sha512"), + auth_hmac_sha512::run, + ); + assert_ne!(replayed, 0, "auth_hmac_sha512 corpus should not be empty"); } #[test] fn replay_auth_kmac256_corpus() { - let replayed = replay_corpus_dir("auth_kmac256", corpus_dir("auth_kmac256"), auth_kmac256::run); - assert_ne!(replayed, 0, "auth_kmac256 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_kmac256", corpus_dir("auth_kmac256"), auth_kmac256::run); + assert_ne!(replayed, 0, "auth_kmac256 corpus should not be empty"); } #[test] fn replay_auth_mlkem512_corpus() { - let replayed = replay_corpus_dir("auth_mlkem512", corpus_dir("auth_mlkem512"), auth_mlkem512::run); - assert_ne!(replayed, 0, "auth_mlkem512 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_mlkem512", corpus_dir("auth_mlkem512"), auth_mlkem512::run); + assert_ne!(replayed, 0, "auth_mlkem512 corpus should not be empty"); } #[test] fn replay_auth_mlkem768_corpus() { - let replayed = replay_corpus_dir("auth_mlkem768", corpus_dir("auth_mlkem768"), auth_mlkem768::run); - assert_ne!(replayed, 0, "auth_mlkem768 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_mlkem768", corpus_dir("auth_mlkem768"), auth_mlkem768::run); + assert_ne!(replayed, 0, "auth_mlkem768 corpus should not be empty"); } #[test] fn replay_auth_mlkem1024_corpus() { - let replayed = replay_corpus_dir("auth_mlkem1024", corpus_dir("auth_mlkem1024"), auth_mlkem1024::run); - assert_ne!(replayed, 0, "auth_mlkem1024 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_mlkem1024", corpus_dir("auth_mlkem1024"), auth_mlkem1024::run); + assert_ne!(replayed, 0, "auth_mlkem1024 corpus should not be empty"); } #[test] fn replay_auth_pbkdf2_corpus() { - let replayed = replay_corpus_dir("auth_pbkdf2", corpus_dir("auth_pbkdf2"), auth_pbkdf2::run); - assert_ne!(replayed, 0, "auth_pbkdf2 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_pbkdf2", corpus_dir("auth_pbkdf2"), auth_pbkdf2::run); + assert_ne!(replayed, 0, "auth_pbkdf2 corpus should not be empty"); } #[test] fn replay_auth_phc_corpus() { - let replayed = replay_corpus_dir("auth_phc", corpus_dir("auth_phc"), auth_phc::run); - assert_ne!(replayed, 0, "auth_phc corpus should not be empty"); + let replayed = replay_corpus_dir("auth_phc", corpus_dir("auth_phc"), auth_phc::run); + assert_ne!(replayed, 0, "auth_phc corpus should not be empty"); } #[test] fn replay_auth_rsa_public_key_corpus() { - let replayed = replay_corpus_dir("auth_rsa_public_key", corpus_dir("auth_rsa_public_key"), auth_rsa_public_key::run); - assert_ne!(replayed, 0, "auth_rsa_public_key corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_rsa_public_key", + corpus_dir("auth_rsa_public_key"), + auth_rsa_public_key::run, + ); + assert_ne!(replayed, 0, "auth_rsa_public_key corpus should not be empty"); } #[test] fn replay_auth_rsa_import_corpus() { - let replayed = replay_corpus_dir("auth_rsa_import", corpus_dir("auth_rsa_import"), auth_rsa_import::run); - assert_ne!(replayed, 0, "auth_rsa_import corpus should not be empty"); + let replayed = replay_corpus_dir("auth_rsa_import", corpus_dir("auth_rsa_import"), auth_rsa_import::run); + assert_ne!(replayed, 0, "auth_rsa_import corpus should not be empty"); } #[test] fn replay_auth_rsa_protocol_corpus() { - let replayed = replay_corpus_dir("auth_rsa_protocol", corpus_dir("auth_rsa_protocol"), auth_rsa_protocol::run); - assert_ne!(replayed, 0, "auth_rsa_protocol corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_rsa_protocol", + corpus_dir("auth_rsa_protocol"), + auth_rsa_protocol::run, + ); + assert_ne!(replayed, 0, "auth_rsa_protocol corpus should not be empty"); } #[test] fn replay_auth_rsa_private_ops_corpus() { - let replayed = replay_corpus_dir("auth_rsa_private_ops", corpus_dir("auth_rsa_private_ops"), auth_rsa_private_ops::run); - assert_ne!(replayed, 0, "auth_rsa_private_ops corpus should not be empty"); + let replayed = replay_corpus_dir( + "auth_rsa_private_ops", + corpus_dir("auth_rsa_private_ops"), + auth_rsa_private_ops::run, + ); + assert_ne!(replayed, 0, "auth_rsa_private_ops corpus should not be empty"); } #[test] fn replay_auth_rsa_verify_corpus() { - let replayed = replay_corpus_dir("auth_rsa_verify", corpus_dir("auth_rsa_verify"), auth_rsa_verify::run); - assert_ne!(replayed, 0, "auth_rsa_verify corpus should not be empty"); + let replayed = replay_corpus_dir("auth_rsa_verify", corpus_dir("auth_rsa_verify"), auth_rsa_verify::run); + assert_ne!(replayed, 0, "auth_rsa_verify corpus should not be empty"); } #[test] fn replay_auth_scrypt_corpus() { - let replayed = replay_corpus_dir("auth_scrypt", corpus_dir("auth_scrypt"), auth_scrypt::run); - assert_ne!(replayed, 0, "auth_scrypt corpus should not be empty"); + let replayed = replay_corpus_dir("auth_scrypt", corpus_dir("auth_scrypt"), auth_scrypt::run); + assert_ne!(replayed, 0, "auth_scrypt corpus should not be empty"); } #[test] fn replay_auth_x25519_corpus() { - let replayed = replay_corpus_dir("auth_x25519", corpus_dir("auth_x25519"), auth_x25519::run); - assert_ne!(replayed, 0, "auth_x25519 corpus should not be empty"); + let replayed = replay_corpus_dir("auth_x25519", corpus_dir("auth_x25519"), auth_x25519::run); + assert_ne!(replayed, 0, "auth_x25519 corpus should not be empty"); } #[test] fn replay_checksum_crc_corpus() { - let replayed = replay_corpus_dir("checksum_crc", corpus_dir("checksum_crc"), checksum_crc::run); - assert_ne!(replayed, 0, "checksum_crc corpus should not be empty"); + let replayed = replay_corpus_dir("checksum_crc", corpus_dir("checksum_crc"), checksum_crc::run); + assert_ne!(replayed, 0, "checksum_crc corpus should not be empty"); } #[test] fn replay_fast_rapidhash_corpus() { - let replayed = replay_corpus_dir("fast_rapidhash", corpus_dir("fast_rapidhash"), fast_rapidhash::run); - assert_ne!(replayed, 0, "fast_rapidhash corpus should not be empty"); + let replayed = replay_corpus_dir("fast_rapidhash", corpus_dir("fast_rapidhash"), fast_rapidhash::run); + assert_ne!(replayed, 0, "fast_rapidhash corpus should not be empty"); } #[test] fn replay_fast_xxh3_corpus() { - let replayed = replay_corpus_dir("fast_xxh3", corpus_dir("fast_xxh3"), fast_xxh3::run); - assert_ne!(replayed, 0, "fast_xxh3 corpus should not be empty"); + let replayed = replay_corpus_dir("fast_xxh3", corpus_dir("fast_xxh3"), fast_xxh3::run); + assert_ne!(replayed, 0, "fast_xxh3 corpus should not be empty"); } #[test] fn replay_hash_ascon_corpus() { - let replayed = replay_corpus_dir("hash_ascon", corpus_dir("hash_ascon"), hash_ascon::run); - assert_ne!(replayed, 0, "hash_ascon corpus should not be empty"); + let replayed = replay_corpus_dir("hash_ascon", corpus_dir("hash_ascon"), hash_ascon::run); + assert_ne!(replayed, 0, "hash_ascon corpus should not be empty"); } #[test] fn replay_hash_ascon_cxof_corpus() { - let replayed = replay_corpus_dir("hash_ascon_cxof", corpus_dir("hash_ascon_cxof"), hash_ascon_cxof::run); - assert_ne!(replayed, 0, "hash_ascon_cxof corpus should not be empty"); + let replayed = replay_corpus_dir("hash_ascon_cxof", corpus_dir("hash_ascon_cxof"), hash_ascon_cxof::run); + assert_ne!(replayed, 0, "hash_ascon_cxof corpus should not be empty"); } #[test] fn replay_hash_blake2b_corpus() { - let replayed = replay_corpus_dir("hash_blake2b", corpus_dir("hash_blake2b"), hash_blake2b::run); - assert_ne!(replayed, 0, "hash_blake2b corpus should not be empty"); + let replayed = replay_corpus_dir("hash_blake2b", corpus_dir("hash_blake2b"), hash_blake2b::run); + assert_ne!(replayed, 0, "hash_blake2b corpus should not be empty"); } #[test] fn replay_hash_blake2s_corpus() { - let replayed = replay_corpus_dir("hash_blake2s", corpus_dir("hash_blake2s"), hash_blake2s::run); - assert_ne!(replayed, 0, "hash_blake2s corpus should not be empty"); + let replayed = replay_corpus_dir("hash_blake2s", corpus_dir("hash_blake2s"), hash_blake2s::run); + assert_ne!(replayed, 0, "hash_blake2s corpus should not be empty"); } #[test] fn replay_hash_blake3_corpus() { - let replayed = replay_corpus_dir("hash_blake3", corpus_dir("hash_blake3"), hash_blake3::run); - assert_ne!(replayed, 0, "hash_blake3 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_blake3", corpus_dir("hash_blake3"), hash_blake3::run); + assert_ne!(replayed, 0, "hash_blake3 corpus should not be empty"); } #[test] fn replay_hash_blake3_derive_corpus() { - let replayed = replay_corpus_dir("hash_blake3_derive", corpus_dir("hash_blake3_derive"), hash_blake3_derive::run); - assert_ne!(replayed, 0, "hash_blake3_derive corpus should not be empty"); + let replayed = replay_corpus_dir( + "hash_blake3_derive", + corpus_dir("hash_blake3_derive"), + hash_blake3_derive::run, + ); + assert_ne!(replayed, 0, "hash_blake3_derive corpus should not be empty"); } #[test] fn replay_hash_blake3_keyed_corpus() { - let replayed = replay_corpus_dir("hash_blake3_keyed", corpus_dir("hash_blake3_keyed"), hash_blake3_keyed::run); - assert_ne!(replayed, 0, "hash_blake3_keyed corpus should not be empty"); + let replayed = replay_corpus_dir( + "hash_blake3_keyed", + corpus_dir("hash_blake3_keyed"), + hash_blake3_keyed::run, + ); + assert_ne!(replayed, 0, "hash_blake3_keyed corpus should not be empty"); } #[test] fn replay_hash_cshake256_corpus() { - let replayed = replay_corpus_dir("hash_cshake256", corpus_dir("hash_cshake256"), hash_cshake256::run); - assert_ne!(replayed, 0, "hash_cshake256 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_cshake256", corpus_dir("hash_cshake256"), hash_cshake256::run); + assert_ne!(replayed, 0, "hash_cshake256 corpus should not be empty"); } #[test] fn replay_hash_sha2_corpus() { - let replayed = replay_corpus_dir("hash_sha2", corpus_dir("hash_sha2"), hash_sha2::run); - assert_ne!(replayed, 0, "hash_sha2 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_sha2", corpus_dir("hash_sha2"), hash_sha2::run); + assert_ne!(replayed, 0, "hash_sha2 corpus should not be empty"); } #[test] fn replay_hash_sha3_corpus() { - let replayed = replay_corpus_dir("hash_sha3", corpus_dir("hash_sha3"), hash_sha3::run); - assert_ne!(replayed, 0, "hash_sha3 corpus should not be empty"); + let replayed = replay_corpus_dir("hash_sha3", corpus_dir("hash_sha3"), hash_sha3::run); + assert_ne!(replayed, 0, "hash_sha3 corpus should not be empty"); } #[test] fn replay_hex_parse_corpus() { - let replayed = replay_corpus_dir("hex_parse", corpus_dir("hex_parse"), hex_parse::run); - assert_ne!(replayed, 0, "hex_parse corpus should not be empty"); + let replayed = replay_corpus_dir("hex_parse", corpus_dir("hex_parse"), hex_parse::run); + assert_ne!(replayed, 0, "hex_parse corpus should not be empty"); } #[test] fn replay_traits_io_corpus() { - let replayed = replay_corpus_dir("traits_io", corpus_dir("traits_io"), traits_io::run); - assert_ne!(replayed, 0, "traits_io corpus should not be empty"); + let replayed = replay_corpus_dir("traits_io", corpus_dir("traits_io"), traits_io::run); + assert_ne!(replayed, 0, "traits_io corpus should not be empty"); } diff --git a/justfile b/justfile index 3da40268..a2dc6e0d 100644 --- a/justfile +++ b/justfile @@ -1,104 +1,49 @@ # Remote dev. Provider mechanics live in ~/dev-machines. -ssh target: - @"$HOME/dev-machines/dev-machine" ssh rscrypto "{{ target }}" +dev_machine := env_var_or_default("DEV_MACHINE_BIN", env_var("HOME") + "/dev-machines/dev-machine") -ssh-check target: - @"$HOME/dev-machines/dev-machine" ssh rscrypto "{{ target }}" --check +ssh target *args="": + @"{{ dev_machine }}" ssh rscrypto "{{ target }}" {{ args }} -ssh-create target *args="": - @"$HOME/dev-machines/dev-machine" create rscrypto "{{ target }}" {{ args }} - -ssh-kill target: - @"$HOME/dev-machines/dev-machine" kill rscrypto "{{ target }}" - -ssh-status target="": - @if [ -n "{{ target }}" ]; then "$HOME/dev-machines/dev-machine" status rscrypto "{{ target }}"; else "$HOME/dev-machines/dev-machine" status rscrypto; fi - -ssh-bootstrap target: - @"$HOME/dev-machines/dev-machine" bootstrap rscrypto "{{ target }}" - -ssh-aws-linux-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-x64 - -ssh-aws-linux-arm64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-arm64 - -ssh-aws-windows-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-windows-x64 - -ssh-azure-linux-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-x64 - -ssh-azure-linux-arm64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-arm64 - -ssh-azure-windows-x64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-x64 +ssh-check target *args="": + @"{{ dev_machine }}" ssh rscrypto "{{ target }}" --check {{ args }} -ssh-azure-windows-arm64: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-arm64 +ssh-preflight target: + @"{{ dev_machine }}" preflight rscrypto "{{ target }}" -ssh-aws-linux-x64-perf: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-x64-perf - -ssh-aws-linux-intel-gnr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-intel-gnr-profile - -ssh-aws-linux-intel-spr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-intel-spr-profile - -ssh-aws-linux-amd-zen5-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-amd-zen5-profile - -ssh-aws-linux-amd-zen4-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-amd-zen4-profile - -ssh-aws-linux-arm64-graviton3-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-arm64-graviton3-profile - -ssh-aws-linux-arm64-graviton4-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-linux-arm64-graviton4-profile - -ssh-azure-linux-intel-gnr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-intel-gnr-profile - -ssh-azure-linux-intel-emr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-intel-emr-profile - -ssh-azure-linux-amd-zen5-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-amd-zen5-profile +ssh-create target *args="": + @"{{ dev_machine }}" create rscrypto "{{ target }}" {{ args }} -ssh-azure-linux-amd-zen4-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-amd-zen4-profile +ssh-start target: + @"{{ dev_machine }}" start rscrypto "{{ target }}" -ssh-azure-linux-arm64-cobalt-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-arm64-cobalt-profile +ssh-deallocate target: + @"{{ dev_machine }}" deallocate rscrypto "{{ target }}" -ssh-azure-linux-arm64-ampere-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-linux-arm64-ampere-profile +ssh-kill target: + @"{{ dev_machine }}" kill rscrypto "{{ target }}" -ssh-azure-windows-amd-zen5-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-amd-zen5-profile +ssh-status target="": + @if [ -n "{{ target }}" ]; then "{{ dev_machine }}" status rscrypto "{{ target }}"; else "{{ dev_machine }}" status rscrypto; fi -ssh-azure-windows-intel-gnr-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-intel-gnr-profile +ssh-bootstrap target profile="": + @if [ -n "{{ profile }}" ]; then "{{ dev_machine }}" bootstrap rscrypto "{{ target }}" "{{ profile }}"; else "{{ dev_machine }}" bootstrap rscrypto "{{ target }}"; fi -ssh-azure-windows-arm64-profile: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-windows-arm64-profile +ssh-just target *args="": + @"{{ dev_machine }}" just rscrypto "{{ target }}" {{ args }} -ssh-aws-test-rdma: - @"$HOME/dev-machines/dev-machine" ssh rscrypto aws-test-rdma +ssh-collect-bench target run_id destination: + @"{{ dev_machine }}" collect-bench rscrypto "{{ target }}" "{{ run_id }}" "{{ destination }}" -ssh-azure-test-rdma: - @"$HOME/dev-machines/dev-machine" ssh rscrypto azure-test-rdma +ssh-list: + @"{{ dev_machine }}" list # Builds build: - cargo build --workspace --all-targets --all-features + cargo build --locked --workspace --all-targets --all-features build-release: - cargo build --workspace --all-targets --all-features --release + cargo build --locked --workspace --all-targets --all-features --release # Checks check *args="": @@ -120,9 +65,9 @@ test-feature-matrix: @scripts/test/test-feature-matrix.sh test-native-api: - cargo test --no-default-features --features 'alloc,aead,ed25519,x25519,ecdsa,ml-kem' --test api_consistency - cargo test --features 'aead,signatures,key-exchange,getrandom' --test api_consistency - cargo test --features 'signatures,key-exchange,getrandom' --test getrandom_smoke + cargo test --locked --no-default-features --features 'alloc,aead,ed25519,x25519,ecdsa,ml-kem' --test api_consistency + cargo test --locked --features 'aead,signatures,key-exchange,getrandom' --test api_consistency + cargo test --locked --features 'signatures,key-exchange,getrandom' --test getrandom_smoke # Tests test *crates="": diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f3cb4461..e81aa497 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,26 +1,4 @@ [toolchain] -channel = "nightly-2026-07-17" -profile = "default" -components = ["clippy", "rustfmt", "miri", "rust-src"] - -targets = [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "wasm32-unknown-unknown", - "wasm32-wasip1", - "x86_64-unknown-linux-musl", - "aarch64-pc-windows-msvc", - "aarch64-unknown-none", - "loongarch64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv32imac-unknown-none-elf", - "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-none-elf", - "s390x-unknown-linux-gnu", - "thumbv6m-none-eabi", - "x86_64-unknown-none", -] +channel = "1.97.1" +profile = "minimal" +components = ["clippy", "rustfmt", "rust-src", "rust-analyzer"] diff --git a/rustfmt.toml b/rustfmt.toml index 50910770..7fd03e83 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,22 +1,7 @@ edition = "2024" style_edition = "2024" -tab_spaces = 2 max_width = 120 hard_tabs = false - -reorder_imports = true -imports_granularity = "Crate" -group_imports = "StdExternalCrate" - -format_code_in_doc_comments = true -format_strings = true -normalize_comments = true -wrap_comments = true -comment_width = 100 - -use_field_init_shorthand = true -use_try_shorthand = true - -error_on_line_overflow = false -error_on_unformatted = false +tab_spaces = 2 +newline_style = "Unix" diff --git a/scripts/README.md b/scripts/README.md index 3bdd2d16..bc6beb14 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -73,6 +73,7 @@ claim in [`docs/secret-lifecycle.md`](../docs/secret-lifecycle.md). | `check/check-win.sh` | `scripts/check/check-all.sh` | | `check/check-linux.sh` | `scripts/check/check-all.sh` | | `check/check-ibm.sh` | `scripts/check/check-all.sh` | +| `check/lint-independent-workspaces.sh` | `scripts/check/check.sh --all` | | `check/zig-cc.sh` | `scripts/check/check-linux.sh`, `scripts/check/check-ibm.sh` | ## Bench Internals @@ -127,10 +128,10 @@ candidate selected after the metadata refresh, verifies the installed version, and refuses downgrades. OPAM uses exact packages from a repository pinned to a full Git commit and verifies package source hashes from that immutable metadata. CT formal reports bind the resulting BINSEC executable by SHA-256. Rustup -receives only the exact toolchain declared in `rust-toolchain.toml`; runner -images must provide rustup, which verifies component downloads against the -exact distribution manifest, because network bootstrap installers are -rejected. +receives only the exact stable or nightly contract declared in +`rust-toolchain.toml` and `.config/toolchains.toml`; runner images must provide +rustup, which verifies component downloads against the exact distribution +manifest, because network bootstrap installers are rejected. ## Results layout diff --git a/scripts/check/check-all.sh b/scripts/check/check-all.sh index 31ad9ff3..b13e6b4f 100755 --- a/scripts/check/check-all.sh +++ b/scripts/check/check-all.sh @@ -10,6 +10,8 @@ source "$SCRIPT_DIR/../lib/common.sh" # shellcheck source=../lib/targets.sh source "$SCRIPT_DIR/../lib/targets.sh" +NIGHTLY_TOOLCHAIN=$("$SCRIPT_DIR/../lib/toolchain.sh" --nightly) + DEFAULT_CONSTRAINED_CRATES=( "rscrypto" ) @@ -77,6 +79,16 @@ crate_supports_alloc() { [[ -f "$manifest" ]] && grep -q '^[[:space:]]*alloc[[:space:]]*=' "$manifest" } +cargo_for_target() { + local target=$1 + shift + if [[ "$target" == riscv32* ]]; then + RUSTUP_TOOLCHAIN="$NIGHTLY_TOOLCHAIN" cargo "$@" + else + cargo "$@" + fi +} + run_constrained_check() { local crate=$1 local target=$2 @@ -97,7 +109,7 @@ run_constrained_check() { args+=(--features "$feature_set") fi - if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" cargo "${args[@]}" >>"$log_file" 2>&1; then + if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" cargo_for_target "$target" "${args[@]}" >>"$log_file" 2>&1; then return 1 fi } @@ -106,7 +118,11 @@ run_constrained_target() { local target=$1 local log_dir=$2 - ensure_target "$target" + if [[ "$target" == riscv32* ]]; then + ensure_target "$target" "$NIGHTLY_TOOLCHAIN" + else + ensure_target "$target" + fi local target_dir="target/cross-check/$target" mkdir -p "$target_dir" @@ -147,7 +163,7 @@ run_constrained_target() { step "$target check (alloc)" for crate in "${alloc_crates[@]}"; do if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo check --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib \ + cargo_for_target "$target" check --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib \ >>"$log_file" 2>&1; then fail show_error "$log_file" @@ -160,7 +176,7 @@ run_constrained_target() { step "$target build (no features)" for crate in "${CONSTRAINED_CRATES[@]}"; do if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo build --locked -p "$crate" --no-default-features --target "$target" --lib --release \ + cargo_for_target "$target" build --locked -p "$crate" --no-default-features --target "$target" --lib --release \ >>"$log_file" 2>&1; then fail show_error "$log_file" @@ -173,7 +189,7 @@ run_constrained_target() { step "$target build (alloc)" for crate in "${alloc_crates[@]}"; do if ! RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo build --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib --release \ + cargo_for_target "$target" build --locked -p "$crate" --no-default-features --features alloc --target "$target" --lib --release \ >>"$log_file" 2>&1; then fail show_error "$log_file" diff --git a/scripts/check/check-ibm.sh b/scripts/check/check-ibm.sh index c9ce4008..140f91ff 100755 --- a/scripts/check/check-ibm.sh +++ b/scripts/check/check-ibm.sh @@ -21,6 +21,7 @@ fi get_crate_flags "$@" export ZIG_CC="$SCRIPT_DIR/zig-cc.sh" +NIGHTLY_TOOLCHAIN=$("$SCRIPT_DIR/../lib/toolchain.sh" --nightly) LOG_DIR=$(mktemp -d) trap 'rm -rf "$LOG_DIR"' EXIT @@ -33,7 +34,7 @@ if [[ ${#IBM_TARGETS[@]} -eq 0 ]]; then fi for target in "${IBM_TARGETS[@]}"; do - ensure_target "$target" + ensure_target "$target" "$NIGHTLY_TOOLCHAIN" done for target in "${IBM_TARGETS[@]}"; do @@ -53,8 +54,8 @@ for i in "${!IBM_TARGETS[@]}"; do ( # shellcheck disable=SC2086 - if ! CC="$ZIG_CC" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" -- -D warnings \ + if ! CC="$ZIG_CC" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" RUSTUP_TOOLCHAIN="$NIGHTLY_TOOLCHAIN" \ + cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" \ >"$log_file" 2>&1; then exit 1 fi diff --git a/scripts/check/check-linux.sh b/scripts/check/check-linux.sh index f35420d3..54ee22d4 100755 --- a/scripts/check/check-linux.sh +++ b/scripts/check/check-linux.sh @@ -21,6 +21,7 @@ fi get_crate_flags "$@" export ZIG_CC="$SCRIPT_DIR/zig-cc.sh" +NIGHTLY_TOOLCHAIN=$("$SCRIPT_DIR/../lib/toolchain.sh" --nightly) LOG_DIR=$(mktemp -d) trap 'rm -rf "$LOG_DIR"' EXIT @@ -33,7 +34,11 @@ if [[ ${#LINUX_TARGETS[@]} -eq 0 ]]; then fi for target in "${LINUX_TARGETS[@]}"; do - ensure_target "$target" + if [[ "$target" == riscv64* ]]; then + ensure_target "$target" "$NIGHTLY_TOOLCHAIN" + else + ensure_target "$target" + fi done for target in "${LINUX_TARGETS[@]}"; do @@ -52,9 +57,13 @@ for i in "${!LINUX_TARGETS[@]}"; do targets[i]="$target" ( + toolchain_env=() + if [[ "$target" == riscv64* ]]; then + toolchain_env=(env "RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN") + fi # shellcheck disable=SC2086 if ! CC="$ZIG_CC" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" -- -D warnings \ + "${toolchain_env[@]}" cargo clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" \ >"$log_file" 2>&1; then exit 1 fi diff --git a/scripts/check/check-win.sh b/scripts/check/check-win.sh index decebde9..33164498 100755 --- a/scripts/check/check-win.sh +++ b/scripts/check/check-win.sh @@ -85,7 +85,7 @@ for i in "${!WIN_TARGETS[@]}"; do # shellcheck disable=SC2086 if ! XWIN_CACHE_DIR="$target_cache_dir" \ CARGO_TARGET_DIR="$target_dir" \ - cargo xwin clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" -- -D warnings \ + cargo xwin clippy $CRATE_FLAGS --lib --all-features --locked --target "$target" \ >"$log_file" 2>&1; then exit 1 fi diff --git a/scripts/check/check.sh b/scripts/check/check.sh index c9995b5d..7dd75b74 100755 --- a/scripts/check/check.sh +++ b/scripts/check/check.sh @@ -145,13 +145,23 @@ fi # Clippy step "Linting" # shellcheck disable=SC2086 -if ! cargo clippy $CRATE_FLAGS --all-targets --all-features --locked -- -D warnings >"$LOG_DIR/clippy.log" 2>&1; then +if ! cargo clippy $CRATE_FLAGS --all-targets --all-features --locked >"$LOG_DIR/clippy.log" 2>&1; then fail show_error "$LOG_DIR/clippy.log" exit 1 fi ok +if [[ "$FULL_WORKSPACE" == true ]]; then + step "Linting independent workspaces" + if ! "$SCRIPT_DIR/lint-independent-workspaces.sh" >"$LOG_DIR/independent-lints.log" 2>&1; then + fail + show_error "$LOG_DIR/independent-lints.log" + exit 1 + fi + ok +fi + # Audit/Deny (workspace only). CI owns this in the dedicated supply-chain lane. if [[ "$FULL_WORKSPACE" == true && "${RSCRYPTO_SKIP_CHECK_SUPPLY_CHAIN:-}" != "1" ]]; then step "Auditing deps" diff --git a/scripts/check/lint-independent-workspaces.sh b/scripts/check/lint-independent-workspaces.sh new file mode 100755 index 00000000..6dac5565 --- /dev/null +++ b/scripts/check/lint-independent-workspaces.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd -P)" +POLICY_MANIFEST="$REPO_ROOT/Cargo.toml" +TARGET_DIR="${RSCRYPTO_INDEPENDENT_LINT_TARGET_DIR:-$REPO_ROOT/target/independent-lints}" +MESSAGE_FORMAT="${RSCRYPTO_LINT_MESSAGE_FORMAT:-human}" +LINT_CAP="${RSCRYPTO_LINT_CAP:-}" + +case "$MESSAGE_FORMAT" in + human | json) ;; + *) echo "RSCRYPTO_LINT_MESSAGE_FORMAT must be human or json" >&2; exit 2 ;; +esac +case "$LINT_CAP" in + "" | warn) ;; + *) echo "RSCRYPTO_LINT_CAP must be empty or warn" >&2; exit 2 ;; +esac + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to resolve independent Cargo workspaces" >&2 + exit 1 +fi + +manifests=() +while IFS= read -r package_manifest; do + if ! metadata=$(cargo metadata \ + --locked \ + --no-deps \ + --format-version 1 \ + --manifest-path "$package_manifest"); then + echo "Failed to resolve Cargo workspace for $package_manifest" >&2 + exit 1 + fi + workspace_root=$(printf '%s\n' "$metadata" | jq -er '.workspace_root') + workspace_root=$(cd "$workspace_root" && pwd -P) + if [[ "$workspace_root" == "$REPO_ROOT" ]]; then + continue + fi + + workspace_manifest="$workspace_root/Cargo.toml" + already_listed=false + for manifest in "${manifests[@]:+${manifests[@]}}"; do + if [[ "$manifest" == "$workspace_manifest" ]]; then + already_listed=true + break + fi + done + if [[ "$already_listed" == false ]]; then + manifests+=("$workspace_manifest") + fi +done < <(find "$REPO_ROOT" -type d -name target -prune -o -type f -name Cargo.toml -print | sort) + +if [[ ${#manifests[@]} -eq 0 ]]; then + echo "No independent Cargo workspaces found through Cargo metadata" + exit 0 +fi + +lint_names() { + local namespace=$1 + awk -v section="[lints.$namespace]" ' + /^\[/ { + active = ($0 == section) + next + } + active { + line = $0 + sub(/^[[:space:]]*/, "", line) + if (line ~ /^[a-z0-9_]+[[:space:]]*=/ && line ~ /"deny"/) { + sub(/[[:space:]]*=.*/, "", line) + print line + } + } + ' "$POLICY_MANIFEST" +} + +lint_flags=() +while IFS= read -r lint; do + [[ -n "$lint" ]] && lint_flags+=("-D$lint") +done < <(lint_names rust) +while IFS= read -r lint; do + [[ -n "$lint" ]] && lint_flags+=("-Dclippy::$lint") +done < <(lint_names clippy) + +if [[ ${#lint_flags[@]} -eq 0 ]]; then + echo "No deny-level Rust or Clippy policy found in $POLICY_MANIFEST" >&2 + exit 1 +fi + +check_cfg_flags=( + '--check-cfg=cfg(miri)' + '--check-cfg=cfg(fuzzing)' + '--check-cfg=cfg(rscrypto_internal_fuzzing)' + '--check-cfg=cfg(target_feature,values("movdiri","movdir64b","serialize"))' +) + +failed=0 +for manifest in "${manifests[@]}"; do + relative_manifest=${manifest#"$REPO_ROOT/"} + if [[ "$MESSAGE_FORMAT" == json ]]; then + echo "Linting independent workspace: $relative_manifest" >&2 + else + echo "Linting independent workspace: $relative_manifest" + fi + + cargo_args=( + clippy + --locked + --manifest-path "$manifest" + --workspace + --all-targets + --all-features + --no-deps + ) + if [[ "$MESSAGE_FORMAT" == json ]]; then + cargo_args+=(--message-format=json) + fi + + compiler_flags=("${lint_flags[@]}" "${check_cfg_flags[@]}") + if [[ -n "$LINT_CAP" ]]; then + compiler_flags+=(--cap-lints "$LINT_CAP") + fi + + if ! CARGO_TARGET_DIR="$TARGET_DIR" cargo "${cargo_args[@]}" -- "${compiler_flags[@]}"; then + failed=1 + fi +done + +exit "$failed" diff --git a/scripts/check/rsa-asm-provenance.sh b/scripts/check/rsa-asm-provenance.sh index 65bd3051..a7941399 100755 --- a/scripts/check/rsa-asm-provenance.sh +++ b/scripts/check/rsa-asm-provenance.sh @@ -156,9 +156,9 @@ verify_rsa_wrapper_fingerprints() { [[ "$actual" == "$expected" ]] \ || die "$wrapper changed without a provenance review" done <<'EOF' -src/auth/rsa_aarch64_asm.rs 65c059365bb76d948d2f09873870ba7e146af15e52e10560bf632d39bdd33834 -src/auth/rsa_aarch64_linux_asm.rs 2b311eb4f253ff458492c76f52f35c0a4d5ae8705774e568ae853e1f6ea1a049 -src/auth/rsa_x86_64_asm.rs 98bc241435de70963c23f0e98431024585f95eed268b773a42d3bd2983b8038e +src/auth/rsa_aarch64_asm.rs dbcd55bc88daa1a9ab1812e964c84bfd4586ae9f6fa9b07dcd28985df6576507 +src/auth/rsa_aarch64_linux_asm.rs 69531bafa963d6d817c6a74f4a0fcaf1d98e713611874f4597768b1a6e0a0e7a +src/auth/rsa_x86_64_asm.rs 5552b8ffa76f1ba43f6aaea1b91a28fb04a02fa167408d925bf5ccec6daefe18 EOF } diff --git a/scripts/ci/changed-test-planning-test.sh b/scripts/ci/changed-test-planning-test.sh index 470aa82f..855e7902 100755 --- a/scripts/ci/changed-test-planning-test.sh +++ b/scripts/ci/changed-test-planning-test.sh @@ -226,7 +226,7 @@ run_test_consumer() { assert_eq "$expected" "$actual" "test commands for $name" } -workspace_test='cargo test --workspace --all-features --lib --tests' +workspace_test='cargo test --locked --workspace --all-features --lib --tests' run_test_consumer planner-failure '' 9 "$workspace_test" run_test_consumer empty-output '' 0 "$workspace_test" run_test_consumer malformed-json '{' 0 "$workspace_test" @@ -238,12 +238,13 @@ run_test_consumer malformed-crates "$empty_crate_selection" 0 "$workspace_test" run_test_consumer malformed-surface "$malformed_surface" 0 "$workspace_test" run_test_consumer valid-empty "$EMPTY_PLAN" 0 '' run_test_consumer valid-workspace "$WORKSPACE_PLAN" 0 "$workspace_test" -run_test_consumer valid-crates "$CRATES_PLAN" 0 $'cargo test -p crate-a --all-features --lib --tests\ncargo test -p crate-b --all-features --lib --tests' +run_test_consumer valid-crates "$CRATES_PLAN" 0 $'cargo test --locked -p crate-a --all-features --lib --tests\ncargo test --locked -p crate-b --all-features --lib --tests' run_test_consumer valid-legacy-workspace "$LEGACY_WORKSPACE_PLAN" 0 "$workspace_test" check_fixture="$TMP_ROOT/check-repository" mkdir -p "$check_fixture/scripts/check" "$check_fixture/scripts/lib" "$check_fixture/scripts/ct" "$check_fixture/scripts/test" cp "$REPO_ROOT/scripts/check/check.sh" "$check_fixture/scripts/check/check.sh" +cp "$REPO_ROOT/scripts/check/lint-independent-workspaces.sh" "$check_fixture/scripts/check/lint-independent-workspaces.sh" cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" "$check_fixture/scripts/lib/" for helper in "$check_fixture/scripts/check/asm-ledger.sh" \ "$check_fixture/scripts/check/check-feature-matrix.sh" \ @@ -289,9 +290,11 @@ run_check_consumer valid-workspace "$WORKSPACE_PLAN" 0 '--workspace' run_check_consumer valid-crates "$CRATES_PLAN" 0 '-p crate-a -p crate-b' check_all_fixture="$TMP_ROOT/check-all-repository" -mkdir -p "$check_all_fixture/scripts/check" "$check_all_fixture/scripts/lib" +mkdir -p "$check_all_fixture/.config" "$check_all_fixture/scripts/check" "$check_all_fixture/scripts/lib" cp "$REPO_ROOT/scripts/check/check-all.sh" "$check_all_fixture/scripts/check/check-all.sh" -cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" "$check_all_fixture/scripts/lib/" +cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" \ + "$REPO_ROOT/scripts/lib/toolchain.sh" "$check_all_fixture/scripts/lib/" +cp "$REPO_ROOT/.config/toolchains.toml" "$check_all_fixture/.config/toolchains.toml" cat >"$check_all_fixture/scripts/lib/targets.sh" <<'EOF' WIN_TARGETS=() LINUX_TARGETS=() diff --git a/scripts/ci/check-action-pins-test.sh b/scripts/ci/check-action-pins-test.sh index 75ddc32c..6518b288 100755 --- a/scripts/ci/check-action-pins-test.sh +++ b/scripts/ci/check-action-pins-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CHECKER="$SCRIPT_DIR/check-action-pins.sh" diff --git a/scripts/ci/check-ci-ownership-test.sh b/scripts/ci/check-ci-ownership-test.sh index 781883e6..5838cf1d 100755 --- a/scripts/ci/check-ci-ownership-test.sh +++ b/scripts/ci/check-ci-ownership-test.sh @@ -128,7 +128,7 @@ expect_failure "$invalid_tool_digest" "direct tool digest is malformed" mutable_tool_url="$TMP_ROOT/mutable-tool-url" make_fixture "$mutable_tool_url" -sed -i.bak 's#/download/v46\.0\.1/#/download/Latest/#' \ +sed -i.bak 's#/download/v47\.0\.3/#/download/Latest/#' \ "$mutable_tool_url/.config/ci-tool-archives.tsv" rm -f "$mutable_tool_url/.config/ci-tool-archives.tsv.bak" expect_failure "$mutable_tool_url" "direct tool URL resolves a mutable release" @@ -166,6 +166,13 @@ printf '\n - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca002889 >>"$unauthenticated_rustup/.github/actions/setup-toolchain/action.yaml" expect_failure "$unauthenticated_rustup" "toolchain setup can run a network bootstrap installer" +inactive_toolchain_contract="$TMP_ROOT/inactive-toolchain-contract" +make_fixture "$inactive_toolchain_contract" +sed -i.bak 's/ "\$GITHUB_ENV"$//' \ + "$inactive_toolchain_contract/.github/actions/setup-toolchain/action.yaml" +rm -f "$inactive_toolchain_contract/.github/actions/setup-toolchain/action.yaml.bak" +expect_failure "$inactive_toolchain_contract" "toolchain contract is installed but not activated" + floating_rail_action="$TMP_ROOT/floating-rail-action" make_fixture "$floating_rail_action" yq eval '(.jobs."rail-plan".steps[] | select(.id == "rail") | .uses) = "loadingalias/cargo-rail-action@v6"' -i \ diff --git a/scripts/ci/check-ci-ownership.sh b/scripts/ci/check-ci-ownership.sh index d83c9e17..5c102499 100755 --- a/scripts/ci/check-ci-ownership.sh +++ b/scripts/ci/check-ci-ownership.sh @@ -344,8 +344,15 @@ release_intent_condition=$(yq eval '.jobs."rail-plan".steps[] | select(.name == [[ "$release_intent_condition" == *"startsWith(github.head_ref, 'rail/release-')"* \ && "$release_intent_condition" == *"github.event.pull_request.head.repo.full_name == github.repository"* ]] \ || fail "only repository-owned Cargo Rail release PRs may consume change intent" -grep -Fq 'scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS"' "$TOOLCHAIN_ACTION" \ +grep -Fq 'scripts/ci/setup-toolchain.sh "$TOOLCHAIN" "$TOOLCHAIN_COMPONENTS" "$GITHUB_ENV"' "$TOOLCHAIN_ACTION" \ || fail "toolchain setup must use the repository-owned rustup policy" +grep -Fq "printf 'RUSTUP_TOOLCHAIN=%s\\n' \"\$toolchain\" >>\"\$github_env\"" "$SETUP_TOOLCHAIN" \ + || fail "toolchain setup must activate the resolved contract for later steps" +grep -Fq 'RUSTUP_TOOLCHAIN="$TOOLCHAIN" rustc --version --verbose' "$TOOLCHAIN_ACTION" \ + || fail "toolchain setup must verify the activated contract without a rust-toolchain override" +if grep -Fq 'rustup default ' "$SETUP_TOOLCHAIN"; then + fail "toolchain setup must not mutate a runner-global default" +fi if grep -Eq '[.]cargo/(bin|[.]crates)|[.]opam' "$SETUP_ACTION"; then fail "CI tool executables and OPAM switches must not be restored from caches" fi @@ -686,7 +693,7 @@ group_musl=$(jq '[.groups.linux[] | select(contains("musl"))] | length' "$MANIFE [[ "$group_musl" -eq 2 ]] || fail "the target manifest must retain both MUSL triples" # shellcheck disable=SC2016 # `$target` is an intentional literal in the workflow contract regex. -[[ $(count_matches 'cargo (check|clippy|build) --target "\$target"' "$CROSS_SCRIPT") -ge 3 ]] \ +[[ $(count_matches 'cargo (check|clippy|build) --locked --target "\$target"' "$CROSS_SCRIPT") -ge 3 ]] \ || fail "MUSL evidence must pass the target triple explicitly to Cargo" echo "CI ownership contract passed" diff --git a/scripts/ci/check-locked-cargo-test.sh b/scripts/ci/check-locked-cargo-test.sh new file mode 100755 index 00000000..8ee6480e --- /dev/null +++ b/scripts/ci/check-locked-cargo-test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHECKER="$SCRIPT_DIR/check-locked-cargo.sh" +TMP_ROOT=$(mktemp -d) +trap 'rm -rf "$TMP_ROOT"' EXIT + +fail() { + echo "locked Cargo inventory test failure: $*" >&2 + exit 1 +} + +mkdir -p "$TMP_ROOT/scripts/ci" "$TMP_ROOT/.zed" +printf '%s\n' 'build:' ' cargo build --locked --workspace' >"$TMP_ROOT/justfile" +"$CHECKER" --root "$TMP_ROOT" || fail "locked command was rejected" + +printf '%s\n' '#!/usr/bin/env bash' 'cargo test --workspace' >"$TMP_ROOT/scripts/ci/example.sh" +if "$CHECKER" --root "$TMP_ROOT" >/dev/null 2>&1; then + fail "unlocked command was accepted" +fi + +printf '%s\n' '#!/usr/bin/env bash' 'cargo test \' ' --locked \' ' --workspace' >"$TMP_ROOT/scripts/ci/example.sh" +"$CHECKER" --root "$TMP_ROOT" || fail "multiline locked command was rejected" + +printf '%s\n' '#!/usr/bin/env bash' '# cargo check --workspace' >"$TMP_ROOT/scripts/ci/example.sh" +"$CHECKER" --root "$TMP_ROOT" || fail "comment was treated as a command" + +printf '%s\n' '[{"label":"check","command":"cargo","args":["check"]}]' >"$TMP_ROOT/.zed/tasks.json" +if "$CHECKER" --root "$TMP_ROOT" >/dev/null 2>&1; then + fail "unlocked Zed Cargo task was accepted" +fi + +printf '%s\n' '[{"label":"check","command":"just","args":["check"]}]' >"$TMP_ROOT/.zed/tasks.json" +"$CHECKER" --root "$TMP_ROOT" || fail "repository-front-door Zed task was rejected" + +echo "Locked Cargo inventory regression tests passed" diff --git a/scripts/ci/check-locked-cargo.sh b/scripts/ci/check-locked-cargo.sh new file mode 100755 index 00000000..ddd1db65 --- /dev/null +++ b/scripts/ci/check-locked-cargo.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="" +if [[ ${1:-} == --root ]]; then + ROOT=${2:?missing path after --root} + shift 2 +fi +if [[ $# -ne 0 ]]; then + echo "usage: check-locked-cargo.sh [--root PATH]" >&2 + exit 2 +fi + +if [[ -z "$ROOT" ]]; then + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +fi + +status=0 +check_statement() { + local file=$1 + local line_number=$2 + local statement=$3 + local trimmed=${statement#"${statement%%[![:space:]]*}"} + + case "$trimmed" in + echo\ * | printf\ * | step\ *) return ;; + esac + + if [[ "$statement" =~ (^|[[:space:]])cargo[[:space:]]+(build|check|clippy|test|bench|rustc|run|doc|miri|nextest)($|[[:space:]]) ]] \ + && [[ "$statement" != *"--locked"* ]]; then + echo "$file:$line_number: routine Cargo command must use --locked" >&2 + echo " $statement" >&2 + status=1 + fi + + if [[ "$statement" =~ (^|[[:space:]])cargo[[:space:]]+llvm-cov[[:space:]]+(nextest|test)($|[[:space:]]) ]] \ + && [[ "$statement" != *"--locked"* ]]; then + echo "$file:$line_number: cargo-llvm-cov execution must use --locked" >&2 + echo " $statement" >&2 + status=1 + fi +} + +while IFS= read -r file; do + statement="" + statement_line=0 + line_number=0 + while IFS= read -r line || [[ -n "$line" ]]; do + line_number=$((line_number + 1)) + if [[ -z "$statement" ]]; then + [[ "$line" =~ ^[[:space:]]*# ]] && continue + statement_line=$line_number + fi + statement+=" ${line%\\}" + if [[ "$line" == *\\ ]]; then + continue + fi + check_statement "${file#"$ROOT"/}" "$statement_line" "$statement" + statement="" + done <"$file" + if [[ -n "$statement" ]]; then + check_statement "${file#"$ROOT"/}" "$statement_line" "$statement" + fi +done < <( + { + [[ -f "$ROOT/justfile" ]] && printf '%s\n' "$ROOT/justfile" + find "$ROOT/scripts" -type f -name '*.sh' \ + ! -name '*-test.sh' ! -name '*-scheduler-test.sh' 2>/dev/null || true + find "$ROOT/.github" -type f \( -name '*.yaml' -o -name '*.yml' \) 2>/dev/null || true + } | LC_ALL=C sort +) + +if [[ -f "$ROOT/.zed/tasks.json" ]]; then + while IFS= read -r task; do + command=$(jq -r '.command' <<<"$task") + args=$(jq -r '(.args // []) | join(" ")' <<<"$task") + check_statement ".zed/tasks.json" 1 "$command $args" + done < <(jq -c '.[] | select((.command // "") == "cargo" or ((.command // "") | startswith("cargo ")))' \ + "$ROOT/.zed/tasks.json") +fi + +exit "$status" diff --git a/scripts/ci/check-worktree-test.sh b/scripts/ci/check-worktree-test.sh index e05ff043..f0ff3788 100755 --- a/scripts/ci/check-worktree-test.sh +++ b/scripts/ci/check-worktree-test.sh @@ -17,6 +17,7 @@ fake_home="$TMP_ROOT/home" command_log="$TMP_ROOT/commands.log" preflight_marker="$TMP_ROOT/locked-metadata-preflight" mkdir -p \ + "$fixture/.config" \ "$fixture/scripts/check" \ "$fixture/scripts/ct" \ "$fixture/scripts/lib" \ @@ -31,12 +32,15 @@ cp \ "$REPO_ROOT/scripts/check/check-feature-matrix.sh" \ "$REPO_ROOT/scripts/check/check-ibm.sh" \ "$REPO_ROOT/scripts/check/check-linux.sh" \ + "$REPO_ROOT/scripts/check/lint-independent-workspaces.sh" \ "$REPO_ROOT/scripts/check/check-win.sh" \ "$REPO_ROOT/scripts/check/check.sh" \ "$REPO_ROOT/scripts/check/zig-cc.sh" \ "$fixture/scripts/check/" cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" \ - "$REPO_ROOT/scripts/lib/feature-profiles.sh" "$fixture/scripts/lib/" + "$REPO_ROOT/scripts/lib/feature-profiles.sh" "$REPO_ROOT/scripts/lib/toolchain.sh" \ + "$fixture/scripts/lib/" +cp "$REPO_ROOT/.config/toolchains.toml" "$fixture/.config/toolchains.toml" cp "$REPO_ROOT/scripts/test/test-feature-matrix.sh" "$fixture/scripts/test/test-feature-matrix.sh" cat >"$fixture/scripts/lib/targets.sh" <<'EOF' @@ -104,6 +108,7 @@ case "${1:-}" in metadata) require_locked "$@" : >"$MOCK_PREFLIGHT_MARKER" + printf '{"workspace_root":"%s"}\n' "$MOCK_REPO_ROOT" ;; rail) if [[ "${2:-}" == "plan" ]]; then diff --git a/scripts/ci/ci-check.sh b/scripts/ci/ci-check.sh index 700023ba..0af74976 100755 --- a/scripts/ci/ci-check.sh +++ b/scripts/ci/ci-check.sh @@ -14,6 +14,9 @@ echo "" echo "🔗 Checking workflow ownership and action pins..." "$SCRIPT_DIR/check-ci-ownership.sh" "$SCRIPT_DIR/check-ci-ownership-test.sh" +"$SCRIPT_DIR/check-locked-cargo.sh" +"$SCRIPT_DIR/check-locked-cargo-test.sh" +"$SCRIPT_DIR/../lib/target-matrix.sh" --validate "$SCRIPT_DIR/check-action-pins.sh" export RSCRYPTO_SKIP_CHECK_SUPPLY_CHAIN=1 @@ -21,7 +24,7 @@ export RSCRYPTO_SKIP_CHECK_SUPPLY_CHAIN=1 echo "" echo "🔨 Building all targets..." -cargo build --workspace --all-targets --all-features +cargo build --locked --workspace --all-targets --all-features echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/scripts/ci/cross-targets.sh b/scripts/ci/cross-targets.sh index 044312e7..01784b29 100755 --- a/scripts/ci/cross-targets.sh +++ b/scripts/ci/cross-targets.sh @@ -36,11 +36,11 @@ for target in "${MUSL_TARGETS[@]}"; do mkdir -p "$target_dir" RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo check --target "$target" --no-default-features --lib + cargo check --locked --target "$target" --no-default-features --lib RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo clippy --target "$target" --lib --all-features -- -D warnings + cargo clippy --locked --target "$target" --lib --all-features RUSTC_WRAPPER="" CARGO_TARGET_DIR="$target_dir" \ - cargo build --target "$target" --no-default-features --features alloc --lib --release + cargo build --locked --target "$target" --no-default-features --features alloc --lib --release done for target in "${NOSTD_TARGETS[@]}" "${WASM_TARGETS[@]}"; do diff --git a/scripts/ci/dependabot-smoke.sh b/scripts/ci/dependabot-smoke.sh index 14ec2b8e..7863c393 100755 --- a/scripts/ci/dependabot-smoke.sh +++ b/scripts/ci/dependabot-smoke.sh @@ -39,7 +39,7 @@ run_root_smoke() { cargo fetch --locked cargo check --locked --workspace --all-targets --all-features cargo test --locked --workspace --all-features --no-run - cargo deny check advisories + cargo deny --locked check advisories } run_manifest_smoke() { diff --git a/scripts/ci/install-tools.sh b/scripts/ci/install-tools.sh index a5dcea6a..e354cd52 100755 --- a/scripts/ci/install-tools.sh +++ b/scripts/ci/install-tools.sh @@ -6,13 +6,13 @@ set -euo pipefail MODE=${1:-standard} -CARGO_NEXTEST_VERSION=0.9.140 +CARGO_NEXTEST_VERSION=0.9.143 CARGO_DENY_VERSION=0.20.2 CARGO_AUDIT_VERSION=0.22.2 -CARGO_RAIL_VERSION=0.20.0 +CARGO_RAIL_VERSION=0.21.0 CARGO_SEMVER_CHECKS_VERSION=0.50.0 -JUST_VERSION=1.57.0 -ZIZMOR_VERSION=1.26.1 +JUST_VERSION=1.58.0 +ZIZMOR_VERSION=1.29.0 CARGO_CRITERION_VERSION=1.1.0 CRITCMP_VERSION=0.1.8 CARGO_FUZZ_VERSION=0.13.2 diff --git a/scripts/ci/mlkem-aarch64-gate.sh b/scripts/ci/mlkem-aarch64-gate.sh index babf284d..20f6ed36 100755 --- a/scripts/ci/mlkem-aarch64-gate.sh +++ b/scripts/ci/mlkem-aarch64-gate.sh @@ -60,22 +60,22 @@ else fi run_step "owned aarch64 NEON NTT scalar oracle" \ - cargo test --lib --features ml-kem ntt_neon -- --nocapture + cargo test --locked --lib --features ml-kem ntt_neon -- --nocapture run_step "owned aarch64 basemul scalar oracle" \ - cargo test --lib --features ml-kem basemul_accumulate -- --nocapture + cargo test --locked --lib --features ml-kem basemul_accumulate -- --nocapture run_step "ML-KEM ACVP FIPS 203 vectors" \ - cargo test --test mlkem_acvp --features ml-kem -- --nocapture + cargo test --locked --test mlkem_acvp --features ml-kem -- --nocapture run_step "ML-KEM operation tests" \ - cargo test --test mlkem_ops --features ml-kem -- --nocapture + cargo test --locked --test mlkem_ops --features ml-kem -- --nocapture run_step "ML-KEM FIPS/property tests" \ - cargo test --test mlkem_properties --features ml-kem -- --nocapture + cargo test --locked --test mlkem_properties --features ml-kem -- --nocapture run_step "ML-KEM fuzz corpus replay" \ - cargo test --manifest-path fuzz/Cargo.toml --test corpus_replay replay_auth_mlkem -- --nocapture + cargo test --locked --manifest-path fuzz/Cargo.toml --test corpus_replay replay_auth_mlkem -- --nocapture if [[ "$(bool_value "${RSCRYPTO_MLKEM_AARCH64_GATE_SKIP_BENCH:-false}")" == "true" ]]; then echo "warning: ML-KEM benchmark gate skipped by RSCRYPTO_MLKEM_AARCH64_GATE_SKIP_BENCH" | tee -a "$LOG_PATH" diff --git a/scripts/ci/native-check.sh b/scripts/ci/native-check.sh index 13742b84..dba8f39a 100755 --- a/scripts/ci/native-check.sh +++ b/scripts/ci/native-check.sh @@ -23,7 +23,7 @@ echo "Native validation: $(rustc -vV | sed -n 's/^host: //p')" echo "" echo "Checking no-default-features library boundary..." -cargo check --workspace --lib --no-default-features +cargo check --locked --workspace --lib --no-default-features TARGET_ARGS=(--lib) if [[ "$ALL_TARGETS" == true ]]; then @@ -32,16 +32,16 @@ fi echo "" echo "Checking all-feature native targets..." -cargo check --workspace "${TARGET_ARGS[@]}" --all-features +cargo check --locked --workspace "${TARGET_ARGS[@]}" --all-features echo "" echo "Linting all-feature native targets..." -cargo clippy --workspace "${TARGET_ARGS[@]}" --all-features -- -D warnings +cargo clippy --locked --workspace "${TARGET_ARGS[@]}" --all-features if [[ "$ALL_TARGETS" == true ]]; then echo "" echo "Building all-feature native targets..." - cargo build --workspace --all-targets --all-features + cargo build --locked --workspace --all-targets --all-features fi echo "Native validation passed" diff --git a/scripts/ci/nostd-wasm-suite.sh b/scripts/ci/nostd-wasm-suite.sh index 0125984f..070bc890 100755 --- a/scripts/ci/nostd-wasm-suite.sh +++ b/scripts/ci/nostd-wasm-suite.sh @@ -57,9 +57,9 @@ run_wasm_runtime_vectors() { export CARGO_TARGET_WASM32_WASIP1_RUNNER="wasmtime" local manifest="tools/wasm-runtime-vectors/Cargo.toml" - cargo run --manifest-path "$manifest" --target "$TARGET" + cargo run --locked --manifest-path "$manifest" --target "$TARGET" RUSTFLAGS="-C target-feature=+simd128" \ - cargo run --manifest-path "$manifest" --target "$TARGET" + cargo run --locked --manifest-path "$manifest" --target "$TARGET" } echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -67,11 +67,11 @@ echo "Cross-compile sweep: $TARGET ($DEPTH)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Always: bare no-default-features. -cargo check --target "$TARGET" --no-default-features --lib -cargo build --target "$TARGET" --no-default-features --lib --release +cargo check --locked --target "$TARGET" --no-default-features --lib +cargo build --locked --target "$TARGET" --no-default-features --lib --release # Always: alloc. -cargo check --target "$TARGET" --no-default-features --features alloc --lib +cargo check --locked --target "$TARGET" --no-default-features --features alloc --lib if [[ "$DEPTH" == "deep" ]]; then # Union of the historical check-all facade matrix and the dedicated weekly @@ -110,22 +110,22 @@ if [[ "$DEPTH" == "deep" ]]; then ) for feature_set in "${FEATURE_SETS[@]}"; do - cargo check --target "$TARGET" --no-default-features --features "$feature_set" --lib + cargo check --locked --target "$TARGET" --no-default-features --features "$feature_set" --lib done # Full no_std release build. - cargo build --target "$TARGET" --no-default-features --features "alloc,checksums,hashes,auth,aead" --lib --release + cargo build --locked --target "$TARGET" --no-default-features --features "alloc,checksums,hashes,auth,aead" --lib --release fi # Target-specific smoke (shallow gets a token extra so each target has >0 feature coverage). if [[ "$DEPTH" == "shallow" ]]; then case "$TARGET" in thumbv6m-none-eabi) - cargo check --target "$TARGET" --no-default-features --features checksums --lib + cargo check --locked --target "$TARGET" --no-default-features --features checksums --lib ;; wasm32-unknown-unknown) - cargo check --target "$TARGET" --no-default-features --features hashes --lib - RUSTFLAGS="-C target-feature=+simd128" cargo check --target "$TARGET" --no-default-features --features hashes --lib + cargo check --locked --target "$TARGET" --no-default-features --features hashes --lib + RUSTFLAGS="-C target-feature=+simd128" cargo check --locked --target "$TARGET" --no-default-features --features hashes --lib ;; esac fi diff --git a/scripts/ci/publish-immutable-release-test.sh b/scripts/ci/publish-immutable-release-test.sh index 57b3828a..94d7c19d 100755 --- a/scripts/ci/publish-immutable-release-test.sh +++ b/scripts/ci/publish-immutable-release-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PUBLISHER="$SCRIPT_DIR/publish-immutable-release.sh" diff --git a/scripts/ci/release-evidence-check-test.sh b/scripts/ci/release-evidence-check-test.sh index 46bfd2f5..afc3a19a 100755 --- a/scripts/ci/release-evidence-check-test.sh +++ b/scripts/ci/release-evidence-check-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" diff --git a/scripts/ci/release-identity-test.sh b/scripts/ci/release-identity-test.sh index 2a32782d..aa43fcab 100755 --- a/scripts/ci/release-identity-test.sh +++ b/scripts/ci/release-identity-test.sh @@ -96,7 +96,7 @@ jq -e --arg commit "$commit" ' and (.release.tag_object | test("^[0-9a-f]{40}$")) and .release.git_commit == $commit and (.release.git_tree | test("^[0-9a-f]{40}$")) - and .toolchain.channel == "nightly-2026-07-17" + and .toolchain.channel == "1.97.1" and (.toolchain.manifest.sha256 | test("^[0-9a-f]{64}$")) and .evidence.git_commit == $commit and .evidence.mode == "exact_commit" diff --git a/scripts/ci/release-preflight.sh b/scripts/ci/release-preflight.sh index 853bf515..7bc2b3e1 100755 --- a/scripts/ci/release-preflight.sh +++ b/scripts/ci/release-preflight.sh @@ -110,7 +110,7 @@ cargo rail config validate --strict cargo rail config migrate --check # Exact-commit Weekly release mode owns exhaustive compiler-backed Cargo graph # assurance. The release evidence gate verifies that named job before publication. -cargo deny check all +cargo deny --locked check all # RustCrypto `rsa` is used only as a dev/test/bench oracle. Production RSA # verification is implemented in `src/auth/rsa.rs`; keep this scoped to the # known Marvin advisory until the oracle dependency is removed or fixed. diff --git a/scripts/ci/repository-controls-evidence-test.sh b/scripts/ci/repository-controls-evidence-test.sh index fbb91a30..c2d44c7a 100755 --- a/scripts/ci/repository-controls-evidence-test.sh +++ b/scripts/ci/repository-controls-evidence-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" diff --git a/scripts/ci/run-bench.sh b/scripts/ci/run-bench.sh index a693ccb3..cec94cf6 100755 --- a/scripts/ci/run-bench.sh +++ b/scripts/ci/run-bench.sh @@ -916,7 +916,7 @@ run_bench_cmd() { bench_features="$(bench_features_for_target "$bench")" cargo_bench="$(bench_binary_for_target "$bench")" - cmd=(cargo bench --profile bench --features "$bench_features" --bench "$cargo_bench") + cmd=(cargo bench --locked --profile bench --features "$bench_features" --bench "$cargo_bench") if [[ -n "$filter" || "${#CRITERION_ARGS[@]}" -gt 0 ]]; then cmd+=(--) if [[ -n "$filter" ]]; then @@ -997,7 +997,7 @@ fi GENERIC_FEATURES="$(bench_features_for_invocation "$BENCHES_INPUT")" echo "Using features: $GENERIC_FEATURES" | tee -a "$LOG_PATH" -cmd=(cargo bench --profile bench --features "$GENERIC_FEATURES") +cmd=(cargo bench --locked --profile bench --features "$GENERIC_FEATURES") if [[ "${#BENCH_FLAGS[@]}" -gt 0 ]]; then cmd+=("${BENCH_FLAGS[@]}") fi diff --git a/scripts/ci/run-rust-job-test.sh b/scripts/ci/run-rust-job-test.sh index 6b0f0769..d5c59373 100755 --- a/scripts/ci/run-rust-job-test.sh +++ b/scripts/ci/run-rust-job-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DISPATCHER="$SCRIPT_DIR/run-rust-job.sh" @@ -292,7 +293,7 @@ grep -Fq \ "$CAPTURE/amx-cargo.args" \ || fail "AMX integration test existence was not checked under the required permission contract" grep -Fq \ - 'RUSTFLAGS=-A unstable-features -C target-feature=+amx-tile,+amx-bf16,+amx-int8' \ + 'RUSTFLAGS=-C target-feature=+amx-tile,+amx-bf16,+amx-int8' \ "$CAPTURE/amx-cargo.args" \ || fail "AMX no_std test existence was not checked with forced AMX target features" diff --git a/scripts/ci/run-rust-job.sh b/scripts/ci/run-rust-job.sh index 9b3eca8e..f0a25a96 100755 --- a/scripts/ci/run-rust-job.sh +++ b/scripts/ci/run-rust-job.sh @@ -70,6 +70,11 @@ run_quality() { just ci-check } +run_msrv() { + cargo check --locked --workspace --lib --no-default-features + cargo check --locked --workspace --lib --all-features +} + run_cargo_graph() { mkdir -p target/cargo-rail cargo rail config validate --strict @@ -97,16 +102,16 @@ run_native() { case "$target" in x86_64-pc-windows-msvc) echo "Windows x86_64 lane: compile smoke plus BLAKE3 ASM runtime vectors" - cargo clippy --workspace --lib --all-features -- -D warnings - cargo test --workspace --all-features --no-run - cargo test --workspace --features blake3 \ + cargo clippy --locked --workspace --lib --all-features + cargo test --locked --workspace --all-features --no-run + cargo test --locked --workspace --features blake3 \ --test blake3_official_vectors \ --test blake3_differential ;; aarch64-pc-windows-msvc) echo "Windows AArch64 lane: compile-only smoke" - cargo clippy --workspace --lib --all-features -- -D warnings - cargo test --workspace --all-features --no-run + cargo clippy --locked --workspace --lib --all-features + cargo test --locked --workspace --all-features --no-run ;; x86_64-unknown-linux-gnu | aarch64-unknown-linux-gnu) bash scripts/ci/native-check.sh --all-targets @@ -161,7 +166,7 @@ run_platform_amx() { # NIGHTLY: Rust target-feature names for AMX remain unstable. This lane # deliberately forces them so the no_std permission gate is executable. - local amx_rustflags="-A unstable-features -C target-feature=+amx-tile,+amx-bf16,+amx-int8" + local amx_rustflags="-C target-feature=+amx-tile,+amx-bf16,+amx-int8" RUSTFLAGS="$amx_rustflags" \ assert_single_libtest \ platform::detect::tests::no_std_linux_x86_64_masks_compile_time_amx_without_a_permission_probe \ @@ -181,13 +186,13 @@ run_supply_chain() { require_one_of supply_chain_mode "$mode" light full if [[ "$mode" == "full" ]]; then - cargo deny check all + cargo deny --locked check all # RustCrypto `rsa` is used only as a dev/test/bench oracle. Production RSA # verification is implemented in `src/auth/rsa.rs`; keep this scoped to the # known Marvin advisory until the oracle dependency is removed or fixed. cargo audit --ignore RUSTSEC-2023-0071 else - cargo deny check advisories + cargo deny --locked check advisories fi } @@ -460,6 +465,7 @@ main() { require_nonempty operation "$operation" case "$operation" in quality) run_quality ;; + msrv) run_msrv ;; cargo-graph) run_cargo_graph ;; feature-contracts) run_feature_contracts ;; native) run_native ;; diff --git a/scripts/ci/setup-toolchain.sh b/scripts/ci/setup-toolchain.sh index beba45ec..688675cf 100755 --- a/scripts/ci/setup-toolchain.sh +++ b/scripts/ci/setup-toolchain.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "usage: setup-toolchain.sh [comma-separated-components]" >&2 +if [[ $# -lt 1 || $# -gt 3 ]]; then + echo "usage: setup-toolchain.sh [comma-separated-components] [github-env]" >&2 exit 2 fi toolchain=$1 components=${2:-} +github_env=${3:-} if [[ ! "$toolchain" =~ ^(nightly|beta)-[0-9]{4}-[0-9]{2}-[0-9]{2}$ \ && ! "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -34,5 +35,7 @@ if [[ -n "$components" ]]; then fi rustup "${install_args[@]}" -rustup default "$toolchain" +if [[ -n "$github_env" ]]; then + printf 'RUSTUP_TOOLCHAIN=%s\n' "$toolchain" >>"$github_env" +fi rustc "+$toolchain" --version --verbose diff --git a/scripts/ci/tool-integrity-test.sh b/scripts/ci/tool-integrity-test.sh index 5daedf29..ef0c6418 100755 --- a/scripts/ci/tool-integrity-test.sh +++ b/scripts/ci/tool-integrity-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" @@ -90,11 +91,11 @@ while [[ $# -gt 0 ]]; do esac done [[ -n "$destination" ]] -root="$destination/wasmtime-v46.0.1-x86_64-linux" +root="$destination/wasmtime-v47.0.3-x86_64-linux" mkdir -p "$root" cat >"$root/wasmtime" <<'EOF' #!/usr/bin/env bash -printf 'wasmtime 46.0.1 (mock)\n' +printf 'wasmtime 47.0.3 (mock)\n' printf 'wasmtime executed\n' >>"$MOCK_EXEC_LOG" EOF chmod +x "$root/wasmtime" @@ -332,6 +333,7 @@ SH cat >"$package_bin/rustc" <<'SH' #!/usr/bin/env bash +printf 'rustc %s\n' "$*" >>"$MOCK_PACKAGE_LOG" printf 'rustc 1.99.0-nightly\ncommit-date: 2026-07-16\n' SH @@ -537,13 +539,13 @@ if HOME="$ct_home" \ fi for contract in \ - 'cargo-nextest =0.9.140' \ + 'cargo-nextest =0.9.143' \ 'cargo-deny =0.20.2' \ 'cargo-audit =0.22.2' \ - 'cargo-rail =0.20.0' \ + 'cargo-rail =0.21.0' \ 'cargo-semver-checks =0.50.0' \ - 'just =1.57.0' \ - 'zizmor =1.26.1' \ + 'just =1.58.0' \ + 'zizmor =1.29.0' \ 'cargo-criterion =1.1.0' \ 'critcmp =0.1.8' \ 'cargo-fuzz =0.13.2' \ @@ -571,12 +573,12 @@ mkdir -p "$cached_home/.cargo/bin" cat >"$cached_home/.cargo/bin/just" <<'SH' #!/usr/bin/env bash printf 'forged exact-version cache executed\n' >>"$MALICIOUS_EXEC_LOG" -printf 'just 1.57.0\n' +printf 'just 1.58.0\n' SH chmod +x "$cached_home/.cargo/bin/just" cat >"$cached_home/.cargo/.crates.toml" <<'EOF' [v1] -"just 1.57.0 (registry+https://github.com/rust-lang/crates.io-index)" = ["just"] +"just 1.58.0 (registry+https://github.com/rust-lang/crates.io-index)" = ["just"] EOF : >"$package_log" HOME="$cached_home" \ @@ -589,7 +591,7 @@ HOME="$cached_home" \ "$REPO_ROOT/scripts/ci/install-tools.sh" minimal >/dev/null [[ ! -e "$malicious_exec" ]] \ || fail "forged exact-version cached binary executed before authenticated replacement" -grep -Fq 'cargo install --registry crates-io just --locked --version =1.57.0 --force' "$package_log" \ +grep -Fq 'cargo install --registry crates-io just --locked --version =1.58.0 --force' "$package_log" \ || fail "forged cache did not trigger a fresh authenticated install" trusted_bin=$(tail -n 1 "$github_path_file") case "$trusted_bin" in @@ -598,14 +600,21 @@ case "$trusted_bin" in esac : >"$package_log" +toolchain_env="$TMP_ROOT/toolchain.env" +: >"$toolchain_env" MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" \ - nightly-2026-07-17 'clippy, rustfmt' >/dev/null + nightly-2026-08-12 'clippy, rustfmt' "$toolchain_env" >/dev/null grep -Fq \ - 'rustup toolchain install nightly-2026-07-17 --profile minimal --no-self-update --component clippy --component rustfmt' \ + 'rustup toolchain install nightly-2026-08-12 --profile minimal --no-self-update --component clippy --component rustfmt' \ "$package_log" || fail "rustup toolchain command was not exact" -grep -Fq 'rustup default nightly-2026-07-17' "$package_log" \ - || fail "rustup did not select the exact toolchain" +grep -Fq 'rustc +nightly-2026-08-12 --version --verbose' "$package_log" \ + || fail "installed toolchain was not verified explicitly" +grep -Fxq 'RUSTUP_TOOLCHAIN=nightly-2026-08-12' "$toolchain_env" \ + || fail "installed toolchain was not activated for later CI steps" +if grep -Fq 'rustup default ' "$package_log"; then + fail "toolchain setup mutated the runner default" +fi if MOCK_PACKAGE_LOG="$package_log" PATH="$package_bin:$PATH" \ "$REPO_ROOT/scripts/ci/setup-toolchain.sh" nightly clippy >/dev/null 2>&1; then fail "mutable rustup channel was accepted" diff --git a/scripts/ct/dudect.sh b/scripts/ct/dudect.sh index 4c9a7a8a..cc454d62 100755 --- a/scripts/ct/dudect.sh +++ b/scripts/ct/dudect.sh @@ -115,7 +115,7 @@ BUILD_TARGET_DIR="$ROOT/target/ct-dudect-build/$TARGET/$PROFILE" if [[ ! -s "$LINKER_COMMAND_PATH" ]]; then rm -rf "$BUILD_TARGET_DIR" fi -CARGO_ARGS=(--locked --manifest-path "$ROOT/tools/ct-dudect/Cargo.toml" --target-dir "$BUILD_TARGET_DIR" --target "$TARGET") +CARGO_ARGS=(--manifest-path "$ROOT/tools/ct-dudect/Cargo.toml" --target-dir "$BUILD_TARGET_DIR" --target "$TARGET") if [[ "$PROFILE" == "release" ]]; then CARGO_ARGS+=(--release) elif [[ "$PROFILE" != "debug" ]]; then @@ -124,7 +124,7 @@ elif [[ "$PROFILE" != "debug" ]]; then fi linker_log_candidate="$(mktemp "$OUT_DIR/.dudect-linker-command.XXXXXXXX")" -cargo rustc "${CARGO_ARGS[@]}" --bin rscrypto-ct-dudect -- --print link-args 2>&1 | tee "$linker_log_candidate" +cargo rustc --locked "${CARGO_ARGS[@]}" --bin rscrypto-ct-dudect -- --print link-args 2>&1 | tee "$linker_log_candidate" link_command_count=$(grep -c '"-o"' "$linker_log_candidate" || true) if [[ "$link_command_count" -gt 1 ]]; then echo "expected at most one DudeCT linker command; found $link_command_count" >&2 diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 0387a3fc..e3b5ce49 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -158,7 +158,27 @@ apply_ci_resource_profile() { ensure_target() { local target=$1 - if ! rustup target list --installed 2>/dev/null | grep -q "^${target}$"; then + local toolchain=${2:-} + if [[ -n "$toolchain" ]]; then + if ! rustup target list --toolchain "$toolchain" --installed 2>/dev/null | grep -q "^${target}$"; then + rustup target add --toolchain "$toolchain" "$target" >/dev/null 2>&1 || true + fi + elif ! rustup target list --installed 2>/dev/null | grep -q "^${target}$"; then rustup target add "$target" >/dev/null 2>&1 || true fi } + +activate_nightly_toolchain() { + local toolchain_script="$COMMON_DIR/toolchain.sh" + local toolchain_contracts="$COMMON_DIR/../../.config/toolchains.toml" + [[ -x "$toolchain_script" ]] || { + echo "ERROR: nightly toolchain resolver not found: $toolchain_script" >&2 + return 1 + } + [[ -f "$toolchain_contracts" ]] || { + echo "ERROR: nightly toolchain authority not found: $toolchain_contracts" >&2 + return 1 + } + RUSTUP_TOOLCHAIN=$("$toolchain_script" --nightly) + export RUSTUP_TOOLCHAIN +} diff --git a/scripts/lib/fuzz-packages.sh b/scripts/lib/fuzz-packages.sh index c9bdf5dd..155a5ced 100644 --- a/scripts/lib/fuzz-packages.sh +++ b/scripts/lib/fuzz-packages.sh @@ -79,6 +79,9 @@ fuzz_in_package() { ( cd "$REPO_ROOT" || exit + if [[ "$subcommand" == build || "$subcommand" == run ]]; then + cargo metadata --locked --no-deps --manifest-path "$package_dir/Cargo.toml" >/dev/null + fi CARGO_TARGET_DIR="$FUZZ_SHARED_TARGET_DIR" cargo fuzz "$subcommand" --fuzz-dir "$package_dir" "$@" ) } diff --git a/scripts/lib/target-matrix.sh b/scripts/lib/target-matrix.sh index 18bda859..969a9100 100755 --- a/scripts/lib/target-matrix.sh +++ b/scripts/lib/target-matrix.sh @@ -25,21 +25,79 @@ if ! command -v jq >/dev/null 2>&1; then fi usage() { - echo "Usage: $0 --format {shell|json} [--key KEY]" >&2 + echo "Usage: $0 --validate | --format {shell|json} [--key KEY]" >&2 exit 1 } FORMAT="" KEY="" +VALIDATE=false while [[ $# -gt 0 ]]; do case "$1" in + --validate) VALIDATE=true; shift ;; --format) FORMAT="$2"; shift 2 ;; --key) KEY="$2"; shift 2 ;; *) usage ;; esac done +if [[ "$VALIDATE" == true && -n "$FORMAT" ]]; then + usage +fi + +validate_manifest() { + jq -e ' + . as $root | + ($root.groups | keys) == ["ibm", "linux", "macos", "no_std", "wasm", "win"] and + all($root.groups[]; type == "array" and length > 0 and . == sort and all(.[]; type == "string" and length > 0)) and + ([$root.groups[][]] | length) == ([$root.groups[][]] | unique | length) and + ($root.ci | type == "array" and length > 0) and + ($root.ci == ($root.ci | sort_by(.name))) and + ([$root.ci[].name] | length) == ([$root.ci[].name] | unique | length) and + all($root.ci[]; + .name as $name | + any($root.groups[][]; . == $name) and + if .type == "runson" then + (keys | sort) == ["name", "pool", "type"] + elif .type == "gha" then + (keys | sort) == ["name", "runner", "type"] + else + false + end + ) + ' "$MANIFEST" >/dev/null || { + echo "ERROR: invalid target matrix schema: $MANIFEST" >&2 + return 1 + } + + local matrix_targets + matrix_targets="$(jq -r '.groups[][]' "$MANIFEST" | LC_ALL=C sort)" + + local projection + for projection in "$REPO_ROOT/.config/rail.toml" "$REPO_ROOT/deny.toml"; do + local projected_targets + projected_targets="$(awk ' + /^targets = \[$/ { in_targets = 1; next } + in_targets && /^\]$/ { exit } + in_targets && match($0, /"[^"]+"/) { + print substr($0, RSTART + 1, RLENGTH - 2) + } + ' "$projection" | LC_ALL=C sort)" + if [[ "$projected_targets" != "$matrix_targets" ]]; then + echo "ERROR: target projection does not match .config/target-matrix.json: $projection" >&2 + diff -u <(printf '%s\n' "$matrix_targets") <(printf '%s\n' "$projected_targets") >&2 || true + return 1 + fi + done +} + +validate_manifest + +if [[ "$VALIDATE" == true ]]; then + exit 0 +fi + [[ -n "$FORMAT" ]] || usage print_shell() { diff --git a/scripts/lib/toolchain.sh b/scripts/lib/toolchain.sh index 9bc20554..48b7e881 100755 --- a/scripts/lib/toolchain.sh +++ b/scripts/lib/toolchain.sh @@ -2,24 +2,46 @@ # Extract toolchain channel from rust-toolchain.toml. # # Usage: -# scripts/lib/toolchain.sh # prints channel -# RUST_TOOLCHAIN=$(scripts/lib/toolchain.sh) +# scripts/lib/toolchain.sh # prints the development channel +# scripts/lib/toolchain.sh --msrv # prints the package MSRV +# scripts/lib/toolchain.sh --nightly # prints the pinned nightly channel set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" TOOLCHAIN_FILE="$REPO_ROOT/rust-toolchain.toml" +CARGO_MANIFEST="$REPO_ROOT/Cargo.toml" +TOOLCHAIN_CONTRACTS="$REPO_ROOT/.config/toolchains.toml" -if [ ! -f "$TOOLCHAIN_FILE" ]; then - echo "ERROR: rust-toolchain.toml not found at $TOOLCHAIN_FILE" >&2 +case "${1:-}" in + "") + SOURCE_FILE="$TOOLCHAIN_FILE" + FIELD_PATTERN='^channel[[:space:]]*=' + ;; + --msrv) + SOURCE_FILE="$CARGO_MANIFEST" + FIELD_PATTERN='^rust-version[[:space:]]*=' + ;; + --nightly) + SOURCE_FILE="$TOOLCHAIN_CONTRACTS" + FIELD_PATTERN='^nightly[[:space:]]*=' + ;; + *) + echo "Usage: $0 [--msrv|--nightly]" >&2 + exit 2 + ;; +esac + +if [[ ! -f "$SOURCE_FILE" ]]; then + echo "ERROR: toolchain authority not found: $SOURCE_FILE" >&2 exit 1 fi -RUST_TOOLCHAIN=$(awk -F'"' '/^channel/ {print $2}' "$TOOLCHAIN_FILE") +RUST_TOOLCHAIN=$(awk -F'"' -v pattern="$FIELD_PATTERN" '$0 ~ pattern { print $2; exit }' "$SOURCE_FILE") -if [ -z "$RUST_TOOLCHAIN" ]; then - echo "ERROR: Could not extract toolchain from $TOOLCHAIN_FILE" >&2 +if [[ -z "$RUST_TOOLCHAIN" ]]; then + echo "ERROR: could not extract toolchain from $SOURCE_FILE" >&2 exit 1 fi diff --git a/scripts/render_perf_chart.rs b/scripts/render_perf_chart.rs index a60efb58..56ccca6a 100644 --- a/scripts/render_perf_chart.rs +++ b/scripts/render_perf_chart.rs @@ -26,6 +26,7 @@ const MUTED: &str = "#7d8590"; const RULE: &str = "#30363d"; const BLUE: &str = "#0090FF"; const TRACK: &str = "#21262d"; +const LOSS: &str = "#FF7B72"; const CHART_TITLE: &str = "rscrypto"; const CHART_SUBTITLE: &str = "Geomean speedups vs fastest matched competitors. Higher is better."; @@ -34,9 +35,10 @@ const SUMMARY_APPLE_LABEL: &str = "Apple Silicon"; const CHECKSUM_TITLE: &str = "Checksums"; const CHECKSUM_COMPETITORS: &str = "- Competitor Crates/Libs: crc-fast, crc, crc32fast, crc32c, crc64fast"; const GROUP_TITLE: &str = "Primitive Geomeans"; +const GROUP_PARITY_NOTE: &str = "1.00x"; const FOOTER_LINUX_RUNNERS: &str = concat!( "- Linux Runners: AMD Zen 4/5; Intel Sapphire Rapids/Ice Lake; ", - "AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x); Rise RISC-V", + "AWS Graviton 3/4; IBM POWER 10 and IBM Z16 (s390x)", ); const FOOTER_MACOS: &str = "- macOS: MBP M1 10-Core, 16GB RAM - Local Dev Box"; const FOOTER_FASTEST_EXTERNAL: &str = @@ -561,22 +563,51 @@ fn render_group_bars(svg: &mut String, data: &ChartData) { let row_y = 398.0; let row_gap = 21.0; let bar_h = 11.0; - let max = 1.60_f64; + let (lo, hi) = group_axis(data); + let span = hi - lo; + let pos = |value: f64| bar_x + ((value.clamp(lo, hi) - lo) / span) * bar_w; + let parity_x = pos(1.0); for (idx, (label, value)) in data.rows.iter().enumerate() { let y = row_y + (idx as f64) * row_gap; - let clamped = value.clamp(1.0, max); - let width = ((clamped - 1.0) / (max - 1.0)) * bar_w; + let value_x_pos = pos(*value); text(svg, label_x, y + 10.0, 13, 700, TEXT, label); svg.push_str(&format!( "" )); + // Bars grow from the 1.00x parity marker: wins extend right, losses extend left. + let (fill_x, fill_w, fill) = if *value >= 1.0 { + (parity_x, value_x_pos - parity_x, BLUE) + } else { + (value_x_pos, parity_x - value_x_pos, LOSS) + }; svg.push_str(&format!( - "" + "" )); mono(svg, value_x, y + 10.0, 15, 850, TITLE, &format!("{value:.2}x")); } + + let rule_top = row_y - 4.0; + let rule_bottom = row_y + (data.rows.len() as f64 - 1.0) * row_gap + bar_h + 4.0; + svg.push_str(&format!( + "" + )); + mono(svg, parity_x - 15.0, rule_bottom + 12.0, 9, 500, MUTED, GROUP_PARITY_NOTE); +} + +/// Axis bounds for the primitive bars: always straddle parity, never clip a bar. +fn group_axis(data: &ChartData) -> (f64, f64) { + let mut lo = 1.0_f64; + let mut hi = 1.0_f64; + for (_, value) in &data.rows { + lo = lo.min(*value); + hi = hi.max(*value); + } + let lo = ((lo - 0.05) * 10.0).floor() / 10.0; + let hi = ((hi + 0.05) * 10.0).ceil() / 10.0; + (lo.min(0.9), hi.max(1.6)) } fn render_evidence_bullets(svg: &mut String) { diff --git a/scripts/test/test-coverage.sh b/scripts/test/test-coverage.sh index deb772ab..b00d513e 100755 --- a/scripts/test/test-coverage.sh +++ b/scripts/test/test-coverage.sh @@ -79,7 +79,7 @@ cargo llvm-cov clean --workspace if [ "$RUN_NEXTEST" = true ]; then echo "━━━ Test Suite Coverage Capture (cargo-nextest) ━━━" - cargo llvm-cov nextest --no-report --workspace --all-features "${NEXTEST_ARGS[@]}" + cargo llvm-cov nextest --locked --no-report --workspace --all-features "${NEXTEST_ARGS[@]}" fi if [ "$RUN_FUZZ" = true ]; then @@ -87,14 +87,14 @@ if [ "$RUN_FUZZ" = true ]; then export RSCRYPTO_FUZZ_REPLAY_MISSING=skip echo "Full fuzz workspace replay" - cargo llvm-cov test --no-report \ + cargo llvm-cov test --locked --no-report \ --manifest-path "$REPO_ROOT/fuzz/Cargo.toml" \ --all-features --test corpus_replay -- --nocapture echo "Scoped fuzz package replay" while IFS= read -r manifest; do echo " -> ${manifest#"$REPO_ROOT"/}" - cargo llvm-cov test --no-report \ + cargo llvm-cov test --locked --no-report \ --manifest-path "$manifest" \ --all-features --test corpus_replay -- --nocapture done < <(find "$REPO_ROOT/fuzz-packages" -mindepth 2 -maxdepth 2 -name Cargo.toml | sort) diff --git a/scripts/test/test-fuzz-asan.sh b/scripts/test/test-fuzz-asan.sh index 06aac483..5f786ef6 100755 --- a/scripts/test/test-fuzz-asan.sh +++ b/scripts/test/test-fuzz-asan.sh @@ -11,6 +11,7 @@ source "$SCRIPT_DIR/../lib/common.sh" source "$SCRIPT_DIR/../lib/fuzz-packages.sh" maybe_disable_sccache +activate_nightly_toolchain PACKAGE_SCOPE="full" @@ -85,6 +86,7 @@ for package_dir in "${SELECTED_FUZZ_PACKAGES[@]:+${SELECTED_FUZZ_PACKAGES[@]}}"; echo "ASan corpus replay package: $(fuzz_package_label "$package_dir")" if ! cargo test \ + --locked \ -Zbuild-std \ --target "$(fuzz_host_target)" \ --manifest-path "$package_dir/Cargo.toml" \ diff --git a/scripts/test/test-fuzz-scheduler-test.sh b/scripts/test/test-fuzz-scheduler-test.sh index 2d1bce45..58a4f964 100755 --- a/scripts/test/test-fuzz-scheduler-test.sh +++ b/scripts/test/test-fuzz-scheduler-test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +unset BASH_ENV SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" @@ -14,10 +15,12 @@ fail() { FIXTURE="$TMP_ROOT/repo" CAPTURE="$TMP_ROOT/capture" BIN="$TMP_ROOT/bin" -mkdir -p "$FIXTURE/scripts/test" "$FIXTURE/scripts/lib" "$FIXTURE/fuzz/corpus/alpha" "$CAPTURE" "$BIN" +mkdir -p "$FIXTURE/.config" "$FIXTURE/scripts/test" "$FIXTURE/scripts/lib" "$FIXTURE/fuzz/corpus/alpha" "$CAPTURE" "$BIN" cp "$REPO_ROOT/scripts/test/test-fuzz.sh" "$FIXTURE/scripts/test/" cp "$REPO_ROOT/scripts/lib/common.sh" "$REPO_ROOT/scripts/lib/rail-plan.sh" \ - "$REPO_ROOT/scripts/lib/fuzz-packages.sh" "$FIXTURE/scripts/lib/" + "$REPO_ROOT/scripts/lib/fuzz-packages.sh" "$REPO_ROOT/scripts/lib/toolchain.sh" \ + "$FIXTURE/scripts/lib/" +cp "$REPO_ROOT/.config/toolchains.toml" "$FIXTURE/.config/toolchains.toml" cat >"$FIXTURE/fuzz/Cargo.toml" <<'EOF' [package] diff --git a/scripts/test/test-fuzz.sh b/scripts/test/test-fuzz.sh index 32f824c7..fab0dded 100755 --- a/scripts/test/test-fuzz.sh +++ b/scripts/test/test-fuzz.sh @@ -29,6 +29,7 @@ source "$SCRIPT_DIR/../lib/common.sh" source "$SCRIPT_DIR/../lib/fuzz-packages.sh" maybe_disable_sccache +activate_nightly_toolchain # Configuration (can be overridden via environment) DURATION_SECS=${RSCRYPTO_FUZZ_DURATION_SECS:-60} diff --git a/scripts/test/test-miri.sh b/scripts/test/test-miri.sh index e17ddd00..a2619b37 100755 --- a/scripts/test/test-miri.sh +++ b/scripts/test/test-miri.sh @@ -34,6 +34,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ echo "" maybe_disable_sccache +activate_nightly_toolchain unset RUSTC_WRAPPER unset CARGO_BUILD_RUSTC_WRAPPER @@ -87,7 +88,7 @@ run_miri_lib_filter() { echo "" echo "━━━ $label ━━━" - cargo miri test --lib --features "$MIRI_FEATURES" "$filter" + cargo miri test --locked --lib --features "$MIRI_FEATURES" "$filter" } run_miri_lib_filter_features() { @@ -100,14 +101,14 @@ run_miri_lib_filter_features() { echo "" echo "━━━ $label ━━━" - listing=$(cargo miri test --lib --features "$features" "$filter" -- --list) + listing=$(cargo miri test --locked --lib --features "$features" "$filter" -- --list) printf '%s\n' "$listing" count=$(printf '%s\n' "$listing" | awk '/: test$/ { count++ } END { print count + 0 }') if [ "$count" -ne "$expected_count" ]; then echo "Expected $expected_count Miri tests matching '$filter'; found $count" >&2 return 1 fi - cargo miri test --lib --features "$features" "$filter" + cargo miri test --locked --lib --features "$features" "$filter" } run_miri_test_target() { @@ -117,7 +118,7 @@ run_miri_test_target() { echo "" echo "━━━ $label ━━━" - cargo miri test --test "$target" --features "$features" + cargo miri test --locked --test "$target" --features "$features" } # Run Miri Tests @@ -152,7 +153,7 @@ case "$MIRI_SCOPE" in ;; exhaustive) echo "Scope: exhaustive lib tests under Miri" - cargo miri test --lib --features "$MIRI_FEATURES" + cargo miri test --locked --lib --features "$MIRI_FEATURES" ;; *) echo "Invalid RSCRYPTO_MIRI_SCOPE: $MIRI_SCOPE" diff --git a/scripts/test/test-rsa-leakage.sh b/scripts/test/test-rsa-leakage.sh index 87207a34..27294421 100755 --- a/scripts/test/test-rsa-leakage.sh +++ b/scripts/test/test-rsa-leakage.sh @@ -21,7 +21,7 @@ echo "Welch t threshold: $RSCRYPTO_RSA_LEAKAGE_T_THRESHOLD" echo "Warmup iterations: $RSCRYPTO_RSA_LEAKAGE_WARMUP" echo "" -cargo test --release --test rsa_leakage --features rsa,diag,getrandom \ +cargo test --locked --release --test rsa_leakage --features rsa,diag,getrandom \ rsa_private_operations_do_not_show_first_order_timing_leakage \ -- --ignored --nocapture diff --git a/scripts/test/test.sh b/scripts/test/test.sh index 1f52f7b6..593ad75f 100755 --- a/scripts/test/test.sh +++ b/scripts/test/test.sh @@ -57,7 +57,7 @@ run_workspace_doctests() { echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Running doctests for entire workspace" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - cargo test --workspace --doc --all-features + cargo test --locked --workspace --doc --all-features } run_crate_doctests() { @@ -77,7 +77,7 @@ run_crate_doctests() { echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" for crate in "${crates[@]}"; do - cargo test -p "$crate" --doc --all-features + cargo test --locked -p "$crate" --doc --all-features done } @@ -135,21 +135,21 @@ if [ ${#CRATES[@]} -gt 0 ]; then echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [ "$HAS_NEXTEST" = true ]; then # shellcheck disable=SC2086 - cargo nextest run $CRATE_FLAGS -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked $CRATE_FLAGS -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_crate_doctests "${CRATES[@]}" else # shellcheck disable=SC2086 - cargo test $CRATE_FLAGS --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked $CRATE_FLAGS --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" fi elif [ "$ALL_FLAG" = true ]; then echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Testing entire workspace" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [ "$HAS_NEXTEST" = true ]; then - cargo nextest run --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_workspace_doctests else - cargo test --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" fi else # Rail-scoped (default): cargo-rail planner selects the affected crates. @@ -173,11 +173,11 @@ else for crate in $affected; do CRATE_FLAGS+=(-p "$crate") done - cargo nextest run "${CRATE_FLAGS[@]}" -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked "${CRATE_FLAGS[@]}" -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_rail_scoped_doctests else for crate in $affected; do - cargo test -p "$crate" --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked -p "$crate" --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" done fi exit 0 @@ -186,9 +186,9 @@ else esac if [ "$HAS_NEXTEST" = true ]; then - cargo nextest run --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" + cargo nextest run --locked --workspace -P "$PROFILE" --all-features --config-file .config/nextest.toml "${NEXTEST_THREAD_ARGS[@]:+${NEXTEST_THREAD_ARGS[@]}}" run_workspace_doctests else - cargo test --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" + cargo test --locked --workspace --all-features "${CARGO_TEST_TARGET_ARGS[@]:+${CARGO_TEST_TARGET_ARGS[@]}}" fi fi diff --git a/src/aead/aegis256.rs b/src/aead/aegis256.rs index 849441d5..efc6c093 100644 --- a/src/aead/aegis256.rs +++ b/src/aead/aegis256.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! AEGIS-256 authenticated encryption (draft-irtf-cfrg-aegis-aead). //! //! AES-round-based AEAD with a 256-bit key, 256-bit nonce, and 128-bit @@ -16,6 +14,7 @@ use core::fmt; ))] use super::targets::AeadBackend; #[cfg(any( + target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "powerpc64", target_endian = "little"), target_arch = "riscv64", @@ -217,31 +216,21 @@ fn finalize(s: &mut State, ad_len: usize, msg_len: usize) -> [u8; TAG_SIZE] { // riscv64 scalar AES backend (Zkne) #[cfg(target_arch = "aarch64")] -#[allow(unsafe_op_in_unsafe_fn)] #[path = "aegis256/aarch64_ce.rs"] mod ce; #[cfg(target_arch = "x86_64")] -#[allow(unsafe_op_in_unsafe_fn)] #[path = "aegis256/x86_64_ni.rs"] mod ni; #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] -#[allow(unsafe_code, unsafe_op_in_unsafe_fn)] #[path = "aegis256/powerpc64_ppc.rs"] mod ppc; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] -#[path = "aegis256/riscv64_vperm.rs"] -mod rv_vperm; -#[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aegis256/riscv64_zkne.rs"] mod rv_zkne; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aegis256/riscv64_zvkned.rs"] mod rv_zvkned; #[cfg(target_arch = "s390x")] -#[allow(unsafe_code)] #[path = "aegis256/s390x_vperm.rs"] mod s390x_vperm; #[cfg(any( @@ -254,20 +243,7 @@ mod s390x_vperm; #[inline] fn resolve_backend() -> AeadBackend { let caps = crate::platform::caps(); - - #[cfg(target_arch = "x86_64")] - { - use crate::platform::caps::x86; - if caps.has(x86::AESNI) && caps.has(x86::AVX) { - return AeadBackend::X86Aesni; - } - AeadBackend::Portable - } - - #[cfg(not(target_arch = "x86_64"))] - { - select_backend(AeadPrimitive::Aegis256, crate::platform::arch(), caps) - } + select_backend(AeadPrimitive::Aegis256, crate::platform::arch(), caps) } // Key @@ -435,6 +411,7 @@ fn encrypt_portable(key: &[u8; KEY_SIZE], nonce: &[u8; NONCE_SIZE], aad: &[u8], } #[cfg(feature = "diag")] +/// Run one portable AEGIS-256 state update and return its diagnostic final tag. #[unsafe(no_mangle)] #[inline(never)] #[must_use] @@ -570,13 +547,6 @@ impl Aead for Aegis256 { return Ok(Aegis256Tag::from_bytes(tag)); } - #[cfg(target_arch = "riscv64")] - if self.backend == AeadBackend::Riscv64Vperm { - // SAFETY: backend resolution confirmed the RISC-V V extension is available. - let tag = unsafe { rv_vperm::encrypt_fused(key, nonce, aad, buffer) }; - return Ok(Aegis256Tag::from_bytes(tag)); - } - #[cfg(target_arch = "s390x")] if self.backend == AeadBackend::S390xVperm { // SAFETY: s390x vperm backend call because: @@ -584,7 +554,6 @@ impl Aead for Aegis256 { // 2. `select_backend` returns `S390xVperm` only when caps include `s390x::VECTOR`. // 3. `key`, `nonce`, `aad`, and `buffer` are valid references from the safe AEAD API. let tag = unsafe { s390x_vperm::encrypt_fused(key, nonce, aad, buffer) }; - #[allow(clippy::needless_return)] return Ok(Aegis256Tag::from_bytes(tag)); } @@ -635,9 +604,6 @@ impl Aead for Aegis256 { } else if self.backend == AeadBackend::Riscv64ScalarCrypto { // SAFETY: backend resolution confirmed scalar AES (`zkne`) is available. unsafe { rv_zkne::decrypt_fused(key, nonce, aad, buffer) } - } else if self.backend == AeadBackend::Riscv64Vperm { - // SAFETY: backend resolution confirmed the RISC-V V extension is available. - unsafe { rv_vperm::decrypt_fused(key, nonce, aad, buffer) } } else { decrypt_portable(key, nonce, aad, buffer) }; @@ -676,10 +642,12 @@ impl Aead for Aegis256 { #[cfg(test)] mod tests { use alloc::{vec, vec::Vec}; - use std::eprintln; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_array, hex_vec as hex}, + }; #[cfg(not(target_arch = "s390x"))] #[inline(always)] @@ -687,18 +655,8 @@ mod tests { super::super::aes_round::aes_enc_round_portable(block, round_key) } - fn hex(s: &str) -> Vec { - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i.strict_add(2)], 16).unwrap()) - .collect() - } - fn hex_block(s: &str) -> [u8; 16] { - let v = hex(s); - let mut out = [0u8; 16]; - out.copy_from_slice(&v); - out + hex_array(s) } // -- AESRound test vector (Appendix A.1) -- @@ -767,15 +725,10 @@ mod tests { // -- Spec test vectors (Appendix A.3) -- fn spec_key() -> Aegis256Key { - Aegis256Key::from_bytes( - hex_block("10010000000000000000000000000000") - .iter() - .chain(hex_block("00000000000000000000000000000000").iter()) - .copied() - .collect::>() - .try_into() - .unwrap(), - ) + Aegis256Key::from_bytes(hex_array(concat!( + "10010000000000000000000000000000", + "00000000000000000000000000000000" + ))) } fn spec_nonce() -> Nonce256 { @@ -793,12 +746,16 @@ mod tests { // Encrypt. let mut buf = msg.to_vec(); - let tag = aead.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("AEGIS-256 specification-vector encryption must succeed"); assert_eq!(&buf, &expected_ct, "ciphertext mismatch"); assert_eq!(tag.as_bytes(), expected_tag.as_slice(), "tag mismatch"); // Decrypt round-trip. - aead.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("AEGIS-256 specification-vector decryption must succeed"); assert_eq!(&buf, msg, "plaintext recovery mismatch"); } @@ -861,8 +818,12 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = []; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("empty AEGIS-256 encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("empty AEGIS-256 decryption must succeed"); } #[test] @@ -873,10 +834,14 @@ mod tests { let plaintext = b"the quick brown fox jumps over the lazy dog"; let mut buf = *plaintext; - let tag = aead.encrypt_in_place(&nonce, b"header", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"header", &mut buf) + .expect("AEGIS-256 encryption with AAD must succeed"); assert_ne!(&buf[..], &plaintext[..]); - aead.decrypt_in_place(&nonce, b"header", &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, b"header", &mut buf, &tag) + .expect("AEGIS-256 decryption with AAD must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -889,10 +854,10 @@ mod tests { let mut buf = []; let tag = aead .encrypt_in_place(&nonce, b"associated data only", &mut buf) - .unwrap(); + .expect("AAD-only AEGIS-256 encryption must succeed"); aead .decrypt_in_place(&nonce, b"associated data only", &mut buf, &tag) - .unwrap(); + .expect("AAD-only AEGIS-256 decryption must succeed"); } #[test] @@ -902,14 +867,18 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"zero me on failure"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = Aegis256Tag::from_bytes(bad_tag); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -920,11 +889,15 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"secret"; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); buf[0] ^= 1; - let result = aead.decrypt_in_place(&nonce, b"", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"", &mut buf, &tag), + Err(OpenError::verification()) + ); assert_eq!(&buf, &[0u8; 6]); } @@ -935,14 +908,18 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"data"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); let mut bad_tag_bytes = tag.to_bytes(); bad_tag_bytes[15] ^= 1; let bad_tag = Aegis256Tag::from_bytes(bad_tag_bytes); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert_eq!(&buf, &[0u8; 4]); } @@ -953,10 +930,14 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"msg"; - let tag = aead.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); - let result = aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -966,11 +947,15 @@ mod tests { let aead = Aegis256::new(&key); let mut buf = *b"nonce test"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AEGIS-256 test setup encryption must succeed"); let wrong_nonce = Nonce256::from_bytes([11; 32]); - let result = aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -981,18 +966,29 @@ mod tests { let pt = b"combined mode"; let mut sealed = vec![0u8; pt.len().strict_add(TAG_SIZE)]; - aead.encrypt(&nonce, b"h", pt.as_slice(), &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"h", pt.as_slice(), &mut sealed) + .expect("combined AEGIS-256 encryption must succeed"); let mut opened = vec![0u8; pt.len()]; - aead.decrypt(&nonce, b"h", &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, b"h", &sealed, &mut opened) + .expect("combined AEGIS-256 decryption must succeed"); assert_eq!(&opened, &pt[..]); } #[test] fn tag_from_slice_rejects_wrong_length() { - assert!(Aegis256::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aegis256::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aegis256::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aegis256::tag_from_slice(&[0u8; 15]).expect_err("short AEGIS-256 tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aegis256::tag_from_slice(&[0u8; 17]).expect_err("long AEGIS-256 tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aegis256::tag_from_slice(&[0u8; 16]).expect("16-byte AEGIS-256 tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] @@ -1006,7 +1002,7 @@ mod tests { let mut buf = plaintext; let tag = aead .encrypt_in_place(&nonce, b"multi-block aad that is longer than one rate block", &mut buf) - .unwrap(); + .expect("multi-block AEGIS-256 encryption must succeed"); aead .decrypt_in_place( &nonce, @@ -1014,7 +1010,7 @@ mod tests { &mut buf, &tag, ) - .unwrap(); + .expect("multi-block AEGIS-256 decryption must succeed"); assert_eq!(buf, plaintext); } @@ -1027,15 +1023,23 @@ mod tests { // Exactly 16 bytes = 1 full block, 0-byte tail. let plaintext = [0x55u8; 16]; let mut buf = plaintext; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("one-block AEGIS-256 encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("one-block AEGIS-256 decryption must succeed"); assert_eq!(buf, plaintext); // Exactly 32 bytes = 2 full blocks, 0-byte tail. let plaintext32 = [0x66u8; 32]; let mut buf32 = plaintext32; - let tag32 = aead.encrypt_in_place(&nonce, b"", &mut buf32).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf32, &tag32).unwrap(); + let tag32 = aead + .encrypt_in_place(&nonce, b"", &mut buf32) + .expect("two-block AEGIS-256 encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf32, &tag32) + .expect("two-block AEGIS-256 decryption must succeed"); assert_eq!(buf32, plaintext32); } @@ -1047,12 +1051,16 @@ mod tests { let aead = Aegis256::new(&key); let aad = b"four-block-test"; - for &size in &[48, 64, 80, 96, 112, 128, 256, 1024, 4096] { - let plaintext: Vec = (0..size).map(|i| (i & 0xFF) as u8).collect(); + for &size in &[48usize, 64, 80, 96, 112, 128, 256, 1024, 4096] { + let plaintext: Vec = (0u8..=u8::MAX).cycle().take(size).collect(); let mut buf = plaintext.clone(); - let tag = aead.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("boundary AEGIS-256 encryption must succeed"); assert_ne!(&buf, &plaintext, "size {size}: ciphertext must differ"); - aead.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("boundary AEGIS-256 decryption must succeed"); assert_eq!(&buf, &plaintext, "size {size}: round-trip failed"); } } @@ -1066,7 +1074,7 @@ mod tests { /// Scalar simulation of vperm: table[index & 0x0F]. fn vperm_scalar(table: &[u8; 16], index: u8) -> u8 { - table[(index & 0x0F) as usize] + table[usize::from(index & 0x0F)] } /// Scalar simulation of vperm with PSHUFB zeroing: returns 0 when bit 7 set. @@ -1074,7 +1082,7 @@ mod tests { if index & 0x80 != 0 { 0 } else { - table[(index & 0x0F) as usize] + table[usize::from(index & 0x0F)] } } @@ -1147,24 +1155,14 @@ mod tests { 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16, ]; - let mut failures = 0u32; - for input in 0u16..256 { - let got = vperm_sbox_scalar(input as u8); + for input in u8::MIN..=u8::MAX { + let got = vperm_sbox_scalar(input); // The Hamburg vperm S-box omits the AES affine constant. // vpaes_sbox(x) = AES_sbox(x) ^ AES_AFFINE for all x. use super::super::aes_round::AES_AFFINE; - let expected = AES_SBOX[input as usize] ^ AES_AFFINE; - if got != expected { - if failures < 16 { - eprintln!( - "vperm S-box mismatch at input 0x{:02X}: got 0x{:02X}, expected 0x{:02X}", - input, got, expected, - ); - } - failures = failures.strict_add(1); - } + let expected = AES_SBOX[usize::from(input)] ^ AES_AFFINE; + assert_eq!(got, expected, "vperm S-box mismatch at input 0x{input:02X}"); } - assert_eq!(failures, 0, "{failures} vperm S-box mismatches out of 256"); } // -- Full vperm AES round validation -- @@ -1180,35 +1178,37 @@ mod tests { // SubBytes via vperm tower field (includes affine constant compensation) use super::super::aes_round::{AES_AFFINE, VPERM_SR as SR}; let mut sb = [0u8; 16]; - for i in 0..16 { - sb[i] = vperm_sbox_scalar(block[i]) ^ AES_AFFINE; + for (substituted, &input) in sb.iter_mut().zip(block) { + *substituted = vperm_sbox_scalar(input) ^ AES_AFFINE; } // ShiftRows let mut sr = [0u8; 16]; - for i in 0..16 { - sr[i] = sb[SR[i] as usize]; + for (shifted, &source) in sr.iter_mut().zip(&SR) { + *shifted = sb[usize::from(source)]; } // MixColumns via xtime decomposition fn xtime(b: u8) -> u8 { - let r = (b as u16) << 1; - (r ^ (if r & 0x100 != 0 { 0x1B } else { 0 })) as u8 + let doubled = u16::from(b).strict_mul(2); + let reduced = if doubled & 0x100 == 0 { doubled } else { doubled ^ 0x11B }; + u8::try_from(reduced).expect("AES polynomial reduction must produce one byte") } let mut mc = [0u8; 16]; - for col in 0..4 { - let c = col * 4; - let (b0, b1, b2, b3) = (sr[c], sr[c + 1], sr[c + 2], sr[c + 3]); - mc[c] = xtime(b0) ^ xtime(b1) ^ b1 ^ b2 ^ b3; - mc[c + 1] = b0 ^ xtime(b1) ^ xtime(b2) ^ b2 ^ b3; - mc[c + 2] = b0 ^ b1 ^ xtime(b2) ^ xtime(b3) ^ b3; - mc[c + 3] = xtime(b0) ^ b0 ^ b1 ^ b2 ^ xtime(b3); + for (source, mixed) in sr.as_chunks::<4>().0.iter().zip(mc.as_chunks_mut::<4>().0) { + let [b0, b1, b2, b3] = *source; + *mixed = [ + xtime(b0) ^ xtime(b1) ^ b1 ^ b2 ^ b3, + b0 ^ xtime(b1) ^ xtime(b2) ^ b2 ^ b3, + b0 ^ b1 ^ xtime(b2) ^ xtime(b3) ^ b3, + xtime(b0) ^ b0 ^ b1 ^ b2 ^ xtime(b3), + ]; } // AddRoundKey let mut result = [0u8; 16]; - for i in 0..16 { - result[i] = mc[i] ^ round_key[i]; + for ((output, &mixed), &key_byte) in result.iter_mut().zip(&mc).zip(round_key) { + *output = mixed ^ key_byte; } result } @@ -1237,9 +1237,9 @@ mod tests { assert_eq!(vperm_aes_round_scalar(&a, &b), portable_aes_round(&a, &b)); // Exhaustive: test all single-byte patterns in position 0 - for val in 0u16..256 { + for val in u8::MIN..=u8::MAX { let mut block = [0u8; 16]; - block[0] = val as u8; + block[0] = val; let key = [0u8; 16]; let got = vperm_aes_round_scalar(&block, &key); let expected = portable_aes_round(&block, &key); diff --git a/src/aead/aegis256/aarch64_ce.rs b/src/aead/aegis256/aarch64_ce.rs index d8262b58..2809fd1d 100644 --- a/src/aead/aegis256/aarch64_ce.rs +++ b/src/aead/aegis256/aarch64_ce.rs @@ -4,18 +4,26 @@ use super::{BLOCK_SIZE, C0, C1, KEY_SIZE, NONCE_SIZE, TAG_SIZE}; #[inline] /// # Safety -/// -/// `bytes` must refer to a valid 16-byte block. +/// Caller must ensure `neon` support is available. unsafe fn load(bytes: &[u8; BLOCK_SIZE]) -> uint8x16_t { - vld1q_u8(bytes.as_ptr()) + // SAFETY: the caller guarantees NEON; `bytes` provides 16 initialized bytes for this unaligned load. + unsafe { vld1q_u8(bytes.as_ptr()) } } #[inline] /// # Safety -/// -/// `out` must refer to a valid writable 16-byte block. +/// Caller must ensure `neon` support is available. unsafe fn store(v: uint8x16_t, out: &mut [u8; BLOCK_SIZE]) { - vst1q_u8(out.as_mut_ptr(), v); + // SAFETY: the caller guarantees NEON; `out` provides exclusive access to 16 bytes for this unaligned store. + unsafe { vst1q_u8(out.as_mut_ptr(), v) }; +} + +#[inline(always)] +fn prefetch_read_l1(ptr: *const u8) { + // SAFETY: `prfm` is an AArch64 cache hint that does not dereference `ptr` or alter architectural state. + unsafe { + core::arch::asm!("prfm pldl1keep, [{ptr}]", ptr = in(reg) ptr, options(nostack, preserves_flags)); + } } // ── Register-based helpers ────────────────────────────────────────────── @@ -27,7 +35,6 @@ unsafe fn store(v: uint8x16_t, out: &mut [u8; BLOCK_SIZE]) { #[target_feature(enable = "aes,neon")] #[inline] -#[allow(clippy::too_many_arguments)] /// # Safety /// /// Caller must ensure `aes` and `neon` support is available and all register @@ -40,8 +47,8 @@ unsafe fn update_regs( s4: &mut uint8x16_t, s5: &mut uint8x16_t, m: uint8x16_t, - zero: uint8x16_t, ) { + let zero = vdupq_n_u8(0); let tmp = *s5; #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos"))] @@ -68,10 +75,10 @@ unsafe fn update_regs( #[inline] /// # Safety /// -/// The provided registers must come from a valid AEGIS-256 state on an -/// `aes` + `neon` capable CPU. +/// Caller must ensure `neon` support is available. unsafe fn keystream_regs(s1: uint8x16_t, s2: uint8x16_t, s3: uint8x16_t, s4: uint8x16_t, s5: uint8x16_t) -> uint8x16_t { - veorq_u8(veorq_u8(s1, s4), veorq_u8(s5, vandq_u8(s2, s3))) + // SAFETY: the caller guarantees NEON support for these register-only operations. + unsafe { veorq_u8(veorq_u8(s1, s4), veorq_u8(s5, vandq_u8(s2, s3))) } } // ── Fused encrypt/decrypt ─────────────────────────────────────────────── @@ -88,41 +95,54 @@ pub(super) unsafe fn encrypt_fused( ) -> [u8; TAG_SIZE] { let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); - let k0 = load(kh0); - let k1 = load(kh1); - let n0 = load(nh0); - let n1 = load(nh1); - let c0 = load(&C0); - let c1 = load(&C1); + // SAFETY: the entrypoint enables NEON; `kh0` is a complete initialized block. + let k0 = unsafe { load(kh0) }; + // SAFETY: the entrypoint enables NEON; `kh1` is a complete initialized block. + let k1 = unsafe { load(kh1) }; + // SAFETY: the entrypoint enables NEON; `nh0` is a complete initialized block. + let n0 = unsafe { load(nh0) }; + // SAFETY: the entrypoint enables NEON; `nh1` is a complete initialized block. + let n1 = unsafe { load(nh1) }; + // SAFETY: the entrypoint enables NEON; `C0` is a complete initialized block. + let c0 = unsafe { load(&C0) }; + // SAFETY: the entrypoint enables NEON; `C1` is a complete initialized block. + let c1 = unsafe { load(&C1) }; let k0_xor_n0 = veorq_u8(k0, n0); let k1_xor_n1 = veorq_u8(k1, n1); - let zero = vdupq_n_u8(0); let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, veorq_u8(k0, c0), veorq_u8(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + } } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - vld1q_u8(aad.as_ptr().add(offset)), - zero, - ); + // SAFETY: the loop bound provides a complete readable block at `offset`; the entrypoint enables AES and NEON. + unsafe { + update_regs( + &mut s0, + &mut s1, + &mut s2, + &mut s3, + &mut s4, + &mut s5, + vld1q_u8(aad.as_ptr().add(offset)), + ); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad), zero); + // SAFETY: `pad` is a complete initialized block, and the entrypoint enables AES and NEON. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + } } let msg_len = buffer.len(); let ptr = buffer.as_mut_ptr(); @@ -131,55 +151,71 @@ pub(super) unsafe fn encrypt_fused( let four_blocks = BLOCK_SIZE.strict_mul(4); let two_blocks = BLOCK_SIZE.strict_mul(2); while offset.strict_add(four_blocks) <= len { - core::arch::asm!("prfm pldl1keep, [{ptr}]", ptr = in(reg) ptr.add(offset.strict_add(192)), options(nostack, preserves_flags)); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = vld1q_u8(ptr.add(offset)); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let xi_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c, zero); - vst1q_u8(ptr.add(offset.strict_add(two_blocks)), veorq_u8(xi_c, z_c)); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let xi_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d, zero); - vst1q_u8( - ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), - veorq_u8(xi_d, z_d), - ); + prefetch_read_l1(ptr.wrapping_add(offset.strict_add(192))); + // SAFETY: the loop bound places all four 16-byte lanes inside the exclusively borrowed buffer. The entrypoint + // enables AES and NEON, and each plaintext lane is loaded before its ciphertext overwrites the same bytes. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let xi_a = vld1q_u8(ptr.add(offset)); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); + let z_c = keystream_regs(s1, s2, s3, s4, s5); + let xi_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); + vst1q_u8(ptr.add(offset.strict_add(two_blocks)), veorq_u8(xi_c, z_c)); + let z_d = keystream_regs(s1, s2, s3, s4, s5); + let xi_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); + vst1q_u8( + ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), + veorq_u8(xi_d, z_d), + ); + } offset = offset.strict_add(four_blocks); } if offset.strict_add(two_blocks) <= len { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = vld1q_u8(ptr.add(offset)); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); + // SAFETY: the branch bound places both lanes inside the exclusively borrowed buffer; AES and NEON are enabled. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let xi_a = vld1q_u8(ptr.add(offset)); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), veorq_u8(xi_a, z_a)); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let xi_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), veorq_u8(xi_b, z_b)); + } offset = offset.strict_add(two_blocks); } if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let xi = vld1q_u8(ptr.add(offset)); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, zero); - vst1q_u8(ptr.add(offset), veorq_u8(xi, z)); + // SAFETY: the branch bound provides one complete writable lane in the buffer; AES and NEON are enabled. + unsafe { + let z = keystream_regs(s1, s2, s3, s4, s5); + let xi = vld1q_u8(ptr.add(offset)); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); + vst1q_u8(ptr.add(offset), veorq_u8(xi, z)); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + // SAFETY: the entrypoint enables NEON for the valid state registers. + let z = unsafe { keystream_regs(s1, s2, s3, s4, s5) }; let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); - let xi = load(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, zero); + // SAFETY: `pad` is a complete initialized block, and the entrypoint enables NEON. + let xi = unsafe { load(&pad) }; + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); + } let mut ct_bytes = [0u8; BLOCK_SIZE]; - store(veorq_u8(xi, z), &mut ct_bytes); + // SAFETY: `ct_bytes` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(veorq_u8(xi, z), &mut ct_bytes) }; buffer[offset..].copy_from_slice(&ct_bytes[..tail_len]); } let ad_bits = (aad.len() as u64).strict_mul(8); @@ -187,13 +223,18 @@ pub(super) unsafe fn encrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); - let t = veorq_u8(s3, load(&len_bytes)); + // SAFETY: `len_bytes` is a complete initialized block, and the entrypoint enables NEON. + let t = unsafe { veorq_u8(s3, load(&len_bytes)) }; for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + } } let tag_vec = veorq_u8(veorq_u8(veorq_u8(s0, s1), veorq_u8(s2, s3)), veorq_u8(s4, s5)); let mut tag = [0u8; TAG_SIZE]; - store(tag_vec, &mut tag); + // SAFETY: `tag` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(tag_vec, &mut tag) }; tag } @@ -209,41 +250,54 @@ pub(super) unsafe fn decrypt_fused( ) -> [u8; TAG_SIZE] { let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); - let k0 = load(kh0); - let k1 = load(kh1); - let n0 = load(nh0); - let n1 = load(nh1); - let c0 = load(&C0); - let c1 = load(&C1); + // SAFETY: the entrypoint enables NEON; `kh0` is a complete initialized block. + let k0 = unsafe { load(kh0) }; + // SAFETY: the entrypoint enables NEON; `kh1` is a complete initialized block. + let k1 = unsafe { load(kh1) }; + // SAFETY: the entrypoint enables NEON; `nh0` is a complete initialized block. + let n0 = unsafe { load(nh0) }; + // SAFETY: the entrypoint enables NEON; `nh1` is a complete initialized block. + let n1 = unsafe { load(nh1) }; + // SAFETY: the entrypoint enables NEON; `C0` is a complete initialized block. + let c0 = unsafe { load(&C0) }; + // SAFETY: the entrypoint enables NEON; `C1` is a complete initialized block. + let c1 = unsafe { load(&C1) }; let k0_xor_n0 = veorq_u8(k0, n0); let k1_xor_n1 = veorq_u8(k1, n1); - let zero = vdupq_n_u8(0); let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, veorq_u8(k0, c0), veorq_u8(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, zero); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + } } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - vld1q_u8(aad.as_ptr().add(offset)), - zero, - ); + // SAFETY: the loop bound provides a complete readable block at `offset`; the entrypoint enables AES and NEON. + unsafe { + update_regs( + &mut s0, + &mut s1, + &mut s2, + &mut s3, + &mut s4, + &mut s5, + vld1q_u8(aad.as_ptr().add(offset)), + ); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad), zero); + // SAFETY: `pad` is a complete initialized block, and the entrypoint enables AES and NEON. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + } } let ct_len = buffer.len(); let ptr = buffer.as_mut_ptr(); @@ -252,71 +306,77 @@ pub(super) unsafe fn decrypt_fused( let four_blocks = BLOCK_SIZE.strict_mul(4); let two_blocks = BLOCK_SIZE.strict_mul(2); while offset.strict_add(four_blocks) <= len { - core::arch::asm!("prfm pldl1keep, [{ptr}]", ptr = in(reg) ptr.add(offset.strict_add(192)), options(nostack, preserves_flags)); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = vld1q_u8(ptr.add(offset)); - let xi_a = veorq_u8(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - let xi_b = veorq_u8(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let ci_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); - let xi_c = veorq_u8(ci_c, z_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c, zero); - vst1q_u8(ptr.add(offset.strict_add(two_blocks)), xi_c); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let ci_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); - let xi_d = veorq_u8(ci_d, z_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d, zero); - vst1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), xi_d); + prefetch_read_l1(ptr.wrapping_add(offset.strict_add(192))); + // SAFETY: the loop bound places all four 16-byte lanes inside the exclusively borrowed buffer. The entrypoint + // enables AES and NEON, and each ciphertext lane is loaded before its plaintext overwrites the same bytes. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let ci_a = vld1q_u8(ptr.add(offset)); + let xi_a = veorq_u8(ci_a, z_a); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), xi_a); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + let xi_b = veorq_u8(ci_b, z_b); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); + let z_c = keystream_regs(s1, s2, s3, s4, s5); + let ci_c = vld1q_u8(ptr.add(offset.strict_add(two_blocks))); + let xi_c = veorq_u8(ci_c, z_c); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); + vst1q_u8(ptr.add(offset.strict_add(two_blocks)), xi_c); + let z_d = keystream_regs(s1, s2, s3, s4, s5); + let ci_d = vld1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)))); + let xi_d = veorq_u8(ci_d, z_d); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); + vst1q_u8(ptr.add(offset.strict_add(two_blocks.strict_add(BLOCK_SIZE))), xi_d); + } offset = offset.strict_add(four_blocks); } if offset.strict_add(two_blocks) <= len { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = vld1q_u8(ptr.add(offset)); - let xi_a = veorq_u8(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a, zero); - vst1q_u8(ptr.add(offset), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); - let xi_b = veorq_u8(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b, zero); - vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); + // SAFETY: the branch bound places both lanes inside the exclusively borrowed buffer; AES and NEON are enabled. + unsafe { + let z_a = keystream_regs(s1, s2, s3, s4, s5); + let ci_a = vld1q_u8(ptr.add(offset)); + let xi_a = veorq_u8(ci_a, z_a); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); + vst1q_u8(ptr.add(offset), xi_a); + let z_b = keystream_regs(s1, s2, s3, s4, s5); + let ci_b = vld1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE))); + let xi_b = veorq_u8(ci_b, z_b); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); + vst1q_u8(ptr.add(offset.strict_add(BLOCK_SIZE)), xi_b); + } offset = offset.strict_add(two_blocks); } if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let ci = vld1q_u8(ptr.add(offset)); - let xi = veorq_u8(ci, z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, zero); - vst1q_u8(ptr.add(offset), xi); + // SAFETY: the branch bound provides one complete writable lane in the buffer; AES and NEON are enabled. + unsafe { + let z = keystream_regs(s1, s2, s3, s4, s5); + let ci = vld1q_u8(ptr.add(offset)); + let xi = veorq_u8(ci, z); + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); + vst1q_u8(ptr.add(offset), xi); + } offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + // SAFETY: the entrypoint enables NEON for the valid state registers. + let z = unsafe { keystream_regs(s1, s2, s3, s4, s5) }; let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); let mut z_bytes = [0u8; BLOCK_SIZE]; - store(z, &mut z_bytes); + // SAFETY: `z_bytes` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(z, &mut z_bytes) }; let mut pt_pad = [0u8; BLOCK_SIZE]; for i in 0..tail_len { pt_pad[i] = pad[i] ^ z_bytes[i]; } - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load(&pt_pad), - zero, - ); + // SAFETY: `pt_pad` is a complete initialized block, and the entrypoint enables AES and NEON. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pt_pad)); + } buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); } let ad_bits = (aad.len() as u64).strict_mul(8); @@ -324,12 +384,17 @@ pub(super) unsafe fn decrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); - let t = veorq_u8(s3, load(&len_bytes)); + // SAFETY: `len_bytes` is a complete initialized block, and the entrypoint enables NEON. + let t = unsafe { veorq_u8(s3, load(&len_bytes)) }; for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, zero); + // SAFETY: the entrypoint establishes AES and NEON for these valid state registers. + unsafe { + update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + } } let tag_vec = veorq_u8(veorq_u8(veorq_u8(s0, s1), veorq_u8(s2, s3)), veorq_u8(s4, s5)); let mut tag = [0u8; TAG_SIZE]; - store(tag_vec, &mut tag); + // SAFETY: `tag` is a complete writable block, and the entrypoint enables NEON. + unsafe { store(tag_vec, &mut tag) }; tag } diff --git a/src/aead/aegis256/powerpc64_ppc.rs b/src/aead/aegis256/powerpc64_ppc.rs index d6d2c1f0..ab3da5af 100644 --- a/src/aead/aegis256/powerpc64_ppc.rs +++ b/src/aead/aegis256/powerpc64_ppc.rs @@ -25,8 +25,8 @@ fn store_be(v: i64x2, out: &mut [u8; 16]) { let arr = v.to_array(); #[cfg(target_endian = "little")] { - let hi = (arr[1] as u64).to_be_bytes(); - let lo = (arr[0] as u64).to_be_bytes(); + let hi = u64::from_ne_bytes(arr[1].to_ne_bytes()).to_be_bytes(); + let lo = u64::from_ne_bytes(arr[0].to_ne_bytes()).to_be_bytes(); out[0..8].copy_from_slice(&hi); out[8..16].copy_from_slice(&lo); } @@ -43,18 +43,19 @@ fn store_be(v: i64x2, out: &mut [u8; 16]) { /// Caller must ensure POWER8 vector crypto support is available. unsafe fn aes_round(block: i64x2, round_key: i64x2) -> i64x2 { let out: i64x2; - asm!( - "vcipher {out}, {block}, {rk}", - out = lateout(vreg) out, - block = in(vreg) block, - rk = in(vreg) round_key, - options(nomem, nostack), - ); + // SAFETY: the caller guarantees POWER8 vector crypto support; the instruction only reads its register operands. + unsafe { + asm!( + "vcipher {out}, {block}, {rk}", + out = lateout(vreg) out, + block = in(vreg) block, + rk = in(vreg) round_key, + options(nomem, nostack), + ) + }; out } -// ── Register-based helpers ────────────────────────────────────────────── - #[inline(always)] fn xor_vec(a: i64x2, b: i64x2) -> i64x2 { let aa = a.to_array(); @@ -69,27 +70,44 @@ fn and_vec(a: i64x2, b: i64x2) -> i64x2 { i64x2::from_array([aa[0] & ba[0], aa[1] & ba[1]]) } -#[inline(always)] -/// # Safety -/// -/// Caller must ensure POWER8 vector crypto support is available and all state -/// registers belong to a valid AEGIS-256 state. -unsafe fn update_regs( - s0: &mut i64x2, - s1: &mut i64x2, - s2: &mut i64x2, - s3: &mut i64x2, - s4: &mut i64x2, - s5: &mut i64x2, - m: i64x2, -) { - let tmp = *s5; - *s5 = aes_round(*s4, *s5); - *s4 = aes_round(*s3, *s4); - *s3 = aes_round(*s2, *s3); - *s2 = aes_round(*s1, *s2); - *s1 = aes_round(*s0, *s1); - *s0 = xor_vec(aes_round(tmp, *s0), m); +type StateMut<'a> = ( + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, + &'a mut i64x2, +); + +#[derive(Clone, Copy)] +struct Power8 { + _private: (), +} + +impl Power8 { + /// Creates a POWER8 vector-crypto capability token. + /// + /// # Safety + /// + /// The current CPU must support POWER8 vector crypto for the token's entire lifetime. + #[inline] + unsafe fn new() -> Self { + Self { _private: () } + } + + #[inline(always)] + fn update(self, (s0, s1, s2, s3, s4, s5): StateMut<'_>, message: i64x2) { + let old_s5 = *s5; + // SAFETY: this module constructs `Power8` only through `new`, whose caller guarantees POWER8 vector crypto. + unsafe { + *s5 = aes_round(*s4, *s5); + *s4 = aes_round(*s3, *s4); + *s3 = aes_round(*s2, *s3); + *s2 = aes_round(*s1, *s2); + *s1 = aes_round(*s0, *s1); + *s0 = xor_vec(aes_round(old_s5, *s0), message); + } + } } #[inline(always)] @@ -97,8 +115,6 @@ fn keystream_regs(s1: i64x2, s2: i64x2, s3: i64x2, s4: i64x2, s5: i64x2) -> i64x xor_vec(xor_vec(s1, s4), xor_vec(s5, and_vec(s2, s3))) } -// ── Fused encrypt/decrypt ─────────────────────────────────────────────── - #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] /// # Safety /// @@ -109,6 +125,8 @@ pub(super) unsafe fn encrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { + // SAFETY: this function's caller guarantees POWER8 vector crypto support. + let power8 = unsafe { Power8::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load_be(kh0); @@ -122,118 +140,63 @@ pub(super) unsafe fn encrypt_fused( let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&tmp)); - offset = offset.strict_add(BLOCK_SIZE); + + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(&pad)); } + let msg_len = buffer.len(); - let len = buffer.len(); - offset = 0; - let four_blocks = BLOCK_SIZE.strict_mul(4); - let two_blocks = BLOCK_SIZE.strict_mul(2); - while offset.strict_add(four_blocks) <= len { - // SAFETY: pointer arithmetic for dcbt prefetch; offset + 256 may exceed - // the buffer but dcbt is a hint and never faults on POWER. - asm!("dcbt 0, {ptr}", ptr = in(reg) buffer.as_ptr().add(offset.strict_add(256)), options(nostack)); - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = load_be(&tmp_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xor_vec(xi_a, z_a), &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = load_be(&tmp_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xor_vec(xi_b, z_b), &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - // block c - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let off_c = offset.strict_add(two_blocks); - let mut tmp_c = [0u8; 16]; - tmp_c.copy_from_slice(&buffer[off_c..off_c.strict_add(BLOCK_SIZE)]); - let xi_c = load_be(&tmp_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - store_be(xor_vec(xi_c, z_c), &mut tmp_c); - buffer[off_c..off_c.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_c); - // block d - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let off_d = offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)); - let mut tmp_d = [0u8; 16]; - tmp_d.copy_from_slice(&buffer[off_d..off_d.strict_add(BLOCK_SIZE)]); - let xi_d = load_be(&tmp_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - store_be(xor_vec(xi_d, z_d), &mut tmp_d); - buffer[off_d..off_d.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_d); - offset = offset.strict_add(four_blocks); - } - if offset.strict_add(two_blocks) <= len { - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = load_be(&tmp_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xor_vec(xi_a, z_a), &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = load_be(&tmp_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xor_vec(xi_b, z_b), &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - offset = offset.strict_add(two_blocks); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = load_be(block); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(xor_vec(plaintext, stream), block); + } } - if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi = load_be(&tmp); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - store_be(xor_vec(xi, z), &mut tmp); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); - offset = offset.strict_add(BLOCK_SIZE); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = load_be(block); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(xor_vec(plaintext, stream), block); } - if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_len = len.strict_sub(offset); + if !tail.is_empty() { + let stream = keystream_regs(s1, s2, s3, s4, s5); let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - let xi = load_be(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - let mut ct_bytes = [0u8; BLOCK_SIZE]; - store_be(xor_vec(xi, z), &mut ct_bytes); - buffer[offset..].copy_from_slice(&ct_bytes[..tail_len]); + let (pad_tail, _) = pad.split_at_mut(tail.len()); + pad_tail.copy_from_slice(tail); + let plaintext = load_be(&pad); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + let mut ciphertext = [0u8; BLOCK_SIZE]; + store_be(xor_vec(plaintext, stream), &mut ciphertext); + let (ciphertext_tail, _) = ciphertext.split_at(tail.len()); + tail.copy_from_slice(ciphertext_tail); } + let ad_bits = (aad.len() as u64).strict_mul(8); let msg_bits = (msg_len as u64).strict_mul(8); let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); + let (ad_len_bytes, msg_len_bytes) = len_bytes.split_at_mut(8); + ad_len_bytes.copy_from_slice(&ad_bits.to_le_bytes()); + msg_len_bytes.copy_from_slice(&msg_bits.to_le_bytes()); let t = xor_vec(s3, load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); let mut tag = [0u8; TAG_SIZE]; @@ -251,6 +214,8 @@ pub(super) unsafe fn decrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { + // SAFETY: this function's caller guarantees POWER8 vector crypto support. + let power8 = unsafe { Power8::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load_be(kh0); @@ -264,121 +229,66 @@ pub(super) unsafe fn decrypt_fused( let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&tmp)); - offset = offset.strict_add(BLOCK_SIZE); + + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load_be(&pad)); } + let ct_len = buffer.len(); - let len = buffer.len(); - offset = 0; - let four_blocks = BLOCK_SIZE.strict_mul(4); - let two_blocks = BLOCK_SIZE.strict_mul(2); - while offset.strict_add(four_blocks) <= len { - // SAFETY: pointer arithmetic for dcbt prefetch; offset + 256 may exceed - // the buffer but dcbt is a hint and never faults on POWER. - asm!("dcbt 0, {ptr}", ptr = in(reg) buffer.as_ptr().add(offset.strict_add(256)), options(nostack)); - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = xor_vec(load_be(&tmp_a), z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xi_a, &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = xor_vec(load_be(&tmp_b), z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xi_b, &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - // block c - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let off_c = offset.strict_add(two_blocks); - let mut tmp_c = [0u8; 16]; - tmp_c.copy_from_slice(&buffer[off_c..off_c.strict_add(BLOCK_SIZE)]); - let xi_c = xor_vec(load_be(&tmp_c), z_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - store_be(xi_c, &mut tmp_c); - buffer[off_c..off_c.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_c); - // block d - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let off_d = offset.strict_add(two_blocks.strict_add(BLOCK_SIZE)); - let mut tmp_d = [0u8; 16]; - tmp_d.copy_from_slice(&buffer[off_d..off_d.strict_add(BLOCK_SIZE)]); - let xi_d = xor_vec(load_be(&tmp_d), z_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - store_be(xi_d, &mut tmp_d); - buffer[off_d..off_d.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_d); - offset = offset.strict_add(four_blocks); - } - if offset.strict_add(two_blocks) <= len { - // block a - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp_a = [0u8; 16]; - tmp_a.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi_a = xor_vec(load_be(&tmp_a), z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - store_be(xi_a, &mut tmp_a); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_a); - // block b - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let off_b = offset.strict_add(BLOCK_SIZE); - let mut tmp_b = [0u8; 16]; - tmp_b.copy_from_slice(&buffer[off_b..off_b.strict_add(BLOCK_SIZE)]); - let xi_b = xor_vec(load_be(&tmp_b), z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - store_be(xi_b, &mut tmp_b); - buffer[off_b..off_b.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp_b); - offset = offset.strict_add(two_blocks); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = xor_vec(load_be(block), stream); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(plaintext, block); + } } - if offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let mut tmp = [0u8; 16]; - tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi = xor_vec(load_be(&tmp), z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - store_be(xi, &mut tmp); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); - offset = offset.strict_add(BLOCK_SIZE); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let plaintext = xor_vec(load_be(block), stream); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store_be(plaintext, block); } - if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_len = len.strict_sub(offset); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - let mut z_bytes = [0u8; BLOCK_SIZE]; - store_be(z, &mut z_bytes); - let mut pt_pad = [0u8; BLOCK_SIZE]; - for i in 0..tail_len { - pt_pad[i] = pad[i] ^ z_bytes[i]; + if !tail.is_empty() { + let stream = keystream_regs(s1, s2, s3, s4, s5); + let mut stream_bytes = [0u8; BLOCK_SIZE]; + store_be(stream, &mut stream_bytes); + let mut plaintext = [0u8; BLOCK_SIZE]; + for ((out, ciphertext), mask) in plaintext.iter_mut().zip(tail.iter()).zip(stream_bytes) { + *out = *ciphertext ^ mask; } - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load_be(&pt_pad)); - buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); + power8.update( + (&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), + load_be(&plaintext), + ); + let (plaintext_tail, _) = plaintext.split_at(tail.len()); + tail.copy_from_slice(plaintext_tail); } + let ad_bits = (aad.len() as u64).strict_mul(8); let ct_bits = (ct_len as u64).strict_mul(8); let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); + let (ad_len_bytes, ct_len_bytes) = len_bytes.split_at_mut(8); + ad_len_bytes.copy_from_slice(&ad_bits.to_le_bytes()); + ct_len_bytes.copy_from_slice(&ct_bits.to_le_bytes()); let t = xor_vec(s3, load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + power8.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); let mut tag = [0u8; TAG_SIZE]; diff --git a/src/aead/aegis256/riscv64_vperm.rs b/src/aead/aegis256/riscv64_vperm.rs deleted file mode 100644 index 4a7e4289..00000000 --- a/src/aead/aegis256/riscv64_vperm.rs +++ /dev/null @@ -1,368 +0,0 @@ -use core::arch::asm; - -use super::{BLOCK_SIZE, Block, C0, C1, KEY_SIZE, NONCE_SIZE, TAG_SIZE, and_block, split_halves, xor_block}; -use crate::aead::aes_round::{ - AES_AFFINE, MC_ROT1, MC_ROT2, VPERM_INV_HI, VPERM_INV_LO, VPERM_IPT_HI, VPERM_IPT_LO, VPERM_SBOT, VPERM_SBOU, - VPERM_SR, XTIME_REDUCE, -}; - -type State = [Block; 6]; - -/// Precomputed Hamburg vperm table block — packed contiguously for -/// offset-based vector loads in the asm block. -#[repr(C, align(16))] -struct VpermTables { - ipt_lo: [u8; 16], // offset 0 - ipt_hi: [u8; 16], // offset 16 - inv_lo: [u8; 16], // offset 32 - inv_hi: [u8; 16], // offset 48 - sbou: [u8; 16], // offset 64 - sbot: [u8; 16], // offset 80 - sr_perm: [u8; 16], // offset 96 - mc_rot1: [u8; 16], // offset 112 - mc_rot2: [u8; 16], // offset 128 - affine: [u8; 16], // offset 144 - xtime: [u8; 16], // offset 160 -} - -impl VpermTables { - #[inline(always)] - fn load() -> Self { - Self { - ipt_lo: VPERM_IPT_LO, - ipt_hi: VPERM_IPT_HI, - inv_lo: VPERM_INV_LO, - inv_hi: VPERM_INV_HI, - sbou: VPERM_SBOU, - sbot: VPERM_SBOT, - sr_perm: VPERM_SR, - mc_rot1: MC_ROT1, - mc_rot2: MC_ROT2, - affine: [AES_AFFINE; 16], - xtime: [XTIME_REDUCE; 16], - } - } -} - -/// Single AES round via Hamburg vperm on RISC-V V: SubBytes + ShiftRows + -/// MixColumns + AddRoundKey. Uses `vrgather.vv` for all S-box nibble lookups. -/// -/// # Safety -/// Requires the RISC-V V extension. -#[target_feature(enable = "v")] -#[inline] -unsafe fn aes_round(block: &Block, round_key: &Block, tables: &VpermTables) -> Block { - let mut out = [0u8; BLOCK_SIZE]; - - // SAFETY: Caller guarantees RISC-V V extension is available. - // The asm block loads the state, tables, and round key from memory, - // performs all computation in vector registers, and stores the result. - // All vrgather indices are masked to 0-15 before lookup — no secret- - // dependent memory access. - unsafe { - asm!( - // ── Setup ────────────────────────────────────────────────────── - "vsetivli zero, 16, e8, m1, ta, ma", - - // Load tables from the VpermTables struct (contiguous, 16B each). - "vle8.v v2, ({tbl})", // IPT_LO (offset 0) - "addi {tmp}, {tbl}, 16", - "vle8.v v3, ({tmp})", // IPT_HI (offset 16) - "addi {tmp}, {tbl}, 32", - "vle8.v v4, ({tmp})", // INV_LO (offset 32) - "addi {tmp}, {tbl}, 48", - "vle8.v v5, ({tmp})", // INV_HI (offset 48) - "addi {tmp}, {tbl}, 64", - "vle8.v v6, ({tmp})", // SBOU (offset 64) - "addi {tmp}, {tbl}, 80", - "vle8.v v7, ({tmp})", // SBOT (offset 80) - "addi {tmp}, {tbl}, 96", - "vle8.v v8, ({tmp})", // SR_PERM (offset 96) - "addi {tmp}, {tbl}, 112", - "vle8.v v9, ({tmp})", // MC_ROT1 (offset 112) - "addi {tmp}, {tbl}, 128", - "vle8.v v10, ({tmp})", // MC_ROT2 (offset 128) - "addi {tmp}, {tbl}, 144", - "vle8.v v11, ({tmp})", // 0x63 (offset 144) - "addi {tmp}, {tbl}, 160", - "vle8.v v12, ({tmp})", // 0x1B (offset 160) - - // Load state and round key. - "vle8.v v0, ({state})", - "vle8.v v1, ({rk})", - - // ── Phase 1: Nibble extraction ───────────────────────────────── - "vand.vi v14, v0, 15", // lo_nib = state & 0x0F - "vsrl.vi v15, v0, 4", // hi_nib = state >> 4 - - // ── Phase 2: Input transform (AES → tower field) ────────────── - "vrgather.vv v16, v2, v14", // ipt_l = IPT_LO[lo_nib] - "vrgather.vv v17, v3, v15", // ipt_h = IPT_HI[hi_nib] - "vxor.vv v14, v16, v17", // x = ipt_l ^ ipt_h - - // ── Phase 3: Re-extract nibbles of transformed value ─────────── - "vand.vi v15, v14, 15", // t_lo = x & 0x0F - "vsrl.vi v16, v14, 4", // t_hi = x >> 4 - - // ── Phase 4: GF(2^4) inverse ────────────────────────────────── - "vrgather.vv v17, v5, v15", // ak = INV_HI[t_lo] - "vxor.vv v18, v16, v15", // j = t_hi ^ t_lo - "vrgather.vv v19, v4, v16", // inv_i = INV_LO[t_hi] - "vxor.vv v20, v19, v17", // iak = inv_i ^ ak - "vrgather.vv v21, v4, v18", // inv_j = INV_LO[j] - "vxor.vv v22, v21, v17", // jak = inv_j ^ ak - - // vperm_z(INV_LO, iak): zero where bit 7 set - "vand.vi v23, v20, 15", // iak & 0x0F - "vrgather.vv v24, v4, v23", // INV_LO[iak & 0x0F] - "vsra.vi v25, v20, 7", // 0xFF where bit 7 set - "vxor.vi v26, v25, -1", // ~mask - "vand.vv v24, v24, v26", // zero masked positions - "vxor.vv v14, v24, v18", // io = inv_iak ^ j - - // vperm_z(INV_LO, jak): zero where bit 7 set - "vand.vi v23, v22, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v22, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v15, v24, v16", // jo = inv_jak ^ t_hi - - // ── Phase 5: Output transform (SubBytes) ────────────────────── - // vperm_z(SBOU, io) - "vand.vi v23, v14, 15", - "vrgather.vv v24, v6, v23", - "vsra.vi v25, v14, 7", - "vxor.vi v26, v25, -1", - "vand.vv v16, v24, v26", // su - - // vperm_z(SBOT, jo) - "vand.vi v23, v15, 15", - "vrgather.vv v24, v7, v23", - "vsra.vi v25, v15, 7", - "vxor.vi v26, v25, -1", - "vand.vv v17, v24, v26", // st - - // sb = su ^ st ^ 0x63 - "vxor.vv v14, v16, v17", - "vxor.vv v14, v14, v11", - - // ── ShiftRows ───────────────────────────────────────────────── - "vrgather.vv v15, v14, v8", // sr = sb permuted - - // ── MixColumns (xtime decomposition) ────────────────────────── - "vrgather.vv v16, v15, v9", // rot1 = column-rotate-by-1 - "vxor.vv v17, v15, v16", // pair = sr ^ rot1 - "vsll.vi v18, v17, 1", // pair << 1 - "vsra.vi v19, v17, 7", // bit-7 mask for xtime reduction - "vand.vv v19, v19, v12", // mask & 0x1B - "vxor.vv v18, v18, v19", // xt = xtime(pair) - "vrgather.vv v19, v17, v10", // rot2_pair = column-rotate-by-2 - "vxor.vv v20, v17, v19", // col_sum = pair ^ rot2_pair - "vxor.vv v14, v15, v20", // sr ^ col_sum - "vxor.vv v14, v14, v18", // mc = sr ^ col_sum ^ xt - - // ── AddRoundKey ─────────────────────────────────────────────── - "vxor.vv v0, v14, v1", - - // ── Store result ────────────────────────────────────────────── - "vse8.v v0, ({out})", - - state = in(reg) block.as_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - out = in(reg) out.as_mut_ptr(), - tmp = out(reg) _, - options(nostack), - ); - } - out -} - -// ── AEGIS-256 state operations ────────────────────────────────────────── - -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn update(s: &mut State, m: &Block, tables: &VpermTables) { - // SAFETY: caller guarantees the RISC-V V extension is available and the - // state blocks are valid local buffers for the Hamburg round function. - unsafe { - let tmp = s[5]; - s[5] = aes_round(&s[4], &s[5], tables); - s[4] = aes_round(&s[3], &s[4], tables); - s[3] = aes_round(&s[2], &s[3], tables); - s[2] = aes_round(&s[1], &s[2], tables); - s[1] = aes_round(&s[0], &s[1], tables); - s[0] = xor_block(&aes_round(&tmp, &s[0], tables), m); - } -} - -#[inline(always)] -fn keystream(s: &State) -> Block { - xor_block(&xor_block(&s[1], &s[4]), &xor_block(&s[5], &and_block(&s[2], &s[3]))) -} - -// ── Fused encrypt/decrypt ───────────────────────────────────────────── - -#[target_feature(enable = "v")] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -pub(super) unsafe fn encrypt_fused( - key: &[u8; KEY_SIZE], - nonce: &[u8; NONCE_SIZE], - aad: &[u8], - buffer: &mut [u8], -) -> [u8; TAG_SIZE] { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // lifetime of this fused operation and all slices are valid Rust references. - unsafe { - let tables = VpermTables::load(); - let (kh0, kh1) = split_halves(key); - let (nh0, nh1) = split_halves(nonce); - let k0_xor_n0 = xor_block(kh0, nh0); - let k1_xor_n1 = xor_block(kh1, nh1); - let mut s: State = [k0_xor_n0, k1_xor_n1, C1, C0, xor_block(kh0, &C0), xor_block(kh1, &C1)]; - - for _ in 0..4 { - update(&mut s, kh0, &tables); - update(&mut s, kh1, &tables); - update(&mut s, &k0_xor_n0, &tables); - update(&mut s, &k1_xor_n1, &tables); - } - - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; BLOCK_SIZE]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update(&mut s, &tmp, &tables); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < aad.len() { - let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update(&mut s, &pad, &tables); - } - - let msg_len = buffer.len(); - let len = buffer.len(); - offset = 0; - while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream(&s); - let mut xi = [0u8; BLOCK_SIZE]; - xi.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - update(&mut s, &xi, &tables); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&xor_block(&xi, &z)); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < len { - let z = keystream(&s); - let tail_len = len.strict_sub(offset); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - update(&mut s, &pad, &tables); - let ct = xor_block(&pad, &z); - buffer[offset..].copy_from_slice(&ct[..tail_len]); - } - - let ad_bits = (aad.len() as u64).strict_mul(8); - let msg_bits = (msg_len as u64).strict_mul(8); - let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); - let t = xor_block(&s[3], &len_bytes); - for _ in 0..7 { - update(&mut s, &t, &tables); - } - - xor_block( - &xor_block(&xor_block(&s[0], &s[1]), &xor_block(&s[2], &s[3])), - &xor_block(&s[4], &s[5]), - ) - } -} - -#[target_feature(enable = "v")] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -pub(super) unsafe fn decrypt_fused( - key: &[u8; KEY_SIZE], - nonce: &[u8; NONCE_SIZE], - aad: &[u8], - buffer: &mut [u8], -) -> [u8; TAG_SIZE] { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // lifetime of this fused operation and all slices are valid Rust references. - unsafe { - let tables = VpermTables::load(); - let (kh0, kh1) = split_halves(key); - let (nh0, nh1) = split_halves(nonce); - let k0_xor_n0 = xor_block(kh0, nh0); - let k1_xor_n1 = xor_block(kh1, nh1); - let mut s: State = [k0_xor_n0, k1_xor_n1, C1, C0, xor_block(kh0, &C0), xor_block(kh1, &C1)]; - - for _ in 0..4 { - update(&mut s, kh0, &tables); - update(&mut s, kh1, &tables); - update(&mut s, &k0_xor_n0, &tables); - update(&mut s, &k1_xor_n1, &tables); - } - - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let mut tmp = [0u8; BLOCK_SIZE]; - tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update(&mut s, &tmp, &tables); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < aad.len() { - let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update(&mut s, &pad, &tables); - } - - let ct_len = buffer.len(); - let len = buffer.len(); - offset = 0; - while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream(&s); - let mut ci = [0u8; BLOCK_SIZE]; - ci.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); - let xi = xor_block(&ci, &z); - update(&mut s, &xi, &tables); - buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&xi); - offset = offset.strict_add(BLOCK_SIZE); - } - if offset < len { - let z = keystream(&s); - let tail_len = len.strict_sub(offset); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..tail_len].copy_from_slice(&buffer[offset..]); - let mut pt_pad = [0u8; BLOCK_SIZE]; - for i in 0..tail_len { - pt_pad[i] = pad[i] ^ z[i]; - } - update(&mut s, &pt_pad, &tables); - buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); - } - - let ad_bits = (aad.len() as u64).strict_mul(8); - let ct_bits = (ct_len as u64).strict_mul(8); - let mut len_bytes = [0u8; BLOCK_SIZE]; - len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); - len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); - let t = xor_block(&s[3], &len_bytes); - for _ in 0..7 { - update(&mut s, &t, &tables); - } - - xor_block( - &xor_block(&xor_block(&s[0], &s[1]), &xor_block(&s[2], &s[3])), - &xor_block(&s[4], &s[5]), - ) - } -} diff --git a/src/aead/aegis256/s390x_vperm.rs b/src/aead/aegis256/s390x_vperm.rs index 11c7fe59..f22d1530 100644 --- a/src/aead/aegis256/s390x_vperm.rs +++ b/src/aead/aegis256/s390x_vperm.rs @@ -48,7 +48,7 @@ fn and_vec(a: i64x2, b: i64x2) -> i64x2 { /// Broadcast a byte to all 16 positions of a vector. #[inline(always)] fn splat_byte(b: u8) -> i64x2 { - let w = u64::from_ne_bytes([b; 8]) as i64; + let w = i64::from_ne_bytes([b; 8]); i64x2::from_array([w, w]) } @@ -297,38 +297,31 @@ impl VpermTables { // ── AEGIS-256 state operations ────────────────────────────────────────── #[target_feature(enable = "vector")] -#[allow(clippy::too_many_arguments)] #[inline] /// # Safety /// /// Caller must ensure the s390x vector facility is available and all state /// registers belong to a valid AEGIS-256 state. -unsafe fn update_regs( - s0: &mut i64x2, - s1: &mut i64x2, - s2: &mut i64x2, - s3: &mut i64x2, - s4: &mut i64x2, - s5: &mut i64x2, - m: i64x2, - tables: &VpermTables, -) { +unsafe fn update_regs(state: &mut [i64x2; 6], m: i64x2, tables: &VpermTables) { // SAFETY: caller guarantees the s390x vector facility is available for // this helper and all state registers are valid local values. unsafe { - let tmp = *s5; - *s5 = aes_round(*s4, *s5, tables); - *s4 = aes_round(*s3, *s4, tables); - *s3 = aes_round(*s2, *s3, tables); - *s2 = aes_round(*s1, *s2, tables); - *s1 = aes_round(*s0, *s1, tables); - *s0 = xor_vec(aes_round(tmp, *s0, tables), m); + let tmp = state[5]; + state[5] = aes_round(state[4], state[5], tables); + state[4] = aes_round(state[3], state[4], tables); + state[3] = aes_round(state[2], state[3], tables); + state[2] = aes_round(state[1], state[2], tables); + state[1] = aes_round(state[0], state[1], tables); + state[0] = xor_vec(aes_round(tmp, state[0], tables), m); } } #[inline(always)] -fn keystream_regs(s1: i64x2, s2: i64x2, s3: i64x2, s4: i64x2, s5: i64x2) -> i64x2 { - xor_vec(xor_vec(s1, s4), xor_vec(s5, and_vec(s2, s3))) +fn keystream_regs(state: &[i64x2; 6]) -> i64x2 { + xor_vec( + xor_vec(state[1], state[4]), + xor_vec(state[5], and_vec(state[2], state[3])), + ) } // ── Fused encrypt/decrypt ───────────────────────────────────────────── @@ -357,64 +350,45 @@ pub(super) unsafe fn encrypt_fused( let c1 = load_be(&C1); let k0_xor_n0 = xor_vec(k0, n0); let k1_xor_n1 = xor_vec(k1, n1); - let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = - (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); + let mut state = [k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)]; for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, &tables); + update_regs(&mut state, k0, &tables); + update_regs(&mut state, k1, &tables); + update_regs(&mut state, k0_xor_n0, &tables); + update_regs(&mut state, k1_xor_n1, &tables); } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { let mut tmp = [0u8; 16]; tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&tmp), - &tables, - ); + update_regs(&mut state, load_be(&tmp), &tables); offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&pad), - &tables, - ); + update_regs(&mut state, load_be(&pad), &tables); } let msg_len = buffer.len(); let len = buffer.len(); offset = 0; while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let mut tmp = [0u8; 16]; tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); let xi = load_be(&tmp); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, &tables); + update_regs(&mut state, xi, &tables); store_be(xor_vec(xi, z), &mut tmp); buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); let xi = load_be(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, &tables); + update_regs(&mut state, xi, &tables); let mut ct_bytes = [0u8; BLOCK_SIZE]; store_be(xor_vec(xi, z), &mut ct_bytes); buffer[offset..].copy_from_slice(&ct_bytes[..tail_len]); @@ -424,11 +398,14 @@ pub(super) unsafe fn encrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&msg_bits.to_le_bytes()); - let t = xor_vec(s3, load_be(&len_bytes)); + let t = xor_vec(state[3], load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, &tables); + update_regs(&mut state, t, &tables); } - let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); + let tag_vec = xor_vec( + xor_vec(xor_vec(state[0], state[1]), xor_vec(state[2], state[3])), + xor_vec(state[4], state[5]), + ); let mut tag = [0u8; TAG_SIZE]; store_be(tag_vec, &mut tag); tag @@ -459,59 +436,40 @@ pub(super) unsafe fn decrypt_fused( let c1 = load_be(&C1); let k0_xor_n0 = xor_vec(k0, n0); let k1_xor_n1 = xor_vec(k1, n1); - let (mut s0, mut s1, mut s2, mut s3, mut s4, mut s5) = - (k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)); + let mut state = [k0_xor_n0, k1_xor_n1, c1, c0, xor_vec(k0, c0), xor_vec(k1, c1)]; for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0, &tables); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1, &tables); + update_regs(&mut state, k0, &tables); + update_regs(&mut state, k1, &tables); + update_regs(&mut state, k0_xor_n0, &tables); + update_regs(&mut state, k1_xor_n1, &tables); } let mut offset = 0usize; while offset.strict_add(BLOCK_SIZE) <= aad.len() { let mut tmp = [0u8; 16]; tmp.copy_from_slice(&aad[offset..offset.strict_add(BLOCK_SIZE)]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&tmp), - &tables, - ); + update_regs(&mut state, load_be(&tmp), &tables); offset = offset.strict_add(BLOCK_SIZE); } if offset < aad.len() { let mut pad = [0u8; BLOCK_SIZE]; pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&pad), - &tables, - ); + update_regs(&mut state, load_be(&pad), &tables); } let ct_len = buffer.len(); let len = buffer.len(); offset = 0; while offset.strict_add(BLOCK_SIZE) <= len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let mut tmp = [0u8; 16]; tmp.copy_from_slice(&buffer[offset..offset.strict_add(BLOCK_SIZE)]); let xi = xor_vec(load_be(&tmp), z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi, &tables); + update_regs(&mut state, xi, &tables); store_be(xi, &mut tmp); buffer[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&tmp); offset = offset.strict_add(BLOCK_SIZE); } if offset < len { - let z = keystream_regs(s1, s2, s3, s4, s5); + let z = keystream_regs(&state); let tail_len = len.strict_sub(offset); let mut pad = [0u8; BLOCK_SIZE]; pad[..tail_len].copy_from_slice(&buffer[offset..]); @@ -521,16 +479,7 @@ pub(super) unsafe fn decrypt_fused( for i in 0..tail_len { pt_pad[i] = pad[i] ^ z_bytes[i]; } - update_regs( - &mut s0, - &mut s1, - &mut s2, - &mut s3, - &mut s4, - &mut s5, - load_be(&pt_pad), - &tables, - ); + update_regs(&mut state, load_be(&pt_pad), &tables); buffer[offset..].copy_from_slice(&pt_pad[..tail_len]); } let ad_bits = (aad.len() as u64).strict_mul(8); @@ -538,11 +487,14 @@ pub(super) unsafe fn decrypt_fused( let mut len_bytes = [0u8; BLOCK_SIZE]; len_bytes[..8].copy_from_slice(&ad_bits.to_le_bytes()); len_bytes[8..].copy_from_slice(&ct_bits.to_le_bytes()); - let t = xor_vec(s3, load_be(&len_bytes)); + let t = xor_vec(state[3], load_be(&len_bytes)); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t, &tables); + update_regs(&mut state, t, &tables); } - let tag_vec = xor_vec(xor_vec(xor_vec(s0, s1), xor_vec(s2, s3)), xor_vec(s4, s5)); + let tag_vec = xor_vec( + xor_vec(xor_vec(state[0], state[1]), xor_vec(state[2], state[3])), + xor_vec(state[4], state[5]), + ); let mut tag = [0u8; TAG_SIZE]; store_be(tag_vec, &mut tag); tag diff --git a/src/aead/aegis256/x86_64_ni.rs b/src/aead/aegis256/x86_64_ni.rs index 3a8c3a5b..32813c71 100644 --- a/src/aead/aegis256/x86_64_ni.rs +++ b/src/aead/aegis256/x86_64_ni.rs @@ -3,66 +3,62 @@ use core::arch::x86_64::*; use super::{BLOCK_SIZE, C0, C1, KEY_SIZE, NONCE_SIZE, TAG_SIZE}; #[inline] -/// # Safety -/// -/// `bytes` must refer to a valid 16-byte block. -unsafe fn load(bytes: &[u8; BLOCK_SIZE]) -> __m128i { - _mm_loadu_si128(bytes.as_ptr().cast()) +fn load(bytes: &[u8; BLOCK_SIZE]) -> __m128i { + // SAFETY: SSE2 is an x86_64 baseline feature; `bytes` provides 16 readable bytes, and the load is unaligned. + unsafe { _mm_loadu_si128(bytes.as_ptr().cast()) } } #[inline] -/// # Safety -/// -/// `out` must refer to a valid writable 16-byte block. -unsafe fn store(v: __m128i, out: &mut [u8; BLOCK_SIZE]) { - _mm_storeu_si128(out.as_mut_ptr().cast(), v); +fn store(value: __m128i, out: &mut [u8; BLOCK_SIZE]) { + // SAFETY: SSE2 is an x86_64 baseline feature; `out` provides 16 writable bytes, and the store is unaligned. + unsafe { _mm_storeu_si128(out.as_mut_ptr().cast(), value) }; } -// ── Register-based helpers ────────────────────────────────────────────── -// -// All 6 AES rounds in an AEGIS-256 update read from the OLD state and are -// mutually independent. Keeping state in 6 local `__m128i` values (rather -// than indexing an array through `&mut State`) lets the register allocator -// pin them to XMM registers across loop iterations, eliminating store-to- -// load round-trips that serialize the OOO pipeline. +type StateMut<'a> = ( + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, + &'a mut __m128i, +); -#[inline] -/// # Safety -/// -/// Caller must ensure AES-NI is available and all register arguments come -/// from a valid AEGIS-256 state. -unsafe fn update_regs( - s0: &mut __m128i, - s1: &mut __m128i, - s2: &mut __m128i, - s3: &mut __m128i, - s4: &mut __m128i, - s5: &mut __m128i, - m: __m128i, -) { - let tmp = *s5; - *s5 = _mm_aesenc_si128(*s4, *s5); - *s4 = _mm_aesenc_si128(*s3, *s4); - *s3 = _mm_aesenc_si128(*s2, *s3); - *s2 = _mm_aesenc_si128(*s1, *s2); - *s1 = _mm_aesenc_si128(*s0, *s1); - *s0 = _mm_xor_si128(_mm_aesenc_si128(tmp, *s0), m); +#[derive(Clone, Copy)] +struct AesNi { + _private: (), } -#[inline] -/// # Safety -/// -/// The provided registers must come from a valid AEGIS-256 state on an -/// AES-NI capable CPU. -unsafe fn keystream_regs(s1: __m128i, s2: __m128i, s3: __m128i, s4: __m128i, s5: __m128i) -> __m128i { - _mm_xor_si128(_mm_xor_si128(s1, s4), _mm_xor_si128(s5, _mm_and_si128(s2, s3))) +impl AesNi { + /// Creates an AES-NI capability token. + /// + /// # Safety + /// + /// The current CPU must support AES-NI and AVX for the token's entire lifetime. + #[inline] + unsafe fn new() -> Self { + Self { _private: () } + } + + #[inline(always)] + fn update(self, (s0, s1, s2, s3, s4, s5): StateMut<'_>, message: __m128i) { + let old_s5 = *s5; + // SAFETY: this module constructs `AesNi` only through `new`, whose caller guarantees AES-NI and AVX support. + unsafe { + *s5 = _mm_aesenc_si128(*s4, *s5); + *s4 = _mm_aesenc_si128(*s3, *s4); + *s3 = _mm_aesenc_si128(*s2, *s3); + *s2 = _mm_aesenc_si128(*s1, *s2); + *s1 = _mm_aesenc_si128(*s0, *s1); + *s0 = _mm_xor_si128(_mm_aesenc_si128(old_s5, *s0), message); + } + } } -// ── Fused encrypt/decrypt ─────────────────────────────────────────────── -// -// Single `#[target_feature]` entry points that keep state in XMM registers -// from init through finalize, eliminating ~15 cycles of stack spills that -// occur when init/aad/encrypt/finalize are separate function calls. +#[inline] +fn keystream(s1: __m128i, s2: __m128i, s3: __m128i, s4: __m128i, s5: __m128i) -> __m128i { + // SAFETY: SSE2 is part of the x86_64 baseline. + unsafe { _mm_xor_si128(_mm_xor_si128(s1, s4), _mm_xor_si128(s5, _mm_and_si128(s2, s3))) } +} #[target_feature(enable = "aes,avx")] /// # Safety @@ -74,7 +70,8 @@ pub(super) unsafe fn encrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { - // ── init ── + // SAFETY: this function's caller guarantees AES-NI and AVX support. + let aesni = unsafe { AesNi::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load(kh0); @@ -94,94 +91,63 @@ pub(super) unsafe fn encrypt_fused( _mm_xor_si128(k1, c1), ); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - // ── aad ── - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let block = _mm_loadu_si128(aad.as_ptr().add(offset).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, block); - offset = offset.strict_add(BLOCK_SIZE); + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(&pad)); } - // ── encrypt ── - // - // Counted loop with raw pointer advancement: eliminates per-iteration - // overflow-check branches (strict_add) from the hot path. The loop count - // is pre-validated via integer division, and all intra-block offsets are - // compile-time constants applied through ptr::add inside this unsafe block. let msg_len = buffer.len(); - let ptr = buffer.as_mut_ptr(); - let n_quads = msg_len / 64; - let mut p = ptr; - for _ in 0..n_quads { - _mm_prefetch(p.add(256).cast::(), _MM_HINT_T0); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = _mm_loadu_si128(p.cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), _mm_xor_si128(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = _mm_loadu_si128(p.add(16).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), _mm_xor_si128(xi_b, z_b)); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let xi_c = _mm_loadu_si128(p.add(32).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - _mm_storeu_si128(p.add(32).cast(), _mm_xor_si128(xi_c, z_c)); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let xi_d = _mm_loadu_si128(p.add(48).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - _mm_storeu_si128(p.add(48).cast(), _mm_xor_si128(xi_d, z_d)); - p = p.add(64); - } - let mut remaining = msg_len.strict_sub(n_quads.strict_mul(64)); - if remaining >= 32 { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let xi_a = _mm_loadu_si128(p.cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), _mm_xor_si128(xi_a, z_a)); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let xi_b = _mm_loadu_si128(p.add(16).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), _mm_xor_si128(xi_b, z_b)); - p = p.add(32); - remaining = remaining.strict_sub(32); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = load(block); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(_mm_xor_si128(plaintext, stream), block); + } } - if remaining >= 16 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let xi = _mm_loadu_si128(p.cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - _mm_storeu_si128(p.cast(), _mm_xor_si128(xi, z)); - remaining = remaining.strict_sub(16); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = load(block); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(_mm_xor_si128(plaintext, stream), block); } - if remaining > 0 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_off = msg_len.strict_sub(remaining); + if !tail.is_empty() { + let stream = keystream(s1, s2, s3, s4, s5); let mut pad = [0u8; BLOCK_SIZE]; - pad[..remaining].copy_from_slice(&buffer[tail_off..]); - let xi = load(&pad); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - let mut ct_bytes = [0u8; BLOCK_SIZE]; - store(_mm_xor_si128(xi, z), &mut ct_bytes); - buffer[tail_off..].copy_from_slice(&ct_bytes[..remaining]); + let (pad_tail, _) = pad.split_at_mut(tail.len()); + pad_tail.copy_from_slice(tail); + let plaintext = load(&pad); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + let mut ciphertext = [0u8; BLOCK_SIZE]; + store(_mm_xor_si128(plaintext, stream), &mut ciphertext); + let (ciphertext_tail, _) = ciphertext.split_at(tail.len()); + tail.copy_from_slice(ciphertext_tail); } - // ── finalize ── let ad_bits = (aad.len() as u64).strict_mul(8); let msg_bits = (msg_len as u64).strict_mul(8); - let len_block = _mm_set_epi64x(msg_bits as i64, ad_bits as i64); + let len_block = _mm_set_epi64x( + i64::from_ne_bytes(msg_bits.to_ne_bytes()), + i64::from_ne_bytes(ad_bits.to_ne_bytes()), + ); let t = _mm_xor_si128(s3, len_block); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = _mm_xor_si128( _mm_xor_si128(_mm_xor_si128(s0, s1), _mm_xor_si128(s2, s3)), @@ -202,7 +168,8 @@ pub(super) unsafe fn decrypt_fused( aad: &[u8], buffer: &mut [u8], ) -> [u8; TAG_SIZE] { - // ── init ── + // SAFETY: this function's caller guarantees AES-NI and AVX support. + let aesni = unsafe { AesNi::new() }; let (kh0, kh1) = super::split_halves(key); let (nh0, nh1) = super::split_halves(nonce); let k0 = load(kh0); @@ -222,102 +189,63 @@ pub(super) unsafe fn decrypt_fused( _mm_xor_si128(k1, c1), ); for _ in 0..4 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k0_xor_n0); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, k1_xor_n1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k0_xor_n0); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), k1_xor_n1); } - // ── aad ── - let mut offset = 0usize; - while offset.strict_add(BLOCK_SIZE) <= aad.len() { - let block = _mm_loadu_si128(aad.as_ptr().add(offset).cast()); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, block); - offset = offset.strict_add(BLOCK_SIZE); + let (aad_blocks, aad_tail) = aad.as_chunks::(); + for block in aad_blocks { + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(block)); } - if offset < aad.len() { + if !aad_tail.is_empty() { let mut pad = [0u8; BLOCK_SIZE]; - pad[..aad.len().strict_sub(offset)].copy_from_slice(&aad[offset..]); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pad)); + let (pad_tail, _) = pad.split_at_mut(aad_tail.len()); + pad_tail.copy_from_slice(aad_tail); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(&pad)); } - // ── decrypt ── - // - // Same counted-loop strategy as encrypt: raw pointer advancement with - // compile-time-constant offsets, zero overflow checks in the hot path. let ct_len = buffer.len(); - let ptr = buffer.as_mut_ptr(); - let n_quads = ct_len / 64; - let mut p = ptr; - for _ in 0..n_quads { - _mm_prefetch(p.add(256).cast::(), _MM_HINT_T0); - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = _mm_loadu_si128(p.cast()); - let xi_a = _mm_xor_si128(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = _mm_loadu_si128(p.add(16).cast()); - let xi_b = _mm_xor_si128(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), xi_b); - let z_c = keystream_regs(s1, s2, s3, s4, s5); - let ci_c = _mm_loadu_si128(p.add(32).cast()); - let xi_c = _mm_xor_si128(ci_c, z_c); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_c); - _mm_storeu_si128(p.add(32).cast(), xi_c); - let z_d = keystream_regs(s1, s2, s3, s4, s5); - let ci_d = _mm_loadu_si128(p.add(48).cast()); - let xi_d = _mm_xor_si128(ci_d, z_d); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_d); - _mm_storeu_si128(p.add(48).cast(), xi_d); - p = p.add(64); - } - let mut remaining = ct_len.strict_sub(n_quads.strict_mul(64)); - if remaining >= 32 { - let z_a = keystream_regs(s1, s2, s3, s4, s5); - let ci_a = _mm_loadu_si128(p.cast()); - let xi_a = _mm_xor_si128(ci_a, z_a); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_a); - _mm_storeu_si128(p.cast(), xi_a); - let z_b = keystream_regs(s1, s2, s3, s4, s5); - let ci_b = _mm_loadu_si128(p.add(16).cast()); - let xi_b = _mm_xor_si128(ci_b, z_b); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi_b); - _mm_storeu_si128(p.add(16).cast(), xi_b); - p = p.add(32); - remaining = remaining.strict_sub(32); + let (quads, remainder) = buffer.as_chunks_mut::<64>(); + for quad in quads { + let (blocks, _) = quad.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = _mm_xor_si128(load(block), stream); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(plaintext, block); + } } - if remaining >= 16 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let ci = _mm_loadu_si128(p.cast()); - let xi = _mm_xor_si128(ci, z); - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, xi); - _mm_storeu_si128(p.cast(), xi); - remaining = remaining.strict_sub(16); + let (blocks, tail) = remainder.as_chunks_mut::(); + for block in blocks { + let stream = keystream(s1, s2, s3, s4, s5); + let plaintext = _mm_xor_si128(load(block), stream); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), plaintext); + store(plaintext, block); } - if remaining > 0 { - let z = keystream_regs(s1, s2, s3, s4, s5); - let tail_off = ct_len.strict_sub(remaining); - let mut pad = [0u8; BLOCK_SIZE]; - pad[..remaining].copy_from_slice(&buffer[tail_off..]); - let mut z_bytes = [0u8; BLOCK_SIZE]; - store(z, &mut z_bytes); - let mut pt_pad = [0u8; BLOCK_SIZE]; - for i in 0..remaining { - pt_pad[i] = pad[i] ^ z_bytes[i]; + if !tail.is_empty() { + let stream = keystream(s1, s2, s3, s4, s5); + let mut stream_bytes = [0u8; BLOCK_SIZE]; + store(stream, &mut stream_bytes); + let mut plaintext = [0u8; BLOCK_SIZE]; + for ((out, ciphertext), mask) in plaintext.iter_mut().zip(tail.iter()).zip(stream_bytes) { + *out = *ciphertext ^ mask; } - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, load(&pt_pad)); - buffer[tail_off..].copy_from_slice(&pt_pad[..remaining]); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), load(&plaintext)); + let (plaintext_tail, _) = plaintext.split_at(tail.len()); + tail.copy_from_slice(plaintext_tail); } - // ── finalize ── let ad_bits = (aad.len() as u64).strict_mul(8); let ct_bits = (ct_len as u64).strict_mul(8); - let len_block = _mm_set_epi64x(ct_bits as i64, ad_bits as i64); + let len_block = _mm_set_epi64x( + i64::from_ne_bytes(ct_bits.to_ne_bytes()), + i64::from_ne_bytes(ad_bits.to_ne_bytes()), + ); let t = _mm_xor_si128(s3, len_block); for _ in 0..7 { - update_regs(&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5, t); + aesni.update((&mut s0, &mut s1, &mut s2, &mut s3, &mut s4, &mut s5), t); } let tag_vec = _mm_xor_si128( _mm_xor_si128(_mm_xor_si128(s0, s1), _mm_xor_si128(s2, s3)), diff --git a/src/aead/aes.rs b/src/aead/aes.rs index e77afab3..1d8c792e 100644 --- a/src/aead/aes.rs +++ b/src/aead/aes.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Portable table-free AES block-cipher core with hardware dispatch. //! //! This module provides AES-128 and AES-256 key expansion and single-block @@ -40,31 +38,23 @@ pub(crate) const EXPANDED_KEY_WORDS_128: usize = 4 * (ROUNDS_128 + 1); // 44 #[path = "aes/aarch64_ce.rs"] mod ce; #[cfg(target_arch = "s390x")] -#[allow(unsafe_code)] #[path = "aes/s390x_km.rs"] mod km; #[cfg(target_arch = "x86_64")] #[path = "aes/x86_64_ni.rs"] mod ni; #[cfg(target_arch = "powerpc64")] -#[allow(unsafe_code)] #[path = "aes/powerpc64_ppc.rs"] mod ppc; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aes/riscv64_aes.rs"] mod rv_aes; -#[cfg(any(target_arch = "riscv64", test))] +#[cfg(any(target_arch = "riscv64", all(test, not(target_arch = "s390x"))))] #[path = "aes/riscv64_fixslice_aes.rs"] mod rv_fixslice_aes; #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] #[path = "aes/riscv64_scalar_aes.rs"] mod rv_scalar_aes; -#[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] -#[path = "aes/riscv64_vperm_aes.rs"] -mod rv_vperm_aes; #[cfg(all( target_arch = "x86_64", target_os = "linux", @@ -113,7 +103,7 @@ pub(crate) struct Aes256EncKey { } enum KeyInner { - #[allow(dead_code)] + #[cfg(not(target_arch = "riscv64"))] PortableRoundKeys([u32; EXPANDED_KEY_WORDS]), #[cfg(target_arch = "x86_64")] X86AesNi(ni::NiRoundKeys), @@ -124,24 +114,21 @@ enum KeyInner { #[cfg(target_arch = "powerpc64")] Power8Crypto(ppc::PpcRoundKeys), #[cfg(target_arch = "riscv64")] - Riscv64ScalarCrypto(rv_scalar_aes::RvScalarRoundKeys), - #[cfg(target_arch = "riscv64")] - Riscv64VectorCrypto(rv_aes::RvRoundKeys), - /// Hamburg vperm via `vrgather.vv` with the table-free portable key schedule. + ScalarCrypto(rv_scalar_aes::RvScalarRoundKeys), #[cfg(target_arch = "riscv64")] - #[allow(dead_code)] // V-only AES is kept for GCM-SIV and explicit diagnostic paths; GCM does not select it yet. - Riscv64Vperm([u32; EXPANDED_KEY_WORDS]), + VectorCrypto(rv_aes::RvRoundKeys), /// Four-block table-free fixslice fallback for scalar RV64 without AES extensions. #[cfg(all(target_arch = "riscv64", feature = "alloc"))] - Riscv64Fixslice(alloc::boxed::Box), + Fixslice(alloc::boxed::Box), /// No-alloc RV64 builds keep the larger fixslice key schedule inline. #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] - Riscv64Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys), + Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys), } impl Drop for Aes256EncKey { fn drop(&mut self) { match &mut self.inner { + #[cfg(not(target_arch = "riscv64"))] KeyInner::PortableRoundKeys(rk) => { // SAFETY: [u32; 60] is layout-compatible with [u8; 240]. crate::traits::ct::zeroize(unsafe { @@ -161,23 +148,15 @@ impl Drop for Aes256EncKey { ppc_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64ScalarCrypto(rv_rk) => { + KeyInner::ScalarCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64VectorCrypto(rv_rk) => { + KeyInner::VectorCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Vperm(rk) => { - // SAFETY: the expanded key is a contiguous `[u32; 60]`, so viewing it as - // a mutable byte slice for zeroization is valid for its exact size. - crate::traits::ct::zeroize(unsafe { - core::slice::from_raw_parts_mut(rk.as_mut_ptr().cast::(), EXPANDED_KEY_WORDS.strict_mul(4)) - }); - } - #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Fixslice(rk) => { + KeyInner::Fixslice(rk) => { rk.zeroize(); } } @@ -195,7 +174,7 @@ pub(crate) struct Aes128EncKey { } enum Key128Inner { - #[allow(dead_code)] + #[cfg(not(target_arch = "riscv64"))] PortableRoundKeys([u32; EXPANDED_KEY_WORDS_128]), #[cfg(target_arch = "x86_64")] X86AesNi(ni::Ni128RoundKeys), @@ -206,24 +185,21 @@ enum Key128Inner { #[cfg(target_arch = "powerpc64")] Power8Crypto(ppc::Ppc128RoundKeys), #[cfg(target_arch = "riscv64")] - Riscv64ScalarCrypto(rv_scalar_aes::RvScalar128RoundKeys), + ScalarCrypto(rv_scalar_aes::RvScalar128RoundKeys), #[cfg(target_arch = "riscv64")] - Riscv64VectorCrypto(rv_aes::Rv128RoundKeys), - /// Hamburg vperm via `vrgather.vv` with the table-free portable key schedule. - #[cfg(target_arch = "riscv64")] - #[allow(dead_code)] // V-only AES is kept for GCM-SIV and explicit diagnostic paths; GCM does not select it yet. - Riscv64Vperm([u32; EXPANDED_KEY_WORDS_128]), + VectorCrypto(rv_aes::Rv128RoundKeys), /// Four-block table-free fixslice fallback for scalar RV64 without AES extensions. #[cfg(all(target_arch = "riscv64", feature = "alloc"))] - Riscv64Fixslice(alloc::boxed::Box), + Fixslice(alloc::boxed::Box), /// No-alloc RV64 builds keep the larger fixslice key schedule inline. #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] - Riscv64Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys), + Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys), } impl Drop for Aes128EncKey { fn drop(&mut self) { match &mut self.inner { + #[cfg(not(target_arch = "riscv64"))] Key128Inner::PortableRoundKeys(rk) => { // SAFETY: [u32; 44] is layout-compatible with [u8; 176]. crate::traits::ct::zeroize(unsafe { @@ -243,22 +219,15 @@ impl Drop for Aes128EncKey { ppc_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64ScalarCrypto(rv_rk) => { + Key128Inner::ScalarCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64VectorCrypto(rv_rk) => { + Key128Inner::VectorCrypto(rv_rk) => { rv_rk.zeroize(); } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Vperm(rk) => { - // SAFETY: [u32; 44] is layout-compatible with [u8; 176]. - crate::traits::ct::zeroize(unsafe { - core::slice::from_raw_parts_mut(rk.as_mut_ptr().cast::(), EXPANDED_KEY_WORDS_128.strict_mul(4)) - }); - } - #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Fixslice(rk) => { + Key128Inner::Fixslice(rk) => { rk.zeroize(); } } @@ -298,7 +267,7 @@ const fn gf256_mul(a: u8, b: u8) -> u8 { prod ^= (prod >> 9).wrapping_mul(0x11b << 1); prod ^= (prod >> 8).wrapping_mul(0x11b); - prod as u8 + prod.to_le_bytes()[0] } /// Square in GF(2^8). Equivalent to `gf256_mul(x, x)` but slightly cheaper. @@ -350,11 +319,8 @@ const fn sbox(x: u8) -> u8 { /// Apply SubBytes to a 32-bit word (four S-box applications). #[inline(always)] const fn sub_word(w: u32) -> u32 { - let b0 = sbox((w >> 24) as u8) as u32; - let b1 = sbox((w >> 16) as u8) as u32; - let b2 = sbox((w >> 8) as u8) as u32; - let b3 = sbox(w as u8) as u32; - (b0 << 24) | (b1 << 16) | (b2 << 8) | b3 + let [b0, b1, b2, b3] = w.to_be_bytes(); + u32::from_be_bytes([sbox(b0), sbox(b1), sbox(b2), sbox(b3)]) } /// Rotate a 32-bit word left by 8 bits. @@ -471,13 +437,13 @@ pub(crate) fn aes128_expand_key_portable(key: &[u8; KEY_SIZE_128]) -> [u32; EXPA #[cfg(all(target_arch = "riscv64", feature = "alloc"))] #[inline] fn riscv64_fixslice_key_inner(key: &[u8; KEY_SIZE]) -> KeyInner { - KeyInner::Riscv64Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixsliceRoundKeys::new(key))) + KeyInner::Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixsliceRoundKeys::new(key))) } #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] #[inline] fn riscv64_fixslice_key_inner(key: &[u8; KEY_SIZE]) -> KeyInner { - KeyInner::Riscv64Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys::new(key)) + KeyInner::Fixslice(rv_fixslice_aes::RvFixsliceRoundKeys::new(key)) } /// Expand a 256-bit AES key into round keys. @@ -529,7 +495,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { let rv_keys = rv_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); return Aes256EncKey { - inner: KeyInner::Riscv64VectorCrypto(rv_keys), + inner: KeyInner::VectorCrypto(rv_keys), }; } if crate::platform::caps().has(crate::platform::caps::riscv::ZKNE) { @@ -537,7 +503,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { let rv_keys = rv_scalar_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); return Aes256EncKey { - inner: KeyInner::Riscv64ScalarCrypto(rv_keys), + inner: KeyInner::ScalarCrypto(rv_keys), }; } Aes256EncKey { @@ -553,7 +519,7 @@ pub(crate) fn aes256_expand_key(key: &[u8; KEY_SIZE]) -> Aes256EncKey { #[cfg(all(target_arch = "riscv64", feature = "alloc"))] #[inline] fn riscv64_fixslice_key_inner_128(key: &[u8; KEY_SIZE_128]) -> Key128Inner { - Key128Inner::Riscv64Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixslice128RoundKeys::new( + Key128Inner::Fixslice(alloc::boxed::Box::new(rv_fixslice_aes::RvFixslice128RoundKeys::new( key, ))) } @@ -561,7 +527,7 @@ fn riscv64_fixslice_key_inner_128(key: &[u8; KEY_SIZE_128]) -> Key128Inner { #[cfg(all(target_arch = "riscv64", not(feature = "alloc")))] #[inline] fn riscv64_fixslice_key_inner_128(key: &[u8; KEY_SIZE_128]) -> Key128Inner { - Key128Inner::Riscv64Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys::new(key)) + Key128Inner::Fixslice(rv_fixslice_aes::RvFixslice128RoundKeys::new(key)) } /// Expand a 128-bit AES key into round keys. @@ -613,7 +579,7 @@ pub(crate) fn aes128_expand_key(key: &[u8; KEY_SIZE_128]) -> Aes128EncKey { let rv_keys = rv_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); return Aes128EncKey { - inner: Key128Inner::Riscv64VectorCrypto(rv_keys), + inner: Key128Inner::VectorCrypto(rv_keys), }; } if crate::platform::caps().has(crate::platform::caps::riscv::ZKNE) { @@ -621,7 +587,7 @@ pub(crate) fn aes128_expand_key(key: &[u8; KEY_SIZE_128]) -> Aes128EncKey { let rv_keys = rv_scalar_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); return Aes128EncKey { - inner: Key128Inner::Riscv64ScalarCrypto(rv_keys), + inner: Key128Inner::ScalarCrypto(rv_keys), }; } Aes128EncKey { @@ -641,7 +607,7 @@ pub(crate) fn aes256_expand_key_riscv_vector(key: &[u8; KEY_SIZE]) -> Aes256EncK let rv_keys = rv_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); Aes256EncKey { - inner: KeyInner::Riscv64VectorCrypto(rv_keys), + inner: KeyInner::VectorCrypto(rv_keys), } } @@ -652,15 +618,7 @@ pub(crate) fn aes256_expand_key_riscv_scalar(key: &[u8; KEY_SIZE]) -> Aes256EncK let rv_keys = rv_scalar_aes::from_portable(&portable_rk); zeroize_expanded_key_words(&mut portable_rk); Aes256EncKey { - inner: KeyInner::Riscv64ScalarCrypto(rv_keys), - } -} - -#[cfg(all(target_arch = "riscv64", feature = "aes-gcm-siv"))] -#[inline] -pub(crate) fn aes256_expand_key_riscv_vperm(key: &[u8; KEY_SIZE]) -> Aes256EncKey { - Aes256EncKey { - inner: KeyInner::Riscv64Vperm(aes256_expand_key_portable(key)), + inner: KeyInner::ScalarCrypto(rv_keys), } } @@ -680,7 +638,7 @@ pub(crate) fn aes128_expand_key_riscv_vector(key: &[u8; KEY_SIZE_128]) -> Aes128 let rv_keys = rv_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); Aes128EncKey { - inner: Key128Inner::Riscv64VectorCrypto(rv_keys), + inner: Key128Inner::VectorCrypto(rv_keys), } } @@ -691,15 +649,7 @@ pub(crate) fn aes128_expand_key_riscv_scalar(key: &[u8; KEY_SIZE_128]) -> Aes128 let rv_keys = rv_scalar_aes::from_portable_128(&portable_rk); zeroize_expanded_key_words_128(&mut portable_rk); Aes128EncKey { - inner: Key128Inner::Riscv64ScalarCrypto(rv_keys), - } -} - -#[cfg(all(target_arch = "riscv64", feature = "aes-gcm-siv"))] -#[inline] -pub(crate) fn aes128_expand_key_riscv_vperm(key: &[u8; KEY_SIZE_128]) -> Aes128EncKey { - Aes128EncKey { - inner: Key128Inner::Riscv64Vperm(aes128_expand_key_portable(key)), + inner: Key128Inner::ScalarCrypto(rv_keys), } } @@ -800,11 +750,32 @@ unsafe fn aarch64_encrypt_blocks_inline(keys: &ce::CeRoundKeys, blocks: &mut [[u } } +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] +#[inline] +fn gcmsiv_derive_keys_fallback(master_ek: &Aes256EncKey, nonce: &[u8; 12]) -> ([u8; 16], [u8; 32]) { + let mut blocks = [[0u8; BLOCK_SIZE]; 6]; + for (counter, block) in (0u32..6).zip(&mut blocks) { + block[..4].copy_from_slice(&counter.to_le_bytes()); + block[4..].copy_from_slice(nonce); + } + aes256_encrypt_blocks_ecb(master_ek, &mut blocks); + + let mut auth_key = [0u8; 16]; + let mut enc_key = [0u8; 32]; + auth_key[..8].copy_from_slice(&blocks[0][..8]); + auth_key[8..].copy_from_slice(&blocks[1][..8]); + enc_key[..8].copy_from_slice(&blocks[2][..8]); + enc_key[8..16].copy_from_slice(&blocks[3][..8]); + enc_key[16..24].copy_from_slice(&blocks[4][..8]); + enc_key[24..].copy_from_slice(&blocks[5][..8]); + crate::traits::ct::zeroize(blocks.as_flattened_mut()); + (auth_key, enc_key) +} + /// Derive AES-256-GCM-SIV per-message keys directly with AES-CE. /// /// # Safety -/// Caller must ensure AES-CE is available and `master_ek` is the AArch64 AES -/// backend variant. +/// Caller must ensure AES-CE is available. #[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,neon")] #[inline] @@ -813,7 +784,7 @@ pub(super) unsafe fn aarch64_gcmsiv_derive_keys_inline( nonce: &[u8; 12], ) -> ([u8; 16], [u8; 32]) { let KeyInner::Aarch64Aes(ce_rk) = &master_ek.inner else { - unreachable!("AArch64 GCM-SIV KDF requires an AES-CE master key"); + return gcmsiv_derive_keys_fallback(master_ek, nonce); }; // SAFETY: direct AES-CE GCM-SIV KDF because: // 1. This function's caller must guarantee AES-CE availability. @@ -930,11 +901,30 @@ unsafe fn aarch64_encrypt_blocks_128_inline(keys: &ce::Ce128RoundKeys, blocks: & } } +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] +#[inline] +fn gcmsiv_derive_keys_128_fallback(master_ek: &Aes128EncKey, nonce: &[u8; 12]) -> ([u8; 16], [u8; 16]) { + let mut blocks = [[0u8; BLOCK_SIZE]; 4]; + for (counter, block) in (0u32..4).zip(&mut blocks) { + block[..4].copy_from_slice(&counter.to_le_bytes()); + block[4..].copy_from_slice(nonce); + } + aes128_encrypt_blocks_ecb(master_ek, &mut blocks); + + let mut auth_key = [0u8; 16]; + let mut enc_key = [0u8; 16]; + auth_key[..8].copy_from_slice(&blocks[0][..8]); + auth_key[8..].copy_from_slice(&blocks[1][..8]); + enc_key[..8].copy_from_slice(&blocks[2][..8]); + enc_key[8..].copy_from_slice(&blocks[3][..8]); + crate::traits::ct::zeroize(blocks.as_flattened_mut()); + (auth_key, enc_key) +} + /// Derive AES-128-GCM-SIV per-message keys directly with AES-CE. /// /// # Safety -/// Caller must ensure AES-CE is available and `master_ek` is the AArch64 AES -/// backend variant. +/// Caller must ensure AES-CE is available. #[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,neon")] #[inline] @@ -943,7 +933,7 @@ pub(super) unsafe fn aarch64_gcmsiv_derive_keys_128_inline( nonce: &[u8; 12], ) -> ([u8; 16], [u8; 16]) { let Key128Inner::Aarch64Aes(ce_rk) = &master_ek.inner else { - unreachable!("AArch64 GCM-SIV KDF requires an AES-CE master key"); + return gcmsiv_derive_keys_128_fallback(master_ek, nonce); }; // SAFETY: direct AES-CE GCM-SIV KDF because: // 1. This function's caller must guarantee AES-CE availability. @@ -1679,6 +1669,7 @@ pub(super) unsafe fn s390x_encrypt_blocks_128_inline(key: &km::Km128Key, blocks: #[inline] pub(crate) fn aes256_encrypt_block(ek: &Aes256EncKey, block: &mut [u8; BLOCK_SIZE]) { match &ek.inner { + #[cfg(not(target_arch = "riscv64"))] KeyInner::PortableRoundKeys(rk) => aes256_encrypt_block_portable(rk, block), #[cfg(target_arch = "x86_64")] KeyInner::X86AesNi(ni_rk) => { @@ -1701,27 +1692,21 @@ pub(crate) fn aes256_encrypt_block(ek: &Aes256EncKey, block: &mut [u8; BLOCK_SIZ unsafe { ppc::encrypt_block(ppc_rk, block) } } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64ScalarCrypto(rv_rk) => { + KeyInner::ScalarCrypto(rv_rk) => { // SAFETY: RvScalar variant is only constructed after runtime detection confirms Zkne. unsafe { rv_scalar_aes::encrypt_block(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64VectorCrypto(rv_rk) => { + KeyInner::VectorCrypto(rv_rk) => { // SAFETY: RvAes variant is only constructed after runtime detection confirms Zvkned. unsafe { rv_aes::encrypt_block(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Vperm(rk) => { - // SAFETY: RvVperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_block(rk, block) } - } - #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64Fixslice(rk) => rv_fixslice_aes::encrypt_block(rk, block), + KeyInner::Fixslice(rk) => rv_fixslice_aes::encrypt_block(rk, block), } } -#[cfg(any(target_arch = "riscv64", test))] -#[allow(dead_code)] +#[cfg(any(target_arch = "riscv64", all(test, not(target_arch = "s390x"))))] #[inline] pub(super) fn aes_enc_round_4_fixslice(blocks: &mut [[u8; BLOCK_SIZE]; 4], round_keys: &[[u8; BLOCK_SIZE]; 4]) { rv_fixslice_aes::cipher_round_4(blocks, round_keys); @@ -1736,6 +1721,7 @@ pub(super) fn aes_enc_round_4_fixslice(blocks: &mut [[u8; BLOCK_SIZE]; 4], round #[inline] pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZE]) { match &ek.inner { + #[cfg(not(target_arch = "riscv64"))] Key128Inner::PortableRoundKeys(rk) => aes128_encrypt_block_portable(rk, block), #[cfg(target_arch = "x86_64")] Key128Inner::X86AesNi(ni_rk) => { @@ -1758,22 +1744,17 @@ pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZ unsafe { ppc::encrypt_block_128(ppc_rk, block) } } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64ScalarCrypto(rv_rk) => { - // SAFETY: Riscv64ScalarCrypto variant is only constructed after runtime detection confirms Zkne. + Key128Inner::ScalarCrypto(rv_rk) => { + // SAFETY: ScalarCrypto is only constructed after runtime detection confirms Zkne. unsafe { rv_scalar_aes::encrypt_block_128(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64VectorCrypto(rv_rk) => { - // SAFETY: Riscv64VectorCrypto variant is only constructed after runtime detection confirms Zvkned. + Key128Inner::VectorCrypto(rv_rk) => { + // SAFETY: VectorCrypto is only constructed after runtime detection confirms Zvkned. unsafe { rv_aes::encrypt_block_128(rv_rk, block) } } #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Vperm(rk) => { - // SAFETY: Riscv64Vperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_block_128(rk, block) } - } - #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64Fixslice(rk) => rv_fixslice_aes::encrypt_block_128(rk, block), + Key128Inner::Fixslice(rk) => rv_fixslice_aes::encrypt_block_128(rk, block), } } @@ -1783,17 +1764,22 @@ pub(crate) fn aes128_encrypt_block(ek: &Aes128EncKey, block: &mut [u8; BLOCK_SIZ /// instruction or the RV64 4-block kernels when available, otherwise calls /// the per-block dispatcher. Used by `riscv64` from the AES-128 CTR paths /// and by AES-128-GCM-SIV key derivation. -#[cfg_attr( - not(any( - target_arch = "aarch64", - target_arch = "powerpc64", - target_arch = "riscv64", - target_arch = "s390x", - feature = "aes-gcm-siv", - test - )), - allow(dead_code) -)] +// Live callers are GCM-SIV key derivation (any arch), the batch CTR path on the +// arches that have a block-batch kernel, and the unit tests below -- `mod aes` +// is also compiled for `aegis256` under `cfg(test)`. +#[cfg(any( + test, + feature = "aes-gcm-siv", + all( + feature = "aes-gcm", + any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x" + ) + ) +))] #[inline] pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BLOCK_SIZE]]) { #[cfg(target_arch = "aarch64")] @@ -1831,17 +1817,17 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64VectorCrypto(rk) = &ek.inner { + if let Key128Inner::VectorCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zvkned AES-128 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64VectorCrypto` is only constructed after runtime detection confirms Zvkned. + // 2. `VectorCrypto` is only constructed after runtime detection confirms Zvkned. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64VectorCrypto` proves Zvkned availability for this key. + // SAFETY: `VectorCrypto` proves Zvkned availability for this key. unsafe { rv_aes::encrypt_4blocks_128(rk, batch) }; offset = offset.strict_add(4); } @@ -1853,17 +1839,17 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64ScalarCrypto(rk) = &ek.inner { + if let Key128Inner::ScalarCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zkne AES-128 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64ScalarCrypto` is only constructed after runtime detection confirms Zkne. + // 2. `ScalarCrypto` is only constructed after runtime detection confirms Zkne. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64ScalarCrypto` proves Zkne availability for this key. + // SAFETY: `ScalarCrypto` proves Zkne availability for this key. unsafe { rv_scalar_aes::encrypt_4blocks_128(rk, batch) }; offset = offset.strict_add(4); } @@ -1875,26 +1861,7 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64Vperm(rk) = &ek.inner { - let mut offset = 0usize; - while offset.strict_add(4) <= blocks.len() { - let batch_slice = &mut blocks[offset..offset.strict_add(4)]; - debug_assert_eq!(batch_slice.len(), 4); - // SAFETY: `batch_slice` is exactly 4 contiguous `[u8; 16]` elements. - let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: Riscv64Vperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_4blocks_128(rk, batch) }; - offset = offset.strict_add(4); - } - while offset < blocks.len() { - // SAFETY: same V-extension guarantee as the wide path above. - unsafe { rv_vperm_aes::encrypt_block_128(rk, &mut blocks[offset]) }; - offset = offset.strict_add(1); - } - return; - } - #[cfg(target_arch = "riscv64")] - if let Key128Inner::Riscv64Fixslice(rk) = &ek.inner { + if let Key128Inner::Fixslice(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; @@ -1928,6 +1895,7 @@ pub(crate) fn aes128_encrypt_blocks_ecb(ek: &Aes128EncKey, blocks: &mut [[u8; BL /// Portable AES-128 block encryption (10 rounds). #[inline] +#[cfg(any(not(target_arch = "riscv64"), test))] fn aes128_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS_128], block: &mut [u8; BLOCK_SIZE]) { // Load state as four big-endian u32 columns. let mut s0 = u32::from_be_bytes([block[0], block[1], block[2], block[3]]); @@ -1973,17 +1941,22 @@ fn aes128_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS_128], block: &mut /// On s390x this issues a single KM instruction for all `blocks`, /// avoiding per-block parameter-block setup overhead. On other platforms /// falls back to per-block dispatch. -#[cfg_attr( - not(any( - target_arch = "aarch64", - target_arch = "powerpc64", - target_arch = "riscv64", - target_arch = "s390x", - feature = "aes-gcm-siv", - test - )), - allow(dead_code) -)] +// Live callers are GCM-SIV key derivation (any arch), the batch CTR path on the +// arches that have a block-batch kernel, and the unit tests below -- `mod aes` +// is also compiled for `aegis256` under `cfg(test)`. +#[cfg(any( + test, + feature = "aes-gcm-siv", + all( + feature = "aes-gcm", + any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x" + ) + ) +))] #[inline] pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BLOCK_SIZE]]) { #[cfg(target_arch = "aarch64")] @@ -2021,17 +1994,17 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64VectorCrypto(rk) = &ek.inner { + if let KeyInner::VectorCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zvkned AES-256 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64VectorCrypto` is only constructed after runtime detection confirms Zvkned. + // 2. `VectorCrypto` is only constructed after runtime detection confirms Zvkned. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64VectorCrypto` proves Zvkned availability for this key. + // SAFETY: `VectorCrypto` proves Zvkned availability for this key. unsafe { rv_aes::encrypt_4blocks(rk, batch) }; offset = offset.strict_add(4); } @@ -2043,17 +2016,17 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64ScalarCrypto(rk) = &ek.inner { + if let KeyInner::ScalarCrypto(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; debug_assert_eq!(batch_slice.len(), 4); // SAFETY: exact four-block RISC-V Zkne AES-256 batch because: // 1. `batch_slice` is sliced to exactly four contiguous `[u8; 16]` elements. - // 2. `Riscv64ScalarCrypto` is only constructed after runtime detection confirms Zkne. + // 2. `ScalarCrypto` is only constructed after runtime detection confirms Zkne. // 3. The mutable borrow is scoped to this loop iteration. let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: `Riscv64ScalarCrypto` proves Zkne availability for this key. + // SAFETY: `ScalarCrypto` proves Zkne availability for this key. unsafe { rv_scalar_aes::encrypt_4blocks(rk, batch) }; offset = offset.strict_add(4); } @@ -2065,28 +2038,7 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL return; } #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64Vperm(rk) = &ek.inner { - let mut offset = 0usize; - while offset.strict_add(4) <= blocks.len() { - let batch_slice = &mut blocks[offset..offset.strict_add(4)]; - debug_assert_eq!(batch_slice.len(), 4); - // SAFETY: `batch_slice` is exactly 4 contiguous `[u8; 16]` elements, so - // reborrowing it as `&mut [[u8; 16]; 4]` preserves layout and bounds. - let batch: &mut [[u8; BLOCK_SIZE]; 4] = unsafe { &mut *batch_slice.as_mut_ptr().cast::<[[u8; BLOCK_SIZE]; 4]>() }; - // SAFETY: RvVperm variant is only constructed after runtime detection confirms V extension. - unsafe { rv_vperm_aes::encrypt_4blocks(rk, batch) }; - offset = offset.strict_add(4); - } - while offset < blocks.len() { - // SAFETY: Same runtime V-extension guarantee as above; tail stays on the - // existing single-block kernel to avoid special-casing 1-3 blocks. - unsafe { rv_vperm_aes::encrypt_block(rk, &mut blocks[offset]) }; - offset = offset.strict_add(1); - } - return; - } - #[cfg(target_arch = "riscv64")] - if let KeyInner::Riscv64Fixslice(rk) = &ek.inner { + if let KeyInner::Fixslice(rk) = &ek.inner { let mut offset = 0usize; while offset.strict_add(4) <= blocks.len() { let batch_slice = &mut blocks[offset..offset.strict_add(4)]; @@ -2120,6 +2072,7 @@ pub(crate) fn aes256_encrypt_blocks_ecb(ek: &Aes256EncKey, blocks: &mut [[u8; BL /// Portable AES-256 block encryption. #[inline] +#[cfg(any(not(target_arch = "riscv64"), test))] fn aes256_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS], block: &mut [u8; BLOCK_SIZE]) { // Load state as four big-endian u32 columns. let mut s0 = u32::from_be_bytes([block[0], block[1], block[2], block[3]]); @@ -2162,12 +2115,14 @@ fn aes256_encrypt_block_portable(rk: &[u32; EXPANDED_KEY_WORDS], block: &mut [u8 /// Extract byte `row` from a big-endian column word. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn col_byte(col: u32, row: usize) -> u8 { - (col >> (24u32.strict_sub((row as u32).strict_mul(8)))) as u8 + (col >> 24usize.strict_sub(row.strict_mul(8))).to_le_bytes()[0] } /// xtime: multiply by x in GF(2^8), i.e. x << 1 with conditional reduction. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn xtime(x: u8) -> u8 { let hi = (x >> 7) & 1; (x << 1) ^ (hi.wrapping_mul(0x1b)) @@ -2178,6 +2133,7 @@ const fn xtime(x: u8) -> u8 { /// Input/output: four column words in big-endian byte order. /// AddRoundKey is done by the caller. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn aes_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, u32) { // After SubBytes + ShiftRows, column j contains: // row 0 from column j, row 1 from (j+1)%4, row 2 from (j+2)%4, row 3 from (j+3)%4 @@ -2211,6 +2167,7 @@ const fn aes_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, u32) { /// Final AES round: SubBytes → ShiftRows (no MixColumns). #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn aes_final_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, u32) { let t0 = (sbox(col_byte(s0, 0)) as u32) << 24 | (sbox(col_byte(s1, 1)) as u32) << 16 @@ -2234,6 +2191,7 @@ const fn aes_final_round(s0: u32, s1: u32, s2: u32, s3: u32) -> (u32, u32, u32, /// MixColumns on a single column [b0, b1, b2, b3]. #[inline(always)] +#[cfg(any(not(target_arch = "riscv64"), test))] const fn mix_column(col: [u8; 4]) -> u32 { let [b0, b1, b2, b3] = col; @@ -2307,10 +2265,7 @@ pub(crate) fn aes256_ctr32_encrypt(ek: &Aes256EncKey, initial_counter: &[u8; BLO #[cfg(target_arch = "riscv64")] if matches!( &ek.inner, - KeyInner::Riscv64VectorCrypto(_) - | KeyInner::Riscv64ScalarCrypto(_) - | KeyInner::Riscv64Vperm(_) - | KeyInner::Riscv64Fixslice(_) + KeyInner::VectorCrypto(_) | KeyInner::ScalarCrypto(_) | KeyInner::Fixslice(_) ) { let iv_suffix: [u8; 12] = { let mut buf = [0u8; 12]; @@ -2367,7 +2322,7 @@ pub(crate) fn aes256_ctr32_encrypt(ek: &Aes256EncKey, initial_counter: &[u8; BLO aes256_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -2416,10 +2371,7 @@ pub(crate) fn aes128_ctr32_encrypt(ek: &Aes128EncKey, initial_counter: &[u8; BLO #[cfg(target_arch = "riscv64")] if matches!( &ek.inner, - Key128Inner::Riscv64VectorCrypto(_) - | Key128Inner::Riscv64ScalarCrypto(_) - | Key128Inner::Riscv64Vperm(_) - | Key128Inner::Riscv64Fixslice(_) + Key128Inner::VectorCrypto(_) | Key128Inner::ScalarCrypto(_) | Key128Inner::Fixslice(_) ) { let iv_suffix: [u8; 12] = { let mut buf = [0u8; 12]; @@ -2476,7 +2428,7 @@ pub(crate) fn aes128_ctr32_encrypt(ek: &Aes128EncKey, initial_counter: &[u8; BLO aes128_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -2528,10 +2480,8 @@ fn aes256_ctr32_be_uses_block_batch(ek: &Aes256EncKey) -> bool { #[cfg(target_arch = "s390x")] KeyInner::S390xMsa(_) => true, #[cfg(target_arch = "riscv64")] - KeyInner::Riscv64VectorCrypto(_) - | KeyInner::Riscv64ScalarCrypto(_) - | KeyInner::Riscv64Vperm(_) - | KeyInner::Riscv64Fixslice(_) => true, + KeyInner::VectorCrypto(_) | KeyInner::ScalarCrypto(_) | KeyInner::Fixslice(_) => true, + #[cfg(not(target_arch = "riscv64"))] _ => false, } } @@ -2555,10 +2505,8 @@ fn aes128_ctr32_be_uses_block_batch(ek: &Aes128EncKey) -> bool { #[cfg(target_arch = "s390x")] Key128Inner::S390xMsa(_) => true, #[cfg(target_arch = "riscv64")] - Key128Inner::Riscv64VectorCrypto(_) - | Key128Inner::Riscv64ScalarCrypto(_) - | Key128Inner::Riscv64Vperm(_) - | Key128Inner::Riscv64Fixslice(_) => true, + Key128Inner::VectorCrypto(_) | Key128Inner::ScalarCrypto(_) | Key128Inner::Fixslice(_) => true, + #[cfg(not(target_arch = "riscv64"))] _ => false, } } @@ -2674,7 +2622,7 @@ pub(crate) fn aes256_ctr32_encrypt_be(ek: &Aes256EncKey, initial_counter: &[u8; aes256_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -2724,13 +2672,13 @@ unsafe fn x86_gcm_ctr_blocks_be_4(iv_words: [u32; 3], ctr: u32) -> core::arch::x let c1 = ctr.wrapping_add(1).swap_bytes(); let c2 = ctr.wrapping_add(2).swap_bytes(); let c3 = ctr.wrapping_add(3).swap_bytes(); - let iv0 = iv_words[0] as i32; - let iv1 = iv_words[1] as i32; - let iv2 = iv_words[2] as i32; - let b0 = _mm_set_epi32(c0 as i32, iv2, iv1, iv0); - let b1 = _mm_set_epi32(c1 as i32, iv2, iv1, iv0); - let b2 = _mm_set_epi32(c2 as i32, iv2, iv1, iv0); - let b3 = _mm_set_epi32(c3 as i32, iv2, iv1, iv0); + let iv0 = iv_words[0].cast_signed(); + let iv1 = iv_words[1].cast_signed(); + let iv2 = iv_words[2].cast_signed(); + let b0 = _mm_set_epi32(c0.cast_signed(), iv2, iv1, iv0); + let b1 = _mm_set_epi32(c1.cast_signed(), iv2, iv1, iv0); + let b2 = _mm_set_epi32(c2.cast_signed(), iv2, iv1, iv0); + let b3 = _mm_set_epi32(c3.cast_signed(), iv2, iv1, iv0); let z = _mm512_zextsi128_si512(b0); let z = _mm512_inserti32x4(z, b1, 1); @@ -2738,6 +2686,10 @@ unsafe fn x86_gcm_ctr_blocks_be_4(iv_words: [u32; 3], ctr: u32) -> core::arch::x _mm512_inserti32x4(z, b3, 3) } +/// Build sixteen big-endian GCM counter blocks in four VAES registers. +/// +/// # Safety +/// Caller must ensure AVX-512F and AVX-512BW are available. #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] #[target_feature(enable = "avx512f,avx512bw")] #[inline] @@ -2752,9 +2704,14 @@ unsafe fn x86_gcm_ctr_blocks_be_16( ) { use core::arch::x86_64::*; - let iv = _mm_set_epi32(0, iv_words[2] as i32, iv_words[1] as i32, iv_words[0] as i32); + let iv = _mm_set_epi32( + 0, + iv_words[2].cast_signed(), + iv_words[1].cast_signed(), + iv_words[0].cast_signed(), + ); let template = _mm512_broadcast_i32x4(iv); - let ctrs = _mm512_set1_epi32(ctr as i32); + let ctrs = _mm512_set1_epi32(ctr.cast_signed()); let bswap = _mm512_broadcast_i32x4(_mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)); let offsets0 = _mm512_set_epi32(3, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0); @@ -2775,54 +2732,21 @@ unsafe fn x86_gcm_ctr_blocks_be_16( #[cfg(all(target_arch = "x86_64", feature = "aes-gcm", test))] #[target_feature(enable = "avx2")] #[inline] +/// # Safety +/// Caller must ensure AVX2 is available. unsafe fn x86_gcm_ctr_blocks_be_2(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_64::__m256i { use core::arch::x86_64::*; - let p0 = iv_words[0] as i32; - let p1 = iv_words[1] as i32; - let p2 = iv_words[2] as i32; - let b0 = _mm_set_epi32(ctr.to_be() as i32, p2, p1, p0); - let b1 = _mm_set_epi32(ctr.wrapping_add(1).to_be() as i32, p2, p1, p0); + let p0 = iv_words[0].cast_signed(); + let p1 = iv_words[1].cast_signed(); + let p2 = iv_words[2].cast_signed(); + let b0 = _mm_set_epi32(ctr.to_be().cast_signed(), p2, p1, p0); + let b1 = _mm_set_epi32(ctr.wrapping_add(1).to_be().cast_signed(), p2, p1, p0); let z = _mm256_castsi128_si256(b0); _mm256_inserti128_si256(z, b1, 1) } -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "avx2")] -#[inline] -#[allow(dead_code)] -unsafe fn x86_gcm_ctr_blocks_be_8_y256( - iv_words: [u32; 3], - ctr: u32, -) -> ( - core::arch::x86_64::__m256i, - core::arch::x86_64::__m256i, - core::arch::x86_64::__m256i, - core::arch::x86_64::__m256i, -) { - use core::arch::x86_64::*; - - let iv = _mm_set_epi32(0, iv_words[2] as i32, iv_words[1] as i32, iv_words[0] as i32); - let template = _mm256_broadcastsi128_si256(iv); - let ctrs = _mm256_set1_epi32(ctr as i32); - let bswap = _mm256_broadcastsi128_si256(_mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)); - - let offsets0 = _mm256_set_epi32(1, 0, 0, 0, 0, 0, 0, 0); - let offsets1 = _mm256_set_epi32(3, 0, 0, 0, 2, 0, 0, 0); - let offsets2 = _mm256_set_epi32(5, 0, 0, 0, 4, 0, 0, 0); - let offsets3 = _mm256_set_epi32(7, 0, 0, 0, 6, 0, 0, 0); - - macro_rules! make { - ($offsets:expr) => {{ - let be = _mm256_shuffle_epi8(_mm256_add_epi32(ctrs, $offsets), bswap); - _mm256_blend_epi32::<0x88>(template, be) - }}; - } - - (make!(offsets0), make!(offsets1), make!(offsets2), make!(offsets3)) -} - /// Build one big-endian GCM counter block directly in an XMM register. /// /// # Safety @@ -2834,10 +2758,10 @@ unsafe fn x86_gcm_ctr_block_be(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_ use core::arch::x86_64::*; _mm_set_epi32( - ctr.swap_bytes() as i32, - iv_words[2] as i32, - iv_words[1] as i32, - iv_words[0] as i32, + ctr.swap_bytes().cast_signed(), + iv_words[2].cast_signed(), + iv_words[1].cast_signed(), + iv_words[0].cast_signed(), ) } @@ -2851,13 +2775,13 @@ unsafe fn x86_gcm_ctr_block_be(iv_words: [u32; 3], ctr: u32) -> core::arch::x86_ unsafe fn x86_gcmsiv_ctr_blocks_le_4(suffix_words: [u32; 3], ctr: u32) -> core::arch::x86_64::__m512i { use core::arch::x86_64::*; - let s0 = suffix_words[0] as i32; - let s1 = suffix_words[1] as i32; - let s2 = suffix_words[2] as i32; - let b0 = _mm_set_epi32(s2, s1, s0, ctr as i32); - let b1 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(1) as i32); - let b2 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(2) as i32); - let b3 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(3) as i32); + let s0 = suffix_words[0].cast_signed(); + let s1 = suffix_words[1].cast_signed(); + let s2 = suffix_words[2].cast_signed(); + let b0 = _mm_set_epi32(s2, s1, s0, ctr.cast_signed()); + let b1 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(1).cast_signed()); + let b2 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(2).cast_signed()); + let b3 = _mm_set_epi32(s2, s1, s0, ctr.wrapping_add(3).cast_signed()); let z = _mm512_zextsi128_si512(b0); let z = _mm512_inserti32x4(z, b1, 1); @@ -3184,7 +3108,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes256_gcm_seal_64x_vaes512_x86_64_linux( @@ -3193,7 +3117,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes256_gcm_seal_16x_vaes512_x86_64_linux( @@ -3202,7 +3126,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -3360,7 +3284,7 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes256_gcm_open_64x_vaes512_x86_64_linux( @@ -3369,7 +3293,7 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes256_gcm_open_16x_vaes512_x86_64_linux( @@ -3378,7 +3302,7 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -3465,39 +3389,33 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_wide_ghash( } } -/// AES-256 CTR encryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. +/// AES-256 CTR encryption fused with PMULL GHASH accumulation for GCM sealing. /// -/// This avoids ZMM data-path pressure on AMD while preserving the fused -/// counter/AES/XOR/GHASH structure used by the 512-bit path. +/// Encrypts `data` in place and returns the GHASH accumulator after the +/// ciphertext has been folded in. The incoming `acc` is normally the GHASH +/// state after AAD processing. /// /// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( +/// Caller must ensure AES-CE and PMULL are available. +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] +#[target_feature(enable = "aes,neon")] +pub(crate) unsafe fn aes256_ctr32_encrypt_be_aarch64_ghash( ek: &Aes256EncKey, initial_counter: &[u8; BLOCK_SIZE], data: &mut [u8], mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], + tables: &Aarch64GcmTables<'_>, ) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-256 AES-GCM sealing because: - // 1. This function's caller guarantees all required x86 target features. - // 2. `data` is a valid mutable byte slice; all pointer arithmetic stays inside checked chunk - // bounds. - // 3. GHASH folds ciphertext registers after encryption, matching GCM authentication semantics. + // SAFETY: fused aarch64 AES-GCM sealing because: + // 1. This function's caller guarantees AES-CE and PMULL availability. + // 2. `data` is a valid mutable byte slice; all chunk processing uses checked slice ranges and the + // tail path handles partial blocks. unsafe { - let ni_rk = match &ek.inner { - KeyInner::X86AesNi(rk) => rk, + let ce_rk = match &ek.inner { + KeyInner::Aarch64Aes(rk) => rk, _ => { aes256_ctr32_encrypt_be(ek, initial_counter, data); - return ghash_ciphertext_fallback(acc, h_polyval, data); + return ghash_ciphertext_fallback(acc, tables.h_polyval, data); } }; @@ -3506,11 +3424,6 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( buf.copy_from_slice(&initial_counter[..12]); buf }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; let mut ctr = u32::from_be_bytes([ initial_counter[12], initial_counter[13], @@ -3519,89 +3432,28 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( ]); let mut offset = 0usize; - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-256-GCM seal kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 15 initialized 128-bit AES-256 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable bytes. - // 4. `h_powers_rev_8` contains exactly the [H^8..H] powers required by the 8-block fold. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes256_gcm_seal_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let p0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c0 = _mm256_xor_si256(p0, ks0); - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), c0); - - let p1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c1 = _mm256_xor_si256(p1, ks1); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(32)).cast(), c1); - - let p2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c2 = _mm256_xor_si256(p2, ks2); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(64)).cast(), c2); - - let p3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - let c3 = _mm256_xor_si256(p3, ks3); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(96)).cast(), c3); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); + if offset.strict_add(128) <= data.len() { + let state = ce::encrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); + acc = state.0; + ctr = state.1; + offset = state.2; } while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let p0 = _mm_loadu_si128(ptr.cast()); - let p1 = _mm_loadu_si128(ptr.add(16).cast()); - let p2 = _mm_loadu_si128(ptr.add(32).cast()); - let p3 = _mm_loadu_si128(ptr.add(48).cast()); - let c0 = _mm_xor_si128(p0, ks0); - let c1 = _mm_xor_si128(p1, ks1); - let c2 = _mm_xor_si128(p2, ks2); - let c3 = _mm_xor_si128(p3, ks3); - _mm_storeu_si128(ptr.cast(), c0); - _mm_storeu_si128(ptr.add(16).cast(), c1); - _mm_storeu_si128(ptr.add(32).cast(), c2); - _mm_storeu_si128(ptr.add(48).cast(), c3); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - + let end = offset.strict_add(64); + let blocks = ce::encrypt_ctr32_be_xor_4blocks_core(ce_rk, &iv_prefix, ctr, &mut data[offset..end]); + acc = super::polyval::aarch64_aggregate_4blocks_inline(acc, tables.h_powers_rev, &blocks); ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); + offset = end; } while offset < data.len() { - let mut counter_block = [0u8; 16]; + let mut counter_block = [0u8; BLOCK_SIZE]; counter_block[..12].copy_from_slice(&iv_prefix); counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); let mut keystream = counter_block; - ni::encrypt_block(ni_rk, &mut keystream); + ce::encrypt_block_core(ce_rk, &mut keystream); let remaining = data.len().strict_sub(offset); if remaining >= BLOCK_SIZE { @@ -3612,7 +3464,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( let ciphertext = xored.to_ne_bytes(); data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&ciphertext); acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); + acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); offset = offset.strict_add(BLOCK_SIZE); } else { let mut block = [0u8; BLOCK_SIZE]; @@ -3623,7 +3475,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( i = i.strict_add(1); } acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); + acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); offset = offset.strict_add(remaining); } ctr = ctr.wrapping_add(1); @@ -3633,34 +3485,33 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_be_y256_ghash( } } -/// AES-256 CTR decryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. +/// AES-256 CTR decryption fused with PMULL GHASH accumulation for GCM open. +/// +/// Folds ciphertext into GHASH, decrypts the same chunk in place, and returns +/// the accumulator. Authentication is still finalized and checked by the +/// caller, which zeroizes the buffer on failure. /// /// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes256_ctr32_decrypt_be_y256_ghash( +/// Caller must ensure AES-CE and PMULL are available. +#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] +#[target_feature(enable = "aes,neon")] +pub(crate) unsafe fn aes256_ctr32_decrypt_be_aarch64_ghash( ek: &Aes256EncKey, initial_counter: &[u8; BLOCK_SIZE], data: &mut [u8], mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], + tables: &Aarch64GcmTables<'_>, ) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-256 AES-GCM opening because: - // 1. This function's caller guarantees all required x86 target features. - // 2. Ciphertext registers are folded into GHASH before plaintext is stored back. - // 3. All pointer arithmetic stays inside checked chunk bounds. + // SAFETY: fused aarch64 AES-GCM opening because: + // 1. This function's caller guarantees AES-CE and PMULL availability. + // 2. `data` is a valid mutable ciphertext slice. Ciphertext blocks are copied before in-place + // decryption, so GHASH always authenticates the original bytes. + // 3. All chunk processing uses checked slice bounds and the tail path handles partial blocks. unsafe { - let ni_rk = match &ek.inner { - KeyInner::X86AesNi(rk) => rk, + let ce_rk = match &ek.inner { + KeyInner::Aarch64Aes(rk) => rk, _ => { - acc = ghash_ciphertext_fallback(acc, h_polyval, data); + acc = ghash_ciphertext_fallback(acc, tables.h_polyval, data); aes256_ctr32_encrypt_be(ek, initial_counter, data); return acc; } @@ -3671,275 +3522,16 @@ pub(crate) unsafe fn aes256_ctr32_decrypt_be_y256_ghash( buf.copy_from_slice(&initial_counter[..12]); buf }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; let mut ctr = u32::from_be_bytes([ initial_counter[12], initial_counter[13], initial_counter[14], initial_counter[15], ]); - let mut offset = 0usize; - - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-256-GCM open kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 15 initialized 128-bit AES-256 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable ciphertext bytes. - // 4. The kernel folds ciphertext into GHASH before storing plaintext. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let c0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), _mm256_xor_si256(c0, ks0)); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(32)).cast(), - _mm256_xor_si256(c1, ks1), - ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(64)).cast(), - _mm256_xor_si256(c2, ks2), - ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(96)).cast(), - _mm256_xor_si256(c3, ks3), - ); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); - } - - while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let c0 = _mm_loadu_si128(ptr.cast()); - let c1 = _mm_loadu_si128(ptr.add(16).cast()); - let c2 = _mm_loadu_si128(ptr.add(32).cast()); - let c3 = _mm_loadu_si128(ptr.add(48).cast()); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - _mm_storeu_si128(ptr.cast(), _mm_xor_si128(c0, ks0)); - _mm_storeu_si128(ptr.add(16).cast(), _mm_xor_si128(c1, ks1)); - _mm_storeu_si128(ptr.add(32).cast(), _mm_xor_si128(c2, ks2)); - _mm_storeu_si128(ptr.add(48).cast(), _mm_xor_si128(c3, ks3)); - - ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); - } - - while offset < data.len() { - let mut counter_block = [0u8; 16]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ni::encrypt_block(ni_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let mut ciphertext = [0u8; BLOCK_SIZE]; - ciphertext.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let plaintext = u128::from_ne_bytes(ciphertext) ^ u128::from_ne_bytes(keystream); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&plaintext.to_ne_bytes()); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - block[..remaining].copy_from_slice(&data[offset..offset.strict_add(remaining)]); - acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - i = i.strict_add(1); - } - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-256 CTR encryption fused with PMULL GHASH accumulation for GCM sealing. -/// -/// Encrypts `data` in place and returns the GHASH accumulator after the -/// ciphertext has been folded in. The incoming `acc` is normally the GHASH -/// state after AAD processing. -/// -/// # Safety -/// Caller must ensure AES-CE and PMULL are available. -#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,neon")] -pub(crate) unsafe fn aes256_ctr32_encrypt_be_aarch64_ghash( - ek: &Aes256EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - tables: &Aarch64GcmTables<'_>, -) -> u128 { - // SAFETY: fused aarch64 AES-GCM sealing because: - // 1. This function's caller guarantees AES-CE and PMULL availability. - // 2. `data` is a valid mutable byte slice; all chunk processing uses checked slice ranges and the - // tail path handles partial blocks. - unsafe { - let ce_rk = match &ek.inner { - KeyInner::Aarch64Aes(rk) => rk, - _ => { - aes256_ctr32_encrypt_be(ek, initial_counter, data); - return ghash_ciphertext_fallback(acc, tables.h_polyval, data); - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let mut offset = 0usize; - - if offset.strict_add(128) <= data.len() { - let state = ce::encrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); - acc = state.0; - ctr = state.1; - offset = state.2; - } - - while offset.strict_add(64) <= data.len() { - let end = offset.strict_add(64); - let blocks = ce::encrypt_ctr32_be_xor_4blocks_core(ce_rk, &iv_prefix, ctr, &mut data[offset..end]); - acc = super::polyval::aarch64_aggregate_4blocks_inline(acc, tables.h_powers_rev, &blocks); - ctr = ctr.wrapping_add(4); - offset = end; - } - - while offset < data.len() { - let mut counter_block = [0u8; BLOCK_SIZE]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ce::encrypt_block_core(ce_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let ks = u128::from_ne_bytes(keystream); - let mut d = [0u8; BLOCK_SIZE]; - d.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - let xored = u128::from_ne_bytes(d) ^ ks; - let ciphertext = xored.to_ne_bytes(); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&ciphertext); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - block[i] = data[offset.strict_add(i)]; - i = i.strict_add(1); - } - acc ^= u128::from_be_bytes(block); - acc = super::polyval::aarch64_clmul128_reduce_inline(acc, tables.h_polyval); - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-256 CTR decryption fused with PMULL GHASH accumulation for GCM open. -/// -/// Folds ciphertext into GHASH, decrypts the same chunk in place, and returns -/// the accumulator. Authentication is still finalized and checked by the -/// caller, which zeroizes the buffer on failure. -/// -/// # Safety -/// Caller must ensure AES-CE and PMULL are available. -#[cfg(all(target_arch = "aarch64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,neon")] -pub(crate) unsafe fn aes256_ctr32_decrypt_be_aarch64_ghash( - ek: &Aes256EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - tables: &Aarch64GcmTables<'_>, -) -> u128 { - // SAFETY: fused aarch64 AES-GCM opening because: - // 1. This function's caller guarantees AES-CE and PMULL availability. - // 2. `data` is a valid mutable ciphertext slice. Ciphertext blocks are copied before in-place - // decryption, so GHASH always authenticates the original bytes. - // 3. All chunk processing uses checked slice bounds and the tail path handles partial blocks. - unsafe { - let ce_rk = match &ek.inner { - KeyInner::Aarch64Aes(rk) => rk, - _ => { - acc = ghash_ciphertext_fallback(acc, tables.h_polyval, data); - aes256_ctr32_encrypt_be(ek, initial_counter, data); - return acc; - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let state = ce::decrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); - acc = state.0; - ctr = state.1; - let mut offset = state.2; + let state = ce::decrypt_ctr32_be_xor_ghash_128b_chunks_core(ce_rk, &iv_prefix, ctr, data, acc, tables); + acc = state.0; + ctr = state.1; + let mut offset = state.2; while offset.strict_add(64) <= data.len() { let end = offset.strict_add(64); @@ -4108,7 +3700,7 @@ pub(crate) fn aes128_ctr32_encrypt_be(ek: &Aes128EncKey, initial_counter: &[u8; aes128_encrypt_blocks_ecb(ek, &mut keystream[..block_count]); let processed = xor_keystream_tail(data, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } } @@ -4641,7 +4233,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes128_gcm_seal_64x_vaes512_x86_64_linux( @@ -4650,7 +4242,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes128_gcm_seal_16x_vaes512_x86_64_linux( @@ -4659,7 +4251,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -4817,7 +4409,7 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_128.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else if data.len() >= 1024 { x86_64_asm::rscrypto_aes128_gcm_open_64x_vaes512_x86_64_linux( @@ -4826,7 +4418,7 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_64.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } else { x86_64_asm::rscrypto_aes128_gcm_open_16x_vaes512_x86_64_linux( @@ -4835,7 +4427,7 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( data.as_mut_ptr(), data.len(), tables.h_powers_rev_32.as_ptr(), - &mut state, + core::ptr::from_mut(&mut state), ); } acc = state.acc(); @@ -4863,351 +4455,22 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_wide_ghash( ); _mm512_storeu_si512( data.as_mut_ptr().add(offset.strict_add(192)).cast(), - _mm512_xor_si512(c3, ks3), - ); - - ctr = ctr.wrapping_add(16); - offset = offset.strict_add(256); - } - - while offset.strict_add(64) <= data.len() { - let ctr_vec = x86_gcm_ctr_blocks_be_4(iv_words, ctr); - let keystream = ni::encrypt_4blocks_128(ni_rk, ctr_vec); - let ciphertext = _mm512_loadu_si512(data.as_ptr().add(offset).cast()); - acc = super::polyval::x86_aggregate_4blocks_be_lanes_inline(acc, h_powers_rev, ciphertext); - _mm512_storeu_si512( - data.as_mut_ptr().add(offset).cast(), - _mm512_xor_si512(ciphertext, keystream), - ); - - ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); - } - - while offset < data.len() { - let mut counter_block = [0u8; 16]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ni::encrypt_block_128(ni_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let mut ciphertext = [0u8; BLOCK_SIZE]; - ciphertext.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let plaintext = u128::from_ne_bytes(ciphertext) ^ u128::from_ne_bytes(keystream); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&plaintext.to_ne_bytes()); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - block[..remaining].copy_from_slice(&data[offset..offset.strict_add(remaining)]); - acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - i = i.strict_add(1); - } - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-128 CTR encryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes128_ctr32_encrypt_be_y256_ghash( - ek: &Aes128EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], -) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-128 AES-GCM sealing because: - // 1. This function's caller guarantees all required x86 target features. - // 2. `data` is a valid mutable byte slice; all pointer arithmetic stays inside checked chunk - // bounds. - // 3. GHASH folds ciphertext registers after encryption, matching GCM authentication semantics. - unsafe { - let ni_rk = match &ek.inner { - Key128Inner::X86AesNi(rk) => rk, - _ => { - aes128_ctr32_encrypt_be(ek, initial_counter, data); - return ghash_ciphertext_fallback(acc, h_polyval, data); - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let mut offset = 0usize; - - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-128-GCM seal kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 11 initialized 128-bit AES-128 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable bytes. - // 4. `h_powers_rev_8` contains exactly the [H^8..H] powers required by the 8-block fold. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes128_gcm_seal_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_128_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let p0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c0 = _mm256_xor_si256(p0, ks0); - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), c0); - - let p1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c1 = _mm256_xor_si256(p1, ks1); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(32)).cast(), c1); - - let p2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c2 = _mm256_xor_si256(p2, ks2); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(64)).cast(), c2); - - let p3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - let c3 = _mm256_xor_si256(p3, ks3); - _mm256_storeu_si256(data.as_mut_ptr().add(offset.strict_add(96)).cast(), c3); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); - } - - while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_128_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let p0 = _mm_loadu_si128(ptr.cast()); - let p1 = _mm_loadu_si128(ptr.add(16).cast()); - let p2 = _mm_loadu_si128(ptr.add(32).cast()); - let p3 = _mm_loadu_si128(ptr.add(48).cast()); - let c0 = _mm_xor_si128(p0, ks0); - let c1 = _mm_xor_si128(p1, ks1); - let c2 = _mm_xor_si128(p2, ks2); - let c3 = _mm_xor_si128(p3, ks3); - _mm_storeu_si128(ptr.cast(), c0); - _mm_storeu_si128(ptr.add(16).cast(), c1); - _mm_storeu_si128(ptr.add(32).cast(), c2); - _mm_storeu_si128(ptr.add(48).cast(), c3); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - - ctr = ctr.wrapping_add(4); - offset = offset.strict_add(64); - } - - while offset < data.len() { - let mut counter_block = [0u8; 16]; - counter_block[..12].copy_from_slice(&iv_prefix); - counter_block[12..16].copy_from_slice(&ctr.to_be_bytes()); - - let mut keystream = counter_block; - ni::encrypt_block_128(ni_rk, &mut keystream); - - let remaining = data.len().strict_sub(offset); - if remaining >= BLOCK_SIZE { - let ks = u128::from_ne_bytes(keystream); - let mut d = [0u8; BLOCK_SIZE]; - d.copy_from_slice(&data[offset..offset.strict_add(BLOCK_SIZE)]); - let xored = u128::from_ne_bytes(d) ^ ks; - let ciphertext = xored.to_ne_bytes(); - data[offset..offset.strict_add(BLOCK_SIZE)].copy_from_slice(&ciphertext); - acc ^= u128::from_be_bytes(ciphertext); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - offset = offset.strict_add(BLOCK_SIZE); - } else { - let mut block = [0u8; BLOCK_SIZE]; - let mut i = 0usize; - while i < remaining { - data[offset.strict_add(i)] ^= keystream[i]; - block[i] = data[offset.strict_add(i)]; - i = i.strict_add(1); - } - acc ^= u128::from_be_bytes(block); - acc = super::polyval::x86_clmul128_reduce_inline(acc, h_polyval); - offset = offset.strict_add(remaining); - } - ctr = ctr.wrapping_add(1); - } - - acc - } -} - -/// AES-128 CTR decryption fused with 256-bit VAES/VPCLMUL GHASH accumulation. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + VPCLMULQDQ + -/// PCLMULQDQ + AES + SSE2 + SSSE3. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "aes,sse2,ssse3,avx2,avx512f,avx512vl,vaes,vpclmulqdq,pclmulqdq")] -#[allow(dead_code)] -pub(crate) unsafe fn aes128_ctr32_decrypt_be_y256_ghash( - ek: &Aes128EncKey, - initial_counter: &[u8; BLOCK_SIZE], - data: &mut [u8], - mut acc: u128, - h_polyval: u128, - h_powers_rev: &[u128; 4], - h_powers_rev_8: &[u128; 8], -) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: fused x86 VAES-128 AES-GCM opening because: - // 1. This function's caller guarantees all required x86 target features. - // 2. Ciphertext registers are folded into GHASH before plaintext is stored back. - // 3. All pointer arithmetic stays inside checked chunk bounds. - unsafe { - let ni_rk = match &ek.inner { - Key128Inner::X86AesNi(rk) => rk, - _ => { - acc = ghash_ciphertext_fallback(acc, h_polyval, data); - aes128_ctr32_encrypt_be(ek, initial_counter, data); - return acc; - } - }; - - let iv_prefix: [u8; 12] = { - let mut buf = [0u8; 12]; - buf.copy_from_slice(&initial_counter[..12]); - buf - }; - let iv_words = [ - u32::from_le_bytes([iv_prefix[0], iv_prefix[1], iv_prefix[2], iv_prefix[3]]), - u32::from_le_bytes([iv_prefix[4], iv_prefix[5], iv_prefix[6], iv_prefix[7]]), - u32::from_le_bytes([iv_prefix[8], iv_prefix[9], iv_prefix[10], iv_prefix[11]]), - ]; - let mut ctr = u32::from_be_bytes([ - initial_counter[12], - initial_counter[13], - initial_counter[14], - initial_counter[15], - ]); - let mut offset = 0usize; - - #[cfg(target_os = "linux")] - if data.len() >= 128 { - let mut state = x86_64_asm::AesGcmX86State::new(acc, ctr); - // SAFETY: external x86-64 VAES-256 AES-128-GCM open kernel because: - // 1. This target-feature function is only entered after VAES, VPCLMULQDQ, AVX2, AVX-512VL, AES-NI, - // PCLMULQDQ, SSE2, and SSSE3 were selected by runtime/backend dispatch. - // 2. `ni_rk.as_ptr()` addresses 11 initialized 128-bit AES-128 round keys. - // 3. `initial_counter` points to the full 16-byte GCM counter block, and `data` is valid for - // `data.len()` mutable ciphertext bytes. - // 4. The kernel folds ciphertext into GHASH before storing plaintext. - // 5. The kernel only processes complete 128-byte chunks and reports the processed byte count so the - // Rust fallback below handles every remaining full/partial tail. - x86_64_asm::rscrypto_aes128_gcm_open_8x_vaes256_x86_64_linux( - ni_rk.as_ptr(), - initial_counter.as_ptr(), - data.as_mut_ptr(), - data.len(), - h_powers_rev_8.as_ptr(), - &mut state, - ); - acc = state.acc(); - ctr = state.ctr; - offset = state.processed; - } - - while offset.strict_add(128) <= data.len() { - let (ctr0, ctr1, ctr2, ctr3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - let (ks0, ks1, ks2, ks3) = ni::encrypt_8blocks_128_y256(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let c0 = _mm256_loadu_si256(data.as_ptr().add(offset).cast()); - let c1 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(32)).cast()); - let c2 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(64)).cast()); - let c3 = _mm256_loadu_si256(data.as_ptr().add(offset.strict_add(96)).cast()); - acc = super::polyval::x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev_8, c0, c1, c2, c3); - - _mm256_storeu_si256(data.as_mut_ptr().add(offset).cast(), _mm256_xor_si256(c0, ks0)); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(32)).cast(), - _mm256_xor_si256(c1, ks1), - ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(64)).cast(), - _mm256_xor_si256(c2, ks2), - ); - _mm256_storeu_si256( - data.as_mut_ptr().add(offset.strict_add(96)).cast(), - _mm256_xor_si256(c3, ks3), - ); - - ctr = ctr.wrapping_add(8); - offset = offset.strict_add(128); - } - - while offset.strict_add(64) <= data.len() { - let ctr0 = x86_gcm_ctr_block_be(iv_words, ctr); - let ctr1 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(1)); - let ctr2 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(2)); - let ctr3 = x86_gcm_ctr_block_be(iv_words, ctr.wrapping_add(3)); - let (ks0, ks1, ks2, ks3) = ni::encrypt_4blocks_128_aesni(ni_rk, ctr0, ctr1, ctr2, ctr3); - - let ptr = data.as_mut_ptr().add(offset); - let c0 = _mm_loadu_si128(ptr.cast()); - let c1 = _mm_loadu_si128(ptr.add(16).cast()); - let c2 = _mm_loadu_si128(ptr.add(32).cast()); - let c3 = _mm_loadu_si128(ptr.add(48).cast()); - acc = super::polyval::x86_pclmul_aggregate_4blocks_be_xmm_inline(acc, h_powers_rev, c0, c1, c2, c3); - _mm_storeu_si128(ptr.cast(), _mm_xor_si128(c0, ks0)); - _mm_storeu_si128(ptr.add(16).cast(), _mm_xor_si128(c1, ks1)); - _mm_storeu_si128(ptr.add(32).cast(), _mm_xor_si128(c2, ks2)); - _mm_storeu_si128(ptr.add(48).cast(), _mm_xor_si128(c3, ks3)); + _mm512_xor_si512(c3, ks3), + ); + + ctr = ctr.wrapping_add(16); + offset = offset.strict_add(256); + } + + while offset.strict_add(64) <= data.len() { + let ctr_vec = x86_gcm_ctr_blocks_be_4(iv_words, ctr); + let keystream = ni::encrypt_4blocks_128(ni_rk, ctr_vec); + let ciphertext = _mm512_loadu_si512(data.as_ptr().add(offset).cast()); + acc = super::polyval::x86_aggregate_4blocks_be_lanes_inline(acc, h_powers_rev, ciphertext); + _mm512_storeu_si512( + data.as_mut_ptr().add(offset).cast(), + _mm512_xor_si512(ciphertext, keystream), + ); ctr = ctr.wrapping_add(4); offset = offset.strict_add(64); @@ -5251,9 +4514,19 @@ pub(crate) unsafe fn aes128_ctr32_decrypt_be_y256_ghash( } } +#[cfg(all(target_arch = "x86_64", target_os = "linux", feature = "aes-gcm-siv"))] +#[inline(always)] +const fn usize_low_u32(value: usize) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + /// AES-256 CTR encryption using VAES-512 for the bulk, AES-NI for the tail. /// /// GCM-SIV variant: counter occupies bytes 0..3 (little-endian). +/// +/// # Safety +/// Caller must ensure AES-NI, SSE2, AVX-512F, AVX-512VL, and VAES are available. #[cfg(all(target_arch = "x86_64", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,sse2,avx512f,avx512vl,vaes")] pub(crate) unsafe fn aes256_ctr32_encrypt_wide(ek: &Aes256EncKey, initial_counter: &[u8; BLOCK_SIZE], data: &mut [u8]) { @@ -5304,7 +4577,7 @@ pub(crate) unsafe fn aes256_ctr32_encrypt_wide(ek: &Aes256EncKey, initial_counte data.as_mut_ptr(), data.len(), ); - ctr = ctr.wrapping_add((processed / BLOCK_SIZE) as u32); + ctr = ctr.wrapping_add(usize_low_u32(processed.strict_div(BLOCK_SIZE))); offset = processed; } @@ -5439,7 +4712,7 @@ pub(crate) unsafe fn aes128_ctr32_encrypt_wide(ek: &Aes128EncKey, initial_counte data.as_mut_ptr(), data.len(), ); - ctr = ctr.wrapping_add((processed / BLOCK_SIZE) as u32); + ctr = ctr.wrapping_add(usize_low_u32(processed.strict_div(BLOCK_SIZE))); offset = processed; } @@ -5544,12 +4817,12 @@ mod tests { 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16, ]; - for (i, &expected) in CANONICAL.iter().enumerate() { + for (input, &expected) in (0u8..=u8::MAX).zip(&CANONICAL) { assert_eq!( - sbox(i as u8), + sbox(input), expected, - "S-box mismatch at index {i:#04x}: got {:#04x}, expected {expected:#04x}", - sbox(i as u8), + "S-box mismatch at input {input:#04x}: got {:#04x}, expected {expected:#04x}", + sbox(input), ); } } @@ -5644,13 +4917,44 @@ mod tests { #[test] fn gf256_inv_exhaustive() { assert_eq!(gf256_inv(0), 0, "inv(0) must be 0 by AES convention"); - for x in 1u16..=255 { - let x = x as u8; + for x in 1u8..=u8::MAX { let inv = gf256_inv(x); assert_eq!(gf256_mul(x, inv), 1, "x={x:#04x}, inv={inv:#04x}: x * inv != 1"); } } + #[cfg(all(target_arch = "aarch64", feature = "aes-gcm-siv"))] + #[test] + fn aarch64_gcmsiv_kdf_portable_fallback_matches_aes_ce() { + if !crate::platform::caps().has(crate::platform::caps::aarch64::AES) { + return; + } + + let nonce = *b"kdf nonce 12"; + let key_128 = [0x39; KEY_SIZE_128]; + let key_256 = [0xa7; KEY_SIZE]; + let accelerated_128 = aes128_expand_key(&key_128); + let accelerated_256 = aes256_expand_key(&key_256); + let portable_128 = Aes128EncKey { + inner: Key128Inner::PortableRoundKeys(aes128_expand_key_portable(&key_128)), + }; + let portable_256 = Aes256EncKey { + inner: KeyInner::PortableRoundKeys(aes256_expand_key_portable(&key_256)), + }; + + // SAFETY: runtime capabilities above confirm AES-CE before both target-feature calls. + let expected_128 = unsafe { aarch64_gcmsiv_derive_keys_128_inline(&accelerated_128, &nonce) }; + // SAFETY: runtime capabilities above confirm AES-CE; the portable key exercises the safe fallback. + let actual_128 = unsafe { aarch64_gcmsiv_derive_keys_128_inline(&portable_128, &nonce) }; + assert_eq!(actual_128, expected_128); + + // SAFETY: runtime capabilities above confirm AES-CE before both target-feature calls. + let expected_256 = unsafe { aarch64_gcmsiv_derive_keys_inline(&accelerated_256, &nonce) }; + // SAFETY: runtime capabilities above confirm AES-CE; the portable key exercises the safe fallback. + let actual_256 = unsafe { aarch64_gcmsiv_derive_keys_inline(&portable_256, &nonce) }; + assert_eq!(actual_256, expected_256); + } + /// AES-256 CTR mode: round-trip (encrypt then decrypt = identity). #[cfg(feature = "aes-gcm-siv")] #[test] @@ -5713,18 +5017,12 @@ mod tests { let mut buf = [0u8; 80]; aes128_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = 3u32.wrapping_add(block_idx as u32); + let ctr = 3u32.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes128_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); - assert_eq!( - &buf[start..end], - &expected, - "AES-128 CTR-BE block {block_idx} keystream mismatch" - ); + assert_eq!(actual, &expected, "AES-128 CTR-BE block {block_idx} keystream mismatch"); } } @@ -5741,16 +5039,13 @@ mod tests { let mut buf = [0u8; 80]; aes128_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = start_ctr.wrapping_add(block_idx as u32); + let ctr = start_ctr.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes128_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); assert_eq!( - &buf[start..end], - &expected, + actual, &expected, "AES-128 CTR-BE wrap block {block_idx} keystream mismatch" ); } @@ -5768,18 +5063,12 @@ mod tests { let mut buf = [0u8; 80]; aes256_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = 7u32.wrapping_add(block_idx as u32); + let ctr = 7u32.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes256_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); - assert_eq!( - &buf[start..end], - &expected, - "CTR-BE block {block_idx} keystream mismatch" - ); + assert_eq!(actual, &expected, "CTR-BE block {block_idx} keystream mismatch"); } } @@ -5796,16 +5085,13 @@ mod tests { let mut buf = [0u8; 80]; aes256_ctr32_encrypt_be(&ek, &iv, &mut buf); - for block_idx in 0..5usize { + for (block_idx, actual) in (0u32..5).zip(buf.as_chunks::().0) { let mut expected = iv; - let ctr = start_ctr.wrapping_add(block_idx as u32); + let ctr = start_ctr.wrapping_add(block_idx); expected[12..16].copy_from_slice(&ctr.to_be_bytes()); aes256_encrypt_block(&ek, &mut expected); - let start = block_idx.strict_mul(BLOCK_SIZE); - let end = start.strict_add(BLOCK_SIZE); assert_eq!( - &buf[start..end], - &expected, + actual, &expected, "AES-256 CTR-BE wrap block {block_idx} keystream mismatch" ); } @@ -5891,7 +5177,9 @@ mod tests { fn fill_aarch64_gcm_test_plaintext(out: &mut [u8; N]) { let mut i = 0usize; while i < N { - out[i] = (i as u8).wrapping_mul(0x3d).wrapping_add(0x47) ^ ((i >> 3) as u8).wrapping_mul(0x91); + let index = i.to_le_bytes()[0]; + let group = (i >> 3).to_le_bytes()[0]; + out[i] = index.wrapping_mul(0x3d).wrapping_add(0x47) ^ group.wrapping_mul(0x91); i = i.strict_add(1); } } @@ -6060,14 +5348,11 @@ mod tests { #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] fn fill_expected_gcm_counter_blocks(iv_prefix: &[u8; 12], ctr: u32, expected: &mut [u8; N]) { debug_assert_eq!(N.strict_rem(BLOCK_SIZE), 0); - let blocks = N.strict_div(BLOCK_SIZE); - let mut block_idx = 0usize; - while block_idx < blocks { - let start = block_idx.strict_mul(BLOCK_SIZE); - expected[start..start.strict_add(12)].copy_from_slice(iv_prefix); - expected[start.strict_add(12)..start.strict_add(16)] - .copy_from_slice(&ctr.wrapping_add(block_idx as u32).to_be_bytes()); - block_idx = block_idx.strict_add(1); + let (blocks, tail) = expected.as_chunks_mut::(); + debug_assert!(tail.is_empty()); + for (block_idx, block) in (0u32..).zip(blocks) { + block[..12].copy_from_slice(iv_prefix); + block[12..].copy_from_slice(&ctr.wrapping_add(block_idx).to_be_bytes()); } } @@ -6146,29 +5431,6 @@ mod tests { out } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[target_feature(enable = "avx2")] - /// # Safety - /// - /// Caller must ensure AVX2 is available before calling this target-feature helper. - unsafe fn x86_gcm_ctr_blocks_be_8_y256_test_bytes(iv_words: [u32; 3], ctr: u32) -> [u8; 128] { - use core::arch::x86_64::*; - - let mut out = [0u8; 128]; - // SAFETY: test-only YMM counter block stores because: - // 1. The caller verified AVX2 before invoking this target-feature helper. - // 2. `out` is exactly 128 writable bytes, matching four contiguous `__m256i` stores. - // 3. The vectors under test are produced directly from the eight-block counter constructor. - unsafe { - let (c0, c1, c2, c3) = x86_gcm_ctr_blocks_be_8_y256(iv_words, ctr); - _mm256_storeu_si256(out.as_mut_ptr().cast(), c0); - _mm256_storeu_si256(out.as_mut_ptr().add(32).cast(), c1); - _mm256_storeu_si256(out.as_mut_ptr().add(64).cast(), c2); - _mm256_storeu_si256(out.as_mut_ptr().add(96).cast(), c3); - } - out - } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] #[test] fn x86_gcm_ctr_block_be_preserves_prefix_and_encodes_counter() { @@ -6263,39 +5525,11 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_gcm_ctr_blocks_be_8_y256_preserves_prefix_and_wraps_counter() { - if !crate::platform::caps().has(crate::platform::caps::x86::AVX2) { - return; - } - - let iv_prefix = *b"ctr wrap iv!"; - let iv_words = x86_gcm_iv_words(&iv_prefix); - - for ctr in [0x0102_0304, u32::MAX - 3] { - let mut expected = [0u8; 128]; - fill_expected_gcm_counter_blocks(&iv_prefix, ctr, &mut expected); - // SAFETY: runtime caps above confirmed AVX2 before calling the target-feature helper. - let actual = unsafe { x86_gcm_ctr_blocks_be_8_y256_test_bytes(iv_words, ctr) }; - assert_eq!( - actual.as_slice(), - expected.as_slice(), - "x86 vectorized YMM GCM counter block mismatch" - ); - } - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - fn x86_y256_gcm_caps_available() -> bool { - let required = crate::platform::caps::x86::VAES_READY - | crate::platform::caps::x86::VPCLMUL_READY - | crate::platform::caps::x86::AVX2 - | crate::platform::caps::x86::AESNI; - crate::platform::caps().has(required) - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn x86_z512_gcm_caps_available() -> bool { let required = crate::platform::caps::x86::VAES_READY | crate::platform::caps::x86::VPCLMUL_READY @@ -6303,20 +5537,11 @@ mod tests { crate::platform::caps().has(required) } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - fn x86_gcm_test_powers() -> (u128, [u128; 4], [u128; 8]) { - let h_polyval = 0x1287_3d5b_fedc_ba09_7654_3210_f0e1_d2c3u128; - let powers = crate::aead::polyval::precompute_powers_8(h_polyval); - ( - h_polyval, - [powers[3], powers[2], powers[1], powers[0]], - [ - powers[7], powers[6], powers[5], powers[4], powers[3], powers[2], powers[1], powers[0], - ], - ) - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] struct X86GcmTestPowers16 { h_polyval: u128, h_powers_rev: [u128; 4], @@ -6329,7 +5554,11 @@ mod tests { h_powers_rev_128: [u128; 128], } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] impl X86GcmTestPowers16 { fn tables<'a>(&'a self) -> X86GcmTables<'a> { X86GcmTables { @@ -6346,7 +5575,11 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn x86_gcm_test_powers_16() -> X86GcmTestPowers16 { let h_polyval = 0x1287_3d5b_fedc_ba09_7654_3210_f0e1_d2c3u128; let powers = crate::aead::polyval::precompute_powers_128(h_polyval); @@ -6371,7 +5604,11 @@ mod tests { } } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] + #[cfg(all( + target_arch = "x86_64", + feature = "aes-gcm", + any(target_os = "linux", target_os = "macos") + ))] fn x86_gcm_wrap_counter_block() -> [u8; 16] { let mut counter = [0u8; 16]; counter[..12].copy_from_slice(b"ctr wrap iv!"); @@ -6379,329 +5616,19 @@ mod tests { counter } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - fn fill_x86_gcm_test_plaintext(out: &mut [u8]) { - let mut i = 0usize; - while i < out.len() { - out[i] = (i as u8).wrapping_mul(0x3d).wrapping_add(0x47) ^ ((i >> 3) as u8).wrapping_mul(0x91); - i = i.strict_add(1); - } - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes128_gcm_y256_encrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes128_expand_key(&[0xA1u8; KEY_SIZE_128]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0xfeed_face_cafe_babe_1020_3040_5060_7080u128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut expected = plaintext; - aes128_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes128_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, expected, - "AES-128 y256 seal ciphertext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-128 y256 seal GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes128_gcm_y256_decrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes128_expand_key(&[0xB2u8; KEY_SIZE_128]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x9ace_0246_8bdf_1357_1122_3344_5566_7788u128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut ciphertext = plaintext; - aes128_ctr32_encrypt_be(&ek, &counter, &mut ciphertext); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &ciphertext); - - let mut actual = ciphertext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes128_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, plaintext, - "AES-128 y256 open plaintext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-128 y256 open GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes256_gcm_y256_encrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes256_expand_key(&[0xC3u8; KEY_SIZE]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x0123_4567_89ab_cdef_fedc_ba98_7654_3210u128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut expected = plaintext; - aes256_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes256_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, expected, - "AES-256 y256 seal ciphertext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-256 y256 seal GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aes256_gcm_y256_decrypt_matches_scalar_across_counter_wrap() { - if !x86_y256_gcm_caps_available() { - return; - } - - let ek = aes256_expand_key(&[0xD4u8; KEY_SIZE]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0xaa55_aa55_55aa_55aa_cc33_cc33_33cc_33ccu128; - - let mut plaintext = [0u8; 128]; - fill_x86_gcm_test_plaintext(&mut plaintext); - let mut ciphertext = plaintext; - aes256_ctr32_encrypt_be(&ek, &counter, &mut ciphertext); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &ciphertext); - - let mut actual = ciphertext; - // SAFETY: runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI before calling - // the y256 target-feature helper. Inputs are fixed-size initialized test buffers. - let actual_acc = unsafe { - aes256_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!( - actual, plaintext, - "AES-256 y256 open plaintext must match scalar CTR across wrap" - ); - assert_eq!( - actual_acc, expected_acc, - "AES-256 y256 open GHASH accumulator must match scalar fold across wrap" - ); - } - - #[cfg(all( - target_arch = "x86_64", - feature = "aes-gcm", - any(target_os = "linux", target_os = "macos", target_os = "windows") - ))] - #[test] - fn x86_aes128_gcm_y256_large_asm_tail_matches_scalar() { - if !x86_y256_gcm_caps_available() { - return; - } - - const LEN: usize = 1057; - - let ek = aes128_expand_key(&[0xE5u8; KEY_SIZE_128]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x3141_5926_5358_9793_2384_6264_3383_2795u128; - - let mut plaintext = [0u8; LEN]; - fill_x86_gcm_test_plaintext(&mut plaintext); - - let mut expected = plaintext; - aes128_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: large x86 AES-128-GCM seal test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The final byte is handled by the shared Rust tail after the ASM path returns. - let actual_acc = unsafe { - aes128_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(actual, expected, "AES-128 large y256/ASM seal ciphertext mismatch"); - assert_eq!( - actual_acc, expected_acc, - "AES-128 large y256/ASM seal GHASH accumulator mismatch" - ); - - let mut opened = actual; - // SAFETY: large x86 AES-128-GCM open test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The helper GHASHes ciphertext before decrypting and leaves the final byte to the Rust tail. - let open_acc = unsafe { - aes128_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut opened, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(opened, plaintext, "AES-128 large y256/ASM open plaintext mismatch"); - assert_eq!( - open_acc, expected_acc, - "AES-128 large y256/ASM open GHASH accumulator mismatch" - ); - } - #[cfg(all( target_arch = "x86_64", feature = "aes-gcm", - any(target_os = "linux", target_os = "macos", target_os = "windows") + any(target_os = "linux", target_os = "macos") ))] - #[test] - fn x86_aes256_gcm_y256_large_asm_tail_matches_scalar() { - if !x86_y256_gcm_caps_available() { - return; + fn fill_x86_gcm_test_plaintext(out: &mut [u8]) { + let mut i = 0usize; + while i < out.len() { + let index = i.to_le_bytes()[0]; + let group = (i >> 3).to_le_bytes()[0]; + out[i] = index.wrapping_mul(0x3d).wrapping_add(0x47) ^ group.wrapping_mul(0x91); + i = i.strict_add(1); } - - const LEN: usize = 1057; - - let ek = aes256_expand_key(&[0xF6u8; KEY_SIZE]); - let counter = x86_gcm_wrap_counter_block(); - let (h_polyval, h_powers_rev, h_powers_rev_8) = x86_gcm_test_powers(); - let seed_acc = 0x2718_2818_2845_9045_2353_6028_7471_3526u128; - - let mut plaintext = [0u8; LEN]; - fill_x86_gcm_test_plaintext(&mut plaintext); - - let mut expected = plaintext; - aes256_ctr32_encrypt_be(&ek, &counter, &mut expected); - let expected_acc = ghash_ciphertext_fallback(seed_acc, h_polyval, &expected); - - let mut actual = plaintext; - // SAFETY: large x86 AES-256-GCM seal test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The final byte is handled by the shared Rust tail after the ASM path returns. - let actual_acc = unsafe { - aes256_ctr32_encrypt_be_y256_ghash( - &ek, - &counter, - &mut actual, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(actual, expected, "AES-256 large y256/ASM seal ciphertext mismatch"); - assert_eq!( - actual_acc, expected_acc, - "AES-256 large y256/ASM seal GHASH accumulator mismatch" - ); - - let mut opened = actual; - // SAFETY: large x86 AES-256-GCM open test because: - // 1. Runtime caps above confirmed VAES + VPCLMULQDQ + AVX2 + AES-NI. - // 2. The 1057-byte input forces the x86 ASM bulk path to process full blocks. - // 3. The helper GHASHes ciphertext before decrypting and leaves the final byte to the Rust tail. - let open_acc = unsafe { - aes256_ctr32_decrypt_be_y256_ghash( - &ek, - &counter, - &mut opened, - seed_acc, - h_polyval, - &h_powers_rev, - &h_powers_rev_8, - ) - }; - - assert_eq!(opened, plaintext, "AES-256 large y256/ASM open plaintext mismatch"); - assert_eq!( - open_acc, expected_acc, - "AES-256 large y256/ASM open GHASH accumulator mismatch" - ); } #[cfg(all( @@ -6834,9 +5761,9 @@ mod tests { let ek = aes128_expand_key(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 6]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(13) ^ (j as u8).wrapping_mul(31) ^ 0xA3; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(13) ^ j.wrapping_mul(31) ^ 0xA3; } } @@ -6855,9 +5782,9 @@ mod tests { let ek = aes256_expand_key(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 6]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(17) ^ (j as u8).wrapping_mul(29) ^ 0x5C; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(17) ^ j.wrapping_mul(29) ^ 0x5C; } } @@ -6871,6 +5798,7 @@ mod tests { } /// FIPS 197 Appendix C.1 against the table-free fixslice AES-128 path. + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_matches_nist_aes128_vector() { let key: [u8; 16] = [ @@ -6889,6 +5817,7 @@ mod tests { assert_eq!(block, expected); } + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_128_4blocks_matches_portable() { let key = [0xC4u8; KEY_SIZE_128]; @@ -6896,9 +5825,9 @@ mod tests { let fixslice = rv_fixslice_aes::RvFixslice128RoundKeys::new(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 4]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(0x47) ^ (j as u8).wrapping_mul(0x6d) ^ 0x9c; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(0x47) ^ j.wrapping_mul(0x6d) ^ 0x9c; } } @@ -6911,6 +5840,7 @@ mod tests { assert_eq!(blocks, expected); } + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_matches_nist_aes256_vector() { let key: [u8; 32] = [ @@ -6930,6 +5860,7 @@ mod tests { assert_eq!(block, expected); } + #[cfg(not(target_arch = "s390x"))] #[test] fn riscv64_fixslice_4blocks_matches_portable() { let key = [0x3cu8; KEY_SIZE]; @@ -6937,9 +5868,9 @@ mod tests { let fixslice = rv_fixslice_aes::RvFixsliceRoundKeys::new(&key); let mut blocks = [[0u8; BLOCK_SIZE]; 4]; - for (i, block) in blocks.iter_mut().enumerate() { - for (j, byte) in block.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(0x31) ^ (j as u8).wrapping_mul(0x57) ^ 0xa6; + for (i, block) in (0u8..).zip(&mut blocks) { + for (j, byte) in (0u8..).zip(block) { + *byte = i.wrapping_mul(0x31) ^ j.wrapping_mul(0x57) ^ 0xa6; } } diff --git a/src/aead/aes/aarch64/asm.rs b/src/aead/aes/aarch64/asm.rs index f5115eb7..e40c9902 100644 --- a/src/aead/aes/aarch64/asm.rs +++ b/src/aead/aes/aarch64/asm.rs @@ -1,7 +1,5 @@ //! AArch64 rscrypto-owned AES-GCM assembly kernels. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_aes_gcm_aarch64_apple_darwin.s")); @@ -19,8 +17,8 @@ impl AesGcmAarch64State { #[inline] pub(super) fn new(acc: u128, ctr: u32) -> Self { Self { - acc_lo: acc as u64, - acc_hi: (acc >> 64) as u64, + acc_lo: u64::try_from(acc & u128::from(u64::MAX)).expect("masked accumulator half fits u64"), + acc_hi: u64::try_from(acc >> 64).expect("shifted accumulator half fits u64"), ctr, _pad: 0, processed: 0, diff --git a/src/aead/aes/aarch64_ce.rs b/src/aead/aes/aarch64_ce.rs index b22c1cd5..ea1250aa 100644 --- a/src/aead/aes/aarch64_ce.rs +++ b/src/aead/aes/aarch64_ce.rs @@ -266,6 +266,11 @@ pub(super) unsafe fn encrypt_6blocks_core(keys: &CeRoundKeys, blocks: &mut [[u8; #[cfg(feature = "aes-gcm")] #[inline] +/// Convert one initialized GHASH byte vector to its big-endian `u128` value. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn ghash_be_u128_from_vec(block: uint8x16_t) -> u128 { // SAFETY: GHASH lane conversion because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -282,16 +287,48 @@ unsafe fn ghash_be_u128_from_vec(block: uint8x16_t) -> u128 { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Split one GHASH accumulator into its low and high 64-bit NEON lanes. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn ghash_u128_to_lanes(x: u128) -> uint64x2_t { + let [ + lo_0, + lo_1, + lo_2, + lo_3, + lo_4, + lo_5, + lo_6, + lo_7, + hi_0, + hi_1, + hi_2, + hi_3, + hi_4, + hi_5, + hi_6, + hi_7, + ] = x.to_le_bytes(); + let lo = u64::from_le_bytes([lo_0, lo_1, lo_2, lo_3, lo_4, lo_5, lo_6, lo_7]); + let hi = u64::from_le_bytes([hi_0, hi_1, hi_2, hi_3, hi_4, hi_5, hi_6, hi_7]); + // SAFETY: GHASH accumulator lane construction because: // 1. The caller is already inside an AArch64 NEON target scope. // 2. `vcreate_u64` initializes one 64-bit lane from an integer value. // 3. `vcombine_u64` builds a fully initialized two-lane vector. - unsafe { vcombine_u64(vcreate_u64(x as u64), vcreate_u64((x >> 64) as u64)) } + unsafe { vcombine_u64(vcreate_u64(lo), vcreate_u64(hi)) } } #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Reduce one 256-bit GHASH carryless product to two Montgomery-form lanes. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass initialized low and high +/// product halves. unsafe fn ghash_mont_reduce_neon(lo: uint64x2_t, hi: uint64x2_t) -> uint64x2_t { // SAFETY: GHASH Montgomery reduction because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -315,6 +352,12 @@ unsafe fn ghash_mont_reduce_neon(lo: uint64x2_t, hi: uint64x2_t) -> uint64x2_t { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Finish the three Karatsuba accumulators and return the reduced GHASH value. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass initialized low, high, and +/// middle product accumulators for the same fold. unsafe fn ghash_finish_products(ll: uint64x2_t, hh: uint64x2_t, mm: uint64x2_t) -> u128 { // SAFETY: GHASH Karatsuba accumulator finalization because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -332,6 +375,12 @@ unsafe fn ghash_finish_products(ll: uint64x2_t, hh: uint64x2_t, mm: uint64x2_t) #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Load one ciphertext block without imposing an alignment requirement. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. `ptr` must remain valid and readable +/// for 16 initialized bytes for the duration of the call; it need not be aligned. unsafe fn gcm_load_ciphertext_block(ptr: *const u8) -> uint8x16_t { // SAFETY: opaque 16-byte ciphertext load for GCM open because: // 1. The caller passes a pointer into a 128-byte chunk already bounds-checked by the enclosing @@ -353,6 +402,12 @@ unsafe fn gcm_load_ciphertext_block(ptr: *const u8) -> uint8x16_t { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Load one GHASH key power as two unaligned 64-bit lanes. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. `power` must remain valid and readable +/// for one initialized `u128` for the duration of the call; it need not be aligned. unsafe fn ghash_load_power(power: *const u128) -> uint64x2_t { // SAFETY: GHASH H-power vector load because: // 1. The caller passes a pointer into a live H-power table. @@ -395,6 +450,11 @@ macro_rules! gcm_schedule_barrier { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Build the fixed 96-bit prefix of an AES-GCM big-endian counter block. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn gcm_ctr32_base(iv_prefix: &[u8; 12]) -> uint8x16_t { let mut block = [0u8; 16]; block[..12].copy_from_slice(iv_prefix); @@ -407,6 +467,11 @@ unsafe fn gcm_ctr32_base(iv_prefix: &[u8; 12]) -> uint8x16_t { #[cfg(feature = "aes-gcm")] #[inline(always)] +/// Insert one AES-GCM counter into the big-endian final lane. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass an initialized counter base. unsafe fn gcm_ctr32_block(base: uint8x16_t, ctr: u32) -> uint8x16_t { // SAFETY: GCM counter lane update because: // 1. The caller is already inside an AArch64 NEON target scope. @@ -417,6 +482,11 @@ unsafe fn gcm_ctr32_block(base: uint8x16_t, ctr: u32) -> uint8x16_t { #[cfg(feature = "aes-gcm-siv")] #[inline(always)] +/// Build the fixed 96-bit suffix of an AES-GCM-SIV little-endian counter block. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available. unsafe fn gcmsiv_ctr32_base(iv_suffix: &[u8; 12]) -> uint8x16_t { let mut block = [0u8; 16]; block[4..16].copy_from_slice(iv_suffix); @@ -429,6 +499,11 @@ unsafe fn gcmsiv_ctr32_base(iv_suffix: &[u8; 12]) -> uint8x16_t { #[cfg(feature = "aes-gcm-siv")] #[inline(always)] +/// Insert one AES-GCM-SIV counter into the little-endian first lane. +/// +/// # Safety +/// +/// The caller must execute this function only when AArch64 NEON is available and pass an initialized counter base. unsafe fn gcmsiv_ctr32_block(base: uint8x16_t, ctr: u32) -> uint8x16_t { // SAFETY: GCM-SIV counter lane update because: // 1. The caller is inside a NEON target scope. @@ -751,6 +826,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Encrypt eight AES-256-GCM blocks while folding the preceding ciphertext group. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, GHASH powers, and preceding ciphertext lanes belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -883,6 +964,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Fold and decrypt eight current AES-256-GCM ciphertext blocks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, and GHASH powers belong to the same GCM key and stream. pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -1071,6 +1158,12 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Encrypt complete 128-byte AES-256-GCM chunks and fold their ciphertext into GHASH. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// counter prefix, accumulator, and GHASH tables belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -1102,7 +1195,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1116,7 +1209,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1157,6 +1250,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Fold and decrypt complete 128-byte AES-256-GCM ciphertext chunks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available and the round keys, counter prefix, accumulator, and +/// GHASH tables belong to the same GCM key and stream. `data` may be shorter than one complete chunk. pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_core( keys: &CeRoundKeys, iv_prefix: &[u8; 12], @@ -1184,7 +1283,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1198,7 +1297,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -1733,6 +1832,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Encrypt eight AES-128-GCM blocks while folding the preceding ciphertext group. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, GHASH powers, and preceding ciphertext lanes belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -1858,6 +1963,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_8blocks_ghash_prev_bytes_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline] +/// Fold and decrypt eight current AES-128-GCM ciphertext blocks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// accumulator, and GHASH powers belong to the same GCM key and stream. pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -2039,6 +2150,12 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_8blocks_ghash_current_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Encrypt complete 128-byte AES-128-GCM chunks and fold their ciphertext into GHASH. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available; `data` contains at least 128 bytes; and the round keys, +/// counter prefix, accumulator, and GHASH tables belong to the same GCM key and stream. pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -2070,7 +2187,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -2084,7 +2201,7 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -2125,6 +2242,12 @@ pub(super) unsafe fn encrypt_ctr32_be_xor_ghash_128b_chunks_128_core( #[cfg(feature = "aes-gcm")] #[target_feature(enable = "aes,neon")] #[inline(never)] +/// Fold and decrypt complete 128-byte AES-128-GCM ciphertext chunks. +/// +/// # Safety +/// +/// The caller must ensure AES, PMULL, and NEON are available and the round keys, counter prefix, accumulator, and +/// GHASH tables belong to the same GCM key and stream. `data` may be shorter than one complete chunk. pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_128_core( keys: &Ce128RoundKeys, iv_prefix: &[u8; 12], @@ -2152,7 +2275,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_128_core( tables.h_powers_rev_16.as_ptr(), tables.h_powers_rev_16_mid.as_ptr(), tables.h_powers_rev_16_pair.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); @@ -2166,7 +2289,7 @@ pub(super) unsafe fn decrypt_ctr32_be_xor_ghash_128b_chunks_128_core( data.as_mut_ptr(), data.len(), tables.h_powers_rev_8.as_ptr(), - &mut state, + &raw mut state, ); if state.processed != 0 { return (state.acc(), state.ctr, state.processed); diff --git a/src/aead/aes/riscv64_fixslice_aes.rs b/src/aead/aes/riscv64_fixslice_aes.rs index 19002124..5c896fe5 100644 --- a/src/aead/aes/riscv64_fixslice_aes.rs +++ b/src/aead/aes/riscv64_fixslice_aes.rs @@ -8,8 +8,6 @@ //! Reference: Adomnicai et al., "Fixslicing AES-like Ciphers", //! . -#![allow(clippy::unreadable_literal)] - use super::{BLOCK_SIZE, KEY_SIZE, KEY_SIZE_128}; type State = [u64; 8]; @@ -27,7 +25,7 @@ impl RvFixsliceRoundKeys { } #[inline] - #[allow(dead_code)] + #[cfg(target_arch = "riscv64")] pub(super) fn zeroize(&mut self) { // SAFETY: `[u64; 120]` is contiguous and valid to view as bytes for its // exact initialized size. @@ -51,7 +49,7 @@ impl RvFixslice128RoundKeys { } #[inline] - #[allow(dead_code)] + #[cfg(target_arch = "riscv64")] pub(super) fn zeroize(&mut self) { // SAFETY: `[u64; 88]` is contiguous and valid to view as bytes for its // exact initialized size. @@ -110,7 +108,6 @@ pub(super) fn encrypt_4blocks(rkeys: &RvFixsliceRoundKeys, blocks: &mut [[u8; BL } #[inline] -#[allow(dead_code)] pub(super) fn cipher_round_4(blocks: &mut [[u8; BLOCK_SIZE]; 4], round_keys: &[[u8; BLOCK_SIZE]; 4]) { let mut state = State::default(); bitslice(&mut state, &blocks[0], &blocks[1], &blocks[2], &blocks[3]); @@ -625,14 +622,15 @@ fn inv_bitslice(input: &[u64]) -> [[u8; BLOCK_SIZE]; 4] { #[rustfmt::skip] fn write_reordered(columns: u64, output: &mut [u8]) { - output[0x0] = (columns ) as u8; - output[0x1] = (columns >> 0x10) as u8; - output[0x2] = (columns >> 0x20) as u8; - output[0x3] = (columns >> 0x30) as u8; - output[0x8] = (columns >> 0x08) as u8; - output[0x9] = (columns >> 0x18) as u8; - output[0xa] = (columns >> 0x28) as u8; - output[0xb] = (columns >> 0x38) as u8; + let bytes = columns.to_le_bytes(); + output[0x0] = bytes[0]; + output[0x1] = bytes[2]; + output[0x2] = bytes[4]; + output[0x3] = bytes[6]; + output[0x8] = bytes[1]; + output[0x9] = bytes[3]; + output[0xa] = bytes[5]; + output[0xb] = bytes[7]; } let mut output = [[0u8; BLOCK_SIZE]; 4]; @@ -691,7 +689,7 @@ fn ror(x: u64, y: u32) -> u64 { #[inline(always)] fn ror_distance(rows: u32, cols: u32) -> u32 { - (rows << 4) + (cols << 2) + rows.strict_shl(4).strict_add(cols.strict_shl(2)) } #[inline(always)] diff --git a/src/aead/aes/riscv64_vperm_aes.rs b/src/aead/aes/riscv64_vperm_aes.rs deleted file mode 100644 index f48333f8..00000000 --- a/src/aead/aes/riscv64_vperm_aes.rs +++ /dev/null @@ -1,617 +0,0 @@ -use core::arch::asm; - -use crate::aead::aes_round::{ - AES_AFFINE, MC_ROT1, MC_ROT2, VPERM_INV_HI, VPERM_INV_LO, VPERM_IPT_HI, VPERM_IPT_LO, VPERM_SBOT, VPERM_SBOU, - VPERM_SR, XTIME_REDUCE, -}; - -/// Precomputed Hamburg vperm table block — contiguous for offset-based loads. -#[repr(C, align(16))] -struct VpermTables { - ipt_lo: [u8; 16], // offset 0 - ipt_hi: [u8; 16], // offset 16 - inv_lo: [u8; 16], // offset 32 - inv_hi: [u8; 16], // offset 48 - sbou: [u8; 16], // offset 64 - sbot: [u8; 16], // offset 80 - sr_perm: [u8; 16], // offset 96 - mc_rot1: [u8; 16], // offset 112 - mc_rot2: [u8; 16], // offset 128 - affine: [u8; 16], // offset 144 - xtime: [u8; 16], // offset 160 -} - -impl VpermTables { - #[inline(always)] - fn load() -> Self { - Self { - ipt_lo: VPERM_IPT_LO, - ipt_hi: VPERM_IPT_HI, - inv_lo: VPERM_INV_LO, - inv_hi: VPERM_INV_HI, - sbou: VPERM_SBOU, - sbot: VPERM_SBOT, - sr_perm: VPERM_SR, - mc_rot1: MC_ROT1, - mc_rot2: MC_ROT2, - affine: [AES_AFFINE; 16], - xtime: [XTIME_REDUCE; 16], - } - } -} - -/// Extract round key bytes from the portable key schedule. -#[inline] -fn round_key_bytes(rk: &[u32; super::EXPANDED_KEY_WORDS], round: usize) -> [u8; 16] { - let off = round.strict_mul(4); - let mut bytes = [0u8; 16]; - bytes[0..4].copy_from_slice(&rk[off].to_be_bytes()); - bytes[4..8].copy_from_slice(&rk[off.strict_add(1)].to_be_bytes()); - bytes[8..12].copy_from_slice(&rk[off.strict_add(2)].to_be_bytes()); - bytes[12..16].copy_from_slice(&rk[off.strict_add(3)].to_be_bytes()); - bytes -} - -/// Extract round key bytes from the AES-128 portable key schedule. -#[inline] -fn round_key_bytes_128(rk: &[u32; super::EXPANDED_KEY_WORDS_128], round: usize) -> [u8; 16] { - let off = round.strict_mul(4); - let mut bytes = [0u8; 16]; - bytes[0..4].copy_from_slice(&rk[off].to_be_bytes()); - bytes[4..8].copy_from_slice(&rk[off.strict_add(1)].to_be_bytes()); - bytes[8..12].copy_from_slice(&rk[off.strict_add(2)].to_be_bytes()); - bytes[12..16].copy_from_slice(&rk[off.strict_add(3)].to_be_bytes()); - bytes -} - -macro_rules! vperm_inner_round_m1 { - ($state:literal, $rk:literal) => { - concat!( - "vand.vi v19, ", - $state, - ", 15\n", - "vsrl.vi v20, ", - $state, - ", 4\n", - "vrgather.vv v21, v8, v19\n", - "vrgather.vv v22, v9, v20\n", - "vxor.vv v19, v21, v22\n", - "vand.vi v20, v19, 15\n", - "vsrl.vi v21, v19, 4\n", - "vrgather.vv v22, v11, v20\n", - "vxor.vv v23, v21, v20\n", - "vrgather.vv v24, v10, v21\n", - "vxor.vv v25, v24, v22\n", - "vrgather.vv v24, v10, v23\n", - "vxor.vv v26, v24, v22\n", - "vand.vi v27, v25, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v25, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v19, v28, v23\n", - "vand.vi v27, v26, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v26, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v20, v28, v21\n", - "vand.vi v27, v19, 15\n", - "vrgather.vv v28, v12, v27\n", - "vsra.vi v29, v19, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v22, v28, v29\n", - "vand.vi v27, v20, 15\n", - "vrgather.vv v30, v13, v27\n", - "vsra.vi v29, v20, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v23, v30, v29\n", - "vxor.vv v19, v22, v23\n", - "vxor.vv v19, v19, v17\n", - "vrgather.vv v20, v19, v14\n", - "vrgather.vv v21, v20, v15\n", - "vxor.vv v22, v20, v21\n", - "vsll.vi v23, v22, 1\n", - "vsra.vi v24, v22, 7\n", - "vand.vv v24, v24, v18\n", - "vxor.vv v23, v23, v24\n", - "vrgather.vv v24, v22, v16\n", - "vxor.vv v25, v22, v24\n", - "vxor.vv v26, v20, v25\n", - "vxor.vv v26, v26, v23\n", - "vxor.vv ", - $state, - ", v26, ", - $rk - ) - }; -} - -macro_rules! vperm_final_round_m1 { - ($state:literal, $rk:literal) => { - concat!( - "vand.vi v19, ", - $state, - ", 15\n", - "vsrl.vi v20, ", - $state, - ", 4\n", - "vrgather.vv v21, v8, v19\n", - "vrgather.vv v22, v9, v20\n", - "vxor.vv v19, v21, v22\n", - "vand.vi v20, v19, 15\n", - "vsrl.vi v21, v19, 4\n", - "vrgather.vv v22, v11, v20\n", - "vxor.vv v23, v21, v20\n", - "vrgather.vv v24, v10, v21\n", - "vxor.vv v25, v24, v22\n", - "vrgather.vv v24, v10, v23\n", - "vxor.vv v26, v24, v22\n", - "vand.vi v27, v25, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v25, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v19, v28, v23\n", - "vand.vi v27, v26, 15\n", - "vrgather.vv v28, v10, v27\n", - "vsra.vi v29, v26, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v28, v28, v29\n", - "vxor.vv v20, v28, v21\n", - "vand.vi v27, v19, 15\n", - "vrgather.vv v28, v12, v27\n", - "vsra.vi v29, v19, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v22, v28, v29\n", - "vand.vi v27, v20, 15\n", - "vrgather.vv v30, v13, v27\n", - "vsra.vi v29, v20, 7\n", - "vxor.vi v29, v29, -1\n", - "vand.vv v23, v30, v29\n", - "vxor.vv v19, v22, v23\n", - "vxor.vv v19, v19, v17\n", - "vrgather.vv v20, v19, v14\n", - "vxor.vv ", - $state, - ", v20, ", - $rk - ) - }; -} - -/// Single inner AES round (SubBytes + ShiftRows + MixColumns + AddRoundKey). -/// Same asm as aegis256::rv_vperm::aes_round. -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_inner_round(block: &[u8; 16], round_key: &[u8; 16], tables: &VpermTables) -> [u8; 16] { - let mut out = [0u8; 16]; - // SAFETY: caller guarantees the RISC-V V extension is available and the - // asm block only reads the provided state/table/key buffers and writes `out`. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - // Load tables. - "vle8.v v2, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v3, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v4, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v5, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v6, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v7, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v8, ({tmp})", - "addi {tmp}, {tbl}, 112", - "vle8.v v9, ({tmp})", - "addi {tmp}, {tbl}, 128", - "vle8.v v10, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v11, ({tmp})", - "addi {tmp}, {tbl}, 160", - "vle8.v v12, ({tmp})", - "vle8.v v0, ({state})", - "vle8.v v1, ({rk})", - // Phase 1: Nibble extraction. - "vand.vi v14, v0, 15", - "vsrl.vi v15, v0, 4", - // Phase 2: Input transform. - "vrgather.vv v16, v2, v14", - "vrgather.vv v17, v3, v15", - "vxor.vv v14, v16, v17", - // Phase 3: Re-extract nibbles. - "vand.vi v15, v14, 15", - "vsrl.vi v16, v14, 4", - // Phase 4: GF(2^4) inverse. - "vrgather.vv v17, v5, v15", - "vxor.vv v18, v16, v15", - "vrgather.vv v19, v4, v16", - "vxor.vv v20, v19, v17", - "vrgather.vv v21, v4, v18", - "vxor.vv v22, v21, v17", - "vand.vi v23, v20, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v20, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v14, v24, v18", - "vand.vi v23, v22, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v22, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v15, v24, v16", - // Phase 5: Output transform. - "vand.vi v23, v14, 15", - "vrgather.vv v24, v6, v23", - "vsra.vi v25, v14, 7", - "vxor.vi v26, v25, -1", - "vand.vv v16, v24, v26", - "vand.vi v23, v15, 15", - "vrgather.vv v24, v7, v23", - "vsra.vi v25, v15, 7", - "vxor.vi v26, v25, -1", - "vand.vv v17, v24, v26", - "vxor.vv v14, v16, v17", - "vxor.vv v14, v14, v11", - // ShiftRows. - "vrgather.vv v15, v14, v8", - // MixColumns. - "vrgather.vv v16, v15, v9", - "vxor.vv v17, v15, v16", - "vsll.vi v18, v17, 1", - "vsra.vi v19, v17, 7", - "vand.vv v19, v19, v12", - "vxor.vv v18, v18, v19", - "vrgather.vv v19, v17, v10", - "vxor.vv v20, v17, v19", - "vxor.vv v14, v15, v20", - "vxor.vv v14, v14, v18", - // AddRoundKey. - "vxor.vv v0, v14, v1", - "vse8.v v0, ({out})", - state = in(reg) block.as_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - out = in(reg) out.as_mut_ptr(), - tmp = out(reg) _, - options(nostack), - ); - } - out -} - -/// Final AES round (SubBytes + ShiftRows + AddRoundKey, no MixColumns). -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_final_round(block: &[u8; 16], round_key: &[u8; 16], tables: &VpermTables) -> [u8; 16] { - let mut out = [0u8; 16]; - // SAFETY: caller guarantees the RISC-V V extension is available and the - // asm block only reads the provided state/table/key buffers and writes `out`. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - // Load tables (only need SubBytes + ShiftRows, no MixColumns). - "vle8.v v2, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v3, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v4, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v5, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v6, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v7, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v8, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v11, ({tmp})", - "vle8.v v0, ({state})", - "vle8.v v1, ({rk})", - // SubBytes (same as inner round). - "vand.vi v14, v0, 15", - "vsrl.vi v15, v0, 4", - "vrgather.vv v16, v2, v14", - "vrgather.vv v17, v3, v15", - "vxor.vv v14, v16, v17", - "vand.vi v15, v14, 15", - "vsrl.vi v16, v14, 4", - "vrgather.vv v17, v5, v15", - "vxor.vv v18, v16, v15", - "vrgather.vv v19, v4, v16", - "vxor.vv v20, v19, v17", - "vrgather.vv v21, v4, v18", - "vxor.vv v22, v21, v17", - "vand.vi v23, v20, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v20, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v14, v24, v18", - "vand.vi v23, v22, 15", - "vrgather.vv v24, v4, v23", - "vsra.vi v25, v22, 7", - "vxor.vi v26, v25, -1", - "vand.vv v24, v24, v26", - "vxor.vv v15, v24, v16", - "vand.vi v23, v14, 15", - "vrgather.vv v24, v6, v23", - "vsra.vi v25, v14, 7", - "vxor.vi v26, v25, -1", - "vand.vv v16, v24, v26", - "vand.vi v23, v15, 15", - "vrgather.vv v24, v7, v23", - "vsra.vi v25, v15, 7", - "vxor.vi v26, v25, -1", - "vand.vv v17, v24, v26", - "vxor.vv v14, v16, v17", - "vxor.vv v14, v14, v11", - // ShiftRows (no MixColumns). - "vrgather.vv v15, v14, v8", - // AddRoundKey. - "vxor.vv v0, v15, v1", - "vse8.v v0, ({out})", - state = in(reg) block.as_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - out = in(reg) out.as_mut_ptr(), - tmp = out(reg) _, - options(nostack), - ); - } - out -} - -/// Four independent inner rounds with one shared table load. -/// -/// This targets the AES-GCM-SIV fixed-cost path, where six ECB blocks are -/// derived up front. Sharing the vperm tables across 4 blocks removes most -/// of the per-call table-load overhead even before we attempt a wider LMUL -/// kernel. -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_inner_round_4(blocks: &mut [[u8; 16]; 4], round_key: &[u8; 16], tables: &VpermTables) { - // SAFETY: caller guarantees the RISC-V V extension is available and all - // block/key/table references are valid for 16-byte vector loads/stores. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - "vle8.v v8, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v9, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v10, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v11, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v12, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v13, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v14, ({tmp})", - "addi {tmp}, {tbl}, 112", - "vle8.v v15, ({tmp})", - "addi {tmp}, {tbl}, 128", - "vle8.v v16, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v17, ({tmp})", - "addi {tmp}, {tbl}, 160", - "vle8.v v18, ({tmp})", - "vle8.v v0, ({b0})", - "vle8.v v1, ({b1})", - "vle8.v v2, ({b2})", - "vle8.v v3, ({b3})", - "vle8.v v4, ({rk})", - vperm_inner_round_m1!("v0", "v4"), - vperm_inner_round_m1!("v1", "v4"), - vperm_inner_round_m1!("v2", "v4"), - vperm_inner_round_m1!("v3", "v4"), - "vse8.v v0, ({b0})", - "vse8.v v1, ({b1})", - "vse8.v v2, ({b2})", - "vse8.v v3, ({b3})", - b0 = in(reg) blocks[0].as_mut_ptr(), - b1 = in(reg) blocks[1].as_mut_ptr(), - b2 = in(reg) blocks[2].as_mut_ptr(), - b3 = in(reg) blocks[3].as_mut_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - tmp = out(reg) _, - options(nostack), - ); - } -} - -/// Four independent final rounds with one shared table load. -#[target_feature(enable = "v")] -#[inline] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -unsafe fn aes_final_round_4(blocks: &mut [[u8; 16]; 4], round_key: &[u8; 16], tables: &VpermTables) { - // SAFETY: caller guarantees the RISC-V V extension is available and all - // block/key/table references are valid for 16-byte vector loads/stores. - unsafe { - asm!( - "vsetivli zero, 16, e8, m1, ta, ma", - "vle8.v v8, ({tbl})", - "addi {tmp}, {tbl}, 16", - "vle8.v v9, ({tmp})", - "addi {tmp}, {tbl}, 32", - "vle8.v v10, ({tmp})", - "addi {tmp}, {tbl}, 48", - "vle8.v v11, ({tmp})", - "addi {tmp}, {tbl}, 64", - "vle8.v v12, ({tmp})", - "addi {tmp}, {tbl}, 80", - "vle8.v v13, ({tmp})", - "addi {tmp}, {tbl}, 96", - "vle8.v v14, ({tmp})", - "addi {tmp}, {tbl}, 144", - "vle8.v v17, ({tmp})", - "vle8.v v0, ({b0})", - "vle8.v v1, ({b1})", - "vle8.v v2, ({b2})", - "vle8.v v3, ({b3})", - "vle8.v v4, ({rk})", - vperm_final_round_m1!("v0", "v4"), - vperm_final_round_m1!("v1", "v4"), - vperm_final_round_m1!("v2", "v4"), - vperm_final_round_m1!("v3", "v4"), - "vse8.v v0, ({b0})", - "vse8.v v1, ({b1})", - "vse8.v v2, ({b2})", - "vse8.v v3, ({b3})", - b0 = in(reg) blocks[0].as_mut_ptr(), - b1 = in(reg) blocks[1].as_mut_ptr(), - b2 = in(reg) blocks[2].as_mut_ptr(), - b3 = in(reg) blocks[3].as_mut_ptr(), - rk = in(reg) round_key.as_ptr(), - tbl = in(reg) tables as *const VpermTables as *const u8, - tmp = out(reg) _, - options(nostack), - ); - } -} - -/// AES-256 full-block encryption (14 rounds) using Hamburg vperm. -/// -/// # Safety -/// Requires the RISC-V V extension. -#[target_feature(enable = "v")] -pub(super) unsafe fn encrypt_block(rk: &[u32; super::EXPANDED_KEY_WORDS], block: &mut [u8; 16]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // full AES-256 block operation and all references are valid Rust buffers. - unsafe { - let tables = VpermTables::load(); - - // Initial AddRoundKey (round 0). - let rk0 = round_key_bytes(rk, 0); - for i in 0..16 { - block[i] ^= rk0[i]; - } - - // Rounds 1-13: full AES round (SubBytes + ShiftRows + MixColumns + AddRoundKey). - let mut round = 1usize; - while round < super::ROUNDS { - let rk_r = round_key_bytes(rk, round); - *block = aes_inner_round(block, &rk_r, &tables); - round = round.strict_add(1); - } - - // Round 14 (final): SubBytes + ShiftRows + AddRoundKey (no MixColumns). - let rk14 = round_key_bytes(rk, super::ROUNDS); - *block = aes_final_round(block, &rk14, &tables); - } -} - -/// Encrypt 4 independent AES-256 blocks using the vperm backend. -/// -/// The 4-block batch is aimed at GCM-SIV key derivation, which produces 6 -/// unrelated ECB inputs up front. Processing 4 of them together amortizes -/// table loads across the hot fixed-cost path while keeping the existing -/// single-block kernel for small tails. -#[target_feature(enable = "v")] -/// # Safety -/// -/// Caller must ensure the CPU supports the RISC-V `v` extension. -pub(super) unsafe fn encrypt_4blocks(rk: &[u32; super::EXPANDED_KEY_WORDS], blocks: &mut [[u8; 16]; 4]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // duration of the batch and all block buffers are valid 16-byte arrays. - unsafe { - let tables = VpermTables::load(); - - let rk0 = round_key_bytes(rk, 0); - for block in blocks.iter_mut() { - for i in 0..16 { - block[i] ^= rk0[i]; - } - } - - let mut round = 1usize; - while round < super::ROUNDS { - let rk_r = round_key_bytes(rk, round); - aes_inner_round_4(blocks, &rk_r, &tables); - round = round.strict_add(1); - } - - let rk14 = round_key_bytes(rk, super::ROUNDS); - aes_final_round_4(blocks, &rk14, &tables); - } -} - -// AES-128 (10 rounds) - -/// AES-128 full-block encryption (10 rounds) using Hamburg vperm. -/// -/// # Safety -/// Requires the RISC-V V extension. -#[target_feature(enable = "v")] -pub(super) unsafe fn encrypt_block_128(rk: &[u32; super::EXPANDED_KEY_WORDS_128], block: &mut [u8; 16]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // full AES-128 block operation and all references are valid Rust buffers. - unsafe { - let tables = VpermTables::load(); - - // Initial AddRoundKey (round 0). - let rk0 = round_key_bytes_128(rk, 0); - for i in 0..16 { - block[i] ^= rk0[i]; - } - - // Rounds 1-9: full AES round (SubBytes + ShiftRows + MixColumns + AddRoundKey). - let mut round = 1usize; - while round < super::ROUNDS_128 { - let rk_r = round_key_bytes_128(rk, round); - *block = aes_inner_round(block, &rk_r, &tables); - round = round.strict_add(1); - } - - // Round 10 (final): SubBytes + ShiftRows + AddRoundKey (no MixColumns). - let rk10 = round_key_bytes_128(rk, super::ROUNDS_128); - *block = aes_final_round(block, &rk10, &tables); - } -} - -/// Encrypt 4 independent AES-128 blocks using the vperm backend. -/// -/// # Safety -/// Caller must ensure the CPU supports the RISC-V `v` extension. -#[target_feature(enable = "v")] -pub(super) unsafe fn encrypt_4blocks_128(rk: &[u32; super::EXPANDED_KEY_WORDS_128], blocks: &mut [[u8; 16]; 4]) { - // SAFETY: caller guarantees the RISC-V V extension is available for the - // duration of the batch and all block buffers are valid 16-byte arrays. - unsafe { - let tables = VpermTables::load(); - - let rk0 = round_key_bytes_128(rk, 0); - for block in blocks.iter_mut() { - for i in 0..16 { - block[i] ^= rk0[i]; - } - } - - let mut round = 1usize; - while round < super::ROUNDS_128 { - let rk_r = round_key_bytes_128(rk, round); - aes_inner_round_4(blocks, &rk_r, &tables); - round = round.strict_add(1); - } - - let rk10 = round_key_bytes_128(rk, super::ROUNDS_128); - aes_final_round_4(blocks, &rk10, &tables); - } -} diff --git a/src/aead/aes/x86_64/asm.rs b/src/aead/aes/x86_64/asm.rs index 9a7caef4..24a09024 100644 --- a/src/aead/aes/x86_64/asm.rs +++ b/src/aead/aes/x86_64/asm.rs @@ -1,13 +1,10 @@ //! Linux x86-64 rscrypto-owned AES-GCM assembly kernels. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_aes_gcm_x86_64_linux.s")); #[repr(C)] -#[allow(dead_code)] pub(super) struct AesGcmX86State { acc_lo: u64, acc_hi: u64, @@ -16,13 +13,12 @@ pub(super) struct AesGcmX86State { pub(super) processed: usize, } -#[allow(dead_code)] impl AesGcmX86State { #[inline] pub(super) fn new(acc: u128, ctr: u32) -> Self { Self { - acc_lo: acc as u64, - acc_hi: (acc >> 64) as u64, + acc_lo: u64::try_from(acc & u128::from(u64::MAX)).expect("masked accumulator half fits u64"), + acc_hi: u64::try_from(acc >> 64).expect("shifted accumulator half fits u64"), ctr, _pad: 0, processed: 0, @@ -36,7 +32,6 @@ impl AesGcmX86State { } unsafe extern "C" { - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_seal_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -46,7 +41,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_open_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -56,7 +50,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_seal_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -66,7 +59,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_open_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -76,7 +68,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_seal_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -86,7 +77,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes128_gcm_open_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -96,7 +86,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_seal_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -106,7 +95,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_open_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -116,7 +104,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_seal_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -126,7 +113,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_open_64x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -136,7 +122,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_seal_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -146,7 +131,6 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] pub(super) fn rscrypto_aes256_gcm_open_128x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -156,47 +140,7 @@ unsafe extern "C" { state: *mut AesGcmX86State, ); - #[allow(dead_code)] - pub(super) fn rscrypto_aes128_gcm_seal_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] - pub(super) fn rscrypto_aes128_gcm_open_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] - pub(super) fn rscrypto_aes256_gcm_seal_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] - pub(super) fn rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux( - round_keys: *const u8, - initial_counter: *const u8, - data: *mut u8, - len: usize, - h_powers_rev_8: *const u128, - state: *mut AesGcmX86State, - ); - - #[allow(dead_code)] + #[cfg(feature = "aes-gcm-siv")] pub(super) fn rscrypto_aes128_gcmsiv_ctr_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, @@ -204,7 +148,7 @@ unsafe extern "C" { len: usize, ) -> usize; - #[allow(dead_code)] + #[cfg(feature = "aes-gcm-siv")] pub(super) fn rscrypto_aes256_gcmsiv_ctr_16x_vaes512_x86_64_linux( round_keys: *const u8, initial_counter: *const u8, diff --git a/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s b/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s index a5fe46de..f5a7d1a7 100644 --- a/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s +++ b/src/aead/aes/x86_64/asm/rscrypto_aes_gcm_x86_64_linux.s @@ -79,53 +79,6 @@ vmovdqu32 zmm3{{k1}}, zmm25 .endm -.macro GCM_COUNTERS_8_SLOW - vpbroadcastd ymm25, r10d - - vbroadcasti32x4 ymm0, XMMWORD PTR [rsi] - vmovdqa64 ymm1, ymm0 - vmovdqa64 ymm2, ymm0 - vmovdqa64 ymm3, ymm0 - - vmovdqa32 ymm24, ymm25 - vpaddd ymm24, ymm24, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y0] - vpshufb ymm24, ymm24, ymm26 - vmovdqu32 ymm0{{k1}}, ymm24 - - vmovdqa32 ymm24, ymm25 - vpaddd ymm24, ymm24, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y1] - vpshufb ymm24, ymm24, ymm26 - vmovdqu32 ymm1{{k1}}, ymm24 - - vmovdqa32 ymm24, ymm25 - vpaddd ymm24, ymm24, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y2] - vpshufb ymm24, ymm24, ymm26 - vmovdqu32 ymm2{{k1}}, ymm24 - - vpaddd ymm25, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_inc_y3] - vpshufb ymm25, ymm25, ymm26 - vmovdqu32 ymm3{{k1}}, ymm25 -.endm - -.macro GCM_COUNTERS_8 - mov eax, r10d - cmp al, 249 - jae 1f - bswap eax - vpbroadcastd ymm24, eax - - vbroadcasti32x4 ymm25, XMMWORD PTR [rsi] - vmovdqu32 ymm25{{k1}}, ymm24 - - vpaddd ymm0, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y0] - vpaddd ymm1, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y1] - vpaddd ymm2, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y2] - vpaddd ymm3, ymm25, YMMWORD PTR [rip + .Lrscrypto_x86_ctr_incbe_y3] - jmp 2f -1: - GCM_COUNTERS_8_SLOW -2: -.endm .macro AES_ROUND off vbroadcasti32x4 zmm27, XMMWORD PTR [rdi + \off] @@ -173,51 +126,6 @@ .endif .endm -.macro AES_Y_ROUND off - vbroadcasti32x4 ymm27, XMMWORD PTR [rdi + \off] - vaesenc ymm0, ymm0, ymm27 - vaesenc ymm1, ymm1, ymm27 - vaesenc ymm2, ymm2, ymm27 - vaesenc ymm3, ymm3, ymm27 -.endm - -.macro AES_Y_START - vbroadcasti32x4 ymm27, XMMWORD PTR [rdi] - vpxord ymm0, ymm0, ymm27 - vpxord ymm1, ymm1, ymm27 - vpxord ymm2, ymm2, ymm27 - vpxord ymm3, ymm3, ymm27 -.endm - -.macro AES_Y_LAST off - vbroadcasti32x4 ymm27, XMMWORD PTR [rdi + \off] - vaesenclast ymm0, ymm0, ymm27 - vaesenclast ymm1, ymm1, ymm27 - vaesenclast ymm2, ymm2, ymm27 - vaesenclast ymm3, ymm3, ymm27 -.endm - -.macro AES_ENCRYPT_8Y aes256 - AES_Y_START - AES_Y_ROUND 16 - AES_Y_ROUND 32 - AES_Y_ROUND 48 - AES_Y_ROUND 64 - AES_Y_ROUND 80 - AES_Y_ROUND 96 - AES_Y_ROUND 112 - AES_Y_ROUND 128 - AES_Y_ROUND 144 - .if \aes256 - AES_Y_ROUND 160 - AES_Y_ROUND 176 - AES_Y_ROUND 192 - AES_Y_ROUND 208 - AES_Y_LAST 224 - .else - AES_Y_LAST 160 - .endif -.endm .macro GHASH_FOLD data, h vpclmulqdq zmm21, \data, \h, 0x00 @@ -263,22 +171,6 @@ vpxorq zmm29, zmm29, zmm22 .endm -.macro GHASH_FOLD_Y data, h - vpclmulqdq ymm21, \data, \h, 0x00 - vpclmulqdq ymm22, \data, \h, 0x11 - - vpshufd ymm23, \data, 0x4e - vpxord ymm23, ymm23, \data - vpshufd ymm24, \h, 0x4e - vpxord ymm24, ymm24, \h - vpclmulqdq ymm23, ymm23, ymm24, 0x00 - vpternlogq ymm23, ymm21, ymm22, 0x96 - - vpslldq ymm24, ymm23, 8 - vpternlogq ymm28, ymm21, ymm24, 0x96 - vpsrldq ymm24, ymm23, 8 - vpternlogq ymm29, ymm22, ymm24, 0x96 -.endm .macro GHASH_PREP d0, d1, d2, d3 vpshufb \d0, \d0, zmm31 @@ -293,19 +185,6 @@ vpxord zmm29, zmm29, zmm29 .endm -.macro GHASH_PREP_Y d0, d1, d2, d3 - vpshufb \d0, \d0, ymm31 - vpshufb \d1, \d1, ymm31 - vpshufb \d2, \d2, ymm31 - vpshufb \d3, \d3, ymm31 - - vpxord ymm30, ymm30, ymm30 - vmovdqu64 xmm30, XMMWORD PTR [r9] - vpxord \d0, \d0, ymm30 - - vpxord ymm28, ymm28, ymm28 - vpxord ymm29, ymm29, ymm29 -.endm .macro GHASH_REDUCE_ASSEMBLE vpxorq zmm20, zmm20, zmm28 @@ -369,37 +248,6 @@ GHASH_REDUCE_STORE .endm -.macro GHASH_REDUCE_Y - vmovdqa64 xmm16, xmm28 - vextracti64x2 xmm17, ymm28, 1 - vpxorq xmm16, xmm16, xmm17 - - vmovdqa64 xmm19, xmm29 - vextracti64x2 xmm17, ymm29, 1 - vpxorq xmm19, xmm19, xmm17 - - vpsllq xmm17, xmm16, 63 - vpsllq xmm18, xmm16, 62 - vpsllq xmm20, xmm16, 57 - vpternlogq xmm17, xmm18, xmm20, 0x96 - vpslldq xmm17, xmm17, 8 - vpxorq xmm16, xmm16, xmm17 - - vpsrlq xmm17, xmm16, 1 - vpsrlq xmm18, xmm16, 2 - vpsrlq xmm20, xmm16, 7 - vpternlogq xmm17, xmm18, xmm20, 0x96 - vpxorq xmm17, xmm17, xmm16 - - vpsllq xmm18, xmm16, 63 - vpsllq xmm20, xmm16, 62 - vpsllq xmm21, xmm16, 57 - vpternlogq xmm18, xmm20, xmm21, 0x96 - vpsrldq xmm18, xmm18, 8 - - vpternlogq xmm17, xmm18, xmm19, 0x96 - vmovdqu64 XMMWORD PTR [r9], xmm17 -.endm .macro GHASH16_REG d0, d1, d2, d3 GHASH16_REG_H \d0, \d1, \d2, \d3, zmm12, zmm13, zmm14, zmm15 @@ -607,14 +455,6 @@ vmovdqu64 zmm19, ZMMWORD PTR [r8 + 1984] .endm -.macro GHASH8Y_REG d0, d1, d2, d3 - GHASH_PREP_Y \d0, \d1, \d2, \d3 - GHASH_FOLD_Y \d0, ymm12 - GHASH_FOLD_Y \d1, ymm13 - GHASH_FOLD_Y \d2, ymm14 - GHASH_FOLD_Y \d3, ymm15 - GHASH_REDUCE_Y -.endm .macro AES_ENCRYPT_16_WITH_GHASH aes256, d0, d1, d2, d3 AES_START @@ -896,34 +736,6 @@ .endif .endm -.macro AES_ENCRYPT_8Y_WITH_GHASH aes256, d0, d1, d2, d3 - AES_Y_START - GHASH_PREP_Y \d0, \d1, \d2, \d3 - AES_Y_ROUND 16 - GHASH_FOLD_Y \d0, ymm12 - AES_Y_ROUND 32 - GHASH_FOLD_Y \d1, ymm13 - AES_Y_ROUND 48 - GHASH_FOLD_Y \d2, ymm14 - AES_Y_ROUND 64 - GHASH_FOLD_Y \d3, ymm15 - AES_Y_ROUND 80 - AES_Y_ROUND 96 - AES_Y_ROUND 112 - AES_Y_ROUND 128 - AES_Y_ROUND 144 - .if \aes256 - AES_Y_ROUND 160 - AES_Y_ROUND 176 - AES_Y_ROUND 192 - AES_Y_ROUND 208 - GHASH_REDUCE_Y - AES_Y_LAST 224 - .else - GHASH_REDUCE_Y - AES_Y_LAST 160 - .endif -.endm .macro AES_GCM_STORE_Z open, d0, d1, d2, d3 .if \open @@ -1573,114 +1385,6 @@ .size \name, . - \name .endm -.macro AES_GCM_8Y_FUNC name, aes256, open - .p2align 5 - .globl \name - .type \name, @function -\name: - mov r10d, DWORD PTR [r9 + 16] - - cmp rcx, 128 - jb .L\name\()_done - - vmovdqu64 ymm31, YMMWORD PTR [rip + .Lrscrypto_x86_gcm_bswap] - vmovdqu64 ymm26, YMMWORD PTR [rip + .Lrscrypto_x86_dword_bswap] - vmovdqu64 ymm12, YMMWORD PTR [r8] - vmovdqu64 ymm13, YMMWORD PTR [r8 + 32] - vmovdqu64 ymm14, YMMWORD PTR [r8 + 64] - vmovdqu64 ymm15, YMMWORD PTR [r8 + 96] - mov eax, 0x88 - kmovw k1, eax - - GCM_COUNTERS_8 - - AES_ENCRYPT_8Y \aes256 - - .if \open - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm8, ymm0, ymm4 - vpxord ymm9, ymm1, ymm5 - vpxord ymm10, ymm2, ymm6 - vpxord ymm11, ymm3, ymm7 - vmovdqu64 YMMWORD PTR [rdx], ymm8 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm9 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm10 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm11 - .else - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm4, ymm4, ymm0 - vpxord ymm5, ymm5, ymm1 - vpxord ymm6, ymm6, ymm2 - vpxord ymm7, ymm7, ymm3 - vmovdqu64 YMMWORD PTR [rdx], ymm4 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm5 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm6 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm7 - .endif - - add rdx, 128 - sub rcx, 128 - add r10d, 8 - add QWORD PTR [r9 + 24], 128 - cmp rcx, 128 - jb .L\name\()_final_ghash - - .p2align 5 -.L\name\()_loop: - GCM_COUNTERS_8 - - AES_ENCRYPT_8Y_WITH_GHASH \aes256, ymm4, ymm5, ymm6, ymm7 - - .if \open - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm8, ymm0, ymm4 - vpxord ymm9, ymm1, ymm5 - vpxord ymm10, ymm2, ymm6 - vpxord ymm11, ymm3, ymm7 - vmovdqu64 YMMWORD PTR [rdx], ymm8 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm9 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm10 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm11 - .else - vmovdqu64 ymm4, YMMWORD PTR [rdx] - vmovdqu64 ymm5, YMMWORD PTR [rdx + 32] - vmovdqu64 ymm6, YMMWORD PTR [rdx + 64] - vmovdqu64 ymm7, YMMWORD PTR [rdx + 96] - vpxord ymm4, ymm4, ymm0 - vpxord ymm5, ymm5, ymm1 - vpxord ymm6, ymm6, ymm2 - vpxord ymm7, ymm7, ymm3 - vmovdqu64 YMMWORD PTR [rdx], ymm4 - vmovdqu64 YMMWORD PTR [rdx + 32], ymm5 - vmovdqu64 YMMWORD PTR [rdx + 64], ymm6 - vmovdqu64 YMMWORD PTR [rdx + 96], ymm7 - .endif - - add rdx, 128 - sub rcx, 128 - add r10d, 8 - add QWORD PTR [r9 + 24], 128 - cmp rcx, 128 - jae .L\name\()_loop - -.L\name\()_final_ghash: - GHASH8Y_REG ymm4, ymm5, ymm6, ymm7 - -.L\name\()_done: - mov DWORD PTR [r9 + 16], r10d - vzeroupper - ret - .size \name, . - \name -.endm AES_GCM_16X_FUNC rscrypto_aes128_gcm_seal_16x_vaes512_x86_64_linux, 0, 0 AES_GCM_16X_FUNC rscrypto_aes128_gcm_open_16x_vaes512_x86_64_linux, 0, 1 @@ -1696,10 +1400,6 @@ AES_GCM_128X_FUNC rscrypto_aes256_gcm_seal_128x_vaes512_x86_64_linux, 1, 0 AES_GCM_128X_FUNC rscrypto_aes256_gcm_open_128x_vaes512_x86_64_linux, 1, 1 AES_GCMSIV_CTR_16X_FUNC rscrypto_aes128_gcmsiv_ctr_16x_vaes512_x86_64_linux, 0 AES_GCMSIV_CTR_16X_FUNC rscrypto_aes256_gcmsiv_ctr_16x_vaes512_x86_64_linux, 1 -AES_GCM_8Y_FUNC rscrypto_aes128_gcm_seal_8x_vaes256_x86_64_linux, 0, 0 -AES_GCM_8Y_FUNC rscrypto_aes128_gcm_open_8x_vaes256_x86_64_linux, 0, 1 -AES_GCM_8Y_FUNC rscrypto_aes256_gcm_seal_8x_vaes256_x86_64_linux, 1, 0 -AES_GCM_8Y_FUNC rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux, 1, 1 .section .rodata.cst64,"aM",@progbits,64 .p2align 6 @@ -1745,23 +1445,3 @@ AES_GCM_8Y_FUNC rscrypto_aes256_gcm_open_8x_vaes256_x86_64_linux, 1, 1 .long 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0 .Lrscrypto_x86_gcmsiv_ctr_inc_z3: .long 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0 - -.p2align 5 -.Lrscrypto_x86_ctr_inc_y0: - .long 0, 0, 0, 0, 0, 0, 0, 1 -.Lrscrypto_x86_ctr_inc_y1: - .long 0, 0, 0, 2, 0, 0, 0, 3 -.Lrscrypto_x86_ctr_inc_y2: - .long 0, 0, 0, 4, 0, 0, 0, 5 -.Lrscrypto_x86_ctr_inc_y3: - .long 0, 0, 0, 6, 0, 0, 0, 7 - -.p2align 5 -.Lrscrypto_x86_ctr_incbe_y0: - .long 0, 0, 0, 0x00000000, 0, 0, 0, 0x01000000 -.Lrscrypto_x86_ctr_incbe_y1: - .long 0, 0, 0, 0x02000000, 0, 0, 0, 0x03000000 -.Lrscrypto_x86_ctr_incbe_y2: - .long 0, 0, 0, 0x04000000, 0, 0, 0, 0x05000000 -.Lrscrypto_x86_ctr_incbe_y3: - .long 0, 0, 0, 0x06000000, 0, 0, 0, 0x07000000 diff --git a/src/aead/aes/x86_64_ni.rs b/src/aead/aes/x86_64_ni.rs index f441cc19..c9fbb60c 100644 --- a/src/aead/aes/x86_64_ni.rs +++ b/src/aead/aes/x86_64_ni.rs @@ -90,6 +90,7 @@ pub(super) unsafe fn expand_key(key: &[u8; 32]) -> NiRoundKeys { /// /// # Safety /// Caller must ensure AVX-512F + AVX-512VL + VAES + AES + SSE2. +#[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,sse2,avx512f,avx512vl,vaes")] #[inline] pub(super) unsafe fn encrypt_4blocks(keys: &NiRoundKeys, blocks: __m512i) -> __m512i { @@ -219,61 +220,6 @@ pub(super) unsafe fn encrypt_16blocks( ) } -/// Encrypt 8 AES-256 blocks as four independent VAES-256 dependency chains. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + AES + SSE2. -#[cfg(feature = "aes-gcm")] -#[target_feature(enable = "aes,sse2,avx2,avx512f,avx512vl,vaes")] -#[inline] -#[allow(dead_code)] -pub(super) unsafe fn encrypt_8blocks_y256( - keys: &NiRoundKeys, - b0: __m256i, - b1: __m256i, - b2: __m256i, - b3: __m256i, -) -> (__m256i, __m256i, __m256i, __m256i) { - let k = &keys.rk; - let rk0 = _mm256_broadcastsi128_si256(k[0]); - let mut s0 = _mm256_xor_si256(b0, rk0); - let mut s1 = _mm256_xor_si256(b1, rk0); - let mut s2 = _mm256_xor_si256(b2, rk0); - let mut s3 = _mm256_xor_si256(b3, rk0); - - macro_rules! round { - ($idx:expr) => {{ - let rk = _mm256_broadcastsi128_si256(k[$idx]); - s0 = _mm256_aesenc_epi128(s0, rk); - s1 = _mm256_aesenc_epi128(s1, rk); - s2 = _mm256_aesenc_epi128(s2, rk); - s3 = _mm256_aesenc_epi128(s3, rk); - }}; - } - - round!(1); - round!(2); - round!(3); - round!(4); - round!(5); - round!(6); - round!(7); - round!(8); - round!(9); - round!(10); - round!(11); - round!(12); - round!(13); - - let rk14 = _mm256_broadcastsi128_si256(k[14]); - ( - _mm256_aesenclast_epi128(s0, rk14), - _mm256_aesenclast_epi128(s1, rk14), - _mm256_aesenclast_epi128(s2, rk14), - _mm256_aesenclast_epi128(s3, rk14), - ) -} - /// Encrypt a single 16-byte block using AES-256 with AES-NI. /// /// # Safety @@ -379,6 +325,7 @@ pub(super) unsafe fn expand_key_128(key: &[u8; 16]) -> Ni128RoundKeys { /// /// # Safety /// Caller must ensure AVX-512F + AVX-512VL + VAES + AES + SSE2. +#[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "aes,sse2,avx512f,avx512vl,vaes")] #[inline] pub(super) unsafe fn encrypt_4blocks_128(keys: &Ni128RoundKeys, blocks: __m512i) -> __m512i { @@ -496,57 +443,6 @@ pub(super) unsafe fn encrypt_16blocks_128( ) } -/// Encrypt 8 AES-128 blocks as four independent VAES-256 dependency chains. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VAES + AES + SSE2. -#[cfg(feature = "aes-gcm")] -#[target_feature(enable = "aes,sse2,avx2,avx512f,avx512vl,vaes")] -#[inline] -#[allow(dead_code)] -pub(super) unsafe fn encrypt_8blocks_128_y256( - keys: &Ni128RoundKeys, - b0: __m256i, - b1: __m256i, - b2: __m256i, - b3: __m256i, -) -> (__m256i, __m256i, __m256i, __m256i) { - let k = &keys.rk; - let rk0 = _mm256_broadcastsi128_si256(k[0]); - let mut s0 = _mm256_xor_si256(b0, rk0); - let mut s1 = _mm256_xor_si256(b1, rk0); - let mut s2 = _mm256_xor_si256(b2, rk0); - let mut s3 = _mm256_xor_si256(b3, rk0); - - macro_rules! round { - ($idx:expr) => {{ - let rk = _mm256_broadcastsi128_si256(k[$idx]); - s0 = _mm256_aesenc_epi128(s0, rk); - s1 = _mm256_aesenc_epi128(s1, rk); - s2 = _mm256_aesenc_epi128(s2, rk); - s3 = _mm256_aesenc_epi128(s3, rk); - }}; - } - - round!(1); - round!(2); - round!(3); - round!(4); - round!(5); - round!(6); - round!(7); - round!(8); - round!(9); - - let rk10 = _mm256_broadcastsi128_si256(k[10]); - ( - _mm256_aesenclast_epi128(s0, rk10), - _mm256_aesenclast_epi128(s1, rk10), - _mm256_aesenclast_epi128(s2, rk10), - _mm256_aesenclast_epi128(s3, rk10), - ) -} - /// Encrypt a single 16-byte block using AES-128 with AES-NI. /// /// # Safety diff --git a/src/aead/aes128gcm.rs b/src/aead/aes128gcm.rs index f3e860fb..a73cc202 100644 --- a/src/aead/aes128gcm.rs +++ b/src/aead/aes128gcm.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! AES-128-GCM public AEAD surface (NIST SP 800-38D). use core::fmt; @@ -277,9 +275,9 @@ fn ghash_update_padded(mut acc: u128, h_polyval: u128, data: &[u8]) -> u128 { #[inline] fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -289,7 +287,7 @@ fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128 acc = polyval::accumulate_4blocks(acc, h_polyval, h_powers_rev, &blocks); } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); acc = polyval::clmul128_reduce(acc, h_polyval); } @@ -380,6 +378,7 @@ fn encrypt_j0_tag(ek: &aes::Aes128EncKey, j0: &[u8; 16], acc: u128) -> [u8; TAG_ } #[cfg(feature = "diag")] +/// Exercise AES-128 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes128gcm_ctr32_be(cipher: &Aes128Gcm, nonce: &Nonce96, plaintext: &[u8; 44]) -> [u8; 16] { let (_, ctr_block) = make_j0_and_ctr(nonce); @@ -389,6 +388,7 @@ pub fn diag_aes128gcm_ctr32_be(cipher: &Aes128Gcm, nonce: &Nonce96, plaintext: & } #[cfg(feature = "diag")] +/// Return the AES-128-GCM GHASH accumulator for diagnostic backend comparison. #[unsafe(no_mangle)] #[inline(never)] #[must_use] @@ -412,6 +412,7 @@ pub fn diag_aes128gcm_ghash(cipher: &Aes128Gcm, aad: &[u8], ciphertext: &[u8]) - } #[cfg(feature = "diag")] +/// Encrypt a diagnostic GHASH accumulator into the final AES-128-GCM tag. #[must_use] pub fn diag_aes128gcm_tag_aes(cipher: &Aes128Gcm, nonce: &Nonce96, acc: &[u8; 16]) -> [u8; 16] { let (j0, _) = make_j0_and_ctr(nonce); @@ -505,9 +506,9 @@ unsafe fn ghash_update_padded_wide_aarch64( data: &[u8], ) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -520,7 +521,7 @@ unsafe fn ghash_update_padded_wide_aarch64( acc = unsafe { polyval::aarch64_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: PMULL carryless multiply because: // 1. This function's caller must guarantee AES-CE/PMULL availability. @@ -549,9 +550,9 @@ unsafe fn ghash_update_padded_wide_aarch64( #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -564,7 +565,7 @@ unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_ acc = unsafe { polyval::ppc_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: POWER8 carryless multiply because: // 1. This function's caller must guarantee POWER8 crypto availability. @@ -775,10 +776,6 @@ impl Aead for Aes128Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM sealing because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper encrypts `buffer` in place and folds the resulting ciphertext into GHASH. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -929,10 +926,6 @@ impl Aead for Aes128Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM open because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper GHASHes ciphertext bytes before decrypting each chunk in place. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -1076,10 +1069,13 @@ impl Drop for Aes128Gcm { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex16}, + }; // NIST SP 800-38D Test Case 1: AES-128-GCM, empty plaintext, empty AAD. // Key: 00000000000000000000000000000000 @@ -1097,10 +1093,14 @@ mod tests { let expected_tag = hex16("58e2fccefa7e3061367f1d57a4e7455a"); let mut buf = vec![]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST empty AES-128-GCM encryption must succeed"); assert_eq!(tag.0, expected_tag, "Tag mismatch on encrypt"); - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST empty AES-128-GCM decryption must succeed"); } // NIST SP 800-38D Test Case 2: AES-128-GCM, 16-byte zero plaintext, empty AAD. @@ -1120,11 +1120,15 @@ mod tests { let expected_tag = hex16("ab6e47d42cec13bdf53a67b21257bddf"); let mut buf = vec![0u8; 16]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST one-block AES-128-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST one-block AES-128-GCM decryption must succeed"); assert_eq!(buf, vec![0u8; 16], "Plaintext mismatch after decrypt"); } @@ -1150,11 +1154,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST multi-block AES-128-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST multi-block AES-128-GCM decryption must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1181,11 +1189,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("NIST AES-128-GCM encryption with AAD must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("NIST AES-128-GCM decryption with AAD must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1197,11 +1209,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = vec![0u8; 16]; - let mut tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let mut tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); tag.0[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1210,14 +1226,17 @@ mod tests { let key = Aes128GcmKey::from_bytes(hex16("feffe9928665731c6d6a8f9467308308")); let nonce = Nonce96::from_bytes(hex12("cafebabefacedbaddecaf888")); let aad = hex_vec("feedfacedeadbeeffeedfacedeadbeefabaddad2"); - let plaintext = hex_vec("d9313225f88406e5a55909c5aff5269a"); let cipher = Aes128Gcm::new(&key); - let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let mut buf = hex_vec("d9313225f88406e5a55909c5aff5269a"); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); - let result = cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Ciphertext tampering should fail verification. @@ -1228,11 +1247,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = vec![0u8; 32]; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); buf[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Detached encrypt/decrypt round-trip. @@ -1245,10 +1268,14 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("AES-128-GCM detached encryption must succeed"); assert_ne!(&buf[..], &plaintext[..]); - cipher.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("AES-128-GCM detached decryption must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -1262,20 +1289,34 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, aad, plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, aad, plaintext, &mut out) + .expect("AES-128-GCM combined encryption must succeed"); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, aad, &out, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, aad, &out, &mut pt_out) + .expect("AES-128-GCM combined decryption must succeed"); assert_eq!(&pt_out[..], &plaintext[..]); } /// `tag_from_slice` rejects wrong-length input. #[test] fn tag_from_slice_rejects_bad_length() { - assert!(Aes128Gcm::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes128Gcm::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes128Gcm::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes128Gcm::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aes128Gcm::tag_from_slice(&[0u8; 15]).expect_err("short AES-128-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128Gcm::tag_from_slice(&[0u8; 17]).expect_err("long AES-128-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128Gcm::tag_from_slice(&[]).expect_err("empty AES-128-GCM tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes128Gcm::tag_from_slice(&[0u8; 16]).expect("16-byte AES-128-GCM tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } /// Decryption with wrong nonce must fail. @@ -1286,11 +1327,15 @@ mod tests { let cipher = Aes128Gcm::new(&key); let mut buf = *b"hello gcm"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); let wrong_nonce = Nonce96::from_bytes([0x08u8; 12]); - let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1302,42 +1347,18 @@ mod tests { let plaintext = *b"zero me on failure"; let mut buf = plaintext; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-128-GCM test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = Aes128GcmTag::from_bytes(bad_tag); - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } - - // --- Hex helpers --- - - fn hex16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - for i in 0..16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out - } } diff --git a/src/aead/aes128gcmsiv.rs b/src/aead/aes128gcmsiv.rs index 2d9cceb4..9796f20f 100644 --- a/src/aead/aes128gcmsiv.rs +++ b/src/aead/aes128gcmsiv.rs @@ -1,15 +1,18 @@ -#![allow(clippy::indexing_slicing)] - //! AES-128-GCM-SIV public AEAD surface (RFC 8452). use core::fmt; #[cfg(target_arch = "x86_64")] use super::polyval::{accumulate_padded_x86, precompute_powers, precompute_powers_16}; -use super::{ - AeadBufferError, Nonce96, OpenError, SealError, aes, polyval, - targets::{AeadBackend, AeadPrimitive, select_backend}, -}; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] +use super::targets::{AeadBackend, AeadPrimitive, select_backend}; +use super::{AeadBufferError, Nonce96, OpenError, SealError, aes, polyval}; use crate::traits::{Aead, ct}; const KEY_SIZE: usize = 16; @@ -85,7 +88,13 @@ define_aead_tag_type!( /// timing claims are configuration- and release-evidence-bound; see `ct.toml`. pub struct Aes128GcmSiv { master_ek: aes::Aes128EncKey, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend: AeadBackend, } @@ -219,12 +228,14 @@ fn compute_tag( } #[cfg(feature = "diag")] +/// Derive the per-nonce authentication and encryption keys for diagnostic comparison. #[must_use] pub fn diag_aes128gcmsiv_derive_keys(cipher: &Aes128GcmSiv, nonce: &Nonce96) -> ([u8; 16], [u8; 16]) { derive_keys(&cipher.master_ek, nonce) } #[cfg(feature = "diag")] +/// Return the AES-128-GCM-SIV POLYVAL digest before nonce and AES tag finalization. #[must_use] pub fn diag_aes128gcmsiv_polyval_digest(auth_key: &[u8; 16], aad: &[u8], plaintext: &[u8]) -> [u8; 16] { let mut pv = polyval::Polyval::new(auth_key); @@ -236,6 +247,7 @@ pub fn diag_aes128gcmsiv_polyval_digest(auth_key: &[u8; 16], aad: &[u8], plainte } #[cfg(feature = "diag")] +/// Encrypt one diagnostic tag block with a raw AES-128 key. #[must_use] pub fn diag_aes128gcmsiv_raw_tag_aes(enc_key: &[u8; 16], block: &[u8; 16]) -> [u8; 16] { let mut out = *block; @@ -253,6 +265,7 @@ pub fn diag_aes128gcmsiv_raw_tag_aes(enc_key: &[u8; 16], block: &[u8; 16]) -> [u } #[cfg(feature = "diag")] +/// Exercise AES-128 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes128gcmsiv_ctr32(enc_key: &[u8; 16], tag: &[u8; 16], plaintext: &[u8; 44]) -> [u8; 16] { let mut counter_block = *tag; @@ -404,7 +417,6 @@ fn expand_key_riscv_for_backend(key: &[u8; 16], backend: AeadBackend) -> aes::Ae match backend { AeadBackend::Riscv64VectorCrypto => aes::aes128_expand_key_riscv_vector(key), AeadBackend::Riscv64ScalarCrypto => aes::aes128_expand_key_riscv_scalar(key), - AeadBackend::Riscv64Vperm => aes::aes128_expand_key_riscv_vperm(key), AeadBackend::Portable => aes::aes128_expand_key_riscv_ttable(key), _ => aes::aes128_expand_key_riscv_ttable(key), } @@ -416,6 +428,13 @@ fn expand_message_key_riscv(enc_key: &[u8; 16], backend: AeadBackend) -> aes::Ae expand_key_riscv_for_backend(enc_key, backend) } +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] #[inline] fn resolve_backend() -> AeadBackend { select_backend( @@ -431,7 +450,7 @@ fn riscv_polyval_backend(backend: AeadBackend) -> RiscvPolyvalBackend { match backend { AeadBackend::Riscv64VectorCrypto => RiscvPolyvalBackend::Vector, AeadBackend::Riscv64ScalarCrypto => RiscvPolyvalBackend::Scalar, - AeadBackend::Portable | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable => { let caps = crate::platform::caps(); if caps.has(crate::platform::caps::riscv::ZBC) || caps.has(crate::platform::caps::riscv::ZBKC) { RiscvPolyvalBackend::Scalar @@ -559,6 +578,12 @@ fn compute_tag_wide( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Seal one message through the fused AArch64 AES-128-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn encrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 16], @@ -705,6 +730,12 @@ unsafe fn encrypt_fused_aarch64( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Open one message through the fused AArch64 AES-128-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn decrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 16], @@ -857,6 +888,11 @@ unsafe fn decrypt_fused_aarch64( // powerpc64 fused encrypt/decrypt (single #[target_feature] scope) +/// Encrypt with the fused POWER8 AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn encrypt_fused_ppc( @@ -986,6 +1022,11 @@ unsafe fn encrypt_fused_ppc( } } +/// Decrypt and authenticate with the fused POWER8 AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn decrypt_fused_ppc( @@ -1122,6 +1163,11 @@ unsafe fn decrypt_fused_ppc( // s390x fused encrypt/decrypt +/// XOR an AES-128 counter stream into `buffer` using s390x CPACF. +/// +/// # Safety +/// +/// The executing CPU must support MSA AES instructions. #[cfg(target_arch = "s390x")] unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 16], counter_block: &mut [u8; 16], buffer: &mut [u8]) { let mut ctr = u32::from_le_bytes([counter_block[0], counter_block[1], counter_block[2], counter_block[3]]); @@ -1147,12 +1193,17 @@ unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 16], counter_block: &mut [ let processed = aes::xor_keystream_tail(buffer, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } counter_block[0..4].copy_from_slice(&ctr.to_le_bytes()); } +/// Encrypt with the fused s390x AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn encrypt_fused_s390x( @@ -1257,6 +1308,11 @@ unsafe fn encrypt_fused_s390x( } } +/// Decrypt and authenticate with the fused s390x AES-128-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn decrypt_fused_s390x( @@ -1376,6 +1432,13 @@ impl Aead for Aes128GcmSiv { type Tag = Aes128GcmSivTag; fn new(key: &Self::Key) -> Self { + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] let backend = resolve_backend(); Self { @@ -1383,6 +1446,13 @@ impl Aead for Aes128GcmSiv { master_ek: expand_key_riscv_for_backend(key.as_bytes(), backend), #[cfg(not(target_arch = "riscv64"))] master_ek: aes::aes128_expand_key(key.as_bytes()), + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend, } } @@ -1460,10 +1530,7 @@ impl Aead for Aes128GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { let tag_bytes = encrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer); return Ok(Aes128GcmSivTag::from_bytes(tag_bytes)); } @@ -1556,10 +1623,7 @@ impl Aead for Aes128GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { return decrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer, tag).map_err(OpenError::from); } _ => {} @@ -1590,10 +1654,13 @@ impl Aead for Aes128GcmSiv { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex16}, + }; /// RFC 8452 Appendix C.1, test 1: empty plaintext, empty AAD. #[test] @@ -1604,11 +1671,15 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; expected_ct_tag.len()]; - cipher.encrypt(&nonce, &[], &[], &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &[], &mut out) + .expect("RFC 8452 empty AES-128-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; 0]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 empty AES-128-GCM-SIV decryption must succeed"); assert!(pt_out.is_empty()); } @@ -1622,11 +1693,15 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &[], &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &plaintext, &mut out) + .expect("RFC 8452 AES-128-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 AES-128-GCM-SIV decryption must succeed"); assert_eq!(pt_out, plaintext); } @@ -1642,11 +1717,15 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("RFC 8452 AES-128-GCM-SIV encryption with AAD must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 AES-128-GCM-SIV decryption with AAD must succeed"); assert_eq!(pt_out, plaintext); } @@ -1659,22 +1738,16 @@ mod tests { let plaintext = hex_vec( "0200000000000000000000000000000003000000000000000000000000000000040000000000000000000000000000000500000000000000000000000000000000", ); - let expected_ct_tag = hex_vec( - "2f5c64059db55ee0fb847ed51300374651a8c75e07ecbef82c1ed1bf48bb1ff5c1cdf0a4a3aa6a17ce7022eafd1129b25aafdca1fa9706ce0d1e23df1ce04c4a8e10cb44ed62b9a3a39d61b8b3eba23586eb6df746a45dd8d6ce80e7f5d33d39", - ); - - // The expected vector above is illustrative; oracle tests at - // tests/aes128gcmsiv_oracle.rs are the authoritative byte-equivalence - // check. This unit test fixes the 4-block + AAD shape against the - // round-trip rather than a hand-transcribed vector. - let _ = expected_ct_tag; - let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("multi-block AES-128-GCM-SIV encryption must succeed"); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &out, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &out, &mut pt_out) + .expect("multi-block AES-128-GCM-SIV decryption must succeed"); assert_eq!(pt_out, plaintext); } @@ -1688,8 +1761,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; 0]; - let result = cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1701,8 +1776,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; 8]; - let result = cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong nonce should fail. @@ -1715,8 +1792,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; 8]; let wrong_nonce = Nonce96::from_bytes(hex12("040000000000000000000000")); - let result = cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Ciphertext tampering should fail verification. @@ -1732,8 +1811,10 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut pt_out = vec![0u8; plaintext.len()]; - let result = cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1746,14 +1827,18 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("AES-128-GCM-SIV test setup encryption must succeed"); let last = out.len().strict_sub(1); out[last] ^= 0xff; let mut pt_out = vec![0xffu8; plaintext.len()]; - let result = cipher.decrypt(&nonce, &aad, &out, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &out, &mut pt_out), + Err(OpenError::verification()) + ); assert!(pt_out.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -1768,57 +1853,51 @@ mod tests { let cipher = Aes128GcmSiv::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-128-GCM-SIV detached encryption must succeed"); assert_ne!(buf, plaintext); - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("AES-128-GCM-SIV detached decryption must succeed"); assert_eq!(buf, plaintext); } /// `tag_from_slice` rejects wrong-length input. #[test] fn aes128gcmsiv_tag_from_slice_rejects_bad_length() { - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes128GcmSiv::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aes128GcmSiv::tag_from_slice(&[0u8; 15]).expect_err("short AES-128-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128GcmSiv::tag_from_slice(&[0u8; 17]).expect_err("long AES-128-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes128GcmSiv::tag_from_slice(&[]).expect_err("empty AES-128-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes128GcmSiv::tag_from_slice(&[0u8; 16]).expect("16-byte AES-128-GCM-SIV tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] #[cfg(target_pointer_width = "64")] fn aes128gcmsiv_input_limit_matches_rfc8452() { for len in [MAX_INPUT_LEN.strict_sub(1), MAX_INPUT_LEN] { - assert!(super::super::try_bounded_length_as_u64(len as usize, MAX_INPUT_LEN).is_ok()); - } - assert!(super::super::try_bounded_length_as_u64(MAX_INPUT_LEN.strict_add(1) as usize, MAX_INPUT_LEN).is_err()); - } - - // --- Hex helpers --- - - fn hex16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - for i in 0..16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out + let platform_len = usize::try_from(len).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(platform_len, MAX_INPUT_LEN), + Ok(len) + ); + } + let too_large = usize::try_from(MAX_INPUT_LEN.strict_add(1)).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(too_large, MAX_INPUT_LEN), + Err(super::super::LengthOverflow) + ); } } diff --git a/src/aead/aes256gcm.rs b/src/aead/aes256gcm.rs index 2c806bc6..ba27f516 100644 --- a/src/aead/aes256gcm.rs +++ b/src/aead/aes256gcm.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! AES-256-GCM public AEAD surface (NIST SP 800-38D). use core::fmt; @@ -275,9 +273,9 @@ fn ghash_update_padded(mut acc: u128, h_polyval: u128, data: &[u8]) -> u128 { #[inline] fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -287,7 +285,7 @@ fn ghash_update_padded_wide(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128 acc = polyval::accumulate_4blocks(acc, h_polyval, h_powers_rev, &blocks); } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); acc = polyval::clmul128_reduce(acc, h_polyval); } @@ -378,6 +376,7 @@ fn encrypt_j0_tag(ek: &aes::Aes256EncKey, j0: &[u8; 16], acc: u128) -> [u8; TAG_ } #[cfg(feature = "diag")] +/// Exercise AES-256 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes256gcm_ctr32_be(cipher: &Aes256Gcm, nonce: &Nonce96, plaintext: &[u8; 44]) -> [u8; 16] { let (_, ctr_block) = make_j0_and_ctr(nonce); @@ -387,6 +386,7 @@ pub fn diag_aes256gcm_ctr32_be(cipher: &Aes256Gcm, nonce: &Nonce96, plaintext: & } #[cfg(feature = "diag")] +/// Return the AES-256-GCM GHASH accumulator for diagnostic backend comparison. #[unsafe(no_mangle)] #[inline(never)] #[must_use] @@ -410,6 +410,7 @@ pub fn diag_aes256gcm_ghash(cipher: &Aes256Gcm, aad: &[u8], ciphertext: &[u8]) - } #[cfg(feature = "diag")] +/// Encrypt a diagnostic GHASH accumulator into the final AES-256-GCM tag. #[must_use] pub fn diag_aes256gcm_tag_aes(cipher: &Aes256Gcm, nonce: &Nonce96, acc: &[u8; 16]) -> [u8; 16] { let (j0, _) = make_j0_and_ctr(nonce); @@ -506,9 +507,9 @@ unsafe fn ghash_update_padded_wide_aarch64( data: &[u8], ) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -521,7 +522,7 @@ unsafe fn ghash_update_padded_wide_aarch64( acc = unsafe { polyval::aarch64_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: PMULL carryless multiply because: // 1. This function's caller must guarantee AES-CE/PMULL availability. @@ -550,9 +551,9 @@ unsafe fn ghash_update_padded_wide_aarch64( #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_rev: &[u128; 4], data: &[u8]) -> u128 { let (full_blocks, remainder) = data.as_chunks::<16>(); - let mut chunks = full_blocks.chunks_exact(4); + let (chunks, trailing_blocks) = full_blocks.as_chunks::<4>(); - for chunk in &mut chunks { + for chunk in chunks { let blocks = [ u128::from_be_bytes(chunk[0]), u128::from_be_bytes(chunk[1]), @@ -565,7 +566,7 @@ unsafe fn ghash_update_padded_wide_ppc(mut acc: u128, h_polyval: u128, h_powers_ acc = unsafe { polyval::ppc_aggregate_4blocks_inline(acc, h_powers_rev, &blocks) }; } - for block in chunks.remainder() { + for block in trailing_blocks { acc ^= u128::from_be_bytes(*block); // SAFETY: POWER8 carryless multiply because: // 1. This function's caller must guarantee POWER8 crypto availability. @@ -778,10 +779,6 @@ impl Aead for Aes256Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM sealing because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper encrypts `buffer` in place and folds the resulting ciphertext into GHASH. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -932,10 +929,6 @@ impl Aead for Aes256Gcm { // 2. `aad` is a valid byte slice; padding is handled inside the helper. let mut acc = GhashAccumulator(unsafe { ghash_update_padded_wide_aarch64(0, h_polyval, &self.h_powers_rev, aad) }); - // SAFETY: fused intrinsic AArch64 AES-GCM open because: - // 1. Backend resolution selected an AES/PMULL backend only after runtime detection confirmed AES-CE - // and PMULL. - // 2. The helper GHASHes ciphertext bytes before decrypting each chunk in place. let tables = aes::Aarch64GcmTables { h_polyval, h_powers_rev: &self.h_powers_rev, @@ -1079,10 +1072,13 @@ impl Drop for Aes256Gcm { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex16, hex32}, + }; // NIST SP 800-38D Test Case 13: AES-256-GCM, empty plaintext, empty AAD. // Key: 0000...00 (32 bytes) @@ -1101,11 +1097,15 @@ mod tests { // Encrypt. let mut buf = vec![]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST empty AES-256-GCM encryption must succeed"); assert_eq!(tag.0, expected_tag, "Tag mismatch on encrypt"); // Decrypt. - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST empty AES-256-GCM decryption must succeed"); } // NIST SP 800-38D Test Case 14: AES-256-GCM, 16-byte plaintext, empty AAD. @@ -1126,12 +1126,16 @@ mod tests { // Encrypt. let mut buf = vec![0u8; 16]; - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST one-block AES-256-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); // Decrypt. - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST one-block AES-256-GCM decryption must succeed"); assert_eq!(buf, vec![0u8; 16], "Plaintext mismatch after decrypt"); } @@ -1160,12 +1164,16 @@ mod tests { // Encrypt. let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("NIST multi-block AES-256-GCM encryption must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); // Decrypt. - cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &[], &mut buf, &tag) + .expect("NIST multi-block AES-256-GCM decryption must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1195,12 +1203,16 @@ mod tests { // Encrypt. let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("NIST AES-256-GCM encryption with AAD must succeed"); assert_eq!(buf, expected_ct, "Ciphertext mismatch"); assert_eq!(tag.0, expected_tag, "Tag mismatch"); // Decrypt. - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("NIST AES-256-GCM decryption with AAD must succeed"); assert_eq!(buf, plaintext, "Plaintext mismatch after decrypt"); } @@ -1212,11 +1224,15 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = vec![0u8; 16]; - let mut tag = cipher.encrypt_in_place(&nonce, &[], &mut buf).unwrap(); + let mut tag = cipher + .encrypt_in_place(&nonce, &[], &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); tag.0[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, &[], &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1227,15 +1243,18 @@ mod tests { )); let nonce = Nonce96::from_bytes(hex12("cafebabefacedbaddecaf888")); let aad = hex_vec("feedfacedeadbeeffeedfacedeadbeefabaddad2"); - let plaintext = hex_vec("d9313225f88406e5a55909c5aff5269a"); let cipher = Aes256Gcm::new(&key); - let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let mut buf = hex_vec("d9313225f88406e5a55909c5aff5269a"); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); // Wrong AAD. - let result = cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"wrong aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Ciphertext tampering should fail verification. @@ -1246,11 +1265,15 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = vec![0u8; 32]; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); buf[0] ^= 1; - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// Detached encrypt/decrypt round-trip. @@ -1263,10 +1286,14 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut buf) + .expect("AES-256-GCM detached encryption must succeed"); assert_ne!(&buf[..], &plaintext[..]); - cipher.decrypt_in_place(&nonce, aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut buf, &tag) + .expect("AES-256-GCM detached decryption must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -1280,56 +1307,34 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, aad, plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, aad, plaintext, &mut out) + .expect("AES-256-GCM combined encryption must succeed"); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, aad, &out, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, aad, &out, &mut pt_out) + .expect("AES-256-GCM combined decryption must succeed"); assert_eq!(&pt_out[..], &plaintext[..]); } /// `tag_from_slice` rejects wrong-length input. #[test] fn tag_from_slice_rejects_bad_length() { - assert!(Aes256Gcm::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes256Gcm::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes256Gcm::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes256Gcm::tag_from_slice(&[0u8; 16]).is_ok()); - } - - // --- Hex helpers --- - - fn hex16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - for i in 0..16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex32(hex: &str) -> [u8; 32] { - let mut out = [0u8; 32]; - for i in 0..32 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out + assert_eq!( + Aes256Gcm::tag_from_slice(&[0u8; 15]).expect_err("short AES-256-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256Gcm::tag_from_slice(&[0u8; 17]).expect_err("long AES-256-GCM tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256Gcm::tag_from_slice(&[]).expect_err("empty AES-256-GCM tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes256Gcm::tag_from_slice(&[0u8; 16]).expect("16-byte AES-256-GCM tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } /// Decryption with wrong nonce must fail. @@ -1340,11 +1345,15 @@ mod tests { let cipher = Aes256Gcm::new(&key); let mut buf = *b"hello gcm"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); let wrong_nonce = Nonce96::from_bytes([0x08u8; 12]); - let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1356,15 +1365,19 @@ mod tests { let plaintext = *b"zero me on failure"; let mut buf = plaintext; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("AES-256-GCM test setup encryption must succeed"); // Corrupt the tag. let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = Aes256GcmTag::from_bytes(bad_tag); - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } } diff --git a/src/aead/aes256gcmsiv.rs b/src/aead/aes256gcmsiv.rs index 57824a7f..14376a2e 100644 --- a/src/aead/aes256gcmsiv.rs +++ b/src/aead/aes256gcmsiv.rs @@ -1,15 +1,18 @@ -#![allow(clippy::indexing_slicing)] - //! AES-256-GCM-SIV public AEAD surface (RFC 8452). use core::fmt; #[cfg(target_arch = "x86_64")] use super::polyval::{accumulate_padded_x86, precompute_powers, precompute_powers_16}; -use super::{ - AeadBufferError, Nonce96, OpenError, SealError, aes, polyval, - targets::{AeadBackend, AeadPrimitive, select_backend}, -}; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] +use super::targets::{AeadBackend, AeadPrimitive, select_backend}; +use super::{AeadBufferError, Nonce96, OpenError, SealError, aes, polyval}; use crate::traits::{Aead, ct}; const KEY_SIZE: usize = 32; @@ -84,7 +87,13 @@ define_aead_tag_type!( /// are configuration- and release-evidence-bound; see `ct.toml`. pub struct Aes256GcmSiv { master_ek: aes::Aes256EncKey, - #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend: AeadBackend, } @@ -227,12 +236,14 @@ fn compute_tag( } #[cfg(feature = "diag")] +/// Derive the per-nonce authentication and encryption keys for diagnostic comparison. #[must_use] pub fn diag_aes256gcmsiv_derive_keys(cipher: &Aes256GcmSiv, nonce: &Nonce96) -> ([u8; 16], [u8; 32]) { derive_keys(&cipher.master_ek, nonce) } #[cfg(feature = "diag")] +/// Encrypt one diagnostic tag block with a raw AES-256 key. #[must_use] pub fn diag_aes256gcmsiv_raw_tag_aes(enc_key: &[u8; 32], block: &[u8; 16]) -> [u8; 16] { let mut out = *block; @@ -250,6 +261,7 @@ pub fn diag_aes256gcmsiv_raw_tag_aes(enc_key: &[u8; 32], block: &[u8; 16]) -> [u } #[cfg(feature = "diag")] +/// Exercise AES-256 counter-mode encryption and fold the fixed diagnostic output to one block. #[must_use] pub fn diag_aes256gcmsiv_ctr32(enc_key: &[u8; 32], tag: &[u8; 16], plaintext: &[u8; 44]) -> [u8; 16] { let mut counter_block = *tag; @@ -401,7 +413,6 @@ fn expand_key_riscv_for_backend(key: &[u8; 32], backend: AeadBackend) -> aes::Ae match backend { AeadBackend::Riscv64VectorCrypto => aes::aes256_expand_key_riscv_vector(key), AeadBackend::Riscv64ScalarCrypto => aes::aes256_expand_key_riscv_scalar(key), - AeadBackend::Riscv64Vperm => aes::aes256_expand_key_riscv_vperm(key), AeadBackend::Portable => aes::aes256_expand_key_riscv_ttable(key), _ => aes::aes256_expand_key_riscv_ttable(key), } @@ -413,6 +424,13 @@ fn expand_message_key_riscv(enc_key: &[u8; 32], backend: AeadBackend) -> aes::Ae expand_key_riscv_for_backend(enc_key, backend) } +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", +))] #[inline] fn resolve_backend() -> AeadBackend { select_backend( @@ -428,7 +446,7 @@ fn riscv_polyval_backend(backend: AeadBackend) -> RiscvPolyvalBackend { match backend { AeadBackend::Riscv64VectorCrypto => RiscvPolyvalBackend::Vector, AeadBackend::Riscv64ScalarCrypto => RiscvPolyvalBackend::Scalar, - AeadBackend::Portable | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable => { let caps = crate::platform::caps(); if caps.has(crate::platform::caps::riscv::ZBC) || caps.has(crate::platform::caps::riscv::ZBKC) { RiscvPolyvalBackend::Scalar @@ -569,6 +587,12 @@ fn compute_tag_wide( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Seal one message through the fused AArch64 AES-256-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn encrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 32], @@ -725,6 +749,12 @@ unsafe fn encrypt_fused_aarch64( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "aes,neon")] +/// Open one message through the fused AArch64 AES-256-GCM-SIV path. +/// +/// # Safety +/// +/// The current CPU must support AArch64 AES, NEON, and PMULL. Callers must establish those +/// capabilities through validated backend selection before entering this function. unsafe fn decrypt_fused_aarch64( auth_key: &mut [u8; 16], enc_key_bytes: &mut [u8; 32], @@ -886,6 +916,11 @@ unsafe fn decrypt_fused_aarch64( // powerpc64 fused encrypt/decrypt (single #[target_feature] scope) +/// Encrypt with the fused POWER8 AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn encrypt_fused_ppc( @@ -1025,6 +1060,11 @@ unsafe fn encrypt_fused_ppc( } } +/// Decrypt and authenticate with the fused POWER8 AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, POWER8 vector, and POWER8 crypto. #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn decrypt_fused_ppc( @@ -1171,6 +1211,11 @@ unsafe fn decrypt_fused_ppc( // s390x fused encrypt/decrypt (#[target_feature(enable = "vector")] for POLYVAL) +/// XOR an AES-256 counter stream into `buffer` using s390x CPACF. +/// +/// # Safety +/// +/// The executing CPU must support MSA AES instructions. #[cfg(target_arch = "s390x")] unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 32], counter_block: &mut [u8; 16], buffer: &mut [u8]) { let mut ctr = u32::from_le_bytes([counter_block[0], counter_block[1], counter_block[2], counter_block[3]]); @@ -1196,12 +1241,17 @@ unsafe fn s390x_ctr32_le_xor_raw(enc_key_bytes: &[u8; 32], counter_block: &mut [ let processed = aes::xor_keystream_tail(buffer, offset, &keystream, block_count); offset = offset.strict_add(processed); - ctr = ctr.wrapping_add(block_count as u32); + ctr = ctr.wrapping_add(u32::from(block_count.to_le_bytes()[0])); } counter_block[0..4].copy_from_slice(&ctr.to_le_bytes()); } +/// Encrypt with the fused s390x AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn encrypt_fused_s390x( @@ -1315,6 +1365,11 @@ unsafe fn encrypt_fused_s390x( } } +/// Decrypt and authenticate with the fused s390x AES-256-GCM-SIV backend. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility and MSA AES instructions. #[cfg(target_arch = "s390x")] #[target_feature(enable = "vector")] unsafe fn decrypt_fused_s390x( @@ -1443,6 +1498,13 @@ impl Aead for Aes256GcmSiv { type Tag = Aes256GcmSivTag; fn new(key: &Self::Key) -> Self { + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] let backend = resolve_backend(); Self { @@ -1450,6 +1512,13 @@ impl Aead for Aes256GcmSiv { master_ek: expand_key_riscv_for_backend(key.as_bytes(), backend), #[cfg(not(target_arch = "riscv64"))] master_ek: aes::aes256_expand_key(key.as_bytes()), + #[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ))] backend, } } @@ -1527,10 +1596,7 @@ impl Aead for Aes256GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { let tag_bytes = encrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer); return Ok(Aes256GcmSivTag::from_bytes(tag_bytes)); } @@ -1623,10 +1689,7 @@ impl Aead for Aes256GcmSiv { #[cfg(target_arch = "riscv64")] { match self.backend { - AeadBackend::Portable - | AeadBackend::Riscv64VectorCrypto - | AeadBackend::Riscv64ScalarCrypto - | AeadBackend::Riscv64Vperm => { + AeadBackend::Portable | AeadBackend::Riscv64VectorCrypto | AeadBackend::Riscv64ScalarCrypto => { return decrypt_riscv(&self.master_ek, self.backend, nonce, aad, buffer, tag).map_err(OpenError::from); } _ => {} @@ -1657,10 +1720,13 @@ impl Aead for Aes256GcmSiv { #[cfg(test)] mod tests { - use alloc::{vec, vec::Vec}; + use alloc::vec; use super::*; - use crate::aead::expert::AeadWithNonce; + use crate::aead::{ + expert::AeadWithNonce, + test_vectors::{hex_vec, hex12, hex32}, + }; /// RFC 8452 Appendix C.2, test case 1: empty plaintext, empty AAD. #[test] @@ -1673,12 +1739,16 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; expected_ct_tag.len()]; - cipher.encrypt(&nonce, &[], &[], &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &[], &mut out) + .expect("RFC 8452 empty AES-256-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); // Decrypt. let mut pt_out = vec![0u8; 0]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 empty AES-256-GCM-SIV decryption must succeed"); assert!(pt_out.is_empty()); } @@ -1697,12 +1767,16 @@ mod tests { // Encrypt. let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("RFC 8452 AES-256-GCM-SIV encryption with AAD must succeed"); assert_eq!(out, expected_ct_tag); // Decrypt. let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 AES-256-GCM-SIV decryption with AAD must succeed"); assert_eq!(pt_out, plaintext); } @@ -1720,11 +1794,15 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("RFC 8452 long AES-256-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out).unwrap(); + cipher + .decrypt(&nonce, &aad, &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 long AES-256-GCM-SIV decryption must succeed"); assert_eq!(pt_out, plaintext); } @@ -1741,8 +1819,10 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut pt_out = vec![0u8; 0]; - let result = cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[], &bad_ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong AAD should fail. @@ -1757,8 +1837,10 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut pt_out = vec![0u8; 8]; // plaintext was 8 bytes // Wrong AAD: 0x02 instead of 0x01. - let result = cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &[0x02], &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Decryption with wrong nonce should fail. @@ -1774,8 +1856,10 @@ mod tests { let mut pt_out = vec![0u8; 8]; // plaintext was 8 bytes // Wrong nonce: 0x04 instead of 0x03. let wrong_nonce = Nonce96::from_bytes(hex12("040000000000000000000000")); - let result = cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&wrong_nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// Ciphertext body tampering should fail verification. @@ -1794,8 +1878,10 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut pt_out = vec![0u8; plaintext.len()]; - let result = cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &ct_tag, &mut pt_out), + Err(OpenError::verification()) + ); } /// On authentication failure, the output buffer must be zeroed. @@ -1810,15 +1896,19 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; plaintext.len().strict_add(TAG_SIZE)]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut out).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut out) + .expect("AES-256-GCM-SIV test setup encryption must succeed"); // Corrupt the tag. let last = out.len().strict_sub(1); out[last] ^= 0xff; let mut pt_out = vec![0xffu8; plaintext.len()]; // fill with non-zero - let result = cipher.decrypt(&nonce, &aad, &out, &mut pt_out); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt(&nonce, &aad, &out, &mut pt_out), + Err(OpenError::verification()) + ); // Buffer must be zeroed even though decryption was attempted. assert!(pt_out.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -1836,31 +1926,53 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut buf = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut buf) + .expect("AES-256-GCM-SIV detached encryption must succeed"); // buf is now ciphertext, tag is separate. assert_ne!(buf, plaintext); - cipher.decrypt_in_place(&nonce, &aad, &mut buf, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &aad, &mut buf, &tag) + .expect("AES-256-GCM-SIV detached decryption must succeed"); assert_eq!(buf, plaintext); } /// `tag_from_slice` rejects wrong-length input. #[test] fn aes256gcmsiv_tag_from_slice_rejects_bad_length() { - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 15]).is_err()); - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 17]).is_err()); - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 0]).is_err()); - assert!(Aes256GcmSiv::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + Aes256GcmSiv::tag_from_slice(&[0u8; 15]).expect_err("short AES-256-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256GcmSiv::tag_from_slice(&[0u8; 17]).expect_err("long AES-256-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + Aes256GcmSiv::tag_from_slice(&[]).expect_err("empty AES-256-GCM-SIV tag must be rejected"), + AeadBufferError::new() + ); + let tag = Aes256GcmSiv::tag_from_slice(&[0u8; 16]).expect("16-byte AES-256-GCM-SIV tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] #[cfg(target_pointer_width = "64")] fn aes256gcmsiv_input_limit_matches_rfc8452() { for len in [MAX_INPUT_LEN.strict_sub(1), MAX_INPUT_LEN] { - assert!(super::super::try_bounded_length_as_u64(len as usize, MAX_INPUT_LEN).is_ok()); - } - assert!(super::super::try_bounded_length_as_u64(MAX_INPUT_LEN.strict_add(1) as usize, MAX_INPUT_LEN).is_err()); + let platform_len = usize::try_from(len).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(platform_len, MAX_INPUT_LEN), + Ok(len) + ); + } + let too_large = usize::try_from(MAX_INPUT_LEN.strict_add(1)).expect("RFC 8452 input limit fits 64-bit usize"); + assert_eq!( + super::super::try_bounded_length_as_u64(too_large, MAX_INPUT_LEN), + Err(super::super::LengthOverflow) + ); } /// RFC 8452 Appendix C.2 vector with a different key (empty PT+AAD). @@ -1875,38 +1987,14 @@ mod tests { let cipher = Aes256GcmSiv::new(&key); let mut out = vec![0u8; expected_ct_tag.len()]; - cipher.encrypt(&nonce, &[], &[], &mut out).unwrap(); + cipher + .encrypt(&nonce, &[], &[], &mut out) + .expect("RFC 8452 alternate-key AES-256-GCM-SIV encryption must succeed"); assert_eq!(out, expected_ct_tag); let mut pt_out = vec![0u8; 0]; - cipher.decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out).unwrap(); - } - - // --- Hex helpers --- - - fn hex32(hex: &str) -> [u8; 32] { - let mut out = [0u8; 32]; - for i in 0..32 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex12(hex: &str) -> [u8; 12] { - let mut out = [0u8; 12]; - for i in 0..12 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - } - out - } - - fn hex_vec(hex: &str) -> Vec { - let mut out = Vec::with_capacity(hex.len() / 2); - let mut i = 0; - while i < hex.len() { - out.push(u8::from_str_radix(&hex[i..i + 2], 16).unwrap()); - i += 2; - } - out + cipher + .decrypt(&nonce, &[], &expected_ct_tag, &mut pt_out) + .expect("RFC 8452 alternate-key AES-256-GCM-SIV decryption must succeed"); } } diff --git a/src/aead/aes_round.rs b/src/aead/aes_round.rs index 09f48810..73538fd8 100644 --- a/src/aead/aes_round.rs +++ b/src/aead/aes_round.rs @@ -23,7 +23,7 @@ const BLOCK_SIZE: usize = 16; /// Input transform, low nibble: maps AES GF(2^8) basis to tower-field basis. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_IPT_LO: [u8; 16] = [ 0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2, 0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA, @@ -31,7 +31,7 @@ pub(crate) const VPERM_IPT_LO: [u8; 16] = [ /// Input transform, high nibble: maps AES GF(2^8) basis to tower-field basis. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_IPT_HI: [u8; 16] = [ 0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C, 0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD, @@ -39,7 +39,7 @@ pub(crate) const VPERM_IPT_HI: [u8; 16] = [ /// GF(2^4) inverse table, low nibble. Entry 0 is `0x80` (infinity sentinel). #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_INV_LO: [u8; 16] = [ 0x80, 0x01, 0x08, 0x0D, 0x0F, 0x06, 0x05, 0x0E, 0x02, 0x0C, 0x0B, 0x0A, 0x09, 0x03, 0x07, 0x04, @@ -47,7 +47,7 @@ pub(crate) const VPERM_INV_LO: [u8; 16] = [ /// GF(2^4) inverse table, high nibble. Entry 0 is `0x80` (infinity sentinel). #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_INV_HI: [u8; 16] = [ 0x80, 0x07, 0x0B, 0x0F, 0x06, 0x0A, 0x04, 0x01, 0x09, 0x08, 0x05, 0x02, 0x0C, 0x0E, 0x0D, 0x03, @@ -56,7 +56,7 @@ pub(crate) const VPERM_INV_HI: [u8; 16] = [ /// SubBytes output table, upper component (inverse-isomorphism + AES affine, /// no MixColumns). `sbou[io] ^ sbot[jo] = AES_SBOX[x] ^ 0x63`. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_SBOU: [u8; 16] = [ 0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0, 0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15, @@ -65,7 +65,7 @@ pub(crate) const VPERM_SBOU: [u8; 16] = [ /// SubBytes output table, lower component (inverse-isomorphism + AES affine, /// no MixColumns). `sbou[io] ^ sbot[jo] = AES_SBOX[x] ^ 0x63`. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_SBOT: [u8; 16] = [ 0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF, 0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E, @@ -73,18 +73,18 @@ pub(crate) const VPERM_SBOT: [u8; 16] = [ /// AES ShiftRows permutation: output byte `i` reads from input byte `SR[i]`. #[rustfmt::skip] -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const VPERM_SR: [u8; 16] = [ 0x00, 0x05, 0x0A, 0x0F, 0x04, 0x09, 0x0E, 0x03, 0x08, 0x0D, 0x02, 0x07, 0x0C, 0x01, 0x06, 0x0B, ]; /// AES affine constant: Hamburg vperm S-box omits this; XOR after SubBytes. -#[cfg(any(test, target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(any(test, target_arch = "s390x"))] pub(crate) const AES_AFFINE: u8 = 0x63; /// AES MixColumns reduction constant: `xtime(b) = (b<<1) ^ ((b>>7) * 0x1B)`. -#[cfg(any(target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(target_arch = "s390x")] pub(crate) const XTIME_REDUCE: u8 = 0x1B; /// Nibble mask: extract low 4 bits of each byte. @@ -94,7 +94,7 @@ pub(crate) const NIBBLE_MASK: u8 = 0x0F; /// MixColumns column-rotate-by-1 permutation: rotates each 4-byte column by 1. /// `[b1,b2,b3,b0, b5,b6,b7,b4, b9,b10,b11,b8, b13,b14,b15,b12]` #[rustfmt::skip] -#[cfg(any(target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(target_arch = "s390x")] pub(crate) const MC_ROT1: [u8; 16] = [ 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04, 0x09, 0x0A, 0x0B, 0x08, 0x0D, 0x0E, 0x0F, 0x0C, @@ -103,7 +103,7 @@ pub(crate) const MC_ROT1: [u8; 16] = [ /// MixColumns column-rotate-by-2 permutation: rotates each 4-byte column by 2. /// `[b2,b3,b0,b1, b6,b7,b4,b5, b10,b11,b8,b9, b14,b15,b12,b13]` #[rustfmt::skip] -#[cfg(any(target_arch = "s390x", target_arch = "riscv64"))] +#[cfg(target_arch = "s390x")] pub(crate) const MC_ROT2: [u8; 16] = [ 0x02, 0x03, 0x00, 0x01, 0x06, 0x07, 0x04, 0x05, 0x0A, 0x0B, 0x08, 0x09, 0x0E, 0x0F, 0x0C, 0x0D, @@ -140,7 +140,9 @@ const fn gf256_mul(a: u8, b: u8) -> u8 { prod ^= (prod >> 9).wrapping_mul(0x11b << 1); prod ^= (prod >> 8).wrapping_mul(0x11b); - prod as u8 + let [low, high] = prod.to_le_bytes(); + debug_assert!(high == 0); + low } #[cfg(any( @@ -194,7 +196,7 @@ const fn sbox(x: u8) -> u8 { ))] #[inline(always)] const fn col_byte(col: u32, row: usize) -> u8 { - (col >> (24u32.strict_sub((row as u32).strict_mul(8)))) as u8 + col.to_be_bytes()[row] } #[cfg(any( @@ -289,6 +291,7 @@ pub(crate) fn aes_enc_round_portable(block: &[u8; BLOCK_SIZE], round_key: &[u8; #[cfg(all(feature = "diag", feature = "aegis256"))] #[must_use] +/// Apply one portable AES encryption round for backend diagnostics. pub fn diag_aes_enc_round_portable(block: &[u8; BLOCK_SIZE], round_key: &[u8; BLOCK_SIZE]) -> [u8; BLOCK_SIZE] { aes_enc_round_portable(block, round_key) } diff --git a/src/aead/ascon128.rs b/src/aead/ascon128.rs index 51cad04a..a6a4cdab 100644 --- a/src/aead/ascon128.rs +++ b/src/aead/ascon128.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Ascon-AEAD128 authenticated encryption (NIST SP 800-232). //! //! Pure Rust, `no_std` implementation with fixed-work, table-free source @@ -26,17 +24,19 @@ const DOMAIN_SEPARATOR: u64 = 0x8000_0000_0000_0000; /// Little-endian padding: set the first free byte at position `n`. #[inline(always)] -const fn pad(n: usize) -> u64 { - 0x01_u64 << (8 * n) +fn pad(n: usize) -> u64 { + let shift = u32::try_from(n.strict_mul(8)).expect("Ascon tail position must fit the word width"); + 0x01_u64.strict_shl(shift) } /// Clear the lowest `n` bytes of `word`. #[inline(always)] -const fn clear(word: u64, n: usize) -> u64 { +fn clear(word: u64, n: usize) -> u64 { if n == 0 { return word; } - word & (u64::MAX << (8 * n)) + let shift = u32::try_from(n.strict_mul(8)).expect("Ascon tail length must fit the word width"); + word & u64::MAX.strict_shl(shift) } /// Load up to 8 bytes little-endian into a u64, zero-padding on the right. @@ -213,14 +213,13 @@ impl AsconAead128 { /// Absorb associated data into the state. fn process_aad(s: &mut [u64; 5], aad: &[u8]) { if !aad.is_empty() { - let mut chunks = aad.chunks_exact(RATE); - for chunk in chunks.by_ref() { + let (chunks, mut rest) = aad.as_chunks::(); + for chunk in chunks { s[0] ^= load_bytes(&chunk[..8]); s[1] ^= load_bytes(&chunk[8..]); permute_8_portable(s); } - let mut rest = chunks.remainder(); let sidx = if rest.len() >= 8 { s[0] ^= load_bytes(&rest[..8]); rest = &rest[8..]; @@ -289,8 +288,8 @@ impl Aead for AsconAead128 { let mut s = self.initialize(nonce); Self::process_aad(&mut s, aad); - let mut blocks = buffer.chunks_exact_mut(RATE); - for block in blocks.by_ref() { + let (blocks, mut tail) = buffer.as_chunks_mut::(); + for block in blocks { s[0] ^= load_bytes(&block[..8]); block[..8].copy_from_slice(&s[0].to_le_bytes()); s[1] ^= load_bytes(&block[8..]); @@ -298,7 +297,6 @@ impl Aead for AsconAead128 { permute_8_portable(&mut s); } - let mut tail = blocks.into_remainder(); let sidx = if tail.len() >= 8 { s[0] ^= load_bytes(&tail[..8]); tail[..8].copy_from_slice(&s[0].to_le_bytes()); @@ -328,8 +326,8 @@ impl Aead for AsconAead128 { let mut s = self.initialize(nonce); Self::process_aad(&mut s, aad); - let mut blocks = buffer.chunks_exact_mut(RATE); - for block in blocks.by_ref() { + let (blocks, mut tail) = buffer.as_chunks_mut::(); + for block in blocks { let c0 = load_bytes(&block[..8]); block[..8].copy_from_slice(&(s[0] ^ c0).to_le_bytes()); s[0] = c0; @@ -339,7 +337,6 @@ impl Aead for AsconAead128 { permute_8_portable(&mut s); } - let mut tail = blocks.into_remainder(); let sidx = if tail.len() >= 8 { let c0 = load_bytes(&tail[..8]); tail[..8].copy_from_slice(&(s[0] ^ c0).to_le_bytes()); @@ -369,6 +366,7 @@ impl Aead for AsconAead128 { } #[cfg(feature = "diag")] +/// Compare a portable Ascon-AEAD128 tag computation with an expected diagnostic tag. #[unsafe(no_mangle)] #[inline(never)] pub fn diag_ascon_aead128_tag_portable( @@ -402,18 +400,24 @@ mod tests { fn assert_matches_oracle(key: [u8; 16], nonce: [u8; 16], aad: &[u8], plaintext: &[u8]) { let aead = AsconAead128::new(&AsconAead128Key::from_bytes(key)); let nonce_typed = Nonce128::from_bytes(nonce); - let oracle = ascon_aead::AsconAead128::new_from_slice(&key).unwrap(); + let oracle = ascon_aead::AsconAead128::new_from_slice(&key).expect("16-byte Ascon oracle key must be accepted"); let oracle_nonce = Array(nonce); let mut ours = plaintext.to_vec(); - let tag = aead.encrypt_in_place(&nonce_typed, aad, &mut ours).unwrap(); + let tag = aead + .encrypt_in_place(&nonce_typed, aad, &mut ours) + .expect("rscrypto Ascon encryption must succeed"); let mut ours_combined = ours.clone(); ours_combined.extend_from_slice(tag.as_bytes()); - let expected = oracle.encrypt(&oracle_nonce, Payload { msg: plaintext, aad }).unwrap(); + let expected = oracle + .encrypt(&oracle_nonce, Payload { msg: plaintext, aad }) + .expect("oracle Ascon encryption must succeed"); assert_eq!(ours_combined, expected, "encryption mismatch"); let mut ours_buf = ours.clone(); - aead.decrypt_in_place(&nonce_typed, aad, &mut ours_buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce_typed, aad, &mut ours_buf, &tag) + .expect("rscrypto Ascon self-decryption must succeed"); assert_eq!(ours_buf, plaintext, "self decrypt mismatch"); let (oracle_ct, oracle_tag) = expected.split_at(expected.len().strict_sub(TAG_SIZE)); @@ -423,9 +427,13 @@ mod tests { &nonce_typed, aad, &mut oracle_buf, - &AsconAead128Tag::from_bytes(oracle_tag.try_into().unwrap()), + &AsconAead128Tag::from_bytes( + oracle_tag + .try_into() + .expect("oracle Ascon output must end in a 16-byte tag"), + ), ) - .unwrap(); + .expect("rscrypto must decrypt the oracle Ascon ciphertext"); assert_eq!(oracle_buf, plaintext, "oracle decrypt mismatch"); } @@ -437,8 +445,12 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = []; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("empty Ascon encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("empty Ascon decryption must succeed"); } #[test] @@ -449,10 +461,14 @@ mod tests { let plaintext = b"the quick brown fox jumps over the lazy dog"; let mut buf = *plaintext; - let tag = aead.encrypt_in_place(&nonce, b"header", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"header", &mut buf) + .expect("Ascon encryption with AAD must succeed"); assert_ne!(&buf[..], &plaintext[..]); - aead.decrypt_in_place(&nonce, b"header", &mut buf, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, b"header", &mut buf, &tag) + .expect("Ascon decryption with AAD must succeed"); assert_eq!(&buf[..], &plaintext[..]); } @@ -465,10 +481,10 @@ mod tests { let mut buf = []; let tag = aead .encrypt_in_place(&nonce, b"associated data only", &mut buf) - .unwrap(); + .expect("AAD-only Ascon encryption must succeed"); aead .decrypt_in_place(&nonce, b"associated data only", &mut buf, &tag) - .unwrap(); + .expect("AAD-only Ascon decryption must succeed"); } #[test] @@ -478,14 +494,18 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"zero me on failure"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("Ascon test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = AsconAead128Tag::from_bytes(bad_tag); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -496,11 +516,15 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"secret"; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("Ascon test setup encryption must succeed"); buf[0] ^= 1; - let result = aead.decrypt_in_place(&nonce, b"", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"", &mut buf, &tag), + Err(OpenError::verification()) + ); // Buffer must be zeroized on failure. assert_eq!(&buf, &[0u8; 6]); } @@ -512,14 +536,18 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"data"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("Ascon test setup encryption must succeed"); let mut bad_tag_bytes = tag.to_bytes(); bad_tag_bytes[15] ^= 1; let bad_tag = AsconAead128Tag::from_bytes(bad_tag_bytes); - let result = aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert_eq!(&buf, &[0u8; 4]); } @@ -530,10 +558,14 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"msg"; - let tag = aead.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("Ascon test setup encryption must succeed"); - let result = aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -543,11 +575,15 @@ mod tests { let aead = AsconAead128::new(&key); let mut buf = *b"nonce test"; - let tag = aead.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("Ascon test setup encryption must succeed"); let wrong_nonce = Nonce128::from_bytes([11; 16]); - let result = aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + aead.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -558,18 +594,29 @@ mod tests { let pt = b"combined mode"; let mut sealed = vec![0u8; pt.len().strict_add(TAG_SIZE)]; - aead.encrypt(&nonce, b"h", pt.as_slice(), &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"h", pt.as_slice(), &mut sealed) + .expect("combined Ascon encryption must succeed"); let mut opened = vec![0u8; pt.len()]; - aead.decrypt(&nonce, b"h", &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, b"h", &sealed, &mut opened) + .expect("combined Ascon decryption must succeed"); assert_eq!(&opened, &pt[..]); } #[test] fn tag_from_slice_rejects_wrong_length() { - assert!(AsconAead128::tag_from_slice(&[0u8; 15]).is_err()); - assert!(AsconAead128::tag_from_slice(&[0u8; 17]).is_err()); - assert!(AsconAead128::tag_from_slice(&[0u8; 16]).is_ok()); + assert_eq!( + AsconAead128::tag_from_slice(&[0u8; 15]).expect_err("short Ascon tag must be rejected"), + AeadBufferError::new() + ); + assert_eq!( + AsconAead128::tag_from_slice(&[0u8; 17]).expect_err("long Ascon tag must be rejected"), + AeadBufferError::new() + ); + let tag = AsconAead128::tag_from_slice(&[0u8; 16]).expect("16-byte Ascon tag must be accepted"); + assert_eq!(tag.as_bytes(), &[0u8; 16]); } #[test] @@ -583,7 +630,7 @@ mod tests { let mut buf = plaintext; let tag = aead .encrypt_in_place(&nonce, b"multi-block aad that is longer than one rate block", &mut buf) - .unwrap(); + .expect("multi-block Ascon encryption must succeed"); aead .decrypt_in_place( &nonce, @@ -591,7 +638,7 @@ mod tests { &mut buf, &tag, ) - .unwrap(); + .expect("multi-block Ascon decryption must succeed"); assert_eq!(buf, plaintext); } @@ -604,15 +651,23 @@ mod tests { // Exactly 8 bytes = 1 full block, 0-byte tail. let plaintext = [0x55u8; 8]; let mut buf = plaintext; - let tag = aead.encrypt_in_place(&nonce, b"", &mut buf).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf, &tag).unwrap(); + let tag = aead + .encrypt_in_place(&nonce, b"", &mut buf) + .expect("one-word Ascon encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf, &tag) + .expect("one-word Ascon decryption must succeed"); assert_eq!(buf, plaintext); // Exactly 16 bytes = 2 full blocks, 0-byte tail. let plaintext16 = [0x66u8; 16]; let mut buf16 = plaintext16; - let tag16 = aead.encrypt_in_place(&nonce, b"", &mut buf16).unwrap(); - aead.decrypt_in_place(&nonce, b"", &mut buf16, &tag16).unwrap(); + let tag16 = aead + .encrypt_in_place(&nonce, b"", &mut buf16) + .expect("one-rate Ascon encryption must succeed"); + aead + .decrypt_in_place(&nonce, b"", &mut buf16, &tag16) + .expect("one-rate Ascon decryption must succeed"); assert_eq!(buf16, plaintext16); } @@ -653,7 +708,7 @@ mod tests { fn differential_multiblock_matches_oracle() { let key = [0x42; 16]; let nonce = [0x24; 16]; - let aad: Vec = (0..48).map(|i| i as u8).collect(); + let aad: Vec = (0u8..48).collect(); let pt: Vec = (0u8..97).map(|i| i.wrapping_mul(17)).collect(); assert_matches_oracle(key, nonce, &aad, &pt); } diff --git a/src/aead/chacha20.rs b/src/aead/chacha20.rs index 091d6374..0a1db21a 100644 --- a/src/aead/chacha20.rs +++ b/src/aead/chacha20.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Portable ChaCha20 and HChaCha20 core. use core::mem; @@ -25,9 +23,11 @@ pub(crate) const POLY1305_KEY_SIZE: usize = 32; const CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574]; -pub(crate) type XorKeystreamFn = fn(&[u8; KEY_SIZE], u32, &[u8; NONCE_SIZE], &mut [u8]); +type XorKeystreamFn = unsafe fn(&[u8; KEY_SIZE], u32, &[u8; NONCE_SIZE], &mut [u8]); +#[cfg(feature = "xchacha20poly1305")] static XCHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH: OnceCache = OnceCache::new(); +#[cfg(feature = "chacha20poly1305")] static CHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH: OnceCache = OnceCache::new(); #[inline(always)] @@ -133,7 +133,9 @@ pub(crate) fn block(key: &[u8; KEY_SIZE], counter: u32, nonce: &[u8; NONCE_SIZE] } let mut out = [0u8; BLOCK_SIZE]; - for (chunk, word) in out.chunks_exact_mut(4).zip(state) { + let (chunks, remainder) = out.as_chunks_mut::<4>(); + debug_assert!(remainder.is_empty()); + for (chunk, word) in chunks.iter_mut().zip(state) { chunk.copy_from_slice(&word.to_le_bytes()); } out @@ -185,19 +187,25 @@ pub(crate) fn xor_keystream( } } - xor_keystream_resolved(primitive)(key, initial_counter, nonce, buffer); + let kernel = xor_keystream_resolved(primitive); + // SAFETY: the counter-range check above covers every 64-byte block, and the resolver returns an accelerated kernel + // only when its complete capability set is available. The portable function also coerces to this unsafe fn type. + unsafe { kernel(key, initial_counter, nonce, buffer) }; Ok(()) } #[inline] -pub(crate) fn xor_keystream_resolved(primitive: AeadPrimitive) -> XorKeystreamFn { +fn xor_keystream_resolved(primitive: AeadPrimitive) -> XorKeystreamFn { match primitive { + #[cfg(feature = "xchacha20poly1305")] AeadPrimitive::XChaCha20Poly1305 => { XCHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH.get_or_init(|| resolve_xor_keystream(primitive)) } + #[cfg(feature = "chacha20poly1305")] AeadPrimitive::ChaCha20Poly1305 => { CHACHA20POLY1305_XOR_KEYSTREAM_DISPATCH.get_or_init(|| resolve_xor_keystream(primitive)) } + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] _ => resolve_xor_keystream(primitive), } } @@ -247,7 +255,7 @@ fn xor_keystream_portable(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u } } -#[cfg_attr(not(all(target_arch = "powerpc64", target_endian = "little")), allow(dead_code))] +#[cfg(all(target_arch = "powerpc64", target_endian = "little"))] pub(crate) fn xor_keystream_first_block_portable( key: &[u8; KEY_SIZE], counter: u32, @@ -265,31 +273,46 @@ pub(crate) fn xor_keystream_first_block_portable( #[cfg(target_arch = "riscv64")] #[inline(always)] -fn simd_u32x4_rotl(value: u32x4, bits: u32) -> u32x4 { - (value << u32x4::splat(bits)) | (value >> u32x4::splat(32u32.wrapping_sub(bits))) +fn simd_u32x4_rotl(value: u32x4) -> u32x4 { + const { assert!(BITS > 0 && BITS < 32) } + let left = core::ops::Shl::shl(value, u32x4::splat(BITS)); + let right = core::ops::Shr::shr(value, u32x4::splat(32u32.strict_sub(BITS))); + core::ops::BitOr::bitor(left, right) +} + +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn simd_u32x4_wrapping_add(left: u32x4, right: u32x4) -> u32x4 { + core::ops::Add::add(left, right) } #[cfg(target_arch = "riscv64")] #[inline(always)] fn simd_u32x4_quarter_round(a: &mut u32x4, b: &mut u32x4, c: &mut u32x4, d: &mut u32x4) { - *a += *b; + *a = simd_u32x4_wrapping_add(*a, *b); *d ^= *a; - *d = simd_u32x4_rotl(*d, 16); + *d = simd_u32x4_rotl::<16>(*d); - *c += *d; + *c = simd_u32x4_wrapping_add(*c, *d); *b ^= *c; - *b = simd_u32x4_rotl(*b, 12); + *b = simd_u32x4_rotl::<12>(*b); - *a += *b; + *a = simd_u32x4_wrapping_add(*a, *b); *d ^= *a; - *d = simd_u32x4_rotl(*d, 8); + *d = simd_u32x4_rotl::<8>(*d); - *c += *d; + *c = simd_u32x4_wrapping_add(*c, *d); *b ^= *c; - *b = simd_u32x4_rotl(*b, 7); + *b = simd_u32x4_rotl::<7>(*b); } #[cfg(target_arch = "riscv64")] +/// Generate and XOR a ChaCha20 stream in four-block RISC-V vector batches. +/// +/// # Safety +/// +/// The caller must ensure that the RISC-V vector extension is available and that `buffer`'s 64-byte block count fits +/// the counter range starting at `initial_counter`. #[target_feature(enable = "v")] unsafe fn xor_keystream_u32x4_impl( key: &[u8; KEY_SIZE], @@ -298,12 +321,14 @@ unsafe fn xor_keystream_u32x4_impl( buffer: &mut [u8], ) { const BLOCKS_PER_BATCH: usize = 4; + const BLOCKS_PER_BATCH_U32: u32 = 4; + const BATCH_SIZE: usize = BLOCK_SIZE.strict_mul(BLOCKS_PER_BATCH); let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { + let (batches, remainder) = buffer.as_chunks_mut::(); + for chunk in batches { debug_assert!( - counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some(), + counter <= u32::MAX.strict_sub(BLOCKS_PER_BATCH_U32.strict_sub(1)), "ChaCha20 block counter overflow" ); @@ -321,9 +346,9 @@ unsafe fn xor_keystream_u32x4_impl( let mut x11 = u32x4::splat(load_u32_le(&key[28..32])); let mut x12 = u32x4::from_array([ counter, - counter.wrapping_add(1), - counter.wrapping_add(2), - counter.wrapping_add(3), + counter.strict_add(1), + counter.strict_add(2), + counter.strict_add(3), ]); let mut x13 = u32x4::splat(load_u32_le(&nonce[0..4])); let mut x14 = u32x4::splat(load_u32_le(&nonce[4..8])); @@ -361,22 +386,22 @@ unsafe fn xor_keystream_u32x4_impl( round = round.strict_add(1); } - x0 += o0; - x1 += o1; - x2 += o2; - x3 += o3; - x4 += o4; - x5 += o5; - x6 += o6; - x7 += o7; - x8 += o8; - x9 += o9; - x10 += o10; - x11 += o11; - x12 += o12; - x13 += o13; - x14 += o14; - x15 += o15; + x0 = simd_u32x4_wrapping_add(x0, o0); + x1 = simd_u32x4_wrapping_add(x1, o1); + x2 = simd_u32x4_wrapping_add(x2, o2); + x3 = simd_u32x4_wrapping_add(x3, o3); + x4 = simd_u32x4_wrapping_add(x4, o4); + x5 = simd_u32x4_wrapping_add(x5, o5); + x6 = simd_u32x4_wrapping_add(x6, o6); + x7 = simd_u32x4_wrapping_add(x7, o7); + x8 = simd_u32x4_wrapping_add(x8, o8); + x9 = simd_u32x4_wrapping_add(x9, o9); + x10 = simd_u32x4_wrapping_add(x10, o10); + x11 = simd_u32x4_wrapping_add(x11, o11); + x12 = simd_u32x4_wrapping_add(x12, o12); + x13 = simd_u32x4_wrapping_add(x13, o13); + x14 = simd_u32x4_wrapping_add(x14, o14); + x15 = simd_u32x4_wrapping_add(x15, o15); let words = [ x0.to_array(), @@ -412,10 +437,9 @@ unsafe fn xor_keystream_u32x4_impl( block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(BLOCKS_PER_BATCH_U32); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -429,7 +453,9 @@ pub(crate) fn hchacha20(key: &[u8; KEY_SIZE], nonce: &[u8; HCHACHA_NONCE_SIZE]) rounds(&mut state); let mut out = [0u8; KEY_SIZE]; - for (chunk, word) in out.chunks_exact_mut(4).zip([ + let (chunks, remainder) = out.as_chunks_mut::<4>(); + debug_assert!(remainder.is_empty()); + for (chunk, word) in chunks.iter_mut().zip([ state[0], state[1], state[2], state[3], state[12], state[13], state[14], state[15], ]) { chunk.copy_from_slice(&word.to_le_bytes()); @@ -462,15 +488,22 @@ mod x86_avx512; #[path = "chacha20/x86_64_ssse3_x4.rs"] mod x86_ssse3_x4; -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", feature = "chacha20poly1305"))] #[inline] -pub(crate) fn xor_keystream_aarch64_neon( +/// Apply the AArch64 NEON ChaCha20 kernel for the interleaved AEAD path. +/// +/// # Safety +/// +/// The caller must verify [`aarch64::NEON`](crate::platform::caps::aarch64::NEON) and ensure that the number of +/// 64-byte blocks in `buffer` does not exhaust the `u32` block counter starting at `initial_counter`. +pub(super) unsafe fn xor_keystream_aarch64_neon( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller establishes the NEON capability and counter-range preconditions required by this backend. + unsafe { aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer) } } // Forced entry points let backend-equivalence tests bypass runtime dispatch. @@ -490,105 +523,120 @@ pub fn diag_chacha20_xor_keystream_portable( /// /// # Safety /// -/// Caller must verify the host has `aarch64::NEON`. Compile-time gated to -/// `target_arch = "aarch64"`. +/// Caller must verify the host has `aarch64::NEON` and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. Compile-time gated to `target_arch = "aarch64"`. #[cfg(all(feature = "diag", target_arch = "aarch64"))] -pub fn diag_chacha20_xor_keystream_aarch64_neon( +pub unsafe fn diag_chacha20_xor_keystream_aarch64_neon( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { aarch64_neon::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the x86_64 AVX2 ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `x86::AVX2`. +/// Caller must verify the host has `x86::AVX2` and that `buffer`'s 64-byte block count fits the counter range starting +/// at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "x86_64"))] -pub fn diag_chacha20_xor_keystream_x86_avx2( +pub unsafe fn diag_chacha20_xor_keystream_x86_avx2( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - x86_avx2::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { x86_avx2::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the x86_64 AVX-512 ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `x86::AVX512F + AVX512VL + AVX512BW`. +/// Caller must verify the host has `x86::AVX512F + AVX512VL + AVX512BW + AVX512DQ` and that `buffer`'s 64-byte block +/// count fits the counter range starting at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "x86_64"))] -pub fn diag_chacha20_xor_keystream_x86_avx512( +pub unsafe fn diag_chacha20_xor_keystream_x86_avx512( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - x86_avx512::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { x86_avx512::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the POWER VSX ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host is `powerpc64le` with VSX. The portable -/// kernel — which has been the correctness oracle since commit `2631aefa` -/// fixed the rotation-amount bug here — must produce identical bytes. +/// Caller must verify the host has `power::POWER8_VECTOR` and that `buffer`'s 64-byte block count fits the counter +/// range starting at `initial_counter`. The portable kernel — which has been the correctness oracle since commit +/// `2631aefa` fixed the rotation-amount bug here — must produce identical bytes. #[cfg(all(feature = "diag", target_arch = "powerpc64", target_endian = "little"))] -pub fn diag_chacha20_xor_keystream_power_vsx( +pub unsafe fn diag_chacha20_xor_keystream_power_vsx( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - power_vsx::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { power_vsx::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the s390x z/Vector ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `s390x::VECTOR`. Same correctness-oracle -/// invariant as POWER VSX above. +/// Caller must verify the host has `s390x::VECTOR` and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. Same correctness-oracle invariant as POWER VSX above. #[cfg(all(feature = "diag", target_arch = "s390x"))] -pub fn diag_chacha20_xor_keystream_s390x_vector( +pub unsafe fn diag_chacha20_xor_keystream_s390x_vector( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - s390x_vector::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { s390x_vector::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the riscv64 RVV ChaCha20 XOR-keystream. /// /// # Safety /// -/// Caller must verify the host has `riscv::V`. +/// Caller must verify the host has `riscv::V` and that `buffer`'s 64-byte block count fits the counter range starting +/// at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "riscv64"))] -pub fn diag_chacha20_xor_keystream_riscv64_vector( +pub unsafe fn diag_chacha20_xor_keystream_riscv64_vector( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - riscv64_vector::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { riscv64_vector::xor_keystream(key, initial_counter, nonce, buffer) }; } /// Run the wasm32 simd128 ChaCha20 XOR-keystream. +/// +/// # Safety +/// +/// Caller must verify the host has `wasm::SIMD128` and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[cfg(all(feature = "diag", target_arch = "wasm32"))] -pub fn diag_chacha20_xor_keystream_wasm_simd128( +pub unsafe fn diag_chacha20_xor_keystream_wasm_simd128( key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8], ) { - wasm_simd128::xor_keystream(key, initial_counter, nonce, buffer); + // SAFETY: the caller contract is exactly the private backend entry's contract. + unsafe { wasm_simd128::xor_keystream(key, initial_counter, nonce, buffer) }; } #[cfg(test)] mod tests { @@ -611,6 +659,17 @@ mod tests { use super::xor_keystream_portable; use super::{KEY_SIZE, NONCE_SIZE, block, xor_keystream}; use crate::aead::targets::AeadPrimitive; + + fn primitive() -> AeadPrimitive { + #[cfg(feature = "chacha20poly1305")] + { + AeadPrimitive::ChaCha20Poly1305 + } + #[cfg(all(not(feature = "chacha20poly1305"), feature = "xchacha20poly1305"))] + { + AeadPrimitive::XChaCha20Poly1305 + } + } #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] @@ -662,10 +721,12 @@ mod tests { let plaintext = *b"chacha20 portable core"; let mut ciphertext = plaintext; - xor_keystream(AeadPrimitive::ChaCha20Poly1305, &key, 1, &nonce, &mut ciphertext).unwrap(); + xor_keystream(primitive(), &key, 1, &nonce, &mut ciphertext) + .expect("test message must fit the ChaCha20 counter range"); assert_ne!(ciphertext, plaintext); - xor_keystream(AeadPrimitive::XChaCha20Poly1305, &key, 1, &nonce, &mut ciphertext).unwrap(); + xor_keystream(primitive(), &key, 1, &nonce, &mut ciphertext) + .expect("test message must fit the XChaCha20 counter range"); assert_eq!(ciphertext, plaintext); } @@ -675,10 +736,58 @@ mod tests { let nonce = [0u8; NONCE_SIZE]; let mut one_block = [0u8; 64]; - assert!(xor_keystream(AeadPrimitive::ChaCha20Poly1305, &key, u32::MAX, &nonce, &mut one_block).is_ok()); + xor_keystream(primitive(), &key, u32::MAX, &nonce, &mut one_block) + .expect("one block at the final counter must fit"); let mut two_blocks = [0u8; 65]; - assert!(xor_keystream(AeadPrimitive::ChaCha20Poly1305, &key, u32::MAX, &nonce, &mut two_blocks).is_err()); + xor_keystream(primitive(), &key, u32::MAX, &nonce, &mut two_blocks) + .expect_err("a second block after the final counter must fail"); + } + + #[cfg(target_arch = "x86_64")] + /// Compare one capability-gated x86 backend with the portable authority and preserve both guard regions. + /// + /// # Safety + /// + /// The caller must establish `kernel`'s CPU features and ensure that the block count implied by the `len` bytes fits + /// the counter range starting at `initial_counter`. + unsafe fn assert_x86_backend_matches_portable( + backend_name: &str, + kernel: unsafe fn(&[u8; KEY_SIZE], u32, &[u8; NONCE_SIZE], &mut [u8]), + key: &[u8; KEY_SIZE], + nonce: &[u8; NONCE_SIZE], + initial_counter: u32, + offset: usize, + len: usize, + ) { + const GUARD_LEN: usize = 64; + + let allocation_len = GUARD_LEN.strict_add(offset).strict_add(len).strict_add(GUARD_LEN); + let start = GUARD_LEN.strict_add(offset); + let mut portable_storage = vec![0xa5; allocation_len]; + let mut accelerated_storage = portable_storage.clone(); + { + let (_, portable_suffix) = portable_storage.split_at_mut(start); + let (portable, _) = portable_suffix.split_at_mut(len); + let (_, accelerated_suffix) = accelerated_storage.split_at_mut(start); + let (accelerated, _) = accelerated_suffix.split_at_mut(len); + + for (index, (portable_byte, accelerated_byte)) in portable.iter_mut().zip(accelerated.iter_mut()).enumerate() { + let [value, ..] = index.strict_mul(29).strict_add(3).to_le_bytes(); + *portable_byte = value; + *accelerated_byte = value; + } + + xor_keystream_portable(key, initial_counter, nonce, portable); + // SAFETY: the caller established the kernel capability and counter range; `accelerated` is the exact initialized + // in-place segment and the surrounding allocation stays live for the call. + unsafe { kernel(key, initial_counter, nonce, accelerated) }; + } + + assert_eq!( + accelerated_storage, portable_storage, + "{backend_name} mismatch at counter={initial_counter}, offset={offset}, len={len}" + ); } #[test] @@ -690,22 +799,48 @@ mod tests { let key = [0x71; KEY_SIZE]; let nonce = [0x19; NONCE_SIZE]; - for len in [ - 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 768, 769, 1023, 1024, 1280, 1281, 1536, 2048, 4096, 8192, + for initial_counter in [0, 0x7fff_ffff, 0x8000_0000] { + for offset in [0usize, 1, 15, 31, 63] { + for len in [ + 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 768, 769, 1023, 1024, 1279, 1280, 1281, 1536, 2048, + 4096, 8192, + ] { + // SAFETY: the test returned unless AVX512_READY is available; these ordinary/sign-boundary counters leave + // room for every selected test length. + unsafe { + assert_x86_backend_matches_portable( + "AVX-512", + super::x86_avx512::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } + } + } + } + for (initial_counter, len) in [ + (u32::MAX.strict_sub(15), 1024usize), + (u32::MAX.strict_sub(3), 256), + (u32::MAX, 64), ] { - let mut portable = vec![0u8; len]; - let mut accelerated = vec![0u8; len]; - let mut index = 0usize; - while index < len { - let value = index.strict_mul(13).strict_add(5) as u8; - portable[index] = value; - accelerated[index] = value; - index = index.strict_add(1); + for offset in [0usize, 1, 15, 31, 63] { + // SAFETY: the test returned unless AVX512_READY is available, and each pair is the exact maximum-valid counter + // for the 16-way, four-way, or portable width. + unsafe { + assert_x86_backend_matches_portable( + "AVX-512", + super::x86_avx512::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } } - - xor_keystream_portable(&key, 3, &nonce, &mut portable); - super::x86_avx512::xor_keystream(&key, 3, &nonce, &mut accelerated); - assert_eq!(accelerated, portable, "AVX-512 mismatch at len={len}"); } } @@ -718,22 +853,48 @@ mod tests { let key = [0x55; KEY_SIZE]; let nonce = [0x33; NONCE_SIZE]; - for len in [ - 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 768, 769, 1023, 1024, 1280, 1281, 2048, 4096, 8192, + for initial_counter in [0, 0x7fff_ffff, 0x8000_0000] { + for offset in [0usize, 1, 15, 31, 63] { + for len in [ + 0usize, 1, 63, 64, 65, 255, 256, 257, 511, 512, 513, 767, 768, 769, 1023, 1024, 1279, 1280, 1281, 2048, 4096, + 8192, + ] { + // SAFETY: the test returned unless AVX2 is available; these ordinary/sign-boundary counters leave room for + // every selected test length. + unsafe { + assert_x86_backend_matches_portable( + "AVX2", + super::x86_avx2::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } + } + } + } + for (initial_counter, len) in [ + (u32::MAX.strict_sub(7), 512usize), + (u32::MAX.strict_sub(3), 256), + (u32::MAX, 64), ] { - let mut portable = vec![0u8; len]; - let mut accelerated = vec![0u8; len]; - let mut index = 0usize; - while index < len { - let value = index.strict_mul(17).strict_add(9) as u8; - portable[index] = value; - accelerated[index] = value; - index = index.strict_add(1); + for offset in [0usize, 1, 15, 31, 63] { + // SAFETY: the test returned unless AVX2 is available, and each pair is the exact maximum-valid counter for + // the eight-way, four-way, or portable width. + unsafe { + assert_x86_backend_matches_portable( + "AVX2", + super::x86_avx2::xor_keystream, + &key, + &nonce, + initial_counter, + offset, + len, + ); + } } - - xor_keystream_portable(&key, 7, &nonce, &mut portable); - super::x86_avx2::xor_keystream(&key, 7, &nonce, &mut accelerated); - assert_eq!(accelerated, portable, "AVX2 mismatch at len={len}"); } } @@ -746,20 +907,32 @@ mod tests { let key = [0x66; KEY_SIZE]; let nonce = [0x11; NONCE_SIZE]; - for len in [0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 257, 768] { - let mut portable = vec![0u8; len]; - let mut accelerated = vec![0u8; len]; - let mut index = 0usize; - while index < len { - let value = index.strict_mul(29).strict_add(3) as u8; - portable[index] = value; - accelerated[index] = value; - index = index.strict_add(1); + for initial_counter in [0, 11, u32::MAX.strict_sub(11)] { + for offset in 0usize..16 { + for len in [0usize, 1, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513, 768] { + let allocation_len = offset.strict_add(len); + let mut portable_storage = vec![0u8; allocation_len]; + let mut accelerated_storage = vec![0u8; allocation_len]; + let (_, portable) = portable_storage.split_at_mut(offset); + let (_, accelerated) = accelerated_storage.split_at_mut(offset); + let mut index = 0usize; + while index < len { + let [value, ..] = index.strict_mul(29).strict_add(3).to_le_bytes(); + portable[index] = value; + accelerated[index] = value; + index = index.strict_add(1); + } + + xor_keystream_portable(&key, initial_counter, &nonce, portable); + // SAFETY: the test capability guard proves NEON, and the selected maximum counter leaves room for every + // selected length. + unsafe { super::aarch64_neon::xor_keystream(&key, initial_counter, &nonce, accelerated) }; + assert_eq!( + accelerated, portable, + "NEON mismatch at counter={initial_counter}, offset={offset}, len={len}" + ); + } } - - xor_keystream_portable(&key, 11, &nonce, &mut portable); - super::aarch64_neon::xor_keystream(&key, 11, &nonce, &mut accelerated); - assert_eq!(accelerated, portable); } } @@ -784,7 +957,8 @@ mod tests { } xor_keystream_portable(&key, 5, &nonce, &mut portable); - super::power_vsx::xor_keystream(&key, 5, &nonce, &mut accelerated); + // SAFETY: the test capability guard proves POWER8 vector support, and counter 5 leaves room for every length. + unsafe { super::power_vsx::xor_keystream(&key, 5, &nonce, &mut accelerated) }; assert_eq!(accelerated, portable, "POWER VSX mismatch at len={len}"); } } @@ -810,7 +984,8 @@ mod tests { } xor_keystream_portable(&key, 9, &nonce, &mut portable); - super::s390x_vector::xor_keystream(&key, 9, &nonce, &mut accelerated); + // SAFETY: the test capability guard proves z/Vector support, and counter 9 leaves room for every length. + unsafe { super::s390x_vector::xor_keystream(&key, 9, &nonce, &mut accelerated) }; assert_eq!(accelerated, portable, "s390x vector mismatch at len={len}"); } } diff --git a/src/aead/chacha20/aarch64_neon.rs b/src/aead/chacha20/aarch64_neon.rs index de857391..1c23e37e 100644 --- a/src/aead/chacha20/aarch64_neon.rs +++ b/src/aead/chacha20/aarch64_neon.rs @@ -1,21 +1,37 @@ use core::arch::aarch64::{ - uint32x4_t, vaddq_u32, vcombine_u32, vdupq_n_u32, veorq_u32, vget_high_u32, vget_low_u32, vld1q_u32, - vreinterpretq_u16_u32, vreinterpretq_u32_u16, vrev32q_u16, vshrq_n_u32, vsliq_n_u32, vst1q_u32, vzip1q_u32, - vzip2q_u32, + uint32x4_t, vaddq_u32, vcombine_u32, vdupq_n_u32, veorq_u32, vget_high_u32, vget_low_u32, vld1q_u8, vld1q_u32, + vreinterpretq_u8_u32, vreinterpretq_u16_u32, vreinterpretq_u32_u8, vreinterpretq_u32_u16, vrev32q_u16, vshrq_n_u32, + vsliq_n_u32, vst1q_u8, vzip1q_u32, vzip2q_u32, }; use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 4; - +const COUNTERS_PER_BATCH: u32 = 4; +const COUNTERS_PER_DOUBLE_BATCH: u32 = 8; + +/// Generate and XOR a ChaCha20 stream with the NEON kernel. +/// +/// # Safety +/// +/// The caller must ensure that NEON is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees NEON is available before this wrapper is chosen because: - // 1. Runtime dispatch selects this module only after `aarch64::NEON` is present. - // 2. `xor_keystream_impl` is annotated with `#[target_feature(enable = "neon")]`. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects `aarch64::NEON`; direct test and diagnostic callers + // establish the same conditions. `xor_keystream_impl` enables NEON. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// # Safety +/// +/// The caller must ensure that NEON is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[target_feature(enable = "neon")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let c0 = vdupq_n_u32(0x6170_7865); @@ -35,9 +51,9 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let n15 = vdupq_n_u32(load_u32_le(&nonce[8..12])); let mut counter = initial_counter; - let mut double_batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH * 2); - for chunk in &mut double_batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH * 2 - 1) as u32).is_some()); + let (double_batches, double_remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH * 2 }>(); + for chunk in double_batches { + debug_assert!(counter.checked_add(COUNTERS_PER_DOUBLE_BATCH.strict_sub(1)).is_some()); let mut x0 = c0; let mut x1 = c1; @@ -159,7 +175,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let ptr = chunk.as_mut_ptr(); // SAFETY: vector transpose and XOR stores because: - // 1. `chunk` is exactly eight ChaCha20 blocks from `chunks_exact_mut`. + // 1. `chunk` is an array of exactly eight ChaCha20 blocks. // 2. The first four-block group starts at `ptr`; the second starts at `ptr + 256`. // 3. NEON is guaranteed by the enclosing `#[target_feature(enable = "neon")]`. unsafe { @@ -175,14 +191,12 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_store_word_group(hi, 12, y12, y13, y14, y15); } - counter = counter.wrapping_add((BLOCKS_PER_BATCH * 2) as u32); + counter = counter.wrapping_add(COUNTERS_PER_DOUBLE_BATCH); } - let mut batches = double_batches - .into_remainder() - .chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + let (batches, remainder) = double_remainder.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = c0; let mut x1 = c1; @@ -269,7 +283,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: let ptr = chunk.as_mut_ptr(); // SAFETY: vector transpose and XOR stores because: - // 1. `chunk` is exactly `BLOCKS_PER_BATCH * BLOCK_SIZE` bytes from `chunks_exact_mut`. + // 1. `chunk` is an array of exactly `BLOCKS_PER_BATCH * BLOCK_SIZE` bytes. // 2. Each call stores four 16-byte word groups at offsets inside the 256-byte chunk. // 3. NEON is guaranteed by the enclosing `#[target_feature(enable = "neon")]`. unsafe { @@ -279,16 +293,20 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_store_word_group(ptr, 12, x12, x13, x14, x15); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } } /// Transpose four word-major ChaCha vectors into four block-major 16-byte word groups. +/// +/// # Safety +/// +/// The caller must ensure NEON is available, `chunk` is valid for exclusive writes across four complete ChaCha20 +/// blocks, and `word_start` is one of 0, 4, 8, or 12. #[inline(always)] unsafe fn xor_store_word_group( chunk: *mut u8, @@ -320,18 +338,25 @@ unsafe fn xor_store_word_group( } } +/// XOR one 16-byte word group into a block of a four-block ChaCha20 batch. +/// +/// # Safety +/// +/// The caller must ensure NEON is available, `chunk` is valid for exclusive writes across four complete ChaCha20 +/// blocks, `block_index < 4`, and `word_start <= 12`. #[inline(always)] unsafe fn xor_store_block_words(chunk: *mut u8, block_index: usize, word_start: usize, keystream: uint32x4_t) { // SAFETY: in-place 16-byte XOR/store because: // 1. Caller guarantees `chunk` points to a full four-block chunk. // 2. `block_index < 4` and `word_start <= 12`, so `block_index * 64 + word_start * 4 + 16` stays // within the 256-byte chunk. - // 3. `vld1q_u32`/`vst1q_u32` support unaligned addresses and the pointer does not escape. + // 3. The byte-typed NEON load/store accepts every address alignment and the pointer does not escape. unsafe { let offset = block_index.strict_mul(BLOCK_SIZE).strict_add(word_start.strict_mul(4)); - let ptr = chunk.add(offset).cast::(); - let plaintext = vld1q_u32(ptr); - vst1q_u32(ptr, veorq_u32(plaintext, keystream)); + let ptr = chunk.add(offset); + let plaintext = vreinterpretq_u32_u8(vld1q_u8(ptr)); + let ciphertext = veorq_u32(plaintext, keystream); + vst1q_u8(ptr, vreinterpretq_u8_u32(ciphertext)); } } diff --git a/src/aead/chacha20/powerpc64_vsx.rs b/src/aead/chacha20/powerpc64_vsx.rs index 151c2b95..52544af7 100644 --- a/src/aead/chacha20/powerpc64_vsx.rs +++ b/src/aead/chacha20/powerpc64_vsx.rs @@ -3,10 +3,23 @@ use core::simd::i64x2; use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 4; +const COUNTERS_PER_BATCH: u32 = 4; +/// Generate and XOR a ChaCha20 stream with the POWER8 vector kernel. +/// +/// # Safety +/// +/// The caller must ensure that POWER8 vector support is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees POWER vector support before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects `power::POWER8_VECTOR`; direct test and diagnostic + // callers establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } @@ -42,14 +55,20 @@ impl RotShifts { } } +/// Generate and XOR a ChaCha20 stream in four-block POWER8 vector batches. +/// +/// # Safety +/// +/// The caller must ensure that POWER8 vector support is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let rot = RotShifts::new(); let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = splat(0x6170_7865); let mut x1 = splat(0x3320_646e); @@ -162,10 +181,9 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -189,6 +207,10 @@ fn splat(val: u32) -> i64x2 { } /// Vector add unsigned word modulo: `vadduwm`. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn vadduwm(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; @@ -206,6 +228,10 @@ unsafe fn vadduwm(a: i64x2, b: i64x2) -> i64x2 { } /// Vector XOR: `vxor`. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec. #[inline(always)] unsafe fn vxor(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; @@ -223,6 +249,10 @@ unsafe fn vxor(a: i64x2, b: i64x2) -> i64x2 { } /// Vector rotate left word: `vrlw`. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec. #[inline(always)] unsafe fn vrlw(value: i64x2, shift: i64x2) -> i64x2 { let out: i64x2; @@ -239,6 +269,11 @@ unsafe fn vrlw(value: i64x2, shift: i64x2) -> i64x2 { out } +/// Apply one vectorized ChaCha20 quarter round. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn quarter_round(a: &mut i64x2, b: &mut i64x2, c: &mut i64x2, d: &mut i64x2, rot: &RotShifts) { // SAFETY: POWER8+ VSX available via enclosing target_feature. diff --git a/src/aead/chacha20/riscv64_vector.rs b/src/aead/chacha20/riscv64_vector.rs index 77516044..2a7376d6 100644 --- a/src/aead/chacha20/riscv64_vector.rs +++ b/src/aead/chacha20/riscv64_vector.rs @@ -1,15 +1,18 @@ use super::{KEY_SIZE, NONCE_SIZE, xor_keystream_u32x4_impl}; -define_target_feature_forwarder! { - pub(super) fn xor_keystream( - key: &[u8; KEY_SIZE], - initial_counter: u32, - nonce: &[u8; NONCE_SIZE], - buffer: &mut [u8] - ) { - feature = "v"; - outer_safety = "backend selection guarantees the vector extension before this wrapper is chosen."; - inner_safety = "the wrapper only reaches this function when the RISC-V vector extension is available."; - call = xor_keystream_u32x4_impl(key, initial_counter, nonce, buffer); - } +/// Generate and XOR a ChaCha20 stream with the RISC-V vector kernel. +/// +/// # Safety +/// +/// The caller must ensure that the RISC-V vector extension is available and that `buffer`'s 64-byte block count fits +/// the counter range starting at `initial_counter`. +#[inline] +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: the caller established the vector capability and counter range required by the shared implementation. + unsafe { xor_keystream_u32x4_impl(key, initial_counter, nonce, buffer) } } diff --git a/src/aead/chacha20/s390x_vector.rs b/src/aead/chacha20/s390x_vector.rs index 1f82eab7..5cb78603 100644 --- a/src/aead/chacha20/s390x_vector.rs +++ b/src/aead/chacha20/s390x_vector.rs @@ -3,19 +3,38 @@ use core::simd::i64x2; use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 4; +const COUNTERS_PER_BATCH: u32 = 4; +/// Generate and XOR a ChaCha20 stream with the z/Vector kernel. +/// +/// # Safety +/// +/// The caller must ensure that the z/Vector facility is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees the z/Vector facility before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects `s390x::VECTOR`; direct test and diagnostic callers + // establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream in four-block z/Vector batches. +/// +/// # Safety +/// +/// The caller must ensure that the z/Vector facility is available and that `buffer`'s 64-byte block count fits the +/// counter range starting at `initial_counter`. #[target_feature(enable = "vector")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); let mut x0 = splat(0x6170_7865); let mut x1 = splat(0x3320_646e); @@ -128,10 +147,9 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -155,6 +173,10 @@ fn splat(val: u32) -> i64x2 { } /// Vector add fullword: `vaf`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vaf(a: i64x2, b: i64x2) -> i64x2 { @@ -173,6 +195,10 @@ unsafe fn vaf(a: i64x2, b: i64x2) -> i64x2 { } /// Vector exclusive OR: `vx`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { @@ -194,6 +220,10 @@ unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { /// /// ChaCha20 quarter rounds rotate LEFT by 16, 12, 8, and 7 bits, and `verll` /// already rotates LEFT. The immediate maps directly to the ChaCha constant. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn rotl32_via_verll(a: i64x2) -> i64x2 { @@ -215,6 +245,11 @@ unsafe fn rotl32_via_verll(a: i64x2) -> i64x2 { out } +/// Apply one vectorized ChaCha20 quarter round. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn quarter_round(a: &mut i64x2, b: &mut i64x2, c: &mut i64x2, d: &mut i64x2) { diff --git a/src/aead/chacha20/wasm32_simd128.rs b/src/aead/chacha20/wasm32_simd128.rs index 128e6290..aa853f1b 100644 --- a/src/aead/chacha20/wasm32_simd128.rs +++ b/src/aead/chacha20/wasm32_simd128.rs @@ -4,18 +4,36 @@ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, xor_keystream_portabl const BLOCKS_PER_BATCH: usize = 4; +/// Generate and XOR a ChaCha20 stream with the wasm SIMD128 kernel. +/// +/// # Safety +/// +/// The caller must ensure that SIMD128 is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees simd128 is available before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and selects this wrapper only when simd128 is available; direct + // test and diagnostic callers establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream in four-block SIMD128 batches. +/// +/// # Safety +/// +/// The caller must ensure that SIMD128 is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[target_feature(enable = "simd128")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + debug_assert!(counter.checked_add(3).is_some()); let mut x0 = u32x4_splat(0x6170_7865); let mut x1 = u32x4_splat(0x3320_646e); @@ -88,26 +106,24 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: x14 = u32x4_add(x14, o14); x15 = u32x4_add(x15, o15); - let mut words = [[0u32; BLOCKS_PER_BATCH]; 16]; - // SAFETY: each destination is a valid four-lane `u32` array for one unaligned `v128` store. - unsafe { - v128_store(words[0].as_mut_ptr() as *mut v128, x0); - v128_store(words[1].as_mut_ptr() as *mut v128, x1); - v128_store(words[2].as_mut_ptr() as *mut v128, x2); - v128_store(words[3].as_mut_ptr() as *mut v128, x3); - v128_store(words[4].as_mut_ptr() as *mut v128, x4); - v128_store(words[5].as_mut_ptr() as *mut v128, x5); - v128_store(words[6].as_mut_ptr() as *mut v128, x6); - v128_store(words[7].as_mut_ptr() as *mut v128, x7); - v128_store(words[8].as_mut_ptr() as *mut v128, x8); - v128_store(words[9].as_mut_ptr() as *mut v128, x9); - v128_store(words[10].as_mut_ptr() as *mut v128, x10); - v128_store(words[11].as_mut_ptr() as *mut v128, x11); - v128_store(words[12].as_mut_ptr() as *mut v128, x12); - v128_store(words[13].as_mut_ptr() as *mut v128, x13); - v128_store(words[14].as_mut_ptr() as *mut v128, x14); - v128_store(words[15].as_mut_ptr() as *mut v128, x15); - } + let words = [ + lanes(x0), + lanes(x1), + lanes(x2), + lanes(x3), + lanes(x4), + lanes(x5), + lanes(x6), + lanes(x7), + lanes(x8), + lanes(x9), + lanes(x10), + lanes(x11), + lanes(x12), + lanes(x13), + lanes(x14), + lanes(x15), + ]; let mut block_index = 0usize; while block_index < BLOCKS_PER_BATCH { @@ -124,7 +140,7 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: block_index = block_index.strict_add(1); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(4); } let remainder = batches.into_remainder(); @@ -133,6 +149,16 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: } } +#[inline(always)] +fn lanes(value: v128) -> [u32; BLOCKS_PER_BATCH] { + [ + u32x4_extract_lane::<0>(value), + u32x4_extract_lane::<1>(value), + u32x4_extract_lane::<2>(value), + u32x4_extract_lane::<3>(value), + ] +} + #[inline(always)] fn quarter_round(a: &mut v128, b: &mut v128, c: &mut v128, d: &mut v128) { *a = u32x4_add(*a, *b); diff --git a/src/aead/chacha20/x86_64_avx2.rs b/src/aead/chacha20/x86_64_avx2.rs index 5165bce0..a6752bfe 100644 --- a/src/aead/chacha20/x86_64_avx2.rs +++ b/src/aead/chacha20/x86_64_avx2.rs @@ -7,13 +7,32 @@ use core::arch::x86_64::{ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, x86_ssse3_x4, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 8; +const COUNTERS_PER_BATCH: u32 = 8; +/// Generate and XOR a ChaCha20 stream with the AVX2 kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available and that `buffer`'s 64-byte block count fits the counter range +/// starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees AVX2 is available before this wrapper is chosen. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects AVX2; direct test and diagnostic callers establish the + // same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream with the AVX2 eight-way kernel and four-way/portable tails. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available and that the number of 64-byte blocks in `buffer` does not exhaust +/// the 32-bit block counter starting at `initial_counter`. #[target_feature(enable = "avx2")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { // vpshufb masks for byte-aligned rotations (16-bit and 8-bit). @@ -25,35 +44,35 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: ); let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); - - let mut x0 = _mm256_set1_epi32(0x6170_7865u32 as i32); - let mut x1 = _mm256_set1_epi32(0x3320_646eu32 as i32); - let mut x2 = _mm256_set1_epi32(0x7962_2d32u32 as i32); - let mut x3 = _mm256_set1_epi32(0x6b20_6574u32 as i32); - let mut x4 = _mm256_set1_epi32(load_u32_le(&key[0..4]) as i32); - let mut x5 = _mm256_set1_epi32(load_u32_le(&key[4..8]) as i32); - let mut x6 = _mm256_set1_epi32(load_u32_le(&key[8..12]) as i32); - let mut x7 = _mm256_set1_epi32(load_u32_le(&key[12..16]) as i32); - let mut x8 = _mm256_set1_epi32(load_u32_le(&key[16..20]) as i32); - let mut x9 = _mm256_set1_epi32(load_u32_le(&key[20..24]) as i32); - let mut x10 = _mm256_set1_epi32(load_u32_le(&key[24..28]) as i32); - let mut x11 = _mm256_set1_epi32(load_u32_le(&key[28..32]) as i32); + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); + + let mut x0 = _mm256_set1_epi32(0x6170_7865u32.cast_signed()); + let mut x1 = _mm256_set1_epi32(0x3320_646eu32.cast_signed()); + let mut x2 = _mm256_set1_epi32(0x7962_2d32u32.cast_signed()); + let mut x3 = _mm256_set1_epi32(0x6b20_6574u32.cast_signed()); + let mut x4 = _mm256_set1_epi32(load_u32_le(&key[0..4]).cast_signed()); + let mut x5 = _mm256_set1_epi32(load_u32_le(&key[4..8]).cast_signed()); + let mut x6 = _mm256_set1_epi32(load_u32_le(&key[8..12]).cast_signed()); + let mut x7 = _mm256_set1_epi32(load_u32_le(&key[12..16]).cast_signed()); + let mut x8 = _mm256_set1_epi32(load_u32_le(&key[16..20]).cast_signed()); + let mut x9 = _mm256_set1_epi32(load_u32_le(&key[20..24]).cast_signed()); + let mut x10 = _mm256_set1_epi32(load_u32_le(&key[24..28]).cast_signed()); + let mut x11 = _mm256_set1_epi32(load_u32_le(&key[28..32]).cast_signed()); let mut x12 = _mm256_setr_epi32( - counter as i32, - counter.wrapping_add(1) as i32, - counter.wrapping_add(2) as i32, - counter.wrapping_add(3) as i32, - counter.wrapping_add(4) as i32, - counter.wrapping_add(5) as i32, - counter.wrapping_add(6) as i32, - counter.wrapping_add(7) as i32, + counter.cast_signed(), + counter.wrapping_add(1).cast_signed(), + counter.wrapping_add(2).cast_signed(), + counter.wrapping_add(3).cast_signed(), + counter.wrapping_add(4).cast_signed(), + counter.wrapping_add(5).cast_signed(), + counter.wrapping_add(6).cast_signed(), + counter.wrapping_add(7).cast_signed(), ); - let mut x13 = _mm256_set1_epi32(load_u32_le(&nonce[0..4]) as i32); - let mut x14 = _mm256_set1_epi32(load_u32_le(&nonce[4..8]) as i32); - let mut x15 = _mm256_set1_epi32(load_u32_le(&nonce[8..12]) as i32); + let mut x13 = _mm256_set1_epi32(load_u32_le(&nonce[0..4]).cast_signed()); + let mut x14 = _mm256_set1_epi32(load_u32_le(&nonce[4..8]).cast_signed()); + let mut x15 = _mm256_set1_epi32(load_u32_le(&nonce[8..12]).cast_signed()); let o0 = x0; let o1 = x1; @@ -108,7 +127,6 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: // to block-major (each pair of YMM registers = one 64-byte block). // // Stage 1: 32-bit interleave. - // SAFETY: AVX2 intrinsics are valid under the enclosing target_feature. let s1_0 = _mm256_unpacklo_epi32(x0, x1); let s1_1 = _mm256_unpackhi_epi32(x0, x1); let s1_2 = _mm256_unpacklo_epi32(x2, x3); @@ -162,19 +180,17 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_block_pair(ptr, 3, 7, s2_3, s2_7, s2_11, s2_15); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); - let mut x4_batches = remainder.chunks_exact_mut(BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH); - for chunk in &mut x4_batches { + let (x4_batches, remainder) = remainder.as_chunks_mut::<{ BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH }>(); + for chunk in x4_batches { // SAFETY: AVX2-capable CPUs provide the SSSE3 instructions used by the // 4-block tail kernel, and `chunk` is exactly 4 ChaCha20 blocks. unsafe { x86_ssse3_x4::xor_blocks(key, counter, nonce, chunk) }; - counter = counter.wrapping_add(x86_ssse3_x4::BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(x86_ssse3_x4::COUNTERS_PER_BATCH); } - let remainder = x4_batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } @@ -182,6 +198,11 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: /// Permute stage-2 results into two complete blocks (lo_idx and hi_idx) and /// XOR+store them in-place. Each block is 64 bytes = 2 × YMM. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available, `buf` is valid for exclusive access to 512 initialized bytes, +/// and `lo_idx` and `hi_idx` are distinct values below eight. #[inline(always)] unsafe fn xor_block_pair( buf: *mut u8, diff --git a/src/aead/chacha20/x86_64_avx512.rs b/src/aead/chacha20/x86_64_avx512.rs index 8f8c49a7..5fc5dbd2 100644 --- a/src/aead/chacha20/x86_64_avx512.rs +++ b/src/aead/chacha20/x86_64_avx512.rs @@ -7,53 +7,72 @@ use core::arch::x86_64::{ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le, x86_ssse3_x4, xor_keystream_portable}; const BLOCKS_PER_BATCH: usize = 16; +const COUNTERS_PER_BATCH: u32 = 16; +/// Generate and XOR a ChaCha20 stream with the AVX-512 kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX512F, AVX512VL, AVX512BW, and AVX512DQ are available and that `buffer`'s 64-byte +/// block count fits the counter range starting at `initial_counter`. #[inline] -pub(super) fn xor_keystream(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { - // SAFETY: Backend selection guarantees the AVX-512 feature set required by this kernel. +pub(super) unsafe fn xor_keystream( + key: &[u8; KEY_SIZE], + initial_counter: u32, + nonce: &[u8; NONCE_SIZE], + buffer: &mut [u8], +) { + // SAFETY: Production validates the counter range and detects AVX512F+VL+BW+DQ; direct test and diagnostic callers + // establish the same conditions. unsafe { xor_keystream_impl(key, initial_counter, nonce, buffer) } } +/// Generate and XOR a ChaCha20 stream with the AVX-512 16-way kernel and four-way/portable tails. +/// +/// # Safety +/// +/// The caller must ensure that AVX512F, AVX512VL, AVX512BW, and AVX512DQ are available and that the number of +/// 64-byte blocks in `buffer` does not exhaust the 32-bit block counter starting at `initial_counter`. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq")] unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], buffer: &mut [u8]) { let mut counter = initial_counter; - let mut batches = buffer.chunks_exact_mut(BLOCK_SIZE * BLOCKS_PER_BATCH); - for chunk in &mut batches { - debug_assert!(counter.checked_add((BLOCKS_PER_BATCH - 1) as u32).is_some()); + let (batches, remainder) = buffer.as_chunks_mut::<{ BLOCK_SIZE * BLOCKS_PER_BATCH }>(); + for chunk in batches { + debug_assert!(counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); - let mut x0 = _mm512_set1_epi32(0x6170_7865u32 as i32); - let mut x1 = _mm512_set1_epi32(0x3320_646eu32 as i32); - let mut x2 = _mm512_set1_epi32(0x7962_2d32u32 as i32); - let mut x3 = _mm512_set1_epi32(0x6b20_6574u32 as i32); - let mut x4 = _mm512_set1_epi32(load_u32_le(&key[0..4]) as i32); - let mut x5 = _mm512_set1_epi32(load_u32_le(&key[4..8]) as i32); - let mut x6 = _mm512_set1_epi32(load_u32_le(&key[8..12]) as i32); - let mut x7 = _mm512_set1_epi32(load_u32_le(&key[12..16]) as i32); - let mut x8 = _mm512_set1_epi32(load_u32_le(&key[16..20]) as i32); - let mut x9 = _mm512_set1_epi32(load_u32_le(&key[20..24]) as i32); - let mut x10 = _mm512_set1_epi32(load_u32_le(&key[24..28]) as i32); - let mut x11 = _mm512_set1_epi32(load_u32_le(&key[28..32]) as i32); + let mut x0 = _mm512_set1_epi32(0x6170_7865u32.cast_signed()); + let mut x1 = _mm512_set1_epi32(0x3320_646eu32.cast_signed()); + let mut x2 = _mm512_set1_epi32(0x7962_2d32u32.cast_signed()); + let mut x3 = _mm512_set1_epi32(0x6b20_6574u32.cast_signed()); + let mut x4 = _mm512_set1_epi32(load_u32_le(&key[0..4]).cast_signed()); + let mut x5 = _mm512_set1_epi32(load_u32_le(&key[4..8]).cast_signed()); + let mut x6 = _mm512_set1_epi32(load_u32_le(&key[8..12]).cast_signed()); + let mut x7 = _mm512_set1_epi32(load_u32_le(&key[12..16]).cast_signed()); + let mut x8 = _mm512_set1_epi32(load_u32_le(&key[16..20]).cast_signed()); + let mut x9 = _mm512_set1_epi32(load_u32_le(&key[20..24]).cast_signed()); + let mut x10 = _mm512_set1_epi32(load_u32_le(&key[24..28]).cast_signed()); + let mut x11 = _mm512_set1_epi32(load_u32_le(&key[28..32]).cast_signed()); let mut x12 = _mm512_setr_epi32( - counter as i32, - counter.wrapping_add(1) as i32, - counter.wrapping_add(2) as i32, - counter.wrapping_add(3) as i32, - counter.wrapping_add(4) as i32, - counter.wrapping_add(5) as i32, - counter.wrapping_add(6) as i32, - counter.wrapping_add(7) as i32, - counter.wrapping_add(8) as i32, - counter.wrapping_add(9) as i32, - counter.wrapping_add(10) as i32, - counter.wrapping_add(11) as i32, - counter.wrapping_add(12) as i32, - counter.wrapping_add(13) as i32, - counter.wrapping_add(14) as i32, - counter.wrapping_add(15) as i32, + counter.cast_signed(), + counter.wrapping_add(1).cast_signed(), + counter.wrapping_add(2).cast_signed(), + counter.wrapping_add(3).cast_signed(), + counter.wrapping_add(4).cast_signed(), + counter.wrapping_add(5).cast_signed(), + counter.wrapping_add(6).cast_signed(), + counter.wrapping_add(7).cast_signed(), + counter.wrapping_add(8).cast_signed(), + counter.wrapping_add(9).cast_signed(), + counter.wrapping_add(10).cast_signed(), + counter.wrapping_add(11).cast_signed(), + counter.wrapping_add(12).cast_signed(), + counter.wrapping_add(13).cast_signed(), + counter.wrapping_add(14).cast_signed(), + counter.wrapping_add(15).cast_signed(), ); - let mut x13 = _mm512_set1_epi32(load_u32_le(&nonce[0..4]) as i32); - let mut x14 = _mm512_set1_epi32(load_u32_le(&nonce[4..8]) as i32); - let mut x15 = _mm512_set1_epi32(load_u32_le(&nonce[8..12]) as i32); + let mut x13 = _mm512_set1_epi32(load_u32_le(&nonce[0..4]).cast_signed()); + let mut x14 = _mm512_set1_epi32(load_u32_le(&nonce[4..8]).cast_signed()); + let mut x15 = _mm512_set1_epi32(load_u32_le(&nonce[8..12]).cast_signed()); let o0 = x0; let o1 = x1; @@ -108,7 +127,6 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: // 16 blocks) to block-major (each register = one complete 64-byte block). // // Stage 1: 32-bit interleave — pairwise unpack adjacent state-word registers. - // SAFETY: AVX-512 intrinsics are valid under the enclosing target_feature. let s1_0 = _mm512_unpacklo_epi32(x0, x1); let s1_1 = _mm512_unpackhi_epi32(x0, x1); let s1_2 = _mm512_unpacklo_epi32(x2, x3); @@ -215,30 +233,35 @@ unsafe fn xor_keystream_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: xor_block(ptr, 15, blk15); } - counter = counter.wrapping_add(BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(COUNTERS_PER_BATCH); } - let remainder = batches.into_remainder(); - let mut x4_batches = remainder.chunks_exact_mut(BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH); - for chunk in &mut x4_batches { + let (x4_batches, remainder) = remainder.as_chunks_mut::<{ BLOCK_SIZE * x86_ssse3_x4::BLOCKS_PER_BATCH }>(); + for chunk in x4_batches { // SAFETY: AVX-512-ready CPUs provide the SSSE3 instructions used by the // 4-block tail kernel, and `chunk` is exactly 4 ChaCha20 blocks. unsafe { x86_ssse3_x4::xor_blocks(key, counter, nonce, chunk) }; - counter = counter.wrapping_add(x86_ssse3_x4::BLOCKS_PER_BATCH as u32); + counter = counter.wrapping_add(x86_ssse3_x4::COUNTERS_PER_BATCH); } - let remainder = x4_batches.into_remainder(); if !remainder.is_empty() { xor_keystream_portable(key, counter, nonce, remainder); } } /// Load 64 bytes of plaintext at block offset `idx`, XOR with keystream block, store. +/// +/// # Safety +/// +/// The caller must ensure that AVX512F is available, `buf` is valid for exclusive access to 1,024 initialized bytes, +/// and `idx` is less than 16. The buffer need not be 64-byte aligned. #[inline(always)] unsafe fn xor_block(buf: *mut u8, idx: usize, keystream: __m512i) { // SAFETY: caller guarantees `buf` points to a 1024-byte chunk and `idx < 16`. unsafe { - let p = buf.add(idx.strict_mul(BLOCK_SIZE)).cast::<__m512i>(); + let p = core::ptr::NonNull::new_unchecked(buf.add(idx.strict_mul(BLOCK_SIZE))) + .cast::<__m512i>() + .as_ptr(); let plaintext = _mm512_loadu_si512(p); _mm512_storeu_si512(p, _mm512_xor_si512(plaintext, keystream)); } diff --git a/src/aead/chacha20/x86_64_ssse3_x4.rs b/src/aead/chacha20/x86_64_ssse3_x4.rs index 7ac1cce2..acfb8b9e 100644 --- a/src/aead/chacha20/x86_64_ssse3_x4.rs +++ b/src/aead/chacha20/x86_64_ssse3_x4.rs @@ -7,7 +7,14 @@ use core::arch::x86_64::{ use super::{BLOCK_SIZE, KEY_SIZE, NONCE_SIZE, load_u32_le}; pub(super) const BLOCKS_PER_BATCH: usize = 4; - +pub(super) const COUNTERS_PER_BATCH: u32 = 4; + +/// XOR four consecutive ChaCha20 blocks with the SSSE3 four-way kernel. +/// +/// # Safety +/// +/// The caller must ensure that SSSE3 and AVX are available, `chunk` is exactly four blocks, and +/// `initial_counter + 3` fits in `u32`. #[inline] pub(super) unsafe fn xor_blocks( key: &[u8; KEY_SIZE], @@ -16,37 +23,45 @@ pub(super) unsafe fn xor_blocks( chunk: &mut [u8], ) { debug_assert_eq!(chunk.len(), BLOCK_SIZE * BLOCKS_PER_BATCH); + debug_assert!(initial_counter.checked_add(COUNTERS_PER_BATCH.strict_sub(1)).is_some()); // SAFETY: callers only reach this helper from x86 SIMD backends whose - // dispatch gates guarantee AVX/SSSE3-capable hardware. + // dispatch or forced-backend contracts guarantee AVX/SSSE3-capable hardware; the assertions above restate the + // exact block and counter bounds. unsafe { xor_blocks_impl(key, initial_counter, nonce, chunk) } } +/// Generate and XOR four consecutive ChaCha20 blocks. +/// +/// # Safety +/// +/// The caller must ensure that SSSE3 and AVX are available, `chunk` is exactly four blocks, and +/// `initial_counter + 3` fits in `u32`. #[target_feature(enable = "ssse3,avx")] unsafe fn xor_blocks_impl(key: &[u8; KEY_SIZE], initial_counter: u32, nonce: &[u8; NONCE_SIZE], chunk: &mut [u8]) { let rot16 = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); let rot8 = _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); - let mut x0 = _mm_set1_epi32(0x6170_7865u32 as i32); - let mut x1 = _mm_set1_epi32(0x3320_646eu32 as i32); - let mut x2 = _mm_set1_epi32(0x7962_2d32u32 as i32); - let mut x3 = _mm_set1_epi32(0x6b20_6574u32 as i32); - let mut x4 = _mm_set1_epi32(load_u32_le(&key[0..4]) as i32); - let mut x5 = _mm_set1_epi32(load_u32_le(&key[4..8]) as i32); - let mut x6 = _mm_set1_epi32(load_u32_le(&key[8..12]) as i32); - let mut x7 = _mm_set1_epi32(load_u32_le(&key[12..16]) as i32); - let mut x8 = _mm_set1_epi32(load_u32_le(&key[16..20]) as i32); - let mut x9 = _mm_set1_epi32(load_u32_le(&key[20..24]) as i32); - let mut x10 = _mm_set1_epi32(load_u32_le(&key[24..28]) as i32); - let mut x11 = _mm_set1_epi32(load_u32_le(&key[28..32]) as i32); + let mut x0 = _mm_set1_epi32(0x6170_7865u32.cast_signed()); + let mut x1 = _mm_set1_epi32(0x3320_646eu32.cast_signed()); + let mut x2 = _mm_set1_epi32(0x7962_2d32u32.cast_signed()); + let mut x3 = _mm_set1_epi32(0x6b20_6574u32.cast_signed()); + let mut x4 = _mm_set1_epi32(load_u32_le(&key[0..4]).cast_signed()); + let mut x5 = _mm_set1_epi32(load_u32_le(&key[4..8]).cast_signed()); + let mut x6 = _mm_set1_epi32(load_u32_le(&key[8..12]).cast_signed()); + let mut x7 = _mm_set1_epi32(load_u32_le(&key[12..16]).cast_signed()); + let mut x8 = _mm_set1_epi32(load_u32_le(&key[16..20]).cast_signed()); + let mut x9 = _mm_set1_epi32(load_u32_le(&key[20..24]).cast_signed()); + let mut x10 = _mm_set1_epi32(load_u32_le(&key[24..28]).cast_signed()); + let mut x11 = _mm_set1_epi32(load_u32_le(&key[28..32]).cast_signed()); let mut x12 = _mm_setr_epi32( - initial_counter as i32, - initial_counter.wrapping_add(1) as i32, - initial_counter.wrapping_add(2) as i32, - initial_counter.wrapping_add(3) as i32, + initial_counter.cast_signed(), + initial_counter.wrapping_add(1).cast_signed(), + initial_counter.wrapping_add(2).cast_signed(), + initial_counter.wrapping_add(3).cast_signed(), ); - let mut x13 = _mm_set1_epi32(load_u32_le(&nonce[0..4]) as i32); - let mut x14 = _mm_set1_epi32(load_u32_le(&nonce[4..8]) as i32); - let mut x15 = _mm_set1_epi32(load_u32_le(&nonce[8..12]) as i32); + let mut x13 = _mm_set1_epi32(load_u32_le(&nonce[0..4]).cast_signed()); + let mut x14 = _mm_set1_epi32(load_u32_le(&nonce[4..8]).cast_signed()); + let mut x15 = _mm_set1_epi32(load_u32_le(&nonce[8..12]).cast_signed()); let o0 = x0; let o1 = x1; @@ -129,6 +144,11 @@ fn transpose_words(w0: __m128i, w1: __m128i, w2: __m128i, w3: __m128i) -> [__m12 } } +/// XOR one generated ChaCha20 block into a selected block of a four-block buffer. +/// +/// # Safety +/// +/// `buf` must be valid for exclusive access to 256 initialized bytes, and `idx` must be less than four. #[inline(always)] unsafe fn xor_block(buf: *mut u8, idx: usize, w03: __m128i, w47: __m128i, w811: __m128i, w1215: __m128i) { // SAFETY: caller guarantees `buf` points to four full ChaCha20 blocks and @@ -142,6 +162,11 @@ unsafe fn xor_block(buf: *mut u8, idx: usize, w03: __m128i, w47: __m128i, w811: } } +/// XOR 16 keystream bytes into an in-place buffer segment. +/// +/// # Safety +/// +/// `ptr` must be valid for reading and exclusively writing 16 initialized bytes. It need not be aligned. #[inline(always)] unsafe fn xor_store(ptr: *mut u8, keystream: __m128i) { // SAFETY: caller guarantees `ptr..ptr+16` is in bounds for unaligned access. diff --git a/src/aead/chacha20poly1305.rs b/src/aead/chacha20poly1305.rs index 0309998f..7faecdd4 100644 --- a/src/aead/chacha20poly1305.rs +++ b/src/aead/chacha20poly1305.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! ChaCha20-Poly1305 public AEAD surface. use core::fmt; @@ -20,7 +18,9 @@ const POWER_SHORT_FAST_MAX: usize = chacha20::BLOCK_SIZE; #[cfg(target_arch = "aarch64")] const AARCH64_INTERLEAVED_MIN: usize = 1024; #[cfg(target_arch = "aarch64")] -const AARCH64_INTERLEAVED_CHUNK: usize = 1024 * 1024; +const AARCH64_INTERLEAVED_BLOCKS: u32 = 16 * 1024; +#[cfg(target_arch = "aarch64")] +const AARCH64_INTERLEAVED_CHUNK: usize = (AARCH64_INTERLEAVED_BLOCKS as usize) * chacha20::BLOCK_SIZE; #[cfg(all( target_arch = "x86_64", target_os = "linux", @@ -323,7 +323,9 @@ impl ChaCha20Poly1305 { return None; } - let tag = x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the diagnostic entry validated the ChaCha20 length bound; the guards above prove a nonempty buffer and + // AVX2+BMI2 availability. + let tag = unsafe { x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; Some(Ok(ChaCha20Poly1305Tag::from_bytes(tag))) } @@ -348,7 +350,9 @@ impl ChaCha20Poly1305 { return None; } - let expected = x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the diagnostic entry validated the ChaCha20 length bound; the guards above prove a nonempty buffer and + // AVX2+BMI2 availability. + let expected = unsafe { x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Some(Err(OpenError::verification())); @@ -377,7 +381,9 @@ impl ChaCha20Poly1305 { return None; } - let tag = x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the public AEAD entry validated the ChaCha20 length bound, and the recommendation gate proves a + // nonempty buffer plus AVX2+BMI2 availability. + let tag = unsafe { x86_64_asm::seal_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; Some(Ok(ChaCha20Poly1305Tag::from_bytes(tag))) } @@ -403,7 +409,9 @@ impl ChaCha20Poly1305 { return None; } - let expected = x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer); + // SAFETY: the public AEAD entry validated the ChaCha20 length bound, and the recommendation gate proves a + // nonempty buffer plus AVX2+BMI2 availability. + let expected = unsafe { x86_64_asm::open_in_place(self.key.as_bytes(), nonce.as_bytes(), aad, buffer) }; if !ct::fixed_eq(&expected, tag.as_bytes()).declassify() { ct::zeroize(buffer); return Some(Err(OpenError::verification())); @@ -488,16 +496,21 @@ impl ChaCha20Poly1305 { authenticator.update_padded_segment(aad); let mut counter = 1u32; - let mut chunks = buffer.chunks_exact_mut(AARCH64_INTERLEAVED_CHUNK); - for chunk in &mut chunks { - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk); + let (chunks, remainder) = buffer.as_chunks_mut::(); + for chunk in chunks { + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; the fixed-size chunks and exact + // counter advance preserve that whole-buffer bound for this segment. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk) }; authenticator.update_padded_segment(chunk); - counter = counter.wrapping_add((AARCH64_INTERLEAVED_CHUNK / chacha20::BLOCK_SIZE) as u32); + counter = counter.wrapping_add(AARCH64_INTERLEAVED_BLOCKS); } - let remainder = chunks.into_remainder(); if !remainder.is_empty() { - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; `counter` tracks the preceding + // full chunks exactly, so the final remainder cannot exhaust the counter range. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder) }; authenticator.update_padded_segment(remainder); } @@ -539,17 +552,22 @@ impl ChaCha20Poly1305 { authenticator.update_padded_segment(aad); let mut counter = 1u32; - let mut chunks = buffer.chunks_exact_mut(AARCH64_INTERLEAVED_CHUNK); - for chunk in &mut chunks { + let (chunks, remainder) = buffer.as_chunks_mut::(); + for chunk in chunks { authenticator.update_padded_segment(chunk); - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk); - counter = counter.wrapping_add((AARCH64_INTERLEAVED_CHUNK / chacha20::BLOCK_SIZE) as u32); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; the fixed-size chunks and exact + // counter advance preserve that whole-buffer bound for this segment. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), chunk) }; + counter = counter.wrapping_add(AARCH64_INTERLEAVED_BLOCKS); } - let remainder = chunks.into_remainder(); if !remainder.is_empty() { authenticator.update_padded_segment(remainder); - chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder); + // SAFETY: the capability gate proves NEON. Every caller reaches this private helper after the public or + // diagnostic length bound, or from a same-module test with a bounded buffer; `counter` tracks the preceding + // full chunks exactly, so the final remainder cannot exhaust the counter range. + unsafe { chacha20::xor_keystream_aarch64_neon(self.key.as_bytes(), counter, nonce.as_bytes(), remainder) }; } let expected = authenticator.finalize(lengths); @@ -622,6 +640,10 @@ impl ChaCha20Poly1305 { } } +/// Encrypts in place without using a platform-specific integrated ChaCha20-Poly1305 assembly entrypoint. +/// +/// Lower-level ChaCha20 and Poly1305 dispatch remains enabled. Returns an error when the input lengths exceed the +/// supported limits. #[cfg(feature = "diag")] pub fn diag_chacha20poly1305_encrypt_in_place_owned( cipher: &ChaCha20Poly1305, @@ -634,6 +656,9 @@ pub fn diag_chacha20poly1305_encrypt_in_place_owned( } #[cfg(all(feature = "diag", target_arch = "x86_64", target_os = "linux"))] +/// Encrypts in place through the Linux x86-64 assembly entrypoint when that entrypoint is available. +/// +/// Returns `None` when the current CPU cannot execute the assembly backend. pub fn diag_chacha20poly1305_encrypt_in_place_x86_64_asm( cipher: &ChaCha20Poly1305, nonce: &Nonce96, @@ -648,6 +673,9 @@ pub fn diag_chacha20poly1305_encrypt_in_place_x86_64_asm( } #[cfg(all(feature = "diag", target_arch = "x86_64", target_os = "linux"))] +/// Authenticates and decrypts in place through the Linux x86-64 assembly entrypoint when it is available. +/// +/// Returns `None` when the current CPU cannot execute the assembly backend. pub fn diag_chacha20poly1305_decrypt_in_place_x86_64_asm( cipher: &ChaCha20Poly1305, nonce: &Nonce96, @@ -662,6 +690,11 @@ pub fn diag_chacha20poly1305_decrypt_in_place_x86_64_asm( cipher.decrypt_in_place_asm_x86_64_forced(nonce, aad, buffer, tag) } +/// Authenticates and decrypts in place without using a platform-specific integrated ChaCha20-Poly1305 assembly +/// entrypoint. +/// +/// Lower-level ChaCha20 and Poly1305 dispatch remains enabled. Authentication failure zeroes `buffer` and returns an +/// opaque verification error; unsupported input lengths also return an error. #[cfg(feature = "diag")] pub fn diag_chacha20poly1305_decrypt_in_place_owned( cipher: &ChaCha20Poly1305, @@ -803,8 +836,12 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"hello chacha"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); - cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("valid input must encrypt"); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buf, &tag) + .expect("matching nonce, AAD, and tag must decrypt"); assert_eq!(&buf, b"hello chacha"); } @@ -815,7 +852,9 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"nonce test"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("valid input must encrypt"); let wrong_nonce = Nonce96::from_bytes([0x08u8; 12]); let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); @@ -829,7 +868,9 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"zero me on failure"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("valid input must encrypt"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; @@ -900,14 +941,15 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut ciphertext = Vec::new(); let expected_tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) - .unwrap(); + .expect("owned path must produce an empty-text tag"); let actual = cipher.decrypt_empty_text_fast(&nonce, &aad, &expected_tag); if aad_len > SMALL_AAD_FAST_MAX { @@ -918,9 +960,10 @@ mod tests { continue; } - actual - .expect("empty decrypt fast path must apply inside its configured gate") - .unwrap(); + assert_eq!( + actual.expect("empty decrypt fast path must apply inside its configured gate"), + Ok(()) + ); let mut bad_tag = expected_tag.to_bytes(); bad_tag[0] ^= 0x80; @@ -947,21 +990,26 @@ mod tests { for plaintext_len in [ 1usize, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 1024, ] { - let plaintext = (0..plaintext_len) - .map(|index| 0x51u8.wrapping_add((index as u8).wrapping_mul(13))) + let plaintext = (0u8..=u8::MAX) + .cycle() + .take(plaintext_len) + .map(|index| 0x51u8.wrapping_add(index.wrapping_mul(13))) .collect::>(); for aad_len in [0usize, 1, 13, 14, 15, 16, 17, 31, 32, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut ciphertext = plaintext.clone(); let tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) - .unwrap(); + .expect("bounded test input must encrypt through the owned path"); let mut actual = ciphertext.clone(); - let actual_tag = x86_64_asm::open_in_place(key.as_bytes(), nonce.as_bytes(), &aad, &mut actual); + // SAFETY: the test returned unless AVX2+BMI2 are available; every selected ciphertext is nonempty and well + // below ChaCha20's 2^32-block limit. + let actual_tag = unsafe { x86_64_asm::open_in_place(key.as_bytes(), nonce.as_bytes(), &aad, &mut actual) }; assert_eq!( actual, plaintext, @@ -984,13 +1032,15 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); for plaintext_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65] { - let plaintext = (0..plaintext_len) - .map(|index| 0x51u8.wrapping_add((index as u8).wrapping_mul(13))) + let plaintext = (0u8..) + .take(plaintext_len) + .map(|index| 0x51u8.wrapping_add(index.wrapping_mul(13))) .collect::>(); for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut actual = plaintext.clone(); @@ -1008,10 +1058,10 @@ mod tests { let mut expected = plaintext.clone(); let expected_tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut expected) - .unwrap(); + .expect("bounded test input must encrypt through the owned path"); let actual_tag = actual_tag .expect("Power short fast path must apply inside its configured gate") - .unwrap(); + .expect("bounded test input must encrypt through the Power fast path"); assert_eq!( actual, expected, @@ -1033,19 +1083,21 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); for plaintext_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65] { - let plaintext = (0..plaintext_len) - .map(|index| 0x51u8.wrapping_add((index as u8).wrapping_mul(13))) + let plaintext = (0u8..) + .take(plaintext_len) + .map(|index| 0x51u8.wrapping_add(index.wrapping_mul(13))) .collect::>(); for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63, 64] { - let aad = (0..aad_len) - .map(|index| 0xa7u8.wrapping_add((index as u8).wrapping_mul(7))) + let aad = (0u8..) + .take(aad_len) + .map(|index| 0xa7u8.wrapping_add(index.wrapping_mul(7))) .collect::>(); let mut ciphertext = plaintext.clone(); let tag = cipher .encrypt_in_place_owned_unchecked(&nonce, &aad, &mut ciphertext) - .unwrap(); + .expect("bounded test input must encrypt through the owned path"); let mut actual = ciphertext.clone(); let actual_result = cipher.decrypt_short_text_power_fast(&nonce, &aad, &mut actual, &tag); @@ -1059,9 +1111,10 @@ mod tests { continue; } - actual_result - .expect("Power short decrypt fast path must apply inside its configured gate") - .unwrap(); + assert_eq!( + actual_result.expect("Power short decrypt fast path must apply inside its configured gate"), + Ok(()) + ); assert_eq!( actual, plaintext, "Power short plaintext mismatch plaintext_len={plaintext_len} aad_len={aad_len}" @@ -1091,7 +1144,9 @@ mod tests { let cipher = ChaCha20Poly1305::new(&key); let mut buf = *b"aad test"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("valid input must encrypt"); let result = cipher.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); assert!(result.is_err()); diff --git a/src/aead/chacha20poly1305/aarch64_asm.rs b/src/aead/chacha20poly1305/aarch64_asm.rs index c2d1efc5..85b7471e 100644 --- a/src/aead/chacha20poly1305/aarch64_asm.rs +++ b/src/aead/chacha20poly1305/aarch64_asm.rs @@ -4,8 +4,6 @@ //! ChaCha20-Poly1305 assembly. This Rust module owns the ABI boundary and //! keeps runtime feature dispatch in the parent AEAD implementation. -#![allow(unsafe_code)] - use core::{arch::global_asm, mem}; use super::KEY_SIZE; @@ -120,7 +118,10 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // empty slices because the length is zero. // 4. `extra_ciphertext` points at 16 initialized bytes and `extra_ciphertext_len` is zero, matching // the AWS-LC seal ABI for callers without extra trailing ciphertext. - // 5. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 5. `data` is 16-byte aligned and matches the assembly input/output union layout; the assembly initializes all + // 16 bytes of `data.out.tag` before the union field is read. + // 6. After that write, the tag replaces the first 16 copied key bytes; the second half remains initialized key + // material and is volatile-zeroed before the stack allocation expires. unsafe { #[cfg(target_os = "macos")] rscrypto_chacha20_poly1305_seal_aarch64_apple_darwin( @@ -129,7 +130,7 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); #[cfg(target_os = "linux")] rscrypto_chacha20_poly1305_seal_aarch64( @@ -138,8 +139,9 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } @@ -161,7 +163,10 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // the assembly routine supports in-place open, matching the AWS-LC ABI. // 3. `aad.as_ptr()` is valid for `aad.len()` bytes, including the conventional dangling pointer for // empty slices because the length is zero. - // 4. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 4. `data` is 16-byte aligned and matches the assembly input/output union layout; the assembly initializes all + // 16 bytes of `data.out.tag` before the union field is read. + // 5. After that write, the tag replaces the first 16 copied key bytes; the second half remains initialized key + // material and is volatile-zeroed before the stack allocation expires. unsafe { #[cfg(target_os = "macos")] rscrypto_chacha20_poly1305_open_aarch64_apple_darwin( @@ -170,7 +175,7 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); #[cfg(target_os = "linux")] rscrypto_chacha20_poly1305_open_aarch64( @@ -179,8 +184,9 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } diff --git a/src/aead/chacha20poly1305/x86_64_asm.rs b/src/aead/chacha20poly1305/x86_64_asm.rs index bbc2351b..55ac6124 100644 --- a/src/aead/chacha20poly1305/x86_64_asm.rs +++ b/src/aead/chacha20poly1305/x86_64_asm.rs @@ -4,8 +4,6 @@ //! ChaCha20-Poly1305 assembly. This Rust module owns the ABI boundary and //! keeps runtime feature dispatch in the parent AEAD implementation. -#![allow(unsafe_code)] - use core::{arch::global_asm, mem}; use super::KEY_SIZE; @@ -76,8 +74,14 @@ unsafe extern "C" { ); } +/// Seal one nonempty buffer with the integrated x86-64 assembly kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 and BMI2 are available and that `buffer` is nonempty and within ChaCha20's +/// 2³²-block limit. #[inline] -pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { +pub(super) unsafe fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { debug_assert!(!buffer.is_empty()); let extra_ciphertext = [0u8; 16]; @@ -99,7 +103,11 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // empty slices because the length is zero. // 4. `extra_ciphertext` points at 16 initialized bytes and `extra_ciphertext_len` is zero, matching // the AWS-LC seal ABI for callers without extra trailing ciphertext. - // 5. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 5. `data` is 16-byte aligned and matches the assembly input/output union layout; `from_mut` preserves that + // allocation's provenance and exclusive writability for the call, which initializes all 16 tag bytes before + // the union output is read. + // 6. The tag replaces the first 16 copied key bytes; the second half remains initialized key material and is + // volatile-zeroed before the stack allocation expires. unsafe { rscrypto_chacha20_poly1305_seal_x86_64( buffer.as_mut_ptr(), @@ -107,14 +115,21 @@ pub(super) fn seal_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } +/// Open one nonempty buffer with the integrated x86-64 assembly kernel. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 and BMI2 are available and that `buffer` is nonempty and within ChaCha20's +/// 2³²-block limit. #[inline] -pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { +pub(super) unsafe fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer: &mut [u8]) -> [u8; 16] { debug_assert!(!buffer.is_empty()); let mut data = OpenData { @@ -131,7 +146,11 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], // the assembly routine supports in-place open, matching the AWS-LC ABI. // 3. `aad.as_ptr()` is valid for `aad.len()` bytes, including the conventional dangling pointer for // empty slices because the length is zero. - // 4. `data` is 16-byte aligned and matches the assembly input/output union layout. + // 4. `data` is 16-byte aligned and matches the assembly input/output union layout; `from_mut` preserves that + // allocation's provenance and exclusive writability for the call, which initializes all 16 tag bytes before + // the union output is read. + // 5. The tag replaces the first 16 copied key bytes; the second half remains initialized key material and is + // volatile-zeroed before the stack allocation expires. unsafe { rscrypto_chacha20_poly1305_open_x86_64( buffer.as_mut_ptr(), @@ -139,8 +158,9 @@ pub(super) fn open_in_place(key: &[u8; KEY_SIZE], nonce: &[u8; 12], aad: &[u8], buffer.len(), aad.as_ptr(), aad.len(), - &mut data, + core::ptr::from_mut(&mut data), ); + crate::traits::ct::zeroize(&mut data.input.key[16..]); data.out.tag } } diff --git a/src/aead/ghash.rs b/src/aead/ghash.rs index 77ef35e6..55470c44 100644 --- a/src/aead/ghash.rs +++ b/src/aead/ghash.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Fixed-schedule, table-free GHASH universal hash (NIST SP 800-38D). //! //! Generated-code timing claims remain configuration- and @@ -52,6 +50,7 @@ pub(crate) fn h_to_polyval(h_bytes: &[u8; KEY_SIZE]) -> u128 { mul_x_polyval(h) } +/// Computes one GHASH block with the portable POLYVAL-domain reduction. #[cfg(feature = "diag")] #[must_use] pub fn diag_ghash_block_portable(h_bytes: &[u8; KEY_SIZE], block: &[u8; KEY_SIZE]) -> [u8; KEY_SIZE] { @@ -129,10 +128,11 @@ impl Ghash { #[cfg(test)] impl Drop for Ghash { fn drop(&mut self) { - // SAFETY: self.acc/self.h are valid, aligned, dereferenceable pointers to initialized memory. + // SAFETY: the raw pointers address initialized, aligned `u128` fields owned + // exclusively by `self` for the duration of `drop`. unsafe { - core::ptr::write_volatile(&mut self.acc, 0); - core::ptr::write_volatile(&mut self.h, 0); + core::ptr::write_volatile(&raw mut self.acc, 0); + core::ptr::write_volatile(&raw mut self.h, 0); } core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } @@ -144,6 +144,8 @@ impl Drop for Ghash { mod tests { use super::*; + const TEST_H: [u8; 16] = 0x66e9_4bd4_ef8a_2c3b_884c_fa59_ca34_2b2eu128.to_be_bytes(); + /// GHASH with empty input should return zero. #[test] fn ghash_empty() { @@ -165,26 +167,26 @@ mod tests { /// Verify update_padded matches manual block-by-block. #[test] fn ghash_padded_matches_manual() { - let h = hex_to_16("66e94bd4ef8a2c3b884cfa59ca342b2e"); let data = b"Hello, World! This is test data for GHASH padding."; // Manual: split into 16-byte blocks, pad last one. - let mut manual = Ghash::new(&h); - let mut offset = 0; - while offset + 16 <= data.len() { - let block: [u8; 16] = data[offset..offset + 16].try_into().unwrap(); + let mut manual = Ghash::new(&TEST_H); + let mut chunks = data.chunks_exact(BLOCK_SIZE); + for chunk in chunks.by_ref() { + let mut block = [0u8; BLOCK_SIZE]; + block.copy_from_slice(chunk); manual.update_block(&block); - offset += 16; } - if offset < data.len() { - let mut block = [0u8; 16]; - block[..data.len() - offset].copy_from_slice(&data[offset..]); + let remainder = chunks.remainder(); + if !remainder.is_empty() { + let mut block = [0u8; BLOCK_SIZE]; + block[..remainder.len()].copy_from_slice(remainder); manual.update_block(&block); } let manual_result = manual.finalize(); // Padded API. - let mut padded = Ghash::new(&h); + let mut padded = Ghash::new(&TEST_H); padded.update_padded(data); let padded_result = padded.finalize(); @@ -213,14 +215,4 @@ mod tests { "mulX(x^127) should reduce to feedback polynomial" ); } - - fn hex_to_16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - let mut i = 0; - while i < 16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - i = i.strict_add(1); - } - out - } } diff --git a/src/aead/introspect.rs b/src/aead/introspect.rs index fe12e5c5..b350c8f8 100644 --- a/src/aead/introspect.rs +++ b/src/aead/introspect.rs @@ -18,8 +18,7 @@ feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305", - feature = "aegis256", - feature = "ascon-aead" + feature = "aegis256" ))] use crate::aead::targets::{AeadPrimitive, select_backend}; pub use crate::platform::DispatchInfo; @@ -29,8 +28,7 @@ pub use crate::platform::DispatchInfo; feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305", - feature = "aegis256", - feature = "ascon-aead" + feature = "aegis256" ))] #[inline] fn backend_for(primitive: AeadPrimitive) -> &'static str { @@ -98,7 +96,10 @@ pub fn aegis256_backend() -> &'static str { #[inline] #[must_use] pub fn ascon_aead128_backend() -> &'static str { - backend_for(AeadPrimitive::AsconAead128) + match crate::platform::arch() { + crate::platform::Arch::Wasm32 | crate::platform::Arch::Wasm64 => "wasm32/portable", + _ => "portable", + } } #[cfg(test)] diff --git a/src/aead/mod.rs b/src/aead/mod.rs index 0ed370d8..366faeee 100644 --- a/src/aead/mod.rs +++ b/src/aead/mod.rs @@ -34,10 +34,10 @@ //! ```toml //! [dependencies] //! # ChaCha20-Poly1305 only -//! rscrypto = { version = "0.5.0", default-features = false, features = ["chacha20poly1305"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["chacha20poly1305"] } //! //! # All AEADs -//! rscrypto = { version = "0.5.0", default-features = false, features = ["aead"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["aead"] } //! ``` //! //! # API Conventions @@ -73,14 +73,6 @@ mod aegis256; all(feature = "aegis256", target_arch = "riscv64"), all(feature = "aegis256", test), ))] -#[cfg_attr( - all( - feature = "aegis256", - target_arch = "riscv64", - not(any(feature = "aes-gcm", feature = "aes-gcm-siv")) - ), - allow(dead_code) -)] mod aes; #[cfg(feature = "aes-gcm")] mod aes128gcm; @@ -95,6 +87,42 @@ mod aes256gcmsiv; all(target_arch = "riscv64", any(feature = "aes-gcm", feature = "aes-gcm-siv")) ))] mod aes_round; +#[cfg(all(test, any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")))] +mod test_vectors { + use alloc::{vec, vec::Vec}; + + #[track_caller] + pub(super) fn hex_array(hex: &str) -> [u8; N] { + let mut out = [0u8; N]; + crate::hex::from_hex(hex, &mut out).expect("AEAD test vector must contain valid hexadecimal"); + out + } + + #[track_caller] + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] + pub(super) fn hex12(hex: &str) -> [u8; 12] { + hex_array(hex) + } + + #[track_caller] + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] + pub(super) fn hex16(hex: &str) -> [u8; 16] { + hex_array(hex) + } + + #[track_caller] + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] + pub(super) fn hex32(hex: &str) -> [u8; 32] { + hex_array(hex) + } + + #[track_caller] + pub(super) fn hex_vec(hex: &str) -> Vec { + let mut out = vec![0u8; hex.as_bytes().chunks_exact(2).len()]; + crate::hex::from_hex(hex, &mut out).expect("AEAD test vector must contain valid hexadecimal"); + out + } +} #[cfg(feature = "ascon-aead")] mod ascon128; #[cfg(any(feature = "chacha20poly1305", feature = "xchacha20poly1305"))] @@ -111,6 +139,33 @@ mod nonce_counter; mod poly1305; #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] mod polyval; +#[cfg(any( + feature = "aes-gcm", + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ), + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) +))] mod targets; #[cfg(feature = "xchacha20poly1305")] mod xchacha20poly1305; @@ -595,36 +650,40 @@ impl From for OpenError { } } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] pub(crate) struct LengthOverflow; const _: () = assert!(usize::BITS <= u64::BITS); -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] pub(crate) struct AeadByteLengths { aad: u64, text: u64, } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] impl AeadByteLengths { + #[cfg(any(feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] pub(crate) const fn from_usize(aad_len: usize, text_len: usize) -> Self { Self { @@ -633,19 +692,13 @@ impl AeadByteLengths { } } - #[cfg_attr( - not(any(feature = "chacha20poly1305", feature = "xchacha20poly1305")), - allow(dead_code) - )] + #[cfg(any(feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] pub(crate) fn try_new(aad_len: usize, text_len: usize) -> Result { Ok(Self::from_usize(aad_len, text_len)) } - #[cfg_attr( - not(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")), - allow(dead_code) - )] + #[cfg(any(feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] pub(crate) fn try_new_bit_lengths(aad_len: usize, text_len: usize) -> Result { const MAX_BIT_ENCODED_BYTES: u64 = u64::MAX / 8; @@ -655,10 +708,7 @@ impl AeadByteLengths { }) } - #[cfg_attr( - not(any(feature = "chacha20poly1305", feature = "xchacha20poly1305")), - allow(dead_code) - )] + #[cfg(any(feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] pub(crate) fn to_le_bytes_block(self) -> [u8; 16] { let mut block = [0u8; 16]; @@ -667,7 +717,7 @@ impl AeadByteLengths { block } - #[cfg_attr(not(feature = "aes-gcm-siv"), allow(dead_code))] + #[cfg(feature = "aes-gcm-siv")] #[inline] pub(crate) fn to_le_bits_block(self) -> [u8; 16] { let aad_bits = self.aad.strict_mul(8); @@ -678,7 +728,7 @@ impl AeadByteLengths { block } - #[cfg_attr(not(feature = "aes-gcm"), allow(dead_code))] + #[cfg(feature = "aes-gcm")] #[inline] pub(crate) fn to_be_bits_block(self) -> [u8; 16] { let aad_bits = self.aad.strict_mul(8); @@ -689,45 +739,38 @@ impl AeadByteLengths { block } - #[cfg_attr( - not(all( - target_arch = "x86_64", - any(feature = "chacha20poly1305", feature = "xchacha20poly1305") - )), - allow(dead_code) - )] #[inline] + #[cfg(all( + any(feature = "chacha20poly1305", feature = "xchacha20poly1305"), + any(test, target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64") + ))] pub(crate) const fn total_at_least(self, minimum: u64) -> bool { if self.aad >= minimum { return true; } - self.text >= minimum - self.aad + self.text >= minimum.strict_sub(self.aad) } } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] #[inline] pub(crate) fn try_length_as_u64(len: usize) -> Result { u64::try_from(len).map_err(|_| LengthOverflow) } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] #[inline] pub(crate) fn try_bounded_length_as_u64(len: usize, max: u64) -> Result { let len = try_length_as_u64(len)?; @@ -737,47 +780,35 @@ pub(crate) fn try_bounded_length_as_u64(len: usize, max: u64) -> Result Result { try_bounded_length_as_u64(len, max).map_err(|_| SealError::too_large()) } -#[cfg_attr( - not(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")), - allow(dead_code) -)] +#[cfg(any(feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] pub(crate) fn seal_bit_lengths(aad_len: usize, text_len: usize) -> Result { AeadByteLengths::try_new_bit_lengths(aad_len, text_len).map_err(|_| SealError::too_large()) } -#[cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305" - )), - allow(dead_code) -)] +#[cfg(any( + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "xchacha20poly1305" +))] #[inline] pub(crate) fn open_bounded_length_as_u64(len: usize, max: u64) -> Result { try_bounded_length_as_u64(len, max).map_err(|_| OpenError::too_large()) } -#[cfg_attr( - not(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256")), - allow(dead_code) -)] +#[cfg(any(feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] pub(crate) fn open_bit_lengths(aad_len: usize, text_len: usize) -> Result { AeadByteLengths::try_new_bit_lengths(aad_len, text_len).map_err(|_| OpenError::too_large()) @@ -787,7 +818,17 @@ pub(crate) fn open_bit_lengths(aad_len: usize, text_len: usize) -> Result` builds 96-bit nonces as: -//! -//! - 32-bit fixed prefix chosen by the caller -//! - 64-bit big-endian invocation counter -//! -//! This follows the deterministic IV shape from SP 800-38D and removes the -//! easiest nonce-reuse footgun from high-volume AES-GCM usage. -//! -//! ```rust -//! use rscrypto::{Aead, Aes256Gcm, Aes256GcmKey, aead::NonceCounter}; -//! -//! let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes([0x42; 32])); -//! let mut counter = NonceCounter::::new(*b"sess"); -//! -//! let mut sealed = [0u8; 4 + Aes256Gcm::TAG_SIZE]; -//! let nonce = counter.encrypt(&cipher, b"hdr", b"data", &mut sealed)?; -//! -//! let mut opened = [0u8; 4]; -//! cipher.decrypt(&nonce, b"hdr", &sealed, &mut opened)?; -//! assert_eq!(&opened, b"data"); -//! # Ok::<(), Box>(()) -//! ``` - use core::{fmt, marker::PhantomData}; use super::{Aes128Gcm, Aes128GcmTag, Aes256Gcm, Aes256GcmTag, Nonce96, SealError}; @@ -98,6 +73,10 @@ impl From for NonceCounterSealError { /// Monotonic deterministic nonce generator for AES-GCM. /// +/// Each 96-bit nonce consists of a caller-selected 32-bit fixed prefix and a +/// 64-bit big-endian invocation counter, following the deterministic IV shape +/// from SP 800-38D. +/// /// The counter is intentionally not `Clone` or `Copy`. One instance owns one /// nonce stream. If you need restart-safe continuation, persist /// [`next_counter`](Self::next_counter) and restore with @@ -108,6 +87,15 @@ pub struct NonceCounter { _cipher: PhantomData Cipher>, } +impl fmt::Debug for NonceCounter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("NonceCounter") + .field("fixed_prefix", &self.fixed_prefix) + .field("next_counter", &self.next) + .finish() + } +} + impl NonceCounter { /// Fixed per-stream prefix length in bytes. pub const FIXED_PREFIX_LEN: usize = FIXED_PREFIX_LEN; @@ -355,22 +343,41 @@ impl NonceCounter { #[cfg(test)] mod tests { + use alloc::format; + use core::marker::PhantomData; + use super::{Aes128Gcm, Aes256Gcm, NonceCounter, NonceCounterSealError}; use crate::{ Aes128GcmKey, Aes256GcmKey, aead::{Nonce96, SealError}, }; + #[test] + fn nonce_counter_debug_does_not_require_cipher_debug() { + struct CipherWithoutDebug; + + let counter = NonceCounter:: { + fixed_prefix: *b"test", + next: 7, + _cipher: PhantomData, + }; + + assert_eq!( + format!("{counter:?}"), + "NonceCounter { fixed_prefix: [116, 101, 115, 116], next_counter: 7 }" + ); + } + #[test] fn aes_gcm_nonce_counter_formats_prefix_and_counter() { let mut counter = NonceCounter::::new(*b"conn"); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("first nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 0]) ); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("second nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 1]) ); assert_eq!(counter.issued(), 2); @@ -382,10 +389,14 @@ mod tests { let mut counter = NonceCounter::::new(*b"sess"); let mut sealed = [0u8; 4 + Aes256Gcm::TAG_SIZE]; - let nonce = counter.encrypt(&cipher, b"hdr", b"data", &mut sealed).unwrap(); + let nonce = counter + .encrypt(&cipher, b"hdr", b"data", &mut sealed) + .expect("valid inputs should seal"); let mut opened = [0u8; 4]; - cipher.decrypt(&nonce, b"hdr", &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, b"hdr", &sealed, &mut opened) + .expect("freshly sealed ciphertext should open"); assert_eq!(&opened, b"data"); assert_eq!(counter.next_counter(), 1); } @@ -396,7 +407,9 @@ mod tests { let mut counter = NonceCounter::::new(*b"bufr"); let mut out = [0u8; 3]; - let err = counter.encrypt(&cipher, b"", b"data", &mut out).unwrap_err(); + let err = counter + .encrypt(&cipher, b"", b"data", &mut out) + .expect_err("undersized output should be rejected"); assert_eq!(err, NonceCounterSealError::from(SealError::buffer())); assert_eq!(counter.next_counter(), 1); } @@ -404,11 +417,14 @@ mod tests { #[test] fn aes_gcm_nonce_counter_exhausts_cleanly() { let mut counter = - NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)).unwrap(); + NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)) + .expect("last permitted counter should be accepted"); - assert!(counter.next_nonce().is_ok()); + counter.next_nonce().expect("last nonce should be available"); assert_eq!(counter.remaining(), 0); - assert!(counter.next_nonce().is_err()); + counter + .next_nonce() + .expect_err("exhausted counter should reject another nonce"); } #[test] @@ -417,13 +433,15 @@ mod tests { // greater than or equal to MAX_MESSAGES. Pin both the equality and the // strictly-greater branch so a future relaxation cannot silently widen // the deterministic-IV budget past the SP 800-38D limit. - assert!(NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES).is_err()); - assert!(NonceCounter::::with_counter(*b"oflw", u64::MAX).is_err()); + NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES) + .expect_err("maximum counter should be rejected"); + NonceCounter::::with_counter(*b"oflw", u64::MAX).expect_err("out-of-range counter should be rejected"); } #[test] fn aes_gcm_nonce_counters_report_resumed_state() { - let counter256 = NonceCounter::::with_counter(*b"r256", 37).unwrap(); + let counter256 = + NonceCounter::::with_counter(*b"r256", 37).expect("in-range counter should be accepted"); assert_eq!(counter256.fixed_prefix(), *b"r256"); assert_eq!(counter256.next_counter(), 37); assert_eq!(counter256.issued(), 37); @@ -432,7 +450,8 @@ mod tests { NonceCounter::::MAX_MESSAGES.strict_sub(37) ); - let counter128 = NonceCounter::::with_counter(*b"r128", 73).unwrap(); + let counter128 = + NonceCounter::::with_counter(*b"r128", 73).expect("in-range counter should be accepted"); assert_eq!(counter128.fixed_prefix(), *b"r128"); assert_eq!(counter128.next_counter(), 73); assert_eq!(counter128.issued(), 73); @@ -448,10 +467,14 @@ mod tests { let mut counter = NonceCounter::::new(*b"sess"); let mut sealed = [0u8; 4 + Aes128Gcm::TAG_SIZE]; - let nonce = counter.encrypt(&cipher, b"hdr", b"data", &mut sealed).unwrap(); + let nonce = counter + .encrypt(&cipher, b"hdr", b"data", &mut sealed) + .expect("valid inputs should seal"); let mut opened = [0u8; 4]; - cipher.decrypt(&nonce, b"hdr", &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, b"hdr", &sealed, &mut opened) + .expect("freshly sealed ciphertext should open"); assert_eq!(&opened, b"data"); assert_eq!(counter.next_counter(), 1); } @@ -461,11 +484,11 @@ mod tests { let mut counter = NonceCounter::::new(*b"conn"); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("first nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 0]) ); assert_eq!( - counter.next_nonce().unwrap(), + counter.next_nonce().expect("second nonce should be available"), Nonce96::from_bytes([b'c', b'o', b'n', b'n', 0, 0, 0, 0, 0, 0, 0, 1]) ); assert_eq!(counter.issued(), 2); @@ -477,7 +500,9 @@ mod tests { let mut counter = NonceCounter::::new(*b"bufr"); let mut out = [0u8; 3]; - let err = counter.encrypt(&cipher, b"", b"data", &mut out).unwrap_err(); + let err = counter + .encrypt(&cipher, b"", b"data", &mut out) + .expect_err("undersized output should be rejected"); assert_eq!(err, NonceCounterSealError::from(SealError::buffer())); assert_eq!(counter.next_counter(), 1); } @@ -485,11 +510,14 @@ mod tests { #[test] fn aes128_gcm_nonce_counter_exhausts_cleanly() { let mut counter = - NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)).unwrap(); + NonceCounter::::with_counter(*b"last", NonceCounter::::MAX_MESSAGES.strict_sub(1)) + .expect("last permitted counter should be accepted"); - assert!(counter.next_nonce().is_ok()); + counter.next_nonce().expect("last nonce should be available"); assert_eq!(counter.remaining(), 0); - assert!(counter.next_nonce().is_err()); + counter + .next_nonce() + .expect_err("exhausted counter should reject another nonce"); } #[test] @@ -497,7 +525,8 @@ mod tests { // Pin both the equality and strictly-greater branch so a future // relaxation cannot silently widen the deterministic-IV budget past // the SP 800-38D limit. - assert!(NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES).is_err()); - assert!(NonceCounter::::with_counter(*b"oflw", u64::MAX).is_err()); + NonceCounter::::with_counter(*b"oflw", NonceCounter::::MAX_MESSAGES) + .expect_err("maximum counter should be rejected"); + NonceCounter::::with_counter(*b"oflw", u64::MAX).expect_err("out-of-range counter should be rejected"); } } diff --git a/src/aead/poly1305.rs b/src/aead/poly1305.rs index 557d52f1..08e0345b 100644 --- a/src/aead/poly1305.rs +++ b/src/aead/poly1305.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Portable Poly1305 core. #[cfg(feature = "std")] @@ -21,8 +19,10 @@ const RISCV64_PAR4_MIN: u64 = 4096; type ComputeBlockFn = fn(&mut State, &[u8; 16], bool); #[cfg(feature = "std")] +#[cfg(feature = "xchacha20poly1305")] static XCHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH: OnceCache = OnceCache::new(); #[cfg(feature = "std")] +#[cfg(feature = "chacha20poly1305")] static CHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH: OnceCache = OnceCache::new(); #[inline] @@ -32,6 +32,29 @@ fn load_u32_le(input: &[u8]) -> u32 { u32::from_le_bytes(bytes) } +#[cfg(any( + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "wasm32", +))] +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + +#[cfg(any( + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "wasm32", +))] +#[inline(always)] +fn add_limb_product(accumulator: u64, left: u32, right: u32) -> u64 { + accumulator.wrapping_add(u64::from(left).wrapping_mul(u64::from(right))) +} + #[cfg(any( all(target_arch = "powerpc64", target_endian = "little"), target_arch = "riscv64", @@ -52,10 +75,10 @@ fn compute_block_scalar_reduction( let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = r1.wrapping_mul(5); + let s2 = r2.wrapping_mul(5); + let s3 = r3.wrapping_mul(5); + let s4 = r4.wrapping_mul(5); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -69,31 +92,31 @@ fn compute_block_scalar_reduction( h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]), h4, s1); + let mut d1 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]), h4, s2); + let mut d2 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]), h4, s3); + let mut d3 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]), h4, s4); + let mut d4 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]), h4, r0); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = low_u32(d0 >> 26); + h0 = low_u32(d0) & LIMB_MASK; + d1 = d1.wrapping_add(u64::from(c)); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = low_u32(d1 >> 26); + h1 = low_u32(d1) & LIMB_MASK; + d2 = d2.wrapping_add(u64::from(c)); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = low_u32(d2 >> 26); + h2 = low_u32(d2) & LIMB_MASK; + d3 = d3.wrapping_add(u64::from(c)); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = low_u32(d3 >> 26); + h3 = low_u32(d3) & LIMB_MASK; + d4 = d4.wrapping_add(u64::from(c)); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = low_u32(d4 >> 26); + h4 = low_u32(d4) & LIMB_MASK; + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -120,12 +143,15 @@ fn compute_block_resolved(primitive: AeadPrimitive) -> ComputeBlockFn { #[cfg(feature = "std")] { match primitive { + #[cfg(feature = "xchacha20poly1305")] AeadPrimitive::XChaCha20Poly1305 => { XCHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH.get_or_init(|| resolve_compute_block(primitive)) } + #[cfg(feature = "chacha20poly1305")] AeadPrimitive::ChaCha20Poly1305 => { CHACHA20POLY1305_COMPUTE_BLOCK_DISPATCH.get_or_init(|| resolve_compute_block(primitive)) } + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] _ => resolve_compute_block(primitive), } } @@ -159,21 +185,84 @@ fn resolve_compute_block(primitive: AeadPrimitive) -> ComputeBlockFn { #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] +/// Absorbs one Poly1305 block with the x86-64 AVX2 multiplier. +/// +/// # Safety +/// +/// The current CPU must support AVX2. `state` must satisfy the internal clamped-key and reduced-accumulator limb +/// bounds established by `State::new` and the Poly1305 block kernels. unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: bool) { use core::arch::x86_64::{__m256i, _mm256_mul_epu32, _mm256_setr_epi32, _mm256_storeu_si256}; + use core::ptr::NonNull; #[inline(always)] fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { - // SAFETY: the enclosing kernel enables AVX2 and the destination array is a valid - // unaligned store target for one `__m256i`. + let mut lanes = [0u64; 4]; + let destination = NonNull::from(&mut lanes).cast::<__m256i>().as_ptr(); + + // SAFETY: this helper is called only from the enclosing AVX2 kernel. `destination` retains the provenance of + // the writable 32-byte `lanes` array, and `_mm256_storeu_si256` permits its 8-byte alignment. unsafe { - let a = _mm256_setr_epi32(lhs[0] as i32, 0, lhs[1] as i32, 0, lhs[2] as i32, 0, lhs[3] as i32, 0); - let b = _mm256_setr_epi32(rhs[0] as i32, 0, rhs[1] as i32, 0, rhs[2] as i32, 0, rhs[3] as i32, 0); + let a = _mm256_setr_epi32( + lhs[0].cast_signed(), + 0, + lhs[1].cast_signed(), + 0, + lhs[2].cast_signed(), + 0, + lhs[3].cast_signed(), + 0, + ); + let b = _mm256_setr_epi32( + rhs[0].cast_signed(), + 0, + rhs[1].cast_signed(), + 0, + rhs[2].cast_signed(), + 0, + rhs[3].cast_signed(), + 0, + ); let products = _mm256_mul_epu32(a, b); - let mut lanes = [0u64; 4]; - _mm256_storeu_si256(lanes.as_mut_ptr() as *mut __m256i, products); - lanes[0] + lanes[1] + lanes[2] + lanes[3] + _mm256_storeu_si256(destination, products); } + + let sum = u128::from(lanes[0]) + .strict_add(u128::from(lanes[1])) + .strict_add(u128::from(lanes[2])) + .strict_add(u128::from(lanes[3])); + debug_assert!(sum <= u128::from(u64::MAX)); + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn sum5_mul(lhs: [u32; 5], rhs: [u32; 5]) -> u64 { + let [l0, l1, l2, l3, l4] = lhs; + let [r0, r1, r2, r3, r4] = rhs; + let sum = + u128::from(sum4_mul([l0, l1, l2, l3], [r0, r1, r2, r3])).strict_add(u128::from(l4).strict_mul(u128::from(r4))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; @@ -184,10 +273,10 @@ unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: b let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -201,33 +290,33 @@ unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: b h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = sum5_mul([h0, h1, h2, h3, h4], [r0, s4, s3, s2, s1]); + let mut d1 = sum5_mul([h0, h1, h2, h3, h4], [r1, r0, s4, s3, s2]); + let mut d2 = sum5_mul([h0, h1, h2, h3, h4], [r2, r1, r0, s4, s3]); + let mut d3 = sum5_mul([h0, h1, h2, h3, h4], [r3, r2, r1, r0, s4]); + let mut d4 = sum5_mul([h0, h1, h2, h3, h4], [r4, r3, r2, r1, r0]); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); - c = h0 >> 26; + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); @@ -236,58 +325,90 @@ unsafe fn compute_block_x86_avx2(state: &mut State, block: &[u8; 16], partial: b #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq")] +/// Absorbs one Poly1305 block with the x86-64 AVX-512 multiplier. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, and AVX-512DQ. `state` must satisfy the internal +/// clamped-key and reduced-accumulator limb bounds established by `State::new` and the Poly1305 block kernels. unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: bool) { use core::arch::x86_64::{__m512i, _mm512_mul_epu32, _mm512_setr_epi32, _mm512_storeu_si512}; + use core::ptr::NonNull; + + #[inline(always)] + fn narrow_sum(value: u128) -> u64 { + debug_assert!(value <= u128::from(u64::MAX)); + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = value.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } #[inline(always)] fn pair_sum4_mul(lhs: [u32; 4], rhs_lo: [u32; 4], rhs_hi: [u32; 4]) -> (u64, u64) { - // SAFETY: the enclosing kernel enables AVX-512F, and the destination array is a valid - // unaligned store target for one `__m512i`. + let mut lanes = [0u64; 8]; + let destination = NonNull::from(&mut lanes).cast::<__m512i>().as_ptr(); + + // SAFETY: this helper is called only from the enclosing AVX-512 kernel. `destination` retains the provenance of + // the writable 64-byte `lanes` array, and `_mm512_storeu_si512` permits its 8-byte alignment. unsafe { let a = _mm512_setr_epi32( - lhs[0] as i32, + lhs[0].cast_signed(), 0, - lhs[1] as i32, + lhs[1].cast_signed(), 0, - lhs[2] as i32, + lhs[2].cast_signed(), 0, - lhs[3] as i32, + lhs[3].cast_signed(), 0, - lhs[0] as i32, + lhs[0].cast_signed(), 0, - lhs[1] as i32, + lhs[1].cast_signed(), 0, - lhs[2] as i32, + lhs[2].cast_signed(), 0, - lhs[3] as i32, + lhs[3].cast_signed(), 0, ); let b = _mm512_setr_epi32( - rhs_lo[0] as i32, + rhs_lo[0].cast_signed(), 0, - rhs_lo[1] as i32, + rhs_lo[1].cast_signed(), 0, - rhs_lo[2] as i32, + rhs_lo[2].cast_signed(), 0, - rhs_lo[3] as i32, + rhs_lo[3].cast_signed(), 0, - rhs_hi[0] as i32, + rhs_hi[0].cast_signed(), 0, - rhs_hi[1] as i32, + rhs_hi[1].cast_signed(), 0, - rhs_hi[2] as i32, + rhs_hi[2].cast_signed(), 0, - rhs_hi[3] as i32, + rhs_hi[3].cast_signed(), 0, ); let products = _mm512_mul_epu32(a, b); - let mut lanes = [0u64; 8]; - _mm512_storeu_si512(lanes.as_mut_ptr() as *mut __m512i, products); - ( - lanes[0] + lanes[1] + lanes[2] + lanes[3], - lanes[4] + lanes[5] + lanes[6] + lanes[7], - ) + _mm512_storeu_si512(destination, products); } + + let low = u128::from(lanes[0]) + .strict_add(u128::from(lanes[1])) + .strict_add(u128::from(lanes[2])) + .strict_add(u128::from(lanes[3])); + let high = u128::from(lanes[4]) + .strict_add(u128::from(lanes[5])) + .strict_add(u128::from(lanes[6])) + .strict_add(u128::from(lanes[7])); + (narrow_sum(low), narrow_sum(high)) } #[inline(always)] @@ -295,6 +416,19 @@ unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: pair_sum4_mul(lhs, rhs, [0; 4]).0 } + #[inline(always)] + fn add_product(base: u64, lhs: u32, rhs: u32) -> u64 { + let sum = u128::from(base).strict_add(u128::from(lhs).strict_mul(u128::from(rhs))); + narrow_sum(sum) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; let r0 = state.r[0]; @@ -303,10 +437,10 @@ unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -324,39 +458,44 @@ unsafe fn compute_block_x86_avx512(state: &mut State, block: &[u8; 16], partial: let (d2_base, d3_base) = pair_sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4], [r3, r2, r1, r0]); let d4_base = single_sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]); - let d0 = d0_base + (u64::from(h4) * u64::from(s1)); - let mut d1 = d1_base + (u64::from(h4) * u64::from(s2)); - let mut d2 = d2_base + (u64::from(h4) * u64::from(s3)); - let mut d3 = d3_base + (u64::from(h4) * u64::from(s4)); - let mut d4 = d4_base + (u64::from(h4) * u64::from(r0)); + let d0 = add_product(d0_base, h4, s1); + let mut d1 = add_product(d1_base, h4, s2); + let mut d2 = add_product(d2_base, h4, s3); + let mut d3 = add_product(d3_base, h4, s4); + let mut d4 = add_product(d4_base, h4, r0); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); - c = h0 >> 26; + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); state.h = [h0, h1, h2, h3, h4]; } +/// Absorbs one full Poly1305 block with the AArch64 NEON multiplier. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partial: bool) { @@ -379,6 +518,35 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia } } + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn sum5_mul(lhs: [u32; 5], rhs: [u32; 5]) -> u64 { + let [l0, l1, l2, l3, l4] = lhs; + let [r0, r1, r2, r3, r4] = rhs; + let sum = + u128::from(sum4_mul([l0, l1, l2, l3], [r0, r1, r2, r3])).strict_add(u128::from(l4).strict_mul(u128::from(r4))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; let r0 = state.r[0]; @@ -387,10 +555,10 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -404,33 +572,33 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = sum5_mul([h0, h1, h2, h3, h4], [r0, s4, s3, s2, s1]); + let mut d1 = sum5_mul([h0, h1, h2, h3, h4], [r1, r0, s4, s3, s2]); + let mut d2 = sum5_mul([h0, h1, h2, h3, h4], [r2, r1, r0, s4, s3]); + let mut d3 = sum5_mul([h0, h1, h2, h3, h4], [r3, r2, r1, r0, s4]); + let mut d4 = sum5_mul([h0, h1, h2, h3, h4], [r4, r3, r2, r1, r0]); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); - c = h0 >> 26; + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); @@ -439,6 +607,12 @@ unsafe fn compute_block_aarch64_neon(state: &mut State, block: &[u8; 16], partia #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] +/// Absorbs one Poly1305 block with the WASM SIMD128 multiplier. +/// +/// # Safety +/// +/// The current WebAssembly instance must support SIMD128. `state` must satisfy the internal clamped-key and +/// reduced-accumulator limb bounds established by `State::new` and the Poly1305 block kernels. unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partial: bool) { use core::arch::wasm32::{i64x2_add, u32x4, u64x2_extmul_high_u32x4, u64x2_extmul_low_u32x4, u64x2_extract_lane}; @@ -449,7 +623,7 @@ unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partia let lo = u64x2_extmul_low_u32x4(a, b); let hi = u64x2_extmul_high_u32x4(a, b); let sum = i64x2_add(lo, hi); - u64x2_extract_lane::<0>(sum) + u64x2_extract_lane::<1>(sum) + u64x2_extract_lane::<0>(sum).wrapping_add(u64x2_extract_lane::<1>(sum)) } let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; @@ -460,10 +634,10 @@ unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partia let r3 = state.r[3]; let r4 = state.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = r1.wrapping_mul(5); + let s2 = r2.wrapping_mul(5); + let s3 = r3.wrapping_mul(5); + let s4 = r4.wrapping_mul(5); let mut h0 = state.h[0]; let mut h1 = state.h[1]; @@ -477,31 +651,31 @@ unsafe fn compute_block_wasm_simd128(state: &mut State, block: &[u8; 16], partia h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]) + (u64::from(h4) * u64::from(s1)); - let mut d1 = sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]) + (u64::from(h4) * u64::from(s2)); - let mut d2 = sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]) + (u64::from(h4) * u64::from(s3)); - let mut d3 = sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]) + (u64::from(h4) * u64::from(s4)); - let mut d4 = sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]) + (u64::from(h4) * u64::from(r0)); + let d0 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r0, s4, s3, s2]), h4, s1); + let mut d1 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r1, r0, s4, s3]), h4, s2); + let mut d2 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r2, r1, r0, s4]), h4, s3); + let mut d3 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r3, r2, r1, r0]), h4, s4); + let mut d4 = add_limb_product(sum4_mul([h0, h1, h2, h3], [r4, r3, r2, r1]), h4, r0); - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); + let mut c = low_u32(d0 >> 26); + h0 = low_u32(d0) & LIMB_MASK; + d1 = d1.wrapping_add(u64::from(c)); - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); + c = low_u32(d1 >> 26); + h1 = low_u32(d1) & LIMB_MASK; + d2 = d2.wrapping_add(u64::from(c)); - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); + c = low_u32(d2 >> 26); + h2 = low_u32(d2) & LIMB_MASK; + d3 = d3.wrapping_add(u64::from(c)); - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); + c = low_u32(d3 >> 26); + h3 = low_u32(d3) & LIMB_MASK; + d4 = d4.wrapping_add(u64::from(c)); - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + c = low_u32(d4 >> 26); + h4 = low_u32(d4) & LIMB_MASK; + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -549,6 +723,39 @@ impl State { #[inline(always)] fn compute_block_portable(&mut self, block: &[u8; 16], partial: bool) { + #[inline(always)] + fn fivefold_limb(limb: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(limb <= MAX_UNSCALED); + + let product = u64::from(limb).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn scalar_dot5(lhs: [u32; 5], rhs: [u32; 5]) -> u64 { + let [l0, l1, l2, l3, l4] = lhs; + let [r0, r1, r2, r3, r4] = rhs; + let sum = u128::from(l0) + .strict_mul(u128::from(r0)) + .strict_add(u128::from(l1).strict_mul(u128::from(r1))) + .strict_add(u128::from(l2).strict_mul(u128::from(r2))) + .strict_add(u128::from(l3).strict_mul(u128::from(r3))) + .strict_add(u128::from(l4).strict_mul(u128::from(r4))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) + } + + #[inline(always)] + fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let hibit = if partial { 0 } else { FULL_BLOCK_HIBIT }; let r0 = self.r[0]; @@ -557,10 +764,10 @@ impl State { let r3 = self.r[3]; let r4 = self.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = fivefold_limb(r1); + let s2 = fivefold_limb(r2); + let s3 = fivefold_limb(r3); + let s4 = fivefold_limb(r4); let mut h0 = self.h[0]; let mut h1 = self.h[1]; @@ -574,53 +781,33 @@ impl State { h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = (u64::from(h0) * u64::from(r0)) - + (u64::from(h1) * u64::from(s4)) - + (u64::from(h2) * u64::from(s3)) - + (u64::from(h3) * u64::from(s2)) - + (u64::from(h4) * u64::from(s1)); - let mut d1 = (u64::from(h0) * u64::from(r1)) - + (u64::from(h1) * u64::from(r0)) - + (u64::from(h2) * u64::from(s4)) - + (u64::from(h3) * u64::from(s3)) - + (u64::from(h4) * u64::from(s2)); - let mut d2 = (u64::from(h0) * u64::from(r2)) - + (u64::from(h1) * u64::from(r1)) - + (u64::from(h2) * u64::from(r0)) - + (u64::from(h3) * u64::from(s4)) - + (u64::from(h4) * u64::from(s3)); - let mut d3 = (u64::from(h0) * u64::from(r3)) - + (u64::from(h1) * u64::from(r2)) - + (u64::from(h2) * u64::from(r1)) - + (u64::from(h3) * u64::from(r0)) - + (u64::from(h4) * u64::from(s4)); - let mut d4 = (u64::from(h0) * u64::from(r4)) - + (u64::from(h1) * u64::from(r3)) - + (u64::from(h2) * u64::from(r2)) - + (u64::from(h3) * u64::from(r1)) - + (u64::from(h4) * u64::from(r0)); - - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); - - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); - - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); - - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); - - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + let d0 = scalar_dot5([h0, h1, h2, h3, h4], [r0, s4, s3, s2, s1]); + let mut d1 = scalar_dot5([h0, h1, h2, h3, h4], [r1, r0, s4, s3, s2]); + let mut d2 = scalar_dot5([h0, h1, h2, h3, h4], [r2, r1, r0, s4, s3]); + let mut d3 = scalar_dot5([h0, h1, h2, h3, h4], [r3, r2, r1, r0, s4]); + let mut d4 = scalar_dot5([h0, h1, h2, h3, h4], [r4, r3, r2, r1, r0]); - c = h0 >> 26; + let mut c = d0 >> 26; + h0 = narrow_limb(d0 & u64::from(LIMB_MASK)); + d1 = d1.strict_add(c); + + c = d1 >> 26; + h1 = narrow_limb(d1 & u64::from(LIMB_MASK)); + d2 = d2.strict_add(c); + + c = d2 >> 26; + h2 = narrow_limb(d2 & u64::from(LIMB_MASK)); + d3 = d3.strict_add(c); + + c = d3 >> 26; + h3 = narrow_limb(d3 & u64::from(LIMB_MASK)); + d4 = d4.strict_add(c); + + c = d4 >> 26; + h4 = narrow_limb(d4 & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(fivefold_limb(narrow_limb(c))); + + let c = h0 >> 26; h0 &= LIMB_MASK; h1 = h1.wrapping_add(c); @@ -629,14 +816,11 @@ impl State { #[cfg(test)] fn update_message(&mut self, message: &[u8], compute_block: ComputeBlockFn) { - let mut blocks = message.chunks_exact(16); - for chunk in &mut blocks { - let mut block = [0u8; 16]; - block.copy_from_slice(chunk); - compute_block(self, &block, false); + let (blocks, remainder) = message.as_chunks::<16>(); + for block in blocks { + compute_block(self, block, false); } - let remainder = blocks.remainder(); if remainder.is_empty() { return; } @@ -648,14 +832,11 @@ impl State { } fn update_padded_segment(&mut self, segment: &[u8], compute_block: ComputeBlockFn) { - let mut blocks = segment.chunks_exact(16); - for chunk in &mut blocks { - let mut block = [0u8; 16]; - block.copy_from_slice(chunk); - compute_block(self, &block, false); + let (blocks, remainder) = segment.as_chunks::<16>(); + for block in blocks { + compute_block(self, block, false); } - let remainder = blocks.remainder(); if remainder.is_empty() { return; } @@ -672,6 +853,22 @@ impl State { #[inline(always)] fn finalize_in_place(&mut self) -> [u8; 16] { + #[inline(always)] + fn fivefold_carry(carry: u32) -> u32 { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!(carry <= MAX_UNSCALED); + + let product = u64::from(carry).strict_mul(5); + let [b0, b1, b2, b3, _, _, _, _] = product.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + + #[inline(always)] + fn low_word(value: u64) -> u32 { + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) + } + let mut h0 = self.h[0]; let mut h1 = self.h[1]; let mut h2 = self.h[2]; @@ -692,7 +889,7 @@ impl State { c = h4 >> 26; h4 &= LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + h0 = h0.wrapping_add(fivefold_carry(c)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -735,14 +932,14 @@ impl State { h2 = (h2 >> 12) | (h3 << 14); h3 = (h3 >> 18) | (h4 << 8); - let mut f = u64::from(h0) + u64::from(self.pad[0]); - h0 = f as u32; - f = u64::from(h1) + u64::from(self.pad[1]) + (f >> 32); - h1 = f as u32; - f = u64::from(h2) + u64::from(self.pad[2]) + (f >> 32); - h2 = f as u32; - f = u64::from(h3) + u64::from(self.pad[3]) + (f >> 32); - h3 = f as u32; + let mut f = u64::from(h0).strict_add(u64::from(self.pad[0])); + h0 = low_word(f); + f = u64::from(h1).strict_add(u64::from(self.pad[1])).strict_add(f >> 32); + h1 = low_word(f); + f = u64::from(h2).strict_add(u64::from(self.pad[2])).strict_add(f >> 32); + h2 = low_word(f); + f = u64::from(h3).strict_add(u64::from(self.pad[3])).strict_add(f >> 32); + h3 = low_word(f); let mut tag = [0u8; 16]; tag[0..4].copy_from_slice(&h0.to_le_bytes()); @@ -761,7 +958,6 @@ pub(crate) fn authenticate(message: &[u8], key: &[u8; 32]) -> [u8; 16] { state.finalize() } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag")), allow(dead_code))] pub(crate) fn authenticate_aead( primitive: AeadPrimitive, aad: &[u8], @@ -774,7 +970,8 @@ pub(crate) fn authenticate_aead( { use crate::platform::caps::x86; if lengths.total_at_least(64) && current_caps().has(x86::AVX2) { - return Ok(avx2_par4::authenticate_aead_par4(aad, ciphertext, key, lengths)); + // SAFETY: this branch verifies AVX2 before selecting the x86-64 parallel kernel. + return Ok(unsafe { avx2_par4::authenticate_aead_par4(aad, ciphertext, key, lengths) }); } } #[cfg(target_arch = "aarch64")] @@ -794,14 +991,11 @@ pub(crate) fn authenticate_aead( authenticate_aead_with(aad, ciphertext, key, compute_block_resolved(primitive), lengths) } -#[cfg_attr( - not(any( - test, - target_arch = "x86_64", - all(target_arch = "powerpc64", target_endian = "little") - )), - allow(dead_code) -)] +#[cfg(any( + test, + target_arch = "x86_64", + all(target_arch = "powerpc64", target_endian = "little") +))] fn authenticate_aead_portable_blocks( aad: &[u8], ciphertext: &[u8], @@ -820,19 +1014,16 @@ fn authenticate_aead_portable_blocks( tag } -#[cfg_attr( - not(any( - test, - target_arch = "x86_64", - all(target_arch = "powerpc64", target_endian = "little") - )), - allow(dead_code) -)] +#[cfg(any( + test, + target_arch = "x86_64", + all(target_arch = "powerpc64", target_endian = "little") +))] pub(crate) fn authenticate_aead_empty_text_portable(aad: &[u8], key: &[u8; 32]) -> [u8; 16] { authenticate_aead_portable_blocks(aad, &[], key, super::AeadByteLengths::from_usize(aad.len(), 0)) } -#[cfg_attr(not(all(target_arch = "powerpc64", target_endian = "little")), allow(dead_code))] +#[cfg(all(target_arch = "powerpc64", target_endian = "little"))] pub(crate) fn authenticate_aead_short_text_portable(aad: &[u8], ciphertext: &[u8], key: &[u8; 32]) -> [u8; 16] { authenticate_aead_portable_blocks( aad, @@ -843,13 +1034,23 @@ pub(crate) fn authenticate_aead_short_text_portable(aad: &[u8], ciphertext: &[u8 } #[cfg(feature = "diag")] +/// Computes a ChaCha20-Poly1305 authenticator through the selected Poly1305 backend. +/// +/// Returns `None` when the associated-data or ciphertext length cannot be encoded by the AEAD construction. pub fn diag_chacha20poly1305_authenticate_aead(aad: &[u8], ciphertext: &[u8], key: &[u8; 32]) -> Option<[u8; 16]> { - authenticate_aead(AeadPrimitive::ChaCha20Poly1305, aad, ciphertext, key).ok() + #[cfg(feature = "chacha20poly1305")] + let primitive = AeadPrimitive::ChaCha20Poly1305; + #[cfg(all(not(feature = "chacha20poly1305"), feature = "xchacha20poly1305"))] + let primitive = AeadPrimitive::XChaCha20Poly1305; + authenticate_aead(primitive, aad, ciphertext, key).ok() } #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Computes a diagnostic Poly1305 tag after one block using the portable backend. +/// +/// `partial` suppresses the full-block high bit for a caller-prepared partial-block encoding. pub fn diag_poly1305_block_portable_digest(key: &[u8; 32], block: &[u8; 16], partial: bool) -> [u8; 16] { let mut state = State::new(key); state.compute_block_portable(block, partial); @@ -861,6 +1062,9 @@ pub fn diag_poly1305_block_portable_digest(key: &[u8; 32], block: &[u8; 16], par target_arch = "aarch64", any(target_os = "linux", target_os = "macos") ))] +/// Computes a ChaCha20-Poly1305 authenticator with the four-lane AArch64 NEON backend. +/// +/// Returns `None` when the associated-data or ciphertext length cannot be encoded by the AEAD construction. pub fn diag_chacha20poly1305_authenticate_aead_aarch64_neon_par4( aad: &[u8], ciphertext: &[u8], @@ -870,7 +1074,6 @@ pub fn diag_chacha20poly1305_authenticate_aead_aarch64_neon_par4( Some(aarch64_neon::authenticate_aead_par4(aad, ciphertext, key, lengths)) } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag", test)), allow(dead_code))] fn authenticate_aead_with( aad: &[u8], ciphertext: &[u8], @@ -894,7 +1097,6 @@ fn authenticate_aead_with( #[path = "poly1305/aarch64_neon.rs"] pub(crate) mod aarch64_neon; #[cfg(target_arch = "x86_64")] -#[allow(unsafe_op_in_unsafe_fn)] #[path = "poly1305/x86_64_avx2_par4.rs"] mod avx2_par4; #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] @@ -921,10 +1123,21 @@ mod tests { use super::authenticate; #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] - use super::{ComputeBlockFn, State, authenticate_aead_with}; + use super::{ComputeBlockFn, authenticate_aead_with}; #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] use crate::aead::AeadByteLengths; use crate::aead::targets::AeadPrimitive; + + fn primitive() -> AeadPrimitive { + #[cfg(feature = "chacha20poly1305")] + { + AeadPrimitive::ChaCha20Poly1305 + } + #[cfg(all(not(feature = "chacha20poly1305"), feature = "xchacha20poly1305"))] + { + AeadPrimitive::XChaCha20Poly1305 + } + } #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; #[cfg(target_arch = "riscv64")] @@ -932,6 +1145,15 @@ mod tests { #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; + fn patterned_bytes(length: usize, factor: usize, offset: usize) -> Vec { + (0..length) + .map(|index| { + let [byte, ..] = index.strict_mul(factor).strict_add(offset).to_le_bytes(); + byte + }) + .collect() + } + #[test] fn poly1305_matches_rfc_8439_section_2_5_2() { let key = [ @@ -965,10 +1187,8 @@ mod tests { 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91, ]; - assert_eq!( - super::authenticate_aead(AeadPrimitive::ChaCha20Poly1305, &aad, &ciphertext, &poly_key).unwrap(), - expected - ); + let actual = super::authenticate_aead(primitive(), &aad, &ciphertext, &poly_key); + assert_eq!(actual, Ok(expected)); } #[test] @@ -976,13 +1196,12 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 14, 15, 16, 17, 31, 32, 33, 63] { - let aad = (0..aad_len) - .map(|index| index.strict_mul(11).strict_add(7) as u8) - .collect::>(); - let expected = super::authenticate_aead(AeadPrimitive::ChaCha20Poly1305, &aad, &[], &key).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let expected = super::authenticate_aead(primitive(), &aad, &[], &key); let actual = super::authenticate_aead_empty_text_portable(&aad, &key); assert_eq!( - actual, expected, + expected, + Ok(actual), "empty-text authentication mismatch at aad_len={aad_len}" ); } @@ -990,8 +1209,8 @@ mod tests { #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] fn authenticate_aead_portable(aad: &[u8], ciphertext: &[u8], key: &[u8; 32]) -> [u8; 16] { - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); - authenticate_aead_with(aad, ciphertext, key, State::compute_block_portable, lengths).unwrap() + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); + super::authenticate_aead_portable_blocks(aad, ciphertext, key, lengths) } #[cfg(any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64"))] @@ -999,16 +1218,12 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 80] { for ciphertext_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 191, 256] { - let aad = (0..aad_len) - .map(|index| index.strict_mul(11).strict_add(7) as u8) - .collect::>(); - let ciphertext = (0..ciphertext_len) - .map(|index| index.strict_mul(17).strict_add(3) as u8) - .collect::>(); + let aad = patterned_bytes(aad_len, 11, 7); + let ciphertext = patterned_bytes(ciphertext_len, 17, 3); let portable = authenticate_aead_portable(&aad, &ciphertext, &key); - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); - let accelerated = authenticate_aead_with(&aad, &ciphertext, &key, backend, lengths).unwrap(); - assert_eq!(accelerated, portable); + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); + let accelerated = authenticate_aead_with(&aad, &ciphertext, &key, backend, lengths); + assert_eq!(accelerated, Ok(portable)); } } } @@ -1063,11 +1278,12 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 80, 128] { for ct_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 191, 256, 1024, 4096] { - let aad: Vec = (0..aad_len).map(|i| i.strict_mul(11).strict_add(7) as u8).collect(); - let ct: Vec = (0..ct_len).map(|i| i.strict_mul(17).strict_add(3) as u8).collect(); - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let ct = patterned_bytes(ct_len, 17, 3); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); - let parallel = super::avx2_par4::authenticate_aead_par4(&aad, &ct, &key, lengths); + // SAFETY: the test returned above unless AVX2 is available. + let parallel = unsafe { super::avx2_par4::authenticate_aead_par4(&aad, &ct, &key, lengths) }; assert_eq!(parallel, portable, "mismatch at aad={aad_len} ct={ct_len}"); } } @@ -1083,9 +1299,9 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 80, 128] { for ct_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 191, 256, 1024, 4096] { - let aad: Vec = (0..aad_len).map(|i| i.strict_mul(11).strict_add(7) as u8).collect(); - let ct: Vec = (0..ct_len).map(|i| i.strict_mul(17).strict_add(3) as u8).collect(); - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let ct = patterned_bytes(ct_len, 17, 3); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::aarch64_neon::authenticate_aead_par4(&aad, &ct, &key, lengths); assert_eq!(parallel, portable, "mismatch at aad={aad_len} ct={ct_len}"); @@ -1103,9 +1319,9 @@ mod tests { let key = [0x5au8; 32]; for aad_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 48, 63, 64, 65, 80, 128] { for ct_len in [0usize, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 191, 256, 1024, 4096] { - let aad: Vec = (0..aad_len).map(|i| i.strict_mul(11).strict_add(7) as u8).collect(); - let ct: Vec = (0..ct_len).map(|i| i.strict_mul(17).strict_add(3) as u8).collect(); - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let aad = patterned_bytes(aad_len, 11, 7); + let ct = patterned_bytes(ct_len, 17, 3); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::riscv64_vector::authenticate_aead_par4(&aad, &ct, &key, lengths); assert_eq!(parallel, portable, "mismatch at aad={aad_len} ct={ct_len}"); @@ -1113,6 +1329,24 @@ mod tests { } } + #[test] + #[cfg(target_arch = "x86_64")] + fn avx2_par4_handles_high_carry_reduction() { + if !crate::platform::caps().has(x86::AVX2) { + return; + } + + let key = [0xffu8; 32]; + let aad = [0xffu8; 257]; + let ct = [0xffu8; 4112]; + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); + + let portable = authenticate_aead_portable(&aad, &ct, &key); + // SAFETY: the test returned above unless AVX2 is available. + let parallel = unsafe { super::avx2_par4::authenticate_aead_par4(&aad, &ct, &key, lengths) }; + assert_eq!(parallel, portable); + } + #[test] #[cfg(target_arch = "aarch64")] fn neon_par4_handles_high_carry_reduction() { @@ -1123,7 +1357,7 @@ mod tests { let key = [0xffu8; 32]; let aad = [0xffu8; 257]; let ct = [0xffu8; 4096]; - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::aarch64_neon::authenticate_aead_par4(&aad, &ct, &key, lengths); @@ -1140,7 +1374,7 @@ mod tests { let key = [0xffu8; 32]; let aad = [0xffu8; 257]; let ct = [0xffu8; 4096]; - let lengths = AeadByteLengths::try_new(aad.len(), ct.len()).unwrap(); + let lengths = AeadByteLengths::from_usize(aad.len(), ct.len()); let portable = authenticate_aead_portable(&aad, &ct, &key); let parallel = super::riscv64_vector::authenticate_aead_par4(&aad, &ct, &key, lengths); @@ -1172,8 +1406,9 @@ mod tests { 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91, ]; - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); - let result = super::avx2_par4::authenticate_aead_par4(&aad, &ciphertext, &poly_key, lengths); + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); + // SAFETY: the test returned above unless AVX2 is available. + let result = unsafe { super::avx2_par4::authenticate_aead_par4(&aad, &ciphertext, &poly_key, lengths) }; assert_eq!(result, expected); } @@ -1202,7 +1437,7 @@ mod tests { 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91, ]; - let lengths = AeadByteLengths::try_new(aad.len(), ciphertext.len()).unwrap(); + let lengths = AeadByteLengths::from_usize(aad.len(), ciphertext.len()); let result = super::aarch64_neon::authenticate_aead_par4(&aad, &ciphertext, &poly_key, lengths); assert_eq!(result, expected); } diff --git a/src/aead/poly1305/aarch64_neon.rs b/src/aead/poly1305/aarch64_neon.rs index bfa49861..34fbf844 100644 --- a/src/aead/poly1305/aarch64_neon.rs +++ b/src/aead/poly1305/aarch64_neon.rs @@ -75,6 +75,12 @@ impl AeadPar4 { tag } + /// Absorbs one AEAD segment, padding its final partial block with zeros. + /// + /// # Safety + /// + /// The current CPU must support AArch64 NEON. `self` must retain the cache and power-table + /// invariants established by [`AeadPar4::new`]. #[target_feature(enable = "neon")] unsafe fn update_padded_segment_neon(&mut self, segment: &[u8]) { let mut offset = 0usize; @@ -89,11 +95,14 @@ impl AeadPar4 { if self.num_cached == 0 { let group_len = segment.len().strict_sub(offset).strict_div(64).strict_mul(64); let group_end = offset.strict_add(group_len); - for group in segment[offset..group_end].chunks_exact(64) { + let (groups, remainder) = segment[offset..group_end].as_chunks::<64>(); + debug_assert!(remainder.is_empty()); + for group in groups { let (blocks, remainder) = group.as_chunks::<16>(); debug_assert!(remainder.is_empty()); + assert_eq!(blocks.len(), 4, "64-byte Poly1305 group must contain four blocks"); let [b0, b1, b2, b3] = blocks else { - unreachable!("64-byte Poly1305 group must split into four blocks"); + return; }; // SAFETY: direct four-block accumulation because: // 1. `group` is a 64-byte exact chunk split into four full Poly1305 blocks. @@ -121,6 +130,12 @@ impl AeadPar4 { } } + /// Caches one complete block and accumulates each full four-block group. + /// + /// # Safety + /// + /// The current CPU must support AArch64 NEON. `self` must retain the cache and power-table + /// invariants established by [`AeadPar4::new`], including `num_cached < 4` on entry. #[inline(always)] unsafe fn push(&mut self, block: [u8; 16]) { self.cached[self.num_cached] = block; @@ -133,7 +148,6 @@ impl AeadPar4 { } } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag", test)), allow(dead_code))] pub(super) fn authenticate_aead_par4( aad: &[u8], ciphertext: &[u8], @@ -146,12 +160,24 @@ pub(super) fn authenticate_aead_par4( authenticator.finalize(lengths) } +/// Accumulates four consecutive full blocks into `state`. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON, and `powers` must contain the first four powers of +/// the same clamped Poly1305 key represented by `state`. #[inline(always)] unsafe fn accumulate_4_blocks(blocks: &[[u8; 16]; 4], state: &mut State, powers: &Powers) { // SAFETY: caller is NEON-enabled and `blocks` contains four full 16-byte AEAD blocks. unsafe { accumulate_4_block_refs([&blocks[0], &blocks[1], &blocks[2], &blocks[3]], state, powers) }; } +/// Accumulates four referenced consecutive full blocks into `state`. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON, and `powers` must contain the first four powers of +/// the same clamped Poly1305 key represented by `state`. #[inline(always)] unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, powers: &Powers) { let h = mul_unreduced(state.h, powers.r4); @@ -166,6 +192,12 @@ unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, pow ]); } +/// Computes the unreduced four-block weighted sum for consecutive blocks. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. `powers` must contain the first four powers of the +/// clamped Poly1305 key whose accumulator will receive the returned sum. #[inline(always)] unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; 5] { let b0 = block_limbs(blocks[0]); @@ -197,21 +229,21 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; lane4(r4[2], r3[2], r2[2], r1[2]), lane4(r4[3], r3[3], r2[3], r1[3]), lane4(r4[4], r3[4], r2[4], r1[4]), - lane4(r4[1] * 5, r3[1] * 5, r2[1] * 5, r1[1] * 5), - lane4(r4[2] * 5, r3[2] * 5, r2[2] * 5, r1[2] * 5), - lane4(r4[3] * 5, r3[3] * 5, r2[3] * 5, r1[3] * 5), - lane4(r4[4] * 5, r3[4] * 5, r2[4] * 5, r1[4] * 5), + lane4_fivefold(r4[1], r3[1], r2[1], r1[1]), + lane4_fivefold(r4[2], r3[2], r2[2], r1[2]), + lane4_fivefold(r4[3], r3[3], r2[3], r1[3]), + lane4_fivefold(r4[4], r3[4], r2[4], r1[4]), ) }; // SAFETY: all lanes are valid NEON vectors in this target-feature-enabled function. unsafe { [ - dot5_sum(x0, x1, x2, x3, x4, r0, s4, s3, s2, s1), - dot5_sum(x0, x1, x2, x3, x4, r1v, r0, s4, s3, s2), - dot5_sum(x0, x1, x2, x3, x4, r2v, r1v, r0, s4, s3), - dot5_sum(x0, x1, x2, x3, x4, r3v, r2v, r1v, r0, s4), - dot5_sum(x0, x1, x2, x3, x4, r4v, r3v, r2v, r1v, r0), + dot5_sum([x0, x1, x2, x3, x4], [r0, s4, s3, s2, s1]), + dot5_sum([x0, x1, x2, x3, x4], [r1v, r0, s4, s3, s2]), + dot5_sum([x0, x1, x2, x3, x4], [r2v, r1v, r0, s4, s3]), + dot5_sum([x0, x1, x2, x3, x4], [r3v, r2v, r1v, r0, s4]), + dot5_sum([x0, x1, x2, x3, x4], [r4v, r3v, r2v, r1v, r0]), ] } } @@ -227,6 +259,11 @@ fn block_limbs(block: &[u8; 16]) -> [u32; 5] { ] } +/// Packs four scalar values into consecutive NEON lanes. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. #[inline(always)] unsafe fn lane4(a: u32, b: u32, c: u32, d: u32) -> uint32x4_t { // SAFETY: caller guarantees NEON is enabled; inputs are scalar lane values. @@ -238,20 +275,37 @@ unsafe fn lane4(a: u32, b: u32, c: u32, d: u32) -> uint32x4_t { } } +/// Multiplies four bounded limbs by five and packs them into consecutive NEON lanes. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. Each input must be at most 858,993,459 so its +/// fivefold value fits in one `u32` lane. #[inline(always)] -#[allow(clippy::too_many_arguments)] -unsafe fn dot5_sum( - x0: uint32x4_t, - x1: uint32x4_t, - x2: uint32x4_t, - x3: uint32x4_t, - x4: uint32x4_t, - y0: uint32x4_t, - y1: uint32x4_t, - y2: uint32x4_t, - y3: uint32x4_t, - y4: uint32x4_t, -) -> u64 { +unsafe fn lane4_fivefold(a: u32, b: u32, c: u32, d: u32) -> uint32x4_t { + const MAX_UNSCALED: u32 = 858_993_459; + debug_assert!([a, b, c, d].into_iter().all(|limb| limb <= MAX_UNSCALED)); + + let a = u64::from(a).strict_mul(5); + let b = u64::from(b).strict_mul(5); + let c = u64::from(c).strict_mul(5); + let d = u64::from(d).strict_mul(5); + + // SAFETY: the caller guarantees NEON. The input bound proves every widened product fits in + // one 32-bit lane, so the two `u64` values encode four consecutive lanes without overlap. + unsafe { vcombine_u32(vcreate_u32((b << 32) | a), vcreate_u32((d << 32) | c)) } +} + +/// Sums five lane-wise products across four parallel Poly1305 streams. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. +#[inline(always)] +unsafe fn dot5_sum(x: [uint32x4_t; 5], y: [uint32x4_t; 5]) -> u64 { + let [x0, x1, x2, x3, x4] = x; + let [y0, y1, y2, y3, y4] = y; + // SAFETY: caller guarantees NEON is enabled and all inputs are valid four-lane vectors. unsafe { let mut lo = vmull_u32(vget_low_u32(x0), vget_low_u32(y0)); @@ -266,6 +320,11 @@ unsafe fn dot5_sum( } } +/// Adds one lane-wise product to the low and high partial sums. +/// +/// # Safety +/// +/// The current CPU must support AArch64 NEON. #[inline(always)] unsafe fn accumulate_mul(lo: &mut uint64x2_t, hi: &mut uint64x2_t, x: uint32x4_t, y: uint32x4_t) { // SAFETY: caller guarantees NEON is enabled and all inputs are valid four-lane vectors. @@ -282,40 +341,46 @@ fn mul_mod(a: [u32; 5], b: [u32; 5]) -> [u32; 5] { #[inline(always)] fn mul_unreduced(a: [u32; 5], b: [u32; 5]) -> [u64; 5] { - let b1_5 = b[1] * 5; - let b2_5 = b[2] * 5; - let b3_5 = b[3] * 5; - let b4_5 = b[4] * 5; + let b0 = u64::from(b[0]); + let b1 = u64::from(b[1]); + let b2 = u64::from(b[2]); + let b3 = u64::from(b[3]); + let b4 = u64::from(b[4]); + let b1_5 = b1.strict_mul(5); + let b2_5 = b2.strict_mul(5); + let b3_5 = b3.strict_mul(5); + let b4_5 = b4.strict_mul(5); [ - (u64::from(a[0]) * u64::from(b[0])) - + (u64::from(a[1]) * u64::from(b4_5)) - + (u64::from(a[2]) * u64::from(b3_5)) - + (u64::from(a[3]) * u64::from(b2_5)) - + (u64::from(a[4]) * u64::from(b1_5)), - (u64::from(a[0]) * u64::from(b[1])) - + (u64::from(a[1]) * u64::from(b[0])) - + (u64::from(a[2]) * u64::from(b4_5)) - + (u64::from(a[3]) * u64::from(b3_5)) - + (u64::from(a[4]) * u64::from(b2_5)), - (u64::from(a[0]) * u64::from(b[2])) - + (u64::from(a[1]) * u64::from(b[1])) - + (u64::from(a[2]) * u64::from(b[0])) - + (u64::from(a[3]) * u64::from(b4_5)) - + (u64::from(a[4]) * u64::from(b3_5)), - (u64::from(a[0]) * u64::from(b[3])) - + (u64::from(a[1]) * u64::from(b[2])) - + (u64::from(a[2]) * u64::from(b[1])) - + (u64::from(a[3]) * u64::from(b[0])) - + (u64::from(a[4]) * u64::from(b4_5)), - (u64::from(a[0]) * u64::from(b[4])) - + (u64::from(a[1]) * u64::from(b[3])) - + (u64::from(a[2]) * u64::from(b[2])) - + (u64::from(a[3]) * u64::from(b[1])) - + (u64::from(a[4]) * u64::from(b[0])), + scalar_dot5(a, [b0, b4_5, b3_5, b2_5, b1_5]), + scalar_dot5(a, [b1, b0, b4_5, b3_5, b2_5]), + scalar_dot5(a, [b2, b1, b0, b4_5, b3_5]), + scalar_dot5(a, [b3, b2, b1, b0, b4_5]), + scalar_dot5(a, [b4, b3, b2, b1, b0]), ] } +#[inline(always)] +fn scalar_dot5(a: [u32; 5], b: [u64; 5]) -> u64 { + let sum = u128::from(a[0]) + .strict_mul(u128::from(b[0])) + .strict_add(u128::from(a[1]).strict_mul(u128::from(b[1]))) + .strict_add(u128::from(a[2]).strict_mul(u128::from(b[2]))) + .strict_add(u128::from(a[3]).strict_mul(u128::from(b[3]))) + .strict_add(u128::from(a[4]).strict_mul(u128::from(b[4]))); + debug_assert!(sum <= u128::from(u64::MAX)); + + let [b0, b1, b2, b3, b4, b5, b6, b7, _, _, _, _, _, _, _, _] = sum.to_le_bytes(); + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]) +} + +#[inline(always)] +fn narrow_limb(value: u64) -> u32 { + debug_assert_eq!(value >> u32::BITS, 0); + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + #[inline(always)] fn reduce_unreduced(mut d: [u64; 5]) -> [u32; 5] { let mut c = d[0] >> 26; @@ -327,20 +392,20 @@ fn reduce_unreduced(mut d: [u64; 5]) -> [u32; 5] { d[2] = d[2].wrapping_add(c); c = d[2] >> 26; - let h2 = (d[2] as u32) & LIMB_MASK; + let h2 = narrow_limb(d[2] & u64::from(LIMB_MASK)); d[3] = d[3].wrapping_add(c); c = d[3] >> 26; - let h3 = (d[3] as u32) & LIMB_MASK; + let h3 = narrow_limb(d[3] & u64::from(LIMB_MASK)); d[4] = d[4].wrapping_add(c); c = d[4] >> 26; - let h4 = (d[4] as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + let h4 = narrow_limb(d[4] & u64::from(LIMB_MASK)); + h0 = h0.wrapping_add(c.strict_mul(5)); c = h0 >> 26; h0 &= u64::from(LIMB_MASK); let h1 = h1_base.wrapping_add(c); - [h0 as u32, h1 as u32, h2, h3, h4] + [narrow_limb(h0), narrow_limb(h1), h2, h3, h4] } diff --git a/src/aead/poly1305/powerpc64_vsx.rs b/src/aead/poly1305/powerpc64_vsx.rs index af35330d..3e20829a 100644 --- a/src/aead/poly1305/powerpc64_vsx.rs +++ b/src/aead/poly1305/powerpc64_vsx.rs @@ -8,6 +8,11 @@ pub(super) fn compute_block(state: &mut State, block: &[u8; 16], partial: bool) unsafe { compute_block_impl(state, block, partial) } } +/// Process one Poly1305 block through the POWER8 vector multiplier. +/// +/// # Safety +/// +/// The executing CPU must support AltiVec, VSX, and POWER8 vector instructions. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) { compute_block_scalar_reduction(state, block, partial, |lhs, rhs| { @@ -20,6 +25,10 @@ unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) /// /// Computes `lhs[0]*rhs[0] + lhs[1]*rhs[1] + lhs[2]*rhs[2] + lhs[3]*rhs[3]` /// using two 128-bit multiply-odd and one 128-bit add. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { // SAFETY: POWER8+ VSX available via enclosing target_feature. @@ -34,11 +43,15 @@ unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { let sum = vaddudm(prod_lo, prod_hi); let lanes = sum.to_array(); - (lanes[0] as u64).wrapping_add(lanes[1] as u64) + u64::from_ne_bytes(lanes[0].to_ne_bytes()).wrapping_add(u64::from_ne_bytes(lanes[1].to_ne_bytes())) } } /// Multiply low 32 bits of each u64 lane → u64: `vmulouw`. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn vmulouw(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; @@ -56,6 +69,10 @@ unsafe fn vmulouw(a: i64x2, b: i64x2) -> i64x2 { } /// Add u64 lanes: `vaddudm`. +/// +/// # Safety +/// +/// The executing CPU must support POWER8 vector instructions. #[inline(always)] unsafe fn vaddudm(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; diff --git a/src/aead/poly1305/riscv64_vector.rs b/src/aead/poly1305/riscv64_vector.rs index dc27d658..2ee5e80c 100644 --- a/src/aead/poly1305/riscv64_vector.rs +++ b/src/aead/poly1305/riscv64_vector.rs @@ -1,4 +1,4 @@ -use core::simd::{i64x2, num::SimdUint, u32x4, u64x4}; +use core::simd::{num::SimdUint, u32x4, u64x2, u64x4}; use super::{FULL_BLOCK_HIBIT, LIMB_MASK, State, compute_block_scalar_reduction, load_u32_le}; use crate::{aead::AeadByteLengths, traits::ct}; @@ -9,6 +9,11 @@ pub(super) fn compute_block(state: &mut State, block: &[u8; 16], partial: bool) unsafe { compute_block_impl(state, block, partial) } } +/// Process one Poly1305 block through the RISC-V vector multiplier. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) { compute_block_scalar_reduction(state, block, partial, |lhs, rhs| { @@ -18,19 +23,23 @@ unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) } /// Vectorized 4-element dot product using two 64-bit RVV lane multiplies. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { - let a_lo = i64x2::from_array([i64::from(lhs[0]), i64::from(lhs[1])]); - let b_lo = i64x2::from_array([i64::from(rhs[0]), i64::from(rhs[1])]); - let prod_lo = a_lo * b_lo; + let a_lo = u64x2::from_array([u64::from(lhs[0]), u64::from(lhs[1])]); + let b_lo = u64x2::from_array([u64::from(rhs[0]), u64::from(rhs[1])]); + let prod_lo = wrapping_mul2(a_lo, b_lo); - let a_hi = i64x2::from_array([i64::from(lhs[2]), i64::from(lhs[3])]); - let b_hi = i64x2::from_array([i64::from(rhs[2]), i64::from(rhs[3])]); - let prod_hi = a_hi * b_hi; + let a_hi = u64x2::from_array([u64::from(lhs[2]), u64::from(lhs[3])]); + let b_hi = u64x2::from_array([u64::from(rhs[2]), u64::from(rhs[3])]); + let prod_hi = wrapping_mul2(a_hi, b_hi); - let sum = prod_lo + prod_hi; + let sum = wrapping_add2(prod_lo, prod_hi); let lanes = sum.to_array(); - (lanes[0] as u64).wrapping_add(lanes[1] as u64) + lanes[0].wrapping_add(lanes[1]) } #[derive(Clone, Copy)] @@ -51,7 +60,6 @@ impl Powers { } } -#[cfg_attr(not(any(feature = "xchacha20poly1305", feature = "diag", test)), allow(dead_code))] pub(super) fn authenticate_aead_par4( aad: &[u8], ciphertext: &[u8], @@ -64,6 +72,12 @@ pub(super) fn authenticate_aead_par4( unsafe { authenticate_aead_par4_impl(aad, ciphertext, key, lengths) } } +/// Authenticate AEAD input four Poly1305 blocks at a time. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, and `lengths` must +/// encode the byte lengths of `aad` and `ciphertext`. #[target_feature(enable = "v")] unsafe fn authenticate_aead_par4_impl( aad: &[u8], @@ -94,17 +108,13 @@ unsafe fn authenticate_aead_par4_impl( if num_cached == 0 { let group_len = segment.len().strict_sub(offset).strict_div(64).strict_mul(64); let group_end = offset.strict_add(group_len); - for group in segment[offset..group_end].chunks_exact(64) { - let (blocks, remainder) = group.as_chunks::<16>(); - debug_assert!(remainder.is_empty()); - let [b0, b1, b2, b3] = blocks else { - unreachable!("64-byte Poly1305 group must split into four blocks"); - }; + for group in segment[offset..group_end].as_chunks::<64>().0 { + let blocks = group.as_chunks::<16>().0; // SAFETY: direct four-block accumulation because: // 1. `group` is a 64-byte exact chunk split into four full Poly1305 blocks. // 2. This entry point is compiled with the RISC-V V target feature. // 3. `num_cached == 0`, so direct accumulation preserves AEAD block order. - unsafe { accumulate_4_block_refs([b0, b1, b2, b3], &mut state, &powers) }; + unsafe { accumulate_4_block_refs([&blocks[0], &blocks[1], &blocks[2], &blocks[3]], &mut state, &powers) }; } offset = group_end; } @@ -154,6 +164,12 @@ unsafe fn authenticate_aead_par4_impl( tag } +/// Queue one full Poly1305 block and fold the queue when it reaches four blocks. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, and `*num_cached` +/// must be less than four. #[inline(always)] unsafe fn push_cached( cached: &mut [[u8; 16]; 4], @@ -173,6 +189,11 @@ unsafe fn push_cached( } } +/// Accumulate four consecutive Poly1305 blocks. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[inline(always)] unsafe fn accumulate_4_blocks(blocks: &[[u8; 16]; 4], state: &mut State, powers: &Powers) { // SAFETY: RVV four-block accumulation because: @@ -181,6 +202,11 @@ unsafe fn accumulate_4_blocks(blocks: &[[u8; 16]; 4], state: &mut State, powers: unsafe { accumulate_4_block_refs([&blocks[0], &blocks[1], &blocks[2], &blocks[3]], state, powers) }; } +/// Accumulate four consecutive Poly1305 blocks supplied by reference. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[inline(always)] unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, powers: &Powers) { let h = mul_unreduced(state.h, powers.r4); @@ -197,6 +223,11 @@ unsafe fn accumulate_4_block_refs(blocks: [&[u8; 16]; 4], state: &mut State, pow ]); } +/// Multiply four spaced messages by descending powers of the Poly1305 key. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[inline(always)] unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; 5] { let b0 = block_limbs(blocks[0]); @@ -241,7 +272,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[1].wrapping_mul(5), r1[1].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r0, s4, s3, s2, s1) + dot5_sum([x0, x1, x2, x3, x4], [r0, s4, s3, s2, s1]) }; let d1 = { let r1v = lane4(r4[1], r3[1], r2[1], r1[1]); @@ -264,7 +295,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[2].wrapping_mul(5), r1[2].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r1v, r0, s4, s3, s2) + dot5_sum([x0, x1, x2, x3, x4], [r1v, r0, s4, s3, s2]) }; let d2 = { let r2v = lane4(r4[2], r3[2], r2[2], r1[2]); @@ -282,7 +313,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[3].wrapping_mul(5), r1[3].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r2v, r1v, r0, s4, s3) + dot5_sum([x0, x1, x2, x3, x4], [r2v, r1v, r0, s4, s3]) }; let d3 = { let r3v = lane4(r4[3], r3[3], r2[3], r1[3]); @@ -295,7 +326,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; r2[4].wrapping_mul(5), r1[4].wrapping_mul(5), ); - dot5_sum(x0, x1, x2, x3, x4, r3v, r2v, r1v, r0, s4) + dot5_sum([x0, x1, x2, x3, x4], [r3v, r2v, r1v, r0, s4]) }; let d4 = { let r4v = lane4(r4[4], r3[4], r2[4], r1[4]); @@ -303,7 +334,7 @@ unsafe fn mul4_spaced_sum_refs(blocks: [&[u8; 16]; 4], powers: &Powers) -> [u64; let r2v = lane4(r4[2], r3[2], r2[2], r1[2]); let r1v = lane4(r4[1], r3[1], r2[1], r1[1]); let r0 = lane4(r4[0], r3[0], r2[0], r1[0]); - dot5_sum(x0, x1, x2, x3, x4, r4v, r3v, r2v, r1v, r0) + dot5_sum([x0, x1, x2, x3, x4], [r4v, r3v, r2v, r1v, r0]) }; [d0, d1, d2, d3, d4] @@ -331,24 +362,32 @@ fn widen(value: u32x4) -> u64x4 { } #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn dot5_sum( - x0: u32x4, - x1: u32x4, - x2: u32x4, - x3: u32x4, - x4: u32x4, - y0: u32x4, - y1: u32x4, - y2: u32x4, - y3: u32x4, - y4: u32x4, -) -> u64 { - let sum = widen(x0) * widen(y0) - + widen(x1) * widen(y1) - + widen(x2) * widen(y2) - + widen(x3) * widen(y3) - + widen(x4) * widen(y4); +fn wrapping_mul2(lhs: u64x2, rhs: u64x2) -> u64x2 { + core::ops::Mul::mul(lhs, rhs) +} + +#[inline(always)] +fn wrapping_add2(lhs: u64x2, rhs: u64x2) -> u64x2 { + core::ops::Add::add(lhs, rhs) +} + +#[inline(always)] +fn wrapping_mul4(lhs: u64x4, rhs: u64x4) -> u64x4 { + core::ops::Mul::mul(lhs, rhs) +} + +#[inline(always)] +fn wrapping_add4(lhs: u64x4, rhs: u64x4) -> u64x4 { + core::ops::Add::add(lhs, rhs) +} + +#[inline(always)] +fn dot5_sum(x: [u32x4; 5], y: [u32x4; 5]) -> u64 { + let mut sum = wrapping_mul4(widen(x[0]), widen(y[0])); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[1]), widen(y[1]))); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[2]), widen(y[2]))); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[3]), widen(y[3]))); + sum = wrapping_add4(sum, wrapping_mul4(widen(x[4]), widen(y[4]))); let lanes = sum.to_array(); lanes[0] .wrapping_add(lanes[1]) @@ -399,20 +438,26 @@ fn reduce_unreduced(mut d: [u64; 5]) -> [u32; 5] { d[2] = d[2].wrapping_add(c); c = d[2] >> 26; - let h2 = (d[2] as u32) & LIMB_MASK; + let h2 = low_u32(d[2]) & LIMB_MASK; d[3] = d[3].wrapping_add(c); c = d[3] >> 26; - let h3 = (d[3] as u32) & LIMB_MASK; + let h3 = low_u32(d[3]) & LIMB_MASK; d[4] = d[4].wrapping_add(c); c = d[4] >> 26; - let h4 = (d[4] as u32) & LIMB_MASK; + let h4 = low_u32(d[4]) & LIMB_MASK; h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= u64::from(LIMB_MASK); let h1 = h1_base.wrapping_add(c); - [h0 as u32, h1 as u32, h2, h3, h4] + [low_u32(h0), low_u32(h1), h2, h3, h4] +} + +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) } diff --git a/src/aead/poly1305/s390x_vector.rs b/src/aead/poly1305/s390x_vector.rs index df8ae069..f8e3cfaa 100644 --- a/src/aead/poly1305/s390x_vector.rs +++ b/src/aead/poly1305/s390x_vector.rs @@ -8,6 +8,11 @@ pub(super) fn compute_block(state: &mut State, block: &[u8; 16], partial: bool) unsafe { compute_block_impl(state, block, partial) } } +/// Process one Poly1305 block through the s390x vector multiplier. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) { compute_block_scalar_reduction(state, block, partial, |lhs, rhs| { @@ -20,6 +25,10 @@ unsafe fn compute_block_impl(state: &mut State, block: &[u8; 16], partial: bool) /// /// Computes `lhs[0]*rhs[0] + lhs[1]*rhs[1] + lhs[2]*rhs[2] + lhs[3]*rhs[3]` /// using two 128-bit multiply-odd and one 128-bit add. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { @@ -35,11 +44,15 @@ unsafe fn sum4_mul(lhs: [u32; 4], rhs: [u32; 4]) -> u64 { let sum = vag(prod_lo, prod_hi); let lanes = sum.to_array(); - (lanes[0] as u64).wrapping_add(lanes[1] as u64) + u64::from_ne_bytes(lanes[0].to_ne_bytes()).wrapping_add(u64::from_ne_bytes(lanes[1].to_ne_bytes())) } } /// Multiply odd-indexed u32 lanes → u64: `vmlof`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { @@ -58,6 +71,10 @@ unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { } /// Add u64 lanes: `vag`. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[inline] #[target_feature(enable = "vector")] unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { diff --git a/src/aead/poly1305/x86_64_avx2_par4.rs b/src/aead/poly1305/x86_64_avx2_par4.rs index 406fb48a..3efd0a1e 100644 --- a/src/aead/poly1305/x86_64_avx2_par4.rs +++ b/src/aead/poly1305/x86_64_avx2_par4.rs @@ -60,25 +60,40 @@ struct SpacedMultiplier4x130 { impl Aligned130 { /// Pack five scalar 26-bit limbs into a `__m256i`. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_limbs(limbs: [u32; 5]) -> Self { - Aligned130(_mm256_setr_epi32( - limbs[0] as i32, - limbs[1] as i32, - limbs[2] as i32, - limbs[3] as i32, - limbs[4] as i32, - 0, - 0, - 0, - )) + // SAFETY: the caller guarantees AVX2 support. + let lanes = unsafe { + _mm256_setr_epi32( + limbs[0].cast_signed(), + limbs[1].cast_signed(), + limbs[2].cast_signed(), + limbs[3].cast_signed(), + limbs[4].cast_signed(), + 0, + 0, + 0, + ) + }; + Aligned130(lanes) } /// Extract five scalar 26-bit limbs. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn into_limbs(self) -> [u32; 5] { let mut buf = [0u32; 8]; - _mm256_storeu_si256(buf.as_mut_ptr() as *mut __m256i, self.0); + let dst = core::ptr::NonNull::from(&mut buf).cast::<__m256i>().as_ptr(); + // SAFETY: the caller guarantees AVX2 support. `dst` retains `buf`'s provenance and points to its entire writable + // 32-byte range; the unaligned store has no stronger alignment requirement. + unsafe { _mm256_storeu_si256(dst, self.0) }; [buf[0], buf[1], buf[2], buf[3], buf[4]] } @@ -86,187 +101,282 @@ impl Aligned130 { /// /// AEAD-only: unconditionally sets the 2¹²⁸ high bit. Not suitable for raw /// Poly1305 where partial blocks omit the hibit. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_block(block: &[u8; 16]) -> Self { - Self::split_to_26bit(_mm256_or_si256( - _mm256_and_si256( - _mm256_castsi128_si256(_mm_loadu_si128(block.as_ptr() as *const _)), - _mm256_set_epi64x(0, 0, -1, -1), - ), - _mm256_set_epi64x(0, 1, 0, 0), - )) + let src = core::ptr::NonNull::from(block).cast::<__m128i>().as_ptr(); + // SAFETY: the caller guarantees AVX2 support. `src` retains `block`'s provenance and exposes its complete + // initialized 16-byte range; the load is unaligned. The nested split performs only AVX2 register operations. + unsafe { + Self::split_to_26bit(_mm256_or_si256( + _mm256_and_si256( + _mm256_castsi128_si256(_mm_loadu_si128(src)), + _mm256_set_epi64x(0, 0, -1, -1), + ), + _mm256_set_epi64x(0, 1, 0, 0), + )) + } } /// Split a 130-bit integer (low 5 words) into 26-bit limbs. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn split_to_26bit(x: __m256i) -> Self { - let xl = _mm256_sllv_epi32(x, _mm256_set_epi32(32, 32, 32, 24, 18, 12, 6, 0)); - let xh = _mm256_permutevar8x32_epi32( - _mm256_srlv_epi32(x, _mm256_set_epi32(32, 32, 32, 2, 8, 14, 20, 26)), - _mm256_set_epi32(6, 5, 4, 3, 2, 1, 0, 7), - ); - Aligned130(_mm256_and_si256( - _mm256_or_si256(xl, xh), - _mm256_set_epi32(0, 0, 0, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff), - )) + // SAFETY: the caller guarantees AVX2 support; all operations are register-only. + unsafe { + let xl = _mm256_sllv_epi32(x, _mm256_set_epi32(32, 32, 32, 24, 18, 12, 6, 0)); + let xh = _mm256_permutevar8x32_epi32( + _mm256_srlv_epi32(x, _mm256_set_epi32(32, 32, 32, 2, 8, 14, 20, 26)), + _mm256_set_epi32(6, 5, 4, 3, 2, 1, 0, 7), + ); + Aligned130(_mm256_and_si256( + _mm256_or_si256(xl, xh), + _mm256_set_epi32(0, 0, 0, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff), + )) + } } + /// Add two aligned values independently in each 32-bit lane. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn add(self, other: Aligned130) -> Aligned130 { - Aligned130(_mm256_add_epi32(self.0, other.0)) + // SAFETY: the caller guarantees AVX2 support. + Aligned130(unsafe { _mm256_add_epi32(self.0, other.0) }) } } // ── PrecomputedMultiplier ──────────────────────────────────────────────── impl PrecomputedMultiplier { + /// Precompute the lane-packed multiplier from the kernel's bounded radix-2²⁶ representation. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_aligned(r: Aligned130) -> Self { - // 5*R limbs: r + (r << 2) = r * 5 - let a_5 = _mm256_permutevar8x32_epi32( - _mm256_add_epi32(r.0, _mm256_slli_epi32(r.0, 2)), - _mm256_set_epi32(4, 3, 2, 1, 1, 1, 1, 1), - ); - let a = _mm256_blend_epi32(r.0, a_5, 0b11100000); - let a_5 = _mm256_permute2x128_si256(a_5, a_5, 0); - PrecomputedMultiplier { a, a_5 } + // SAFETY: the caller guarantees AVX2 support; all operations are register-only. + unsafe { + // 5*R limbs: r + (r << 2) = r * 5 + let a_5 = _mm256_permutevar8x32_epi32( + _mm256_add_epi32(r.0, _mm256_slli_epi32(r.0, 2)), + _mm256_set_epi32(4, 3, 2, 1, 1, 1, 1, 1), + ); + let a = _mm256_blend_epi32(r.0, a_5, 0b11100000); + let a_5 = _mm256_permute2x128_si256(a_5, a_5, 0); + PrecomputedMultiplier { a, a_5 } + } } } // ── Single multiply: Aligned130 × PrecomputedMultiplier → Unreduced130 ── +/// Multiply one bounded radix-2²⁶ value by a precomputed multiplier. +/// +/// The low five lanes of `x` must be below 2²⁸, and `r` must be derived from reduced multiplier limbs below 2²⁷. +/// These bounds keep the precomputed fivefold lanes below 2³⁰ and every five-product coefficient below 2⁶¹. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn mul_single(x: Aligned130, r: PrecomputedMultiplier) -> Unreduced130 { - let x = x.0; - let y = r.a; - let z = r.a_5; - - // v0 = [t3, t2, t1, t0] — accumulate 5 products per limb. - let mut v0 = _mm256_mul_epu32( - _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(4, 3, 2, 1)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(7, 7, 7, 7)), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every 64-bit lane sum below 2^61. + unsafe { + let x = x.0; + let y = r.a; + let z = r.a_5; + + // v0 = [t3, t2, t1, t0] — accumulate 5 products per limb. + let mut v0 = _mm256_mul_epu32( + _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(4, 3, 2, 1)), + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(7, 7, 7, 7)), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(3, 2, 1, 0)), + _mm256_broadcastd_epi32(_mm256_castsi256_si128(y)), + ), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(1, 1, 3, 3)), + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(2, 1, 6, 5)), + ), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permute4x64_epi64(x, imm8(1, 0, 0, 2)), + _mm256_blend_epi32(_mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 1, 1)), z, 0x03), + ), + ); + v0 = _mm256_add_epi64( + v0, + _mm256_mul_epu32( + _mm256_permute4x64_epi64(x, imm8(0, 2, 2, 1)), + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(3, 6, 5, 6)), + ), + ); + + // v1 = [_, _, _, t4] + let mut v1 = _mm256_mul_epu32( _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(3, 2, 1, 0)), - _mm256_broadcastd_epi32(_mm256_castsi256_si128(y)), - ), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( - _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(1, 1, 3, 3)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(2, 1, 6, 5)), - ), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( - _mm256_permute4x64_epi64(x, imm8(1, 0, 0, 2)), - _mm256_blend_epi32(_mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 1, 1)), z, 0x03), - ), - ); - v0 = _mm256_add_epi64( - v0, - _mm256_mul_epu32( - _mm256_permute4x64_epi64(x, imm8(0, 2, 2, 1)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(3, 6, 5, 6)), - ), - ); - - // v1 = [_, _, _, t4] - let mut v1 = _mm256_mul_epu32( - _mm256_permutevar8x32_epi32(x, _mm256_set_epi64x(3, 2, 1, 0)), - _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 3, 4)), - ); - v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(1, 0, 3, 2))); - v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(0, 0, 0, 1))); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(_mm256_permute4x64_epi64(x, imm8(0, 0, 0, 2)), y)); - - Unreduced130 { v0, v1 } + _mm256_permutevar8x32_epi32(y, _mm256_set_epi64x(1, 2, 3, 4)), + ); + v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(1, 0, 3, 2))); + v1 = _mm256_add_epi64(v1, _mm256_permute4x64_epi64(v1, imm8(0, 0, 0, 1))); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(_mm256_permute4x64_epi64(x, imm8(0, 0, 0, 2)), y)); + + Unreduced130 { v0, v1 } + } } // ── Unreduced130 carry chain and reduction ─────────────────────────────── /// Carry: propagate bits >26 from v0 into v1. +/// +/// The inputs must be bounded unreduced coefficients produced by this kernel's multiplication or preceding fold. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn adc_single(v1: __m256i, v0: __m256i) -> (__m256i, __m256i) { - let v0 = _mm256_add_epi64( - _mm256_and_si256(v0, _mm256_set_epi64x(-1, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff)), - _mm256_permute4x64_epi64( - _mm256_srlv_epi64(v0, _mm256_set_epi64x(64, 26, 26, 26)), - imm8(2, 1, 0, 3), - ), - ); - let v1 = _mm256_add_epi64( - v1, - _mm256_permute4x64_epi64(_mm256_srli_epi64(v0, 26), imm8(2, 1, 0, 3)), - ); - let chain = _mm256_and_si256(v0, _mm256_set_epi64x(0x3ff_ffff, -1, -1, -1)); - (v1, chain) + // SAFETY: the caller guarantees AVX2 support; the kernel bounds keep every propagated carry and lane sum in u64. + unsafe { + let v0 = _mm256_add_epi64( + _mm256_and_si256(v0, _mm256_set_epi64x(-1, 0x3ff_ffff, 0x3ff_ffff, 0x3ff_ffff)), + _mm256_permute4x64_epi64( + _mm256_srlv_epi64(v0, _mm256_set_epi64x(64, 26, 26, 26)), + imm8(2, 1, 0, 3), + ), + ); + let v1 = _mm256_add_epi64( + v1, + _mm256_permute4x64_epi64(_mm256_srli_epi64(v0, 26), imm8(2, 1, 0, 3)), + ); + let chain = _mm256_and_si256(v0, _mm256_set_epi64x(0x3ff_ffff, -1, -1, -1)); + (v1, chain) + } } /// Reduce modulo 2¹³⁰ − 5: fold top limb back into bottom. +/// +/// The inputs must be the bounded carry-propagated representation returned by [`adc_single`]. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn red_single(v1: __m256i, v0: __m256i) -> (__m256i, __m256i) { - let t = _mm256_srlv_epi64(v1, _mm256_set_epi64x(64, 64, 64, 26)); - let red_0 = _mm256_add_epi64(_mm256_add_epi64(v0, t), _mm256_slli_epi64(t, 2)); - let red_1 = _mm256_and_si256(v1, _mm256_set_epi64x(0, 0, 0, 0x3ff_ffff)); - (red_1, red_0) + // SAFETY: the caller guarantees AVX2 support; the bounded top carry and its fivefold fold fit in every u64 lane. + unsafe { + let t = _mm256_srlv_epi64(v1, _mm256_set_epi64x(64, 64, 64, 26)); + let red_0 = _mm256_add_epi64(_mm256_add_epi64(v0, t), _mm256_slli_epi64(t, 2)); + let red_1 = _mm256_and_si256(v1, _mm256_set_epi64x(0, 0, 0, 0x3ff_ffff)); + (red_1, red_0) + } } impl Unreduced130 { + /// Carry-reduce this bounded convolution result into the kernel's radix-2²⁶ representation. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn reduce(self) -> Aligned130 { - let (v1, v0) = adc_single(self.v1, self.v0); - let (v1, v0) = red_single(v1, v0); - let (v1, v0) = adc_single(v1, v0); - // Switch from 64-bit to 32-bit limbs. - Aligned130(_mm256_blend_epi32( - _mm256_permutevar8x32_epi32(v0, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), - _mm256_permutevar8x32_epi32(v1, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), - 0x90, - )) + // SAFETY: the caller guarantees AVX2 support. `self` is a bounded kernel convolution, satisfying both nested + // helpers' carry/fold contracts; the final operations only repack the resulting register lanes. + unsafe { + let (v1, v0) = adc_single(self.v1, self.v0); + let (v1, v0) = red_single(v1, v0); + let (v1, v0) = adc_single(v1, v0); + // Switch from 64-bit to 32-bit limbs. + Aligned130(_mm256_blend_epi32( + _mm256_permutevar8x32_epi32(v0, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), + _mm256_permutevar8x32_epi32(v1, _mm256_set_epi32(0, 6, 4, 0, 6, 4, 2, 0)), + 0x90, + )) + } } } // ── Aligned4x130 ──────────────────────────────────────────────────────── impl Aligned4x130 { + /// Load four consecutive full blocks for parallel processing. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_blocks(src: &[[u8; 16]; 4]) -> Self { - // SAFETY: `[[u8; 16]; 4]` is 64 contiguous bytes; two 32-byte loads are valid. - let ptr = src.as_ptr() as *const __m256i; - let blocks_01 = _mm256_loadu_si256(ptr); - let blocks_23 = _mm256_loadu_si256(ptr.add(1)); - Self::from_loaded_blocks(blocks_01, blocks_23) + let ptr = core::ptr::NonNull::from(src).cast::<__m256i>().as_ptr(); + // SAFETY: the caller guarantees AVX2 support. `ptr` retains `src`'s provenance; `src` contains 64 initialized + // readable bytes, so the unaligned 32-byte loads at offsets 0 and 32 are valid. The nested helper is register-only. + unsafe { + let blocks_01 = _mm256_loadu_si256(ptr); + let blocks_23 = _mm256_loadu_si256(ptr.add(1)); + Self::from_loaded_blocks(blocks_01, blocks_23) + } } /// Interleave 4 blocks into 20 packed 26-bit limbs across 3 vectors. + /// + /// `blocks_01` and `blocks_23` must contain four consecutive full blocks in that order. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn from_loaded_blocks(blocks_01: __m256i, blocks_23: __m256i) -> Self { - let mask_26 = _mm256_set1_epi32(0x3ff_ffff); - let set_hibit = _mm256_set1_epi32(1 << 24); + // SAFETY: the caller guarantees AVX2 support; all operations are register-only. + unsafe { + let mask_26 = _mm256_set1_epi32(0x3ff_ffff); + let set_hibit = _mm256_set1_epi32(1 << 24); - let a0 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); - let a1 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); + let a0 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); + let a1 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi64(blocks_01, blocks_23), imm8(3, 1, 2, 0)); - let v2 = _mm256_or_si256(_mm256_srli_epi64(a0, 40), set_hibit); - let a2 = _mm256_or_si256(_mm256_srli_epi64(a1, 46), _mm256_slli_epi64(a0, 18)); + let v2 = _mm256_or_si256(_mm256_srli_epi64(a0, 40), set_hibit); + let a2 = _mm256_or_si256(_mm256_srli_epi64(a1, 46), _mm256_slli_epi64(a0, 18)); - let v1 = _mm256_and_si256(_mm256_blend_epi32(_mm256_srli_epi64(a1, 26), a2, 0xAA), mask_26); - let v0 = _mm256_and_si256(_mm256_blend_epi32(a1, _mm256_slli_epi64(a2, 26), 0xAA), mask_26); + let v1 = _mm256_and_si256(_mm256_blend_epi32(_mm256_srli_epi64(a1, 26), a2, 0xAA), mask_26); + let v0 = _mm256_and_si256(_mm256_blend_epi32(a1, _mm256_slli_epi64(a2, 26), 0xAA), mask_26); - Aligned4x130 { v0, v1, v2 } + Aligned4x130 { v0, v1, v2 } + } } + /// Add four aligned values independently in each 32-bit lane. + /// + /// Both inputs must satisfy the kernel's loose-limb bounds, which keep every lane sum in `u32`. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn add(self, other: Aligned4x130) -> Aligned4x130 { - Aligned4x130 { - v0: _mm256_add_epi32(self.v0, other.v0), - v1: _mm256_add_epi32(self.v1, other.v1), - v2: _mm256_add_epi32(self.v2, other.v2), + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep all three lane sums in u32. + unsafe { + Aligned4x130 { + v0: _mm256_add_epi32(self.v0, other.v0), + v1: _mm256_add_epi32(self.v1, other.v1), + v2: _mm256_add_epi32(self.v2, other.v2), + } } } } @@ -274,203 +384,243 @@ impl Aligned4x130 { // ── 4-way parallel multiply ────────────────────────────────────────────── /// Multiply 4 values by the same R: `(x0·R, x1·R, x2·R, x3·R)`. +/// +/// `x` and `r` must satisfy the kernel's loose-radix bounds. Those bounds keep every five-product coefficient below +/// 2⁶¹. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn mul_4x130(x: &Aligned4x130, r: PrecomputedMultiplier) -> Unreduced4x130 { - let mut x = *x; - let y = r.a; - let z = r.a_5; - let ord = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every 64-bit lane sum below 2^61. + unsafe { + let mut x = *x; + let y = r.a; + let z = r.a_5; + let ord = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); - let mut t0 = _mm256_permute4x64_epi64(y, imm8(0, 0, 0, 0)); - let mut t1 = _mm256_permute4x64_epi64(y, imm8(1, 1, 1, 1)); + let mut t0 = _mm256_permute4x64_epi64(y, imm8(0, 0, 0, 0)); + let mut t1 = _mm256_permute4x64_epi64(y, imm8(1, 1, 1, 1)); - let mut v0 = _mm256_mul_epu32(x.v0, t0); - let mut v1 = _mm256_mul_epu32(x.v1, t0); - let mut v4 = _mm256_mul_epu32(x.v2, t0); - let mut v2 = _mm256_mul_epu32(x.v0, t1); - let mut v3 = _mm256_mul_epu32(x.v1, t1); + let mut v0 = _mm256_mul_epu32(x.v0, t0); + let mut v1 = _mm256_mul_epu32(x.v1, t0); + let mut v4 = _mm256_mul_epu32(x.v2, t0); + let mut v2 = _mm256_mul_epu32(x.v0, t1); + let mut v3 = _mm256_mul_epu32(x.v1, t1); - t0 = _mm256_permutevar8x32_epi32(t0, ord); - t1 = _mm256_permutevar8x32_epi32(t1, ord); + t0 = _mm256_permutevar8x32_epi32(t0, ord); + t1 = _mm256_permutevar8x32_epi32(t1, ord); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t1)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t1)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t1)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t1)); - let mut t2 = _mm256_permute4x64_epi64(y, imm8(2, 2, 2, 2)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t2)); + let mut t2 = _mm256_permute4x64_epi64(y, imm8(2, 2, 2, 2)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t2)); - x.v0 = _mm256_permutevar8x32_epi32(x.v0, ord); - x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); - t2 = _mm256_permutevar8x32_epi32(t2, ord); + x.v0 = _mm256_permutevar8x32_epi32(x.v0, ord); + x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); + t2 = _mm256_permutevar8x32_epi32(t2, ord); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t2)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, t2)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t0)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t0)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t2)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, t2)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, t0)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, t0)); - t0 = _mm256_permutevar8x32_epi32(t0, ord); - t1 = _mm256_permutevar8x32_epi32(t1, ord); + t0 = _mm256_permutevar8x32_epi32(t0, ord); + t1 = _mm256_permutevar8x32_epi32(t1, ord); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, t0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, t0)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t1)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, t0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, t0)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, t1)); - t0 = _mm256_permute4x64_epi64(y, imm8(3, 3, 3, 3)); + t0 = _mm256_permute4x64_epi64(y, imm8(3, 3, 3, 3)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, t0)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, t0)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, t0)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, t0)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, t0)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, t0)); - t0 = _mm256_permutevar8x32_epi32(t0, ord); + t0 = _mm256_permutevar8x32_epi32(t0, ord); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, t0)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, t0)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, t0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, t0)); - x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); + x.v1 = _mm256_permutevar8x32_epi32(x.v1, ord); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t0)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, z)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, t0)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, z)); - Unreduced4x130 { v0, v1, v2, v3, v4 } + Unreduced4x130 { v0, v1, v2, v3, v4 } + } } // ── Spaced multiply ───────────────────────────────────────────────────── /// Multiply lane i by R^(4−i): `(x0·R⁴, x1·R³, x2·R², x3·R¹)`. +/// +/// `x` and `m` must satisfy the kernel's loose-radix bounds and represent matching streams and powers of one key. +/// Those bounds keep every five-product coefficient below 2⁶¹. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn mul_spaced(x: Aligned4x130, m: SpacedMultiplier4x130) -> Unreduced4x130 { - let mut x = x; - let r1 = m.r1.a; - - let v0u = _mm256_unpacklo_epi32(m.v0, m.v1); - let v1u = _mm256_unpackhi_epi32(m.v0, m.v1); - - let ord_a = _mm256_set_epi32(1, 0, 6, 7, 2, 0, 3, 1); - let m_r_0 = _mm256_blend_epi32( - _mm256_permutevar8x32_epi32(r1, ord_a), - _mm256_permutevar8x32_epi32(v0u, ord_a), - 0b00111111, - ); - let ord_b = _mm256_set_epi32(3, 2, 4, 5, 2, 0, 3, 1); - let m_r_2 = _mm256_blend_epi32( - _mm256_permutevar8x32_epi32(r1, ord_b), - _mm256_permutevar8x32_epi32(v1u, ord_b), - 0b00111111, - ); - let ord_c = _mm256_set_epi32(1, 4, 6, 6, 2, 4, 3, 5); - let m_r_4 = _mm256_blend_epi32( - _mm256_blend_epi32( - _mm256_permutevar8x32_epi32(r1, ord_c), - _mm256_permutevar8x32_epi32(v1u, ord_c), - 0b00010000, - ), - _mm256_permutevar8x32_epi32(v0u, ord_c), - 0b00101111, - ); - - let mut v0 = _mm256_mul_epu32(x.v0, m_r_0); - let mut v1 = _mm256_mul_epu32(x.v1, m_r_0); - let mut v2 = _mm256_mul_epu32(x.v0, m_r_2); - let mut v3 = _mm256_mul_epu32(x.v1, m_r_2); - let mut v4 = _mm256_mul_epu32(x.v0, m_r_4); - - let swap = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); - let m_r_1 = _mm256_permutevar8x32_epi32(m_r_0, swap); - let m_r_3 = _mm256_permutevar8x32_epi32(m_r_2, swap); - - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_r_1)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_r_1)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_3)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_3)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v2, m_r_0)); - - x.v0 = _mm256_permutevar8x32_epi32(x.v0, swap); - - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, m_r_0)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_1)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, m_r_2)); - - let m_5r_3 = _mm256_add_epi32(m_r_3, _mm256_slli_epi32(m_r_3, 2)); - let m_5r_4 = _mm256_add_epi32(m_r_4, _mm256_slli_epi32(m_r_4, 2)); - - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, m_5r_3)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_4)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_5r_4)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, m_5r_3)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, m_5r_4)); - - x.v1 = _mm256_permutevar8x32_epi32(x.v1, swap); - - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, m_5r_3)); - v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_5r_4)); - v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, m_r_0)); - v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_1)); - - let m_5r_1 = _mm256_permutevar8x32_epi32(m_5r_4, swap); - let m_5r_2 = _mm256_permutevar8x32_epi32(m_5r_3, swap); - - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_2)); - v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, m_5r_1)); - v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, m_5r_2)); - - Unreduced4x130 { v0, v1, v2, v3, v4 } + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every 64-bit lane sum below 2^61. + unsafe { + let mut x = x; + let r1 = m.r1.a; + + let v0u = _mm256_unpacklo_epi32(m.v0, m.v1); + let v1u = _mm256_unpackhi_epi32(m.v0, m.v1); + + let ord_a = _mm256_set_epi32(1, 0, 6, 7, 2, 0, 3, 1); + let m_r_0 = _mm256_blend_epi32( + _mm256_permutevar8x32_epi32(r1, ord_a), + _mm256_permutevar8x32_epi32(v0u, ord_a), + 0b00111111, + ); + let ord_b = _mm256_set_epi32(3, 2, 4, 5, 2, 0, 3, 1); + let m_r_2 = _mm256_blend_epi32( + _mm256_permutevar8x32_epi32(r1, ord_b), + _mm256_permutevar8x32_epi32(v1u, ord_b), + 0b00111111, + ); + let ord_c = _mm256_set_epi32(1, 4, 6, 6, 2, 4, 3, 5); + let m_r_4 = _mm256_blend_epi32( + _mm256_blend_epi32( + _mm256_permutevar8x32_epi32(r1, ord_c), + _mm256_permutevar8x32_epi32(v1u, ord_c), + 0b00010000, + ), + _mm256_permutevar8x32_epi32(v0u, ord_c), + 0b00101111, + ); + + let mut v0 = _mm256_mul_epu32(x.v0, m_r_0); + let mut v1 = _mm256_mul_epu32(x.v1, m_r_0); + let mut v2 = _mm256_mul_epu32(x.v0, m_r_2); + let mut v3 = _mm256_mul_epu32(x.v1, m_r_2); + let mut v4 = _mm256_mul_epu32(x.v0, m_r_4); + + let swap = _mm256_set_epi32(6, 7, 4, 5, 2, 3, 0, 1); + let m_r_1 = _mm256_permutevar8x32_epi32(m_r_0, swap); + let m_r_3 = _mm256_permutevar8x32_epi32(m_r_2, swap); + + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_r_1)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_r_1)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_3)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_3)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v2, m_r_0)); + + x.v0 = _mm256_permutevar8x32_epi32(x.v0, swap); + + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v0, m_r_0)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v0, m_r_1)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v0, m_r_2)); + + let m_5r_3 = _mm256_add_epi32(m_r_3, _mm256_slli_epi32(m_r_3, 2)); + let m_5r_4 = _mm256_add_epi32(m_r_4, _mm256_slli_epi32(m_r_4, 2)); + + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v0, m_5r_3)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_4)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v0, m_5r_4)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v2, m_5r_3)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v2, m_5r_4)); + + x.v1 = _mm256_permutevar8x32_epi32(x.v1, swap); + + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v1, m_5r_3)); + v2 = _mm256_add_epi64(v2, _mm256_mul_epu32(x.v1, m_5r_4)); + v3 = _mm256_add_epi64(v3, _mm256_mul_epu32(x.v1, m_r_0)); + v4 = _mm256_add_epi64(v4, _mm256_mul_epu32(x.v1, m_r_1)); + + let m_5r_1 = _mm256_permutevar8x32_epi32(m_5r_4, swap); + let m_5r_2 = _mm256_permutevar8x32_epi32(m_5r_3, swap); + + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v1, m_5r_2)); + v0 = _mm256_add_epi64(v0, _mm256_mul_epu32(x.v2, m_5r_1)); + v1 = _mm256_add_epi64(v1, _mm256_mul_epu32(x.v2, m_5r_2)); + + Unreduced4x130 { v0, v1, v2, v3, v4 } + } } // ── Unreduced4x130 ────────────────────────────────────────────────────── impl Unreduced4x130 { /// Carry-reduce 4 values in parallel back to 26-bit limbs. + /// + /// `self` must contain bounded convolution coefficients produced by [`mul_4x130`] or [`mul_spaced`]. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn reduce(self) -> Aligned4x130 { - let mask_26 = _mm256_set1_epi64x(0x3ff_ffff); - - let adc = |x1: __m256i, x0: __m256i| -> (__m256i, __m256i) { - let y1 = _mm256_add_epi64(x1, _mm256_srli_epi64(x0, 26)); - let y0 = _mm256_and_si256(x0, mask_26); - (y1, y0) - }; - let red = |x4: __m256i, x0: __m256i| -> (__m256i, __m256i) { - let y0 = _mm256_add_epi64(x0, _mm256_mul_epu32(_mm256_srli_epi64(x4, 26), _mm256_set1_epi64x(5))); - let y4 = _mm256_and_si256(x4, mask_26); - (y4, y0) - }; - - let (r1, r0) = adc(self.v1, self.v0); - let (r4, r3) = adc(self.v4, self.v3); - let (r2, r1) = adc(self.v2, r1); - let (r4, r0) = red(r4, r0); - let (r3, r2) = adc(r3, r2); - let (r1, r0) = adc(r1, r0); - let (r4, r3) = adc(r4, r3); - - Aligned4x130 { - v0: _mm256_blend_epi32(r0, _mm256_slli_epi64(r2, 32), 0b10101010), - v1: _mm256_blend_epi32(r1, _mm256_slli_epi64(r3, 32), 0b10101010), - v2: r4, + // SAFETY: the caller guarantees AVX2 support. The bounded coefficients keep every carry, fivefold fold, and lane + // sum in u64; both closures and the final packing are register-only. + unsafe { + let mask_26 = _mm256_set1_epi64x(0x3ff_ffff); + + let adc = |x1: __m256i, x0: __m256i| -> (__m256i, __m256i) { + let y1 = _mm256_add_epi64(x1, _mm256_srli_epi64(x0, 26)); + let y0 = _mm256_and_si256(x0, mask_26); + (y1, y0) + }; + let red = |x4: __m256i, x0: __m256i| -> (__m256i, __m256i) { + let y0 = _mm256_add_epi64(x0, _mm256_mul_epu32(_mm256_srli_epi64(x4, 26), _mm256_set1_epi64x(5))); + let y4 = _mm256_and_si256(x4, mask_26); + (y4, y0) + }; + + let (r1, r0) = adc(self.v1, self.v0); + let (r4, r3) = adc(self.v4, self.v3); + let (r2, r1) = adc(self.v2, r1); + let (r4, r0) = red(r4, r0); + let (r3, r2) = adc(r3, r2); + let (r1, r0) = adc(r1, r0); + let (r4, r3) = adc(r4, r3); + + Aligned4x130 { + v0: _mm256_blend_epi32(r0, _mm256_slli_epi64(r2, 32), 0b10101010), + v1: _mm256_blend_epi32(r1, _mm256_slli_epi64(r3, 32), 0b10101010), + v2: r4, + } } } /// Horizontal sum of 4 lanes into a single `Unreduced130`. + /// + /// `self` must contain the bounded four-stream convolution produced by [`mul_spaced`]. The resulting coefficient + /// sums remain below 2⁶³. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn sum(self) -> Unreduced130 { - let lo01 = _mm256_add_epi64( - _mm256_unpackhi_epi64(self.v0, self.v1), - _mm256_unpacklo_epi64(self.v0, self.v1), - ); - let lo23 = _mm256_add_epi64( - _mm256_unpackhi_epi64(self.v2, self.v3), - _mm256_unpacklo_epi64(self.v2, self.v3), - ); - let v0 = _mm256_add_epi64( - _mm256_inserti128_si256(lo01, _mm256_castsi256_si128(lo23), 1), - _mm256_inserti128_si256(lo23, _mm256_extracti128_si256(lo01, 1), 0), - ); - let v4 = _mm256_add_epi64(self.v4, _mm256_permute4x64_epi64(self.v4, imm8(1, 0, 3, 2))); - let v1 = _mm256_add_epi64(v4, _mm256_permute4x64_epi64(v4, imm8(0, 0, 0, 1))); - Unreduced130 { v0, v1 } + // SAFETY: the caller guarantees AVX2 support; the documented bounds keep every horizontal lane sum in u64. + unsafe { + let lo01 = _mm256_add_epi64( + _mm256_unpackhi_epi64(self.v0, self.v1), + _mm256_unpacklo_epi64(self.v0, self.v1), + ); + let lo23 = _mm256_add_epi64( + _mm256_unpackhi_epi64(self.v2, self.v3), + _mm256_unpacklo_epi64(self.v2, self.v3), + ); + let v0 = _mm256_add_epi64( + _mm256_inserti128_si256(lo01, _mm256_castsi256_si128(lo23), 1), + _mm256_inserti128_si256(lo23, _mm256_extracti128_si256(lo01, 1), 0), + ); + let v4 = _mm256_add_epi64(self.v4, _mm256_permute4x64_epi64(self.v4, imm8(1, 0, 3, 2))); + let v1 = _mm256_add_epi64(v4, _mm256_permute4x64_epi64(v4, imm8(0, 0, 0, 1))); + Unreduced130 { v0, v1 } + } } } @@ -478,24 +628,34 @@ impl Unreduced4x130 { impl SpacedMultiplier4x130 { /// Compute `(multiplier, R⁴)` from `(R¹, R²)`. + /// + /// `r1` and `r2` must be reduced consecutive powers of the same clamped Poly1305 key. + /// + /// # Safety + /// + /// The caller must ensure that AVX2 is available. #[inline(always)] unsafe fn new(r1: PrecomputedMultiplier, r2: PrecomputedMultiplier) -> (Self, PrecomputedMultiplier) { - let r3 = mul_single(Aligned130(r2.a), r1).reduce(); - let r4 = mul_single(Aligned130(r2.a), r2).reduce(); - - let v0 = _mm256_blend_epi32( - r3.0, - _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 3, 1, 0, 0, 0, 0, 0)), - 0b11100000, - ); - let v1 = _mm256_blend_epi32( - r4.0, - _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 2, 0, 0, 0, 0, 0, 0)), - 0b11100000, - ); - - let m = SpacedMultiplier4x130 { v0, v1, r1 }; - (m, PrecomputedMultiplier::from_aligned(r4)) + // SAFETY: the caller guarantees AVX2 support. Matching bounded powers satisfy both nested multiplication/reduction + // contracts; the remaining operations only pack those register values. + unsafe { + let r3 = mul_single(Aligned130(r2.a), r1).reduce(); + let r4 = mul_single(Aligned130(r2.a), r2).reduce(); + + let v0 = _mm256_blend_epi32( + r3.0, + _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 3, 1, 0, 0, 0, 0, 0)), + 0b11100000, + ); + let v1 = _mm256_blend_epi32( + r4.0, + _mm256_permutevar8x32_epi32(r2.a, _mm256_set_epi32(4, 2, 0, 0, 0, 0, 0, 0)), + 0b11100000, + ); + + let m = SpacedMultiplier4x130 { v0, v1, r1 }; + (m, PrecomputedMultiplier::from_aligned(r4)) + } } } @@ -509,21 +669,16 @@ struct Par4State { r4: PrecomputedMultiplier, } -/// Authenticate `(aad, ciphertext)` using 4-way parallel Poly1305. +/// Authenticate `(aad, ciphertext)` using the AVX2 4-way Poly1305 kernel. /// /// Uses its own AVX2 kernel — ignores the per-block `ComputeBlockFn` dispatch. -pub(super) fn authenticate_aead_par4( - aad: &[u8], - ciphertext: &[u8], - key: &[u8; 32], - lengths: AeadByteLengths, -) -> [u8; 16] { - // SAFETY: caller verified AVX2 capability via `current_caps().has(x86::AVX2)`. - unsafe { authenticate_aead_par4_avx2(aad, ciphertext, key, lengths) } -} - +/// `lengths` must encode the lengths of `aad` and `ciphertext`. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available. #[target_feature(enable = "avx2")] -unsafe fn authenticate_aead_par4_avx2( +pub(super) unsafe fn authenticate_aead_par4( aad: &[u8], ciphertext: &[u8], key: &[u8; 32], @@ -532,9 +687,14 @@ unsafe fn authenticate_aead_par4_avx2( let state = State::new(key); // Precompute R¹, R² as AVX2 multipliers. - let r = Aligned130::from_limbs(state.r); - let r1 = PrecomputedMultiplier::from_aligned(r); - let r2 = PrecomputedMultiplier::from_aligned(mul_single(Aligned130(r1.a), r1).reduce()); + // SAFETY: the caller guarantees AVX2. `State::new` produces a clamped multiplier, and each nested helper preserves + // the bounded representation while deriving consecutive powers from that same key. + let (r1, r2) = unsafe { + let r = Aligned130::from_limbs(state.r); + let r1 = PrecomputedMultiplier::from_aligned(r); + let r2 = PrecomputedMultiplier::from_aligned(mul_single(Aligned130(r1.a), r1).reduce()); + (r1, r2) + }; // 4-block accumulator (initialized on first 4-block group). let mut acc: Option = None; @@ -543,60 +703,76 @@ unsafe fn authenticate_aead_par4_avx2( // Reimplements padded-segment logic from `update_padded_segment` for 4-way batching. for segment in [aad, ciphertext] { - let mut chunks = segment.chunks_exact(16); - for chunk in &mut chunks { - let mut block = [0u8; 16]; - block.copy_from_slice(chunk); - num_cached = push_block(block, &mut cached, num_cached, &mut acc, r1, r2); + let (chunks, rem) = segment.as_chunks::<16>(); + for chunk in chunks { + // SAFETY: the caller guarantees AVX2. The loop produces consecutive full blocks; `push_block` preserves + // `num_cached < 4` and keeps the cache, powers, and accumulator in the same key stream. + num_cached = unsafe { push_block(*chunk, &mut cached, num_cached, &mut acc, r1, r2) }; } - let rem = chunks.remainder(); if !rem.is_empty() { let mut block = [0u8; 16]; block[..rem.len()].copy_from_slice(rem); - num_cached = push_block(block, &mut cached, num_cached, &mut acc, r1, r2); + // SAFETY: the caller guarantees AVX2. Zero padding completes the segment's final full AEAD block, and the + // established cache and matching-key invariants still hold. + num_cached = unsafe { push_block(block, &mut cached, num_cached, &mut acc, r1, r2) }; } } // Process lengths block. let length_block = lengths.to_le_bytes_block(); - num_cached = push_block(length_block, &mut cached, num_cached, &mut acc, r1, r2); + // SAFETY: the caller guarantees AVX2. The length encoding is the final full AEAD block, and the established cache + // and matching-key invariants still hold. + num_cached = unsafe { push_block(length_block, &mut cached, num_cached, &mut acc, r1, r2) }; // Finalize: merge 4 lanes, process remaining blocks. - let mut p: Option = acc.map(|s| mul_spaced(s.poly, s.spaced).sum().reduce()); + let mut p: Option = acc.map(|s| { + // SAFETY: the caller guarantees AVX2, and `accumulate_4_blocks` constructed `poly` and `spaced` from matching + // bounded powers in this key stream. + unsafe { mul_spaced(s.poly, s.spaced).sum().reduce() } + }); // 2-block tail. if num_cached >= 2 { - let mut c0 = Aligned130::from_block(&cached[0]); - let c1 = Aligned130::from_block(&cached[1]); - if let Some(pv) = p { - c0 = c0.add(pv); - } - let a = mul_single(c0, r2); - let b = mul_single(c1, r1); - p = Some( - Unreduced130 { - v0: _mm256_add_epi64(a.v0, b.v0), - v1: _mm256_add_epi64(a.v1, b.v1), + // SAFETY: the caller guarantees AVX2. The cache invariant proves slots 0 and 1 are consecutive full blocks; + // `p`, `r2`, and `r1` are bounded values from this key stream, and the two products preserve block order. + unsafe { + let mut c0 = Aligned130::from_block(&cached[0]); + let c1 = Aligned130::from_block(&cached[1]); + if let Some(pv) = p { + c0 = c0.add(pv); } - .reduce(), - ); - cached[0] = cached[2]; - num_cached = num_cached.strict_sub(2); + let a = mul_single(c0, r2); + let b = mul_single(c1, r1); + p = Some( + Unreduced130 { + v0: _mm256_add_epi64(a.v0, b.v0), + v1: _mm256_add_epi64(a.v1, b.v1), + } + .reduce(), + ); + cached[0] = cached[2]; + num_cached = num_cached.strict_sub(2); + } } // 1-block tail. if num_cached == 1 { - let mut c = Aligned130::from_block(&cached[0]); - if let Some(pv) = p { - c = c.add(pv); + // SAFETY: the caller guarantees AVX2. The cache invariant proves slot 0 is the final full block, while `p` and + // `r1` are bounded values from the same key stream. + unsafe { + let mut c = Aligned130::from_block(&cached[0]); + if let Some(pv) = p { + c = c.add(pv); + } + p = Some(mul_single(c, r1).reduce()); } - p = Some(mul_single(c, r1).reduce()); } // Convert AVX2 result back to scalar and finalize. let mut final_state = state; if let Some(pv) = p { - final_state.h = pv.into_limbs(); + // SAFETY: the caller guarantees AVX2, and `pv` is the bounded result of this kernel's reduction chain. + final_state.h = unsafe { pv.into_limbs() }; } let tag = final_state.finalize(); ct::zeroize(cached.as_flattened_mut()); @@ -604,6 +780,11 @@ unsafe fn authenticate_aead_par4_avx2( } /// Cache one block; flush a 4-block group when full. Returns updated `num_cached`. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available, `num_cached < 4`, the occupied cache prefix and `block` are +/// consecutive full AEAD-padded blocks, and `r1`, `r2`, and any accumulator belong to the same bounded key stream. #[inline(always)] unsafe fn push_block( block: [u8; 16], @@ -616,7 +797,9 @@ unsafe fn push_block( cached[num_cached] = block; let n = num_cached.strict_add(1); if n == 4 { - accumulate_4_blocks(cached, acc, r1, r2); + // SAFETY: the caller guarantees AVX2 and a matching cache, multiplier, and accumulator. `n == 4` proves that + // this write completed the four-block group before it is accumulated. + unsafe { accumulate_4_blocks(cached, acc, r1, r2) }; 0 } else { n @@ -624,6 +807,11 @@ unsafe fn push_block( } /// Process a full 4-block group into the parallel accumulator. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available, `cached` contains four consecutive full AEAD-padded blocks, and +/// `r1`, `r2`, and any existing accumulator are bounded powers and state derived from the same clamped Poly1305 key. #[inline(always)] unsafe fn accumulate_4_blocks( cached: &[[u8; 16]; 4], @@ -631,15 +819,19 @@ unsafe fn accumulate_4_blocks( r1: PrecomputedMultiplier, r2: PrecomputedMultiplier, ) { - let blocks = Aligned4x130::from_blocks(cached); - if let Some(ref mut s) = *acc { - s.poly = mul_4x130(&s.poly, s.r4).reduce().add(blocks); - } else { - let (spaced, r4) = SpacedMultiplier4x130::new(r1, r2); - *acc = Some(Par4State { - poly: blocks, - spaced, - r4, - }); + // SAFETY: the caller guarantees AVX2 and the matching-key block, multiplier, and accumulator invariants required + // by each nested helper. All operations are fixed-width loads or register-only arithmetic. + unsafe { + let blocks = Aligned4x130::from_blocks(cached); + if let Some(ref mut s) = *acc { + s.poly = mul_4x130(&s.poly, s.r4).reduce().add(blocks); + } else { + let (spaced, r4) = SpacedMultiplier4x130::new(r1, r2); + *acc = Some(Par4State { + poly: blocks, + spaced, + r4, + }); + } } } diff --git a/src/aead/polyval.rs b/src/aead/polyval.rs index f6f65c08..cefe894b 100644 --- a/src/aead/polyval.rs +++ b/src/aead/polyval.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! Fixed-schedule, table-free POLYVAL universal hash (RFC 8452). //! //! Generated-code timing claims remain configuration- and @@ -31,6 +29,32 @@ pub(crate) const KEY_SIZE: usize = 16; #[cfg(test)] const POLY: u128 = (1u128 << 127) | (1u128 << 126) | (1u128 << 121) | 1; +#[inline(always)] +fn split_u128_le(value: u128) -> (u64, u64) { + let [ + lo_0, + lo_1, + lo_2, + lo_3, + lo_4, + lo_5, + lo_6, + lo_7, + hi_0, + hi_1, + hi_2, + hi_3, + hi_4, + hi_5, + hi_6, + hi_7, + ] = value.to_le_bytes(); + ( + u64::from_le_bytes([lo_0, lo_1, lo_2, lo_3, lo_4, lo_5, lo_6, lo_7]), + u64::from_le_bytes([hi_0, hi_1, hi_2, hi_3, hi_4, hi_5, hi_6, hi_7]), + ) +} + // x86_64 PCLMULQDQ backend #[cfg(target_arch = "x86_64")] @@ -47,8 +71,8 @@ mod pclmul { pub(super) unsafe fn clmul128_reduce(a: u128, b: u128) -> u128 { // SAFETY: target_feature gate guarantees PCLMULQDQ + SSE2. unsafe { - let a_xmm = _mm_loadu_si128((&a as *const u128).cast()); - let b_xmm = _mm_loadu_si128((&b as *const u128).cast()); + let a_xmm = _mm_loadu_si128(core::ptr::from_ref(&a).cast()); + let b_xmm = _mm_loadu_si128(core::ptr::from_ref(&b).cast()); // Schoolbook 128×128 → 256-bit product (4 PCLMULQDQ instructions). let lo = _mm_clmulepi64_si128(a_xmm, b_xmm, 0x00); // a_lo × b_lo @@ -65,7 +89,7 @@ mod pclmul { let result = mont_reduce_sse2(lo_128, hi_128); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -83,14 +107,14 @@ mod pclmul { // fixed-size stack/reference values. unsafe { let b0 = acc ^ blocks[0]; - let d0 = _mm_loadu_si128((&b0 as *const u128).cast()); - let d1 = _mm_loadu_si128((&blocks[1] as *const u128).cast()); - let d2 = _mm_loadu_si128((&blocks[2] as *const u128).cast()); - let d3 = _mm_loadu_si128((&blocks[3] as *const u128).cast()); - let h0 = _mm_loadu_si128((&h_powers_rev[0] as *const u128).cast()); - let h1 = _mm_loadu_si128((&h_powers_rev[1] as *const u128).cast()); - let h2 = _mm_loadu_si128((&h_powers_rev[2] as *const u128).cast()); - let h3 = _mm_loadu_si128((&h_powers_rev[3] as *const u128).cast()); + let d0 = _mm_loadu_si128(core::ptr::from_ref(&b0).cast()); + let d1 = _mm_loadu_si128(core::ptr::from_ref(&blocks[1]).cast()); + let d2 = _mm_loadu_si128(core::ptr::from_ref(&blocks[2]).cast()); + let d3 = _mm_loadu_si128(core::ptr::from_ref(&blocks[3]).cast()); + let h0 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[0]).cast()); + let h1 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[1]).cast()); + let h2 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[2]).cast()); + let h3 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[3]).cast()); aggregate_xmms([d0, d1, d2, d3], [h0, h1, h2, h3]) } @@ -118,20 +142,22 @@ mod pclmul { // fixed-size references, and PSHUFB only shuffles bytes within each register. unsafe { let reverse_bytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - let acc_xmm = _mm_loadu_si128((&acc as *const u128).cast()); + let acc_xmm = _mm_loadu_si128(core::ptr::from_ref(&acc).cast()); let d0 = _mm_xor_si128(_mm_shuffle_epi8(raw0, reverse_bytes), acc_xmm); let d1 = _mm_shuffle_epi8(raw1, reverse_bytes); let d2 = _mm_shuffle_epi8(raw2, reverse_bytes); let d3 = _mm_shuffle_epi8(raw3, reverse_bytes); - let h0 = _mm_loadu_si128((&h_powers_rev[0] as *const u128).cast()); - let h1 = _mm_loadu_si128((&h_powers_rev[1] as *const u128).cast()); - let h2 = _mm_loadu_si128((&h_powers_rev[2] as *const u128).cast()); - let h3 = _mm_loadu_si128((&h_powers_rev[3] as *const u128).cast()); + let h0 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[0]).cast()); + let h1 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[1]).cast()); + let h2 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[2]).cast()); + let h3 = _mm_loadu_si128(core::ptr::from_ref(&h_powers_rev[3]).cast()); aggregate_xmms([d0, d1, d2, d3], [h0, h1, h2, h3]) } } + /// # Safety + /// Caller must ensure PCLMULQDQ and SSE2 are available. #[target_feature(enable = "pclmulqdq,sse2")] #[inline] unsafe fn aggregate_xmms(data: [__m128i; 4], h: [__m128i; 4]) -> u128 { @@ -159,7 +185,7 @@ mod pclmul { let result = mont_reduce_sse2(lo_sum, hi_sum); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -168,7 +194,7 @@ mod pclmul { /// /// Equivalent to the portable `mont_reduce` but uses SSE2 lane-parallel shifts. #[inline] - pub(super) unsafe fn mont_reduce_sse2(lo: __m128i, hi: __m128i) -> __m128i { + pub(super) fn mont_reduce_sse2(lo: __m128i, hi: __m128i) -> __m128i { // SAFETY: caller guarantees SSE2 availability via target_feature chain. unsafe { // Phase 1: Compute left-shift contribution from both lo lanes. @@ -204,6 +230,7 @@ mod pclmul { #[cfg(target_arch = "aarch64")] mod pmull { + use super::split_u128_le; use core::arch::aarch64::*; /// Core PMULL multiply + reduce — `#[inline(always)]` for guaranteed inlining. @@ -212,15 +239,18 @@ mod pmull { /// features on the immediate caller. Combined with `#[inline(always)]`, /// the function body is inlined into the caller — no function call boundary, /// no register spills. + /// + /// # Safety + /// + /// The current CPU must support NEON and PMULL. The caller must enter through + /// an `aes,neon` target-feature scope selected by validated runtime capabilities. #[target_feature(enable = "neon", enable = "aes")] #[inline] pub(super) unsafe fn clmul128_reduce_core(a: u128, b: u128) -> u128 { // SAFETY: caller guarantees NEON + PMULL via target_feature chain. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = split_u128_le(a); + let (b_lo, b_hi) = split_u128_le(b); // Karatsuba 128×128 → 256-bit product (3 PMULL instructions). let ll = vreinterpretq_u64_p128(vmull_p64(a_lo, b_lo)); @@ -266,6 +296,11 @@ mod pmull { /// Uses NEON lane-parallel shifts (`vshlq_n_u64`, `vshrq_n_u64`) and /// `vextq_u64` for cross-lane propagation — structurally identical to /// the SSE2 `mont_reduce_sse2` path. + /// + /// # Safety + /// + /// The caller must execute in a NEON target-feature scope. Every bit pattern + /// in `lo` and `hi` is a valid unreduced field product. #[inline] unsafe fn mont_reduce_neon(lo: uint64x2_t, hi: uint64x2_t) -> uint64x2_t { // SAFETY: caller guarantees NEON availability via target_feature chain. @@ -300,6 +335,11 @@ mod pmull { /// Computes `(acc ^ b0) * H^4 ^ b1 * H^3 ^ b2 * H^2 ^ b3 * H` using /// 12 independent `vmull_p64` instructions that the OOO core on Neoverse /// V1/V2 (2 crypto pipes) can schedule freely, then a single reduction. + /// + /// # Safety + /// + /// The current CPU must support NEON and PMULL. The caller must enter through + /// an `aes,neon` target-feature scope selected by validated runtime capabilities. #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "neon", enable = "aes")] #[inline] @@ -312,34 +352,26 @@ mod pmull { let b3 = blocks[3]; // 12 vmull_p64: 4 blocks × 3 Karatsuba products. - let b0_lo = b0 as u64; - let b0_hi = (b0 >> 64) as u64; - let h0_lo = h_powers_rev[0] as u64; - let h0_hi = (h_powers_rev[0] >> 64) as u64; + let (b0_lo, b0_hi) = split_u128_le(b0); + let (h0_lo, h0_hi) = split_u128_le(h_powers_rev[0]); let ll0 = vreinterpretq_u64_p128(vmull_p64(b0_lo, h0_lo)); let hh0 = vreinterpretq_u64_p128(vmull_p64(b0_hi, h0_hi)); let mm0 = vreinterpretq_u64_p128(vmull_p64(b0_lo ^ b0_hi, h0_lo ^ h0_hi)); - let b1_lo = b1 as u64; - let b1_hi = (b1 >> 64) as u64; - let h1_lo = h_powers_rev[1] as u64; - let h1_hi = (h_powers_rev[1] >> 64) as u64; + let (b1_lo, b1_hi) = split_u128_le(b1); + let (h1_lo, h1_hi) = split_u128_le(h_powers_rev[1]); let ll1 = vreinterpretq_u64_p128(vmull_p64(b1_lo, h1_lo)); let hh1 = vreinterpretq_u64_p128(vmull_p64(b1_hi, h1_hi)); let mm1 = vreinterpretq_u64_p128(vmull_p64(b1_lo ^ b1_hi, h1_lo ^ h1_hi)); - let b2_lo = b2 as u64; - let b2_hi = (b2 >> 64) as u64; - let h2_lo = h_powers_rev[2] as u64; - let h2_hi = (h_powers_rev[2] >> 64) as u64; + let (b2_lo, b2_hi) = split_u128_le(b2); + let (h2_lo, h2_hi) = split_u128_le(h_powers_rev[2]); let ll2 = vreinterpretq_u64_p128(vmull_p64(b2_lo, h2_lo)); let hh2 = vreinterpretq_u64_p128(vmull_p64(b2_hi, h2_hi)); let mm2 = vreinterpretq_u64_p128(vmull_p64(b2_lo ^ b2_hi, h2_lo ^ h2_hi)); - let b3_lo = b3 as u64; - let b3_hi = (b3 >> 64) as u64; - let h3_lo = h_powers_rev[3] as u64; - let h3_hi = (h_powers_rev[3] >> 64) as u64; + let (b3_lo, b3_hi) = split_u128_le(b3); + let (h3_lo, h3_hi) = split_u128_le(h_powers_rev[3]); let ll3 = vreinterpretq_u64_p128(vmull_p64(b3_lo, h3_lo)); let hh3 = vreinterpretq_u64_p128(vmull_p64(b3_hi, h3_hi)); let mm3 = vreinterpretq_u64_p128(vmull_p64(b3_lo ^ b3_hi, h3_lo ^ h3_hi)); @@ -363,6 +395,11 @@ mod pmull { } /// 8-block GHASH aggregate from big-endian ciphertext lanes already held in NEON registers. + /// + /// # Safety + /// + /// The current CPU must support NEON and PMULL. The caller must enter through + /// an `aes,neon` target-feature scope selected by validated runtime capabilities. #[cfg(feature = "aes-gcm")] #[target_feature(enable = "neon", enable = "aes")] #[inline] @@ -375,6 +412,12 @@ mod pmull { // GHASH lanes to the little-endian POLYVAL-domain lane representation. unsafe { #[inline(always)] + /// Loads one little-endian field element into two NEON lanes. + /// + /// # Safety + /// + /// `power` must be valid to read one initialized `u128`. The caller must + /// execute in a NEON target-feature scope. unsafe fn load_power(power: *const u128) -> uint64x2_t { // SAFETY: caller passes a pointer into `h_powers_rev`; `vld1q_u64` // accepts arbitrary alignment and reads exactly one initialized u128. @@ -382,10 +425,16 @@ mod pmull { } #[inline(always)] + /// Places one little-endian field element into two NEON lanes. + /// + /// # Safety + /// + /// The caller must execute in a NEON target-feature scope. unsafe fn u128_to_lanes(x: u128) -> uint64x2_t { + let (lo, hi) = split_u128_le(x); // SAFETY: caller is inside a NEON target scope. `vcreate_u64` // initializes one 64-bit lane and `vcombine_u64` builds the pair. - unsafe { vcombine_u64(vcreate_u64(x as u64), vcreate_u64((x >> 64) as u64)) } + unsafe { vcombine_u64(vcreate_u64(lo), vcreate_u64(hi)) } } let mut ll = vdupq_n_u64(0); @@ -447,7 +496,6 @@ mod pmull { // s390x VGFM backend (Galois field multiply) #[cfg(target_arch = "s390x")] -#[allow(unsafe_code)] mod s390x_vgfm { use core::{arch::asm, simd::i64x2}; @@ -455,11 +503,14 @@ mod s390x_vgfm { /// /// Places operands in the low lane (element 1) with the high lane zeroed, /// so VGFM computes `0*0 XOR a*b = a*b`. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn mul64(a: u64, b: u64) -> i64x2 { - let va = i64x2::from_array([0, a as i64]); - let vb = i64x2::from_array([0, b as i64]); + let va = i64x2::from_array([0, a.cast_signed()]); + let vb = i64x2::from_array([0, b.cast_signed()]); // SAFETY: Caller guarantees z/Vector facility is available. unsafe { let out: i64x2; @@ -475,6 +526,9 @@ mod s390x_vgfm { } /// Per-lane left shift of both 64-bit elements. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn veslg(a: i64x2) -> i64x2 { @@ -493,6 +547,9 @@ mod s390x_vgfm { } /// Per-lane logical right shift of both 64-bit elements. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vesrlg(a: i64x2) -> i64x2 { @@ -514,6 +571,9 @@ mod s390x_vgfm { /// /// - `vsldb(v, zero, 8)`: moves low lane to high, zeros low = `v << 64` /// - `vsldb(zero, v, 8)`: moves high lane to low, zeros high = `v >> 64` + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vsldb(a: i64x2, b: i64x2) -> i64x2 { @@ -537,6 +597,9 @@ mod s390x_vgfm { /// Structurally identical to `mont_reduce_sse2` / `mont_reduce_neon`, /// using s390x VESLG/VESRLG for per-lane shifts and VSLDB for cross-lane /// byte shifts. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn mont_reduce(lo: i64x2, hi: i64x2) -> i64x2 { @@ -564,15 +627,16 @@ mod s390x_vgfm { } /// Core VGFM multiply + reduce — `#[inline(always)]` for guaranteed inlining. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[target_feature(enable = "vector")] #[inline] pub(super) unsafe fn clmul128_reduce_core(a: u128, b: u128) -> u128 { // SAFETY: caller guarantees z/Vector availability via target_feature chain. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); let zero = i64x2::from_array([0, 0]); // Karatsuba: 3 VGFM multiplies. @@ -587,7 +651,7 @@ mod s390x_vgfm { let result = mont_reduce(lo_128, hi_128); let arr = result.to_array(); - ((arr[0] as u64 as u128) << 64) | (arr[1] as u64 as u128) + (u128::from(arr[0].cast_unsigned()) << 64) | u128::from(arr[1].cast_unsigned()) } } @@ -607,6 +671,9 @@ mod s390x_vgfm { /// /// 12 independent VGFM (4 × 3 Karatsuba), then one vector /// Montgomery reduction. + /// + /// # Safety + /// Caller must ensure the z/Vector facility is available. #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "vector")] #[inline] @@ -617,36 +684,32 @@ mod s390x_vgfm { let b1 = blocks[1]; let b2 = blocks[2]; let b3 = blocks[3]; + let (b0_lo, b0_hi) = super::split_u128_le(b0); + let (b1_lo, b1_hi) = super::split_u128_le(b1); + let (b2_lo, b2_hi) = super::split_u128_le(b2); + let (b3_lo, b3_hi) = super::split_u128_le(b3); + let (h0_lo, h0_hi) = super::split_u128_le(h_powers_rev[0]); + let (h1_lo, h1_hi) = super::split_u128_le(h_powers_rev[1]); + let (h2_lo, h2_hi) = super::split_u128_le(h_powers_rev[2]); + let (h3_lo, h3_hi) = super::split_u128_le(h_powers_rev[3]); let zero = i64x2::from_array([0, 0]); // 12 VGFM: 4 blocks × 3 Karatsuba multiplies. - let v0_0 = mul64(b0 as u64, h_powers_rev[0] as u64); - let v1_0 = mul64((b0 >> 64) as u64, (h_powers_rev[0] >> 64) as u64); - let v2_0 = mul64( - b0 as u64 ^ (b0 >> 64) as u64, - h_powers_rev[0] as u64 ^ (h_powers_rev[0] >> 64) as u64, - ); + let v0_0 = mul64(b0_lo, h0_lo); + let v1_0 = mul64(b0_hi, h0_hi); + let v2_0 = mul64(b0_lo ^ b0_hi, h0_lo ^ h0_hi); - let v0_1 = mul64(b1 as u64, h_powers_rev[1] as u64); - let v1_1 = mul64((b1 >> 64) as u64, (h_powers_rev[1] >> 64) as u64); - let v2_1 = mul64( - b1 as u64 ^ (b1 >> 64) as u64, - h_powers_rev[1] as u64 ^ (h_powers_rev[1] >> 64) as u64, - ); + let v0_1 = mul64(b1_lo, h1_lo); + let v1_1 = mul64(b1_hi, h1_hi); + let v2_1 = mul64(b1_lo ^ b1_hi, h1_lo ^ h1_hi); - let v0_2 = mul64(b2 as u64, h_powers_rev[2] as u64); - let v1_2 = mul64((b2 >> 64) as u64, (h_powers_rev[2] >> 64) as u64); - let v2_2 = mul64( - b2 as u64 ^ (b2 >> 64) as u64, - h_powers_rev[2] as u64 ^ (h_powers_rev[2] >> 64) as u64, - ); + let v0_2 = mul64(b2_lo, h2_lo); + let v1_2 = mul64(b2_hi, h2_hi); + let v2_2 = mul64(b2_lo ^ b2_hi, h2_lo ^ h2_hi); - let v0_3 = mul64(b3 as u64, h_powers_rev[3] as u64); - let v1_3 = mul64((b3 >> 64) as u64, (h_powers_rev[3] >> 64) as u64); - let v2_3 = mul64( - b3 as u64 ^ (b3 >> 64) as u64, - h_powers_rev[3] as u64 ^ (h_powers_rev[3] >> 64) as u64, - ); + let v0_3 = mul64(b3_lo, h3_lo); + let v1_3 = mul64(b3_hi, h3_hi); + let v2_3 = mul64(b3_lo ^ b3_hi, h3_lo ^ h3_hi); // XOR all 4 Karatsuba intermediates (i64x2 vector XOR). let v0 = v0_0 ^ v0_1 ^ v0_2 ^ v0_3; @@ -659,7 +722,7 @@ mod s390x_vgfm { let result = mont_reduce(lo_128, hi_128); let arr = result.to_array(); - ((arr[0] as u64 as u128) << 64) | (arr[1] as u64 as u128) + (u128::from(arr[0].cast_unsigned()) << 64) | u128::from(arr[1].cast_unsigned()) } } } @@ -667,7 +730,6 @@ mod s390x_vgfm { // powerpc64 VPMSUMD backend (polynomial multiply-sum doubleword) #[cfg(target_arch = "powerpc64")] -#[allow(unsafe_code)] mod ppc_vpmsum { use core::{arch::asm, simd::i64x2}; @@ -675,11 +737,14 @@ mod ppc_vpmsum { /// /// Match the existing POWER checksum kernels' lane convention: /// lane 0 carries the low 64 bits, lane 1 the high 64 bits. + /// + /// # Safety + /// Caller must ensure POWER8 vector crypto is available. #[inline] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] unsafe fn mul64(a: u64, b: u64) -> (u64, u64) { - let va = i64x2::from_array([a as i64, 0]); - let vb = i64x2::from_array([b as i64, 0]); + let va = i64x2::from_array([a.cast_signed(), 0]); + let vb = i64x2::from_array([b.cast_signed(), 0]); // SAFETY: Caller guarantees POWER8 crypto availability. unsafe { let out: i64x2; @@ -691,20 +756,21 @@ mod ppc_vpmsum { options(nomem, nostack, pure), ); let [lo, hi] = out.to_array(); - (lo as u64, hi as u64) + (lo.cast_unsigned(), hi.cast_unsigned()) } } /// Core VPMSUMD multiply + reduce — `#[inline(always)]` for guaranteed inlining. + /// + /// # Safety + /// Caller must ensure POWER8 vector crypto is available. #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] #[inline] pub(super) unsafe fn clmul128_reduce_core(a: u128, b: u128) -> u128 { // SAFETY: caller guarantees POWER8 crypto availability via target_feature chain. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); // Karatsuba: 3 vpmsumd multiplies. let (v0_lo, v0_hi) = mul64(a_lo, b_lo); @@ -735,6 +801,9 @@ mod ppc_vpmsum { /// /// 12 independent `vpmsumd` (4 × 3 Karatsuba), then one scalar /// Montgomery reduction. + /// + /// # Safety + /// Caller must ensure POWER8 vector crypto is available. #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv"))] #[target_feature(enable = "altivec,vsx,power8-vector,power8-crypto")] #[inline] @@ -745,35 +814,31 @@ mod ppc_vpmsum { let b1 = blocks[1]; let b2 = blocks[2]; let b3 = blocks[3]; + let (b0_lo, b0_hi) = super::split_u128_le(b0); + let (b1_lo, b1_hi) = super::split_u128_le(b1); + let (b2_lo, b2_hi) = super::split_u128_le(b2); + let (b3_lo, b3_hi) = super::split_u128_le(b3); + let (h0_lo, h0_hi) = super::split_u128_le(h_powers_rev[0]); + let (h1_lo, h1_hi) = super::split_u128_le(h_powers_rev[1]); + let (h2_lo, h2_hi) = super::split_u128_le(h_powers_rev[2]); + let (h3_lo, h3_hi) = super::split_u128_le(h_powers_rev[3]); // 12 vpmsumd: 4 blocks × 3 Karatsuba multiplies. - let (z0_0l, z0_0h) = mul64(b0 as u64, h_powers_rev[0] as u64); - let (z1_0l, z1_0h) = mul64((b0 >> 64) as u64, (h_powers_rev[0] >> 64) as u64); - let (z2_0l, z2_0h) = mul64( - b0 as u64 ^ (b0 >> 64) as u64, - h_powers_rev[0] as u64 ^ (h_powers_rev[0] >> 64) as u64, - ); + let (z0_0l, z0_0h) = mul64(b0_lo, h0_lo); + let (z1_0l, z1_0h) = mul64(b0_hi, h0_hi); + let (z2_0l, z2_0h) = mul64(b0_lo ^ b0_hi, h0_lo ^ h0_hi); - let (z0_1l, z0_1h) = mul64(b1 as u64, h_powers_rev[1] as u64); - let (z1_1l, z1_1h) = mul64((b1 >> 64) as u64, (h_powers_rev[1] >> 64) as u64); - let (z2_1l, z2_1h) = mul64( - b1 as u64 ^ (b1 >> 64) as u64, - h_powers_rev[1] as u64 ^ (h_powers_rev[1] >> 64) as u64, - ); + let (z0_1l, z0_1h) = mul64(b1_lo, h1_lo); + let (z1_1l, z1_1h) = mul64(b1_hi, h1_hi); + let (z2_1l, z2_1h) = mul64(b1_lo ^ b1_hi, h1_lo ^ h1_hi); - let (z0_2l, z0_2h) = mul64(b2 as u64, h_powers_rev[2] as u64); - let (z1_2l, z1_2h) = mul64((b2 >> 64) as u64, (h_powers_rev[2] >> 64) as u64); - let (z2_2l, z2_2h) = mul64( - b2 as u64 ^ (b2 >> 64) as u64, - h_powers_rev[2] as u64 ^ (h_powers_rev[2] >> 64) as u64, - ); + let (z0_2l, z0_2h) = mul64(b2_lo, h2_lo); + let (z1_2l, z1_2h) = mul64(b2_hi, h2_hi); + let (z2_2l, z2_2h) = mul64(b2_lo ^ b2_hi, h2_lo ^ h2_hi); - let (z0_3l, z0_3h) = mul64(b3 as u64, h_powers_rev[3] as u64); - let (z1_3l, z1_3h) = mul64((b3 >> 64) as u64, (h_powers_rev[3] >> 64) as u64); - let (z2_3l, z2_3h) = mul64( - b3 as u64 ^ (b3 >> 64) as u64, - h_powers_rev[3] as u64 ^ (h_powers_rev[3] >> 64) as u64, - ); + let (z0_3l, z0_3h) = mul64(b3_lo, h3_lo); + let (z1_3l, z1_3h) = mul64(b3_hi, h3_hi); + let (z2_3l, z2_3h) = mul64(b3_lo ^ b3_hi, h3_lo ^ h3_hi); // XOR all 4 Karatsuba intermediates. let z0_lo = z0_0l ^ z0_1l ^ z0_2l ^ z0_3l; @@ -794,7 +859,6 @@ mod ppc_vpmsum { // riscv64 Zvbc backend (vector carryless multiply) #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] mod rv_clmul { use core::arch::asm; @@ -802,6 +866,9 @@ mod rv_clmul { /// /// Uses the clobber-only vreg workaround: data shuttled through GPRs /// and memory; vector registers referenced by explicit names. + /// + /// # Safety + /// Caller must ensure the V and Zvbc extensions are available. #[inline] #[target_feature(enable = "v", enable = "zvbc")] unsafe fn mul64(a: u64, b: u64) -> (u64, u64) { @@ -840,10 +907,8 @@ mod rv_clmul { pub(super) unsafe fn clmul128_reduce(a: u128, b: u128) -> u128 { // SAFETY: target_feature gate guarantees Zvbc availability. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); // Karatsuba: 3 multiplies. let (v0_lo, v0_hi) = mul64(a_lo, b_lo); @@ -869,13 +934,15 @@ mod rv_clmul { // riscv64 Zbc backend (scalar carryless multiply) #[cfg(target_arch = "riscv64")] -#[allow(unsafe_code)] mod rv_scalar_clmul { use core::arch::asm; /// 64×64→128 carryless multiply using scalar Zbc (clmul + clmulh). /// /// Identical encoding to Zbkc — dispatch checks either cap at runtime. + /// + /// # Safety + /// Caller must ensure the Zbc or Zbkc extension is available. #[inline] #[target_feature(enable = "zbc")] unsafe fn mul64(a: u64, b: u64) -> (u64, u64) { @@ -909,10 +976,8 @@ mod rv_scalar_clmul { // SAFETY: target_feature gate guarantees Zbc availability. mul64 calls // are safe within this target_feature scope. unsafe { - let a_lo = a as u64; - let a_hi = (a >> 64) as u64; - let b_lo = b as u64; - let b_hi = (b >> 64) as u64; + let (a_lo, a_hi) = super::split_u128_le(a); + let (b_lo, b_hi) = super::split_u128_le(b); // Karatsuba: 3 multiplies instead of 4. let (v0_lo, v0_hi) = mul64(a_lo, b_lo); @@ -981,7 +1046,7 @@ mod vpclmul { let result = super::pclmul::mont_reduce_sse2(lo_sum, hi_sum); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -1039,57 +1104,7 @@ mod vpclmul { let result = super::pclmul::mont_reduce_sse2(lo, hi); let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); - out - } - } - - /// Reduce eight POLYVAL-domain lanes using 256-bit VPCLMULQDQ. - /// - /// # Safety - /// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + - /// PCLMULQDQ + SSE2. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] - #[inline] - #[allow(dead_code)] - unsafe fn aggregate_8_lanes_256(data: [__m256i; 4], h_powers_rev: &[u128; 8]) -> u128 { - // SAFETY: x86 VPCLMUL 8-block aggregation because: - // 1. This function's caller guarantees all required target features. - // 2. `data` contains eight initialized POLYVAL-domain lanes in order. - // 3. `h_powers_rev` contains exactly [H^8, H^7, ..., H], matching the lanes. - unsafe { - let mut lo_sum = _mm256_setzero_si256(); - let mut hi_sum = _mm256_setzero_si256(); - - macro_rules! fold_lanes { - ($data:expr, $power_offset:expr) => {{ - let h_vec = _mm256_loadu_si256(h_powers_rev.as_ptr().add($power_offset).cast()); - let lo = _mm256_clmulepi64_epi128($data, h_vec, 0x00); - let hi = _mm256_clmulepi64_epi128($data, h_vec, 0x11); - let data_mid = _mm256_xor_si256($data, _mm256_shuffle_epi32::<0x4e>($data)); - let h_mid = _mm256_xor_si256(h_vec, _mm256_shuffle_epi32::<0x4e>(h_vec)); - let mid = _mm256_xor_si256( - _mm256_xor_si256(_mm256_clmulepi64_epi128(data_mid, h_mid, 0x00), lo), - hi, - ); - - lo_sum = _mm256_xor_si256(lo_sum, _mm256_xor_si256(lo, _mm256_bslli_epi128(mid, 8))); - hi_sum = _mm256_xor_si256(hi_sum, _mm256_xor_si256(hi, _mm256_bsrli_epi128(mid, 8))); - }}; - } - - fold_lanes!(data[0], 0); - fold_lanes!(data[1], 2); - fold_lanes!(data[2], 4); - fold_lanes!(data[3], 6); - - let lo = _mm_xor_si128(_mm256_castsi256_si128(lo_sum), _mm256_extracti128_si256(lo_sum, 1)); - let hi = _mm_xor_si128(_mm256_castsi256_si128(hi_sum), _mm256_extracti128_si256(hi_sum, 1)); - - let result = super::pclmul::mont_reduce_sse2(lo, hi); - let mut out = 0u128; - _mm_storeu_si128((&mut out as *mut u128).cast(), result); + _mm_storeu_si128(core::ptr::from_mut(&mut out).cast(), result); out } } @@ -1123,28 +1138,6 @@ mod vpclmul { _mm512_shuffle_epi8(raw, reverse_bytes) } - /// Convert two big-endian GHASH lanes into POLYVAL-domain lanes. - /// - /// # Safety - /// Caller must ensure AVX2 is available. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2")] - #[inline] - #[allow(dead_code)] - unsafe fn be_lanes_256(raw: __m256i) -> __m256i { - let reverse_bytes = _mm256_set_epi32( - 0x0001_0203, - 0x0405_0607, - 0x0809_0a0b, - 0x0c0d_0e0f, - 0x0001_0203, - 0x0405_0607, - 0x0809_0a0b, - 0x0c0d_0e0f, - ); - _mm256_shuffle_epi8(raw, reverse_bytes) - } - /// Convert four big-endian GHASH lanes and XOR `acc` into the first lane. /// /// # Safety @@ -1158,30 +1151,11 @@ mod vpclmul { // 2. `raw` contains four initialized big-endian GHASH lanes and `acc` is initialized. unsafe { let data = be_lanes(raw); - let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128((&acc as *const u128).cast())); + let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128(core::ptr::from_ref(&acc).cast())); _mm512_xor_si512(data, acc_lane) } } - /// Convert two big-endian GHASH lanes and XOR `acc` into the first lane. - /// - /// # Safety - /// Caller must ensure AVX2 and SSE2 are available. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2,sse2")] - #[inline] - #[allow(dead_code)] - unsafe fn be_lanes_256_with_acc(acc: u128, raw: __m256i) -> __m256i { - // SAFETY: direct GHASH lane conversion because: - // 1. This function's caller guarantees the required x86 target features. - // 2. `raw` contains two initialized big-endian GHASH lanes and `acc` is initialized. - unsafe { - let data = be_lanes_256(raw); - let acc_lane = _mm256_castsi128_si256(_mm_loadu_si128((&acc as *const u128).cast())); - _mm256_xor_si256(data, acc_lane) - } - } - /// Load four big-endian GHASH blocks and XOR `acc` into the first lane. /// /// # Safety @@ -1272,7 +1246,7 @@ mod vpclmul { // representation expected by `aggregate_lanes`. unsafe { let data = _mm512_loadu_si512(block_ptr.cast()); - let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128((&acc as *const u128).cast())); + let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128(core::ptr::from_ref(&acc).cast())); aggregate_lanes(_mm512_xor_si512(data, acc_lane), h_powers_rev) } } @@ -1292,7 +1266,7 @@ mod vpclmul { // 3. Only the first lane receives the incoming accumulator, matching the POLYVAL recurrence. unsafe { let data0 = _mm512_loadu_si512(block_ptr.cast()); - let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128((&acc as *const u128).cast())); + let acc_lane = _mm512_zextsi128_si512(_mm_loadu_si128(core::ptr::from_ref(&acc).cast())); aggregate_16_lanes( [ _mm512_xor_si512(data0, acc_lane), @@ -1368,39 +1342,6 @@ mod vpclmul { ) } } - - /// Process 8 big-endian GHASH lanes already resident in 256-bit SIMD registers. - /// - /// # Safety - /// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + - /// PCLMULQDQ + SSE2. - #[cfg(feature = "aes-gcm")] - #[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] - #[allow(dead_code)] - pub(super) unsafe fn aggregate_8blocks_be_lanes_256( - acc: u128, - h_powers_rev: &[u128; 8], - raw0: __m256i, - raw1: __m256i, - raw2: __m256i, - raw3: __m256i, - ) -> u128 { - // SAFETY: direct-lane GHASH aggregation because: - // 1. This function's caller guarantees all required x86 target features. - // 2. `raw*` contain eight initialized 16-byte ciphertext lanes in memory byte order. - // 3. Only the first lane receives the incoming accumulator, matching GHASH recurrence. - unsafe { - aggregate_8_lanes_256( - [ - be_lanes_256_with_acc(acc, raw0), - be_lanes_256(raw1), - be_lanes_256(raw2), - be_lanes_256(raw3), - ], - h_powers_rev, - ) - } - } } // x86_64: inline helpers for fused GCM paths @@ -1561,29 +1502,6 @@ pub(super) unsafe fn x86_aggregate_16blocks_be_lanes_inline( unsafe { vpclmul::aggregate_16blocks_be_lanes(acc, h_powers_rev, raw0, raw1, raw2, raw3) } } -/// 8-block VPCLMULQDQ aggregate helper for big-endian GHASH lanes already in 256-bit registers. -/// -/// # Safety -/// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + -/// PCLMULQDQ + SSE2 are available. -#[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] -#[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] -#[inline] -pub(super) unsafe fn x86_aggregate_8blocks_be_lanes_256_inline( - acc: u128, - h_powers_rev: &[u128; 8], - raw0: core::arch::x86_64::__m256i, - raw1: core::arch::x86_64::__m256i, - raw2: core::arch::x86_64::__m256i, - raw3: core::arch::x86_64::__m256i, -) -> u128 { - // SAFETY: direct-lane VPCLMUL aggregation because: - // 1. Caller guarantees all required target features. - // 2. `h_powers_rev` and `raw*` are initialized inputs matching the backend contract. - // 3. The helper folds the incoming accumulator only into the first lane. - unsafe { vpclmul::aggregate_8blocks_be_lanes_256(acc, h_powers_rev, raw0, raw1, raw2, raw3) } -} - // aarch64: inline helper for fused paths (#[target_feature] + #[inline(always)]) /// PMULL-based 128×128 carryless multiply + Montgomery reduce, guaranteed @@ -1848,6 +1766,8 @@ pub(super) fn clmul128_reduce(a: u128, b: u128) -> u128 { clmul(a, b) } +/// Multiplies two little-endian POLYVAL field elements with the portable +/// carryless-multiply and reduction implementation. #[cfg(all(feature = "diag", feature = "aes-gcm-siv"))] #[must_use] pub fn diag_polyval_reduce_portable(a: &[u8; 16], b: &[u8; 16]) -> [u8; 16] { @@ -1968,7 +1888,8 @@ pub(super) fn precompute_powers_128(h: u128) -> [u128; 128] { pub(super) fn precompute_powers_16_mid(h_powers_rev_16: &[u128; 16]) -> [u128; 16] { core::array::from_fn(|i| { let h = h_powers_rev_16[i]; - ((h as u64) ^ ((h >> 64) as u64)) as u128 + let (lo, hi) = split_u128_le(h); + (lo ^ hi) as u128 }) } @@ -1985,10 +1906,8 @@ pub(super) fn precompute_powers_16_pair(h_powers_rev_16: &[u128; 16]) -> [u128; let lane = i % 3; let h0 = h_powers_rev_16[pair.strict_mul(2)]; let h1 = h_powers_rev_16[pair.strict_mul(2).strict_add(1)]; - let lo0 = h0 as u64; - let hi0 = (h0 >> 64) as u64; - let lo1 = h1 as u64; - let hi1 = (h1 >> 64) as u64; + let (lo0, hi0) = split_u128_le(h0); + let (lo1, hi1) = split_u128_le(h1); let low = match lane { 0 => lo0, @@ -2251,10 +2170,11 @@ impl Polyval { #[cfg(feature = "aes-gcm-siv")] impl Drop for Polyval { fn drop(&mut self) { - // SAFETY: self.acc/self.h are valid, aligned, dereferenceable pointers to initialized memory. + // SAFETY: the raw pointers address initialized, aligned `u128` fields owned + // exclusively by `self` for the duration of `drop`. unsafe { - core::ptr::write_volatile(&mut self.acc, 0); - core::ptr::write_volatile(&mut self.h, 0); + core::ptr::write_volatile(&raw mut self.acc, 0); + core::ptr::write_volatile(&raw mut self.h, 0); } core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } @@ -2313,10 +2233,8 @@ fn bmul64(x: u64, y: u64) -> u64 { /// Uses Karatsuba decomposition: 3 sub-products × 2 (normal + bit-reversed /// for the high halves) = 6 bmul64 calls = 96 integer multiplies. pub(super) fn clmul128(a: u128, b: u128) -> [u64; 4] { - let a0 = a as u64; - let a1 = (a >> 64) as u64; - let b0 = b as u64; - let b1 = (b >> 64) as u64; + let (a0, a1) = split_u128_le(a); + let (b0, b1) = split_u128_le(b); // Karatsuba middle term operands. let a2 = a0 ^ a1; @@ -2395,21 +2313,24 @@ pub(super) fn mont_reduce(v: [u64; 4]) -> u128 { mod tests { use super::*; + const RFC_H: [u8; 16] = 0x2562_9347_5892_4276_1d31_f826_ba4b_757bu128.to_be_bytes(); + const RFC_X1: [u8; 16] = 0x4f4f_9566_8c83_dfb6_4017_62bb_2d01_a262u128.to_be_bytes(); + const RFC_X2: [u8; 16] = 0xd1a2_4ddd_2721_d006_bbe4_5f20_d3c9_f362u128.to_be_bytes(); + #[cfg(feature = "aes-gcm-siv")] + const RFC_EXPECTED: [u8; 16] = 0xf7a3_b47b_8461_19fa_e5b7_866c_f5e5_b77eu128.to_be_bytes(); + const AGGREGATE_X3: [u8; 16] = 0x0100_0000_0000_0000_0000_0000_0000_000fu128.to_be_bytes(); + const AGGREGATE_X4: [u8; 16] = 0xabcd_ef01_2345_6789_abcd_ef01_2345_6789u128.to_be_bytes(); + /// RFC 8452 Appendix A: POLYVAL test vector. #[cfg(feature = "aes-gcm-siv")] #[test] fn polyval_rfc8452_appendix_a() { - let h = hex_to_16("25629347589242761d31f826ba4b757b"); - let x1 = hex_to_16("4f4f95668c83dfb6401762bb2d01a262"); - let x2 = hex_to_16("d1a24ddd2721d006bbe45f20d3c9f362"); - let expected = hex_to_16("f7a3b47b846119fae5b7866cf5e5b77e"); - - let mut pv = Polyval::new(&h); - pv.update_block(&x1); - pv.update_block(&x2); + let mut pv = Polyval::new(&RFC_H); + pv.update_block(&RFC_X1); + pv.update_block(&RFC_X2); let result = pv.finalize(); - assert_eq!(result, expected, "POLYVAL mismatch"); + assert_eq!(result, RFC_EXPECTED, "POLYVAL mismatch"); } /// POLYVAL with empty input should return zero. @@ -2436,26 +2357,26 @@ mod tests { #[cfg(feature = "aes-gcm-siv")] #[test] fn polyval_padded_matches_manual() { - let h = hex_to_16("25629347589242761d31f826ba4b757b"); let data = b"Hello, World! This is test data for POLYVAL padding."; // Manual: split into 16-byte blocks, pad last one. - let mut manual = Polyval::new(&h); - let mut offset = 0; - while offset + 16 <= data.len() { - let block: [u8; 16] = data[offset..offset + 16].try_into().unwrap(); + let mut manual = Polyval::new(&RFC_H); + let mut chunks = data.chunks_exact(BLOCK_SIZE); + for chunk in chunks.by_ref() { + let mut block = [0u8; BLOCK_SIZE]; + block.copy_from_slice(chunk); manual.update_block(&block); - offset += 16; } - if offset < data.len() { - let mut block = [0u8; 16]; - block[..data.len() - offset].copy_from_slice(&data[offset..]); + let remainder = chunks.remainder(); + if !remainder.is_empty() { + let mut block = [0u8; BLOCK_SIZE]; + block[..remainder.len()].copy_from_slice(remainder); manual.update_block(&block); } let manual_result = manual.finalize(); // Padded API. - let mut padded = Polyval::new(&h); + let mut padded = Polyval::new(&RFC_H); padded.update_padded(data); let padded_result = padded.finalize(); @@ -2498,8 +2419,9 @@ mod tests { fn clmul128_by_one() { let val: u128 = 0x7b75_4bba_26f8_311d_7642_9258_4793_6225; let v = clmul128(1, val); - assert_eq!(v[0], val as u64, "v0 should be val_lo"); - assert_eq!(v[1], (val >> 64) as u64, "v1 should be val_hi"); + let (val_lo, val_hi) = split_u128_le(val); + assert_eq!(v[0], val_lo, "v0 should be val_lo"); + assert_eq!(v[1], val_hi, "v1 should be val_hi"); assert_eq!(v[2], 0, "v2 should be 0"); assert_eq!(v[3], 0, "v3 should be 0"); } @@ -2508,7 +2430,8 @@ mod tests { /// in the low half gives x^128 * x^{-128} = 1. #[test] fn mont_reduce_of_poly() { - let v = [POLY as u64, (POLY >> 64) as u64, 0u64, 0u64]; + let (poly_lo, poly_hi) = split_u128_le(POLY); + let v = [poly_lo, poly_hi, 0u64, 0u64]; let result = mont_reduce(v); assert_eq!(result, 1, "mont_reduce(POLY) should be 1"); } @@ -2516,7 +2439,7 @@ mod tests { /// Verify precompute_powers produces correct powers of H. #[test] fn precompute_powers_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers(h); assert_eq!(powers[0], h, "powers[0] should be H"); assert_eq!(powers[1], clmul128_reduce(h, h), "powers[1] should be H^2"); @@ -2527,7 +2450,7 @@ mod tests { /// Verify precompute_powers_16 extends the same power chain to H^16. #[test] fn precompute_powers_16_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_16(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2545,7 +2468,7 @@ mod tests { /// Verify precompute_powers_32 extends the same power chain to H^32. #[test] fn precompute_powers_32_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_32(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2563,7 +2486,7 @@ mod tests { /// Verify precompute_powers_64 extends the same power chain to H^64. #[test] fn precompute_powers_64_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_64(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2581,7 +2504,7 @@ mod tests { /// Verify precompute_powers_128 extends the same power chain to H^128. #[test] fn precompute_powers_128_correct() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_128(h); assert_eq!(powers[0], h, "powers[0] should be H"); @@ -2599,16 +2522,15 @@ mod tests { /// Verify accumulate_4blocks matches sequential block-by-block processing. #[test] fn accumulate_4blocks_matches_sequential() { - let h_bytes = hex_to_16("25629347589242761d31f826ba4b757b"); - let h = u128::from_le_bytes(h_bytes); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers(h); let h_powers_rev = [powers[3], powers[2], powers[1], powers[0]]; let blocks = [ - u128::from_le_bytes(hex_to_16("4f4f95668c83dfb6401762bb2d01a262")), - u128::from_le_bytes(hex_to_16("d1a24ddd2721d006bbe45f20d3c9f362")), - u128::from_le_bytes(hex_to_16("0100000000000000000000000000000f")), - u128::from_le_bytes(hex_to_16("abcdef0123456789abcdef0123456789")), + u128::from_le_bytes(RFC_X1), + u128::from_le_bytes(RFC_X2), + u128::from_le_bytes(AGGREGATE_X3), + u128::from_le_bytes(AGGREGATE_X4), ]; let acc = 0x42u128; @@ -2631,15 +2553,18 @@ mod tests { /// Verify accumulate_16blocks matches sequential block-by-block processing. #[test] fn accumulate_16blocks_matches_sequential() { - let h_bytes = hex_to_16("25629347589242761d31f826ba4b757b"); - let h = u128::from_le_bytes(h_bytes); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_16(h); let h_powers_rev = core::array::from_fn(|i| powers[15usize.strict_sub(i)]); - let blocks = core::array::from_fn(|i| { - let lane = (i as u128).wrapping_add(1); - 0x4f4f_9566_8c83_dfb6_4017_62bb_2d01_a262u128.wrapping_mul(lane) - ^ 0xd1a2_4ddd_2721_d006_bbe4_5f20_d3c9_f362u128.rotate_left(i as u32) - }); + let mut blocks = [0u128; 16]; + let mut lane = 1u128; + let mut rotation = 0u32; + for block in &mut blocks { + *block = 0x4f4f_9566_8c83_dfb6_4017_62bb_2d01_a262u128.wrapping_mul(lane) + ^ 0xd1a2_4ddd_2721_d006_bbe4_5f20_d3c9_f362u128.rotate_left(rotation); + lane = lane.wrapping_add(1); + rotation = rotation.strict_add(1); + } let acc = 0x42u128; let mut seq = acc; @@ -2654,76 +2579,11 @@ mod tests { assert_eq!(wide, seq, "16-block aggregate must match sequential processing"); } - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[target_feature(enable = "avx2,avx512f,avx512vl,vpclmulqdq,pclmulqdq,sse2")] - /// # Safety - /// - /// Caller must ensure AVX2 + AVX-512F + AVX-512VL + VPCLMULQDQ + - /// PCLMULQDQ + SSE2 are available. - unsafe fn x86_aggregate_8blocks_be_lanes_256_test_call( - acc: u128, - h_powers_rev: &[u128; 8], - bytes: &[u8; 128], - ) -> u128 { - use core::arch::x86_64::*; - - // SAFETY: test-only x86 lane aggregation because: - // 1. The caller verified the CPU features required by this target-feature helper. - // 2. `bytes` is exactly 128 initialized bytes, so all four 32-byte loads are in bounds. - // 3. The loaded lanes are passed directly to the helper under test. - unsafe { - let raw0 = _mm256_loadu_si256(bytes.as_ptr().cast()); - let raw1 = _mm256_loadu_si256(bytes.as_ptr().add(32).cast()); - let raw2 = _mm256_loadu_si256(bytes.as_ptr().add(64).cast()); - let raw3 = _mm256_loadu_si256(bytes.as_ptr().add(96).cast()); - x86_aggregate_8blocks_be_lanes_256_inline(acc, h_powers_rev, raw0, raw1, raw2, raw3) - } - } - - #[cfg(all(target_arch = "x86_64", feature = "aes-gcm"))] - #[test] - fn x86_aggregate_8blocks_be_lanes_256_matches_sequential() { - let required = crate::platform::caps::x86::VPCLMUL_READY | crate::platform::caps::x86::AVX2; - if !crate::platform::caps().has(required) { - return; - } - - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); - let powers = precompute_powers_8(h); - let h_powers_rev = core::array::from_fn(|i| powers[7usize.strict_sub(i)]); - let acc = 0x1122_3344_5566_7788_99aa_bbcc_ddee_ff00u128; - - let mut bytes = [0u8; 128]; - let mut i = 0usize; - while i < bytes.len() { - bytes[i] = i.wrapping_mul(37).wrapping_add(19) as u8; - i = i.strict_add(1); - } - - let mut expected = acc; - let mut offset = 0usize; - while offset < bytes.len() { - let mut block = [0u8; 16]; - block.copy_from_slice(&bytes[offset..offset.strict_add(16)]); - expected ^= u128::from_be_bytes(block); - expected = clmul128_reduce(expected, h); - offset = offset.strict_add(16); - } - - // SAFETY: Runtime caps above confirmed AVX2 + VPCLMUL_READY before calling the target-feature - // helper. The byte array and H-power table are fully initialized. - let wide = unsafe { x86_aggregate_8blocks_be_lanes_256_test_call(acc, &h_powers_rev, &bytes) }; - assert_eq!( - wide, expected, - "8-block 256-bit VPCLMUL GHASH aggregate must match sequential fold" - ); - } - /// Verify the x86 padded wide path matches scalar POLYVAL over boundary sizes. #[cfg(all(target_arch = "x86_64", feature = "aes-gcm-siv"))] #[test] fn accumulate_padded_x86_matches_sequential_boundaries() { - let h = u128::from_le_bytes(hex_to_16("25629347589242761d31f826ba4b757b")); + let h = u128::from_le_bytes(RFC_H); let powers = precompute_powers_16(h); let h_powers_rev = [powers[3], powers[2], powers[1], powers[0]]; let h_powers_rev_16 = core::array::from_fn(|i| powers[15usize.strict_sub(i)]); @@ -2736,7 +2596,7 @@ mod tests { let mut data = [0u8; 321]; let mut i = 0usize; while i < len { - data[i] = i.wrapping_mul(37).wrapping_add(19) as u8; + data[i] = i.to_le_bytes()[0].wrapping_mul(37).wrapping_add(19); i = i.strict_add(1); } @@ -2771,14 +2631,4 @@ mod tests { let result = accumulate_4blocks(0, h, &h_powers_rev, &blocks); assert_eq!(result, 0); } - - fn hex_to_16(hex: &str) -> [u8; 16] { - let mut out = [0u8; 16]; - let mut i = 0; - while i < 16 { - out[i] = u8::from_str_radix(&hex[2 * i..2 * i + 2], 16).unwrap(); - i = i.strict_add(1); - } - out - } } diff --git a/src/aead/targets.rs b/src/aead/targets.rs index 709e29d9..cf3e2c85 100644 --- a/src/aead/targets.rs +++ b/src/aead/targets.rs @@ -5,86 +5,240 @@ //! //! Backend selection is derived from detected CPU capabilities. +#[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] +use crate::platform::caps::wasm; use crate::platform::{ Arch, Caps, - caps::{aarch64, power, riscv, s390x, wasm, x86}, + caps::{aarch64, power, riscv, s390x, x86}, }; /// AEAD primitives on the public surface. -#[allow(dead_code)] // Reduced-feature test builds can compile only byte wrappers, not live dispatch. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] -pub enum AeadPrimitive { +pub(super) enum AeadPrimitive { + #[cfg(any(test, feature = "xchacha20poly1305"))] XChaCha20Poly1305, + #[cfg(any(test, feature = "chacha20poly1305"))] ChaCha20Poly1305, + #[cfg(any( + test, + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aes256GcmSiv, + #[cfg(any(test, feature = "aes-gcm"))] Aes256Gcm, + #[cfg(any(test, feature = "aes-gcm"))] Aes128Gcm, + #[cfg(any( + test, + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aes128GcmSiv, - AsconAead128, + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aegis256, } /// Backend classes selected by live dispatch. -#[allow(dead_code)] // Some architecture-specific variants are only constructed on their target. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] -pub enum AeadBackend { +pub(super) enum AeadBackend { Portable, WasmPortable, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] WasmSimd128, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] X86Avx2, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] X86Avx512, + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] X86Aesni, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] X86AesniPclmul, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] X86VaesVpclmul, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Aarch64Neon, + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Aarch64Aes, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Aarch64AesPmull, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Aarch64Sve2AesPmull, + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] S390xMsa, /// Hamburg vperm AES rounds for AEGIS using register-only z/Vector VPERM. /// Used on s390x z13+ where no single-round AES instruction exists. + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] S390xVperm, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] S390xVector, + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Power8Crypto, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] PowerVector, + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Riscv64ScalarCrypto, + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Riscv64VectorCrypto, + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Riscv64Vector, - /// Hamburg vperm AES via register-only `vrgather.vv` operations. - /// Kept as an explicit backend, but not selected for V-only RISC-V without - /// target-native evidence supporting that dispatch policy. - Riscv64Vperm, } impl AeadBackend { /// Stable backend label for diagnostics and future benchmark grouping. - #[allow(dead_code)] // Used by `diag`; reduced-feature builds can compile dispatch without introspection. + #[cfg(feature = "diag")] #[must_use] - pub const fn name(self) -> &'static str { + pub(super) const fn name(self) -> &'static str { match self { Self::Portable => "portable", Self::WasmPortable => "wasm32/portable", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::WasmSimd128 => "wasm32/simd128", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::X86Avx2 => "x86_64/avx2", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::X86Avx512 => "x86_64/avx512", + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Self::X86Aesni => "x86_64/aesni", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::X86AesniPclmul => "x86_64/aesni+pclmul", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::X86VaesVpclmul => "x86_64/vaes+vpclmul", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::Aarch64Neon => "aarch64/neon", + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Self::Aarch64Aes => "aarch64/aes", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Aarch64AesPmull => "aarch64/aes+pmull", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Aarch64Sve2AesPmull => "aarch64/sve2+aes+pmull", + #[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::S390xMsa => "s390x/msa", + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] Self::S390xVperm => "s390x/vperm", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::S390xVector => "s390x/vector", + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Power8Crypto => "powerpc64/crypto", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::PowerVector => "powerpc64/vector", + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Riscv64ScalarCrypto => "riscv64/scalar-crypto", + #[cfg(any(test, feature = "aegis256", feature = "aes-gcm", feature = "aes-gcm-siv"))] Self::Riscv64VectorCrypto => "riscv64/vector-crypto", + #[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] Self::Riscv64Vector => "riscv64/vector", - Self::Riscv64Vperm => "riscv64/vperm", } } } @@ -93,21 +247,50 @@ impl AeadBackend { /// and current dispatch policy. /// /// SIMD classes without accepted target-native evidence resolve to `portable`. -#[allow(dead_code)] // Some leaf AEAD builds compile target policy without live dispatch on the host. #[must_use] -pub fn select_backend(primitive: AeadPrimitive, arch: Arch, caps: Caps) -> AeadBackend { +pub(super) fn select_backend(primitive: AeadPrimitive, arch: Arch, caps: Caps) -> AeadBackend { match primitive { - AeadPrimitive::XChaCha20Poly1305 | AeadPrimitive::ChaCha20Poly1305 => select_chacha_backend(arch, caps), - AeadPrimitive::Aes256GcmSiv | AeadPrimitive::Aes256Gcm | AeadPrimitive::Aes128Gcm | AeadPrimitive::Aes128GcmSiv => { - select_gcm_backend(arch, caps) - } - AeadPrimitive::AsconAead128 => select_ascon_backend(arch), + #[cfg(any(test, feature = "xchacha20poly1305"))] + AeadPrimitive::XChaCha20Poly1305 => select_chacha_backend(arch, caps), + #[cfg(any(test, feature = "chacha20poly1305"))] + AeadPrimitive::ChaCha20Poly1305 => select_chacha_backend(arch, caps), + #[cfg(any( + test, + all( + feature = "aes-gcm-siv", + any( + feature = "diag", + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] + AeadPrimitive::Aes256GcmSiv | AeadPrimitive::Aes128GcmSiv => select_gcm_backend(arch, caps), + #[cfg(any(test, feature = "aes-gcm"))] + AeadPrimitive::Aes256Gcm | AeadPrimitive::Aes128Gcm => select_gcm_backend(arch, caps), + #[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) + ))] AeadPrimitive::Aegis256 => select_aegis_backend(arch, caps), } } +#[cfg(any(test, feature = "chacha20poly1305", feature = "xchacha20poly1305"))] #[inline] -#[allow(dead_code)] // Only used when a ChaCha20-Poly1305 feature is enabled. fn select_chacha_backend(arch: Arch, caps: Caps) -> AeadBackend { match arch { Arch::X86_64 => { @@ -158,8 +341,8 @@ fn select_chacha_backend(arch: Arch, caps: Caps) -> AeadBackend { } } +#[cfg(any(test, feature = "aes-gcm", feature = "aes-gcm-siv"))] #[inline] -#[allow(dead_code)] // Only used when an AES-GCM-family feature needs live dispatch on this target. fn select_gcm_backend(arch: Arch, caps: Caps) -> AeadBackend { match arch { Arch::X86_64 => { @@ -214,24 +397,28 @@ fn select_gcm_backend(arch: Arch, caps: Caps) -> AeadBackend { } } +#[cfg(any( + test, + all( + feature = "aegis256", + any( + feature = "diag", + target_arch = "aarch64", + all(target_arch = "powerpc64", target_endian = "little"), + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64", + ) + ) +))] #[inline] -#[allow(dead_code)] // Only used by Ascon-AEAD dispatch diagnostics. -const fn select_ascon_backend(arch: Arch) -> AeadBackend { - match arch { - Arch::Wasm32 | Arch::Wasm64 => AeadBackend::WasmPortable, - _ => AeadBackend::Portable, - } -} - -#[inline] -#[allow(dead_code)] // Only used when AEGIS-256 needs target-policy dispatch on this target. fn select_aegis_backend(arch: Arch, caps: Caps) -> AeadBackend { // VAES-256 is intentionally not used for AEGIS-256. Its six dependent state // lanes require cross-lane shuffles in the packed representation; see the // XMM-state path in aegis256.rs. match arch { Arch::X86_64 => { - if caps.has(x86::AESNI) { + if caps.has(x86::AESNI) && caps.has(x86::AVX) { AeadBackend::X86Aesni } else { AeadBackend::Portable @@ -382,6 +569,10 @@ mod tests { ); assert_eq!( select_backend(AeadPrimitive::Aegis256, Arch::X86_64, x86::AESNI), + AeadBackend::Portable + ); + assert_eq!( + select_backend(AeadPrimitive::Aegis256, Arch::X86_64, x86::AESNI | x86::AVX), AeadBackend::X86Aesni ); assert_eq!( @@ -562,16 +753,4 @@ mod tests { AeadBackend::WasmPortable ); } - - #[test] - fn ascon_stays_portable_until_simd_policy_is_accepted() { - assert_eq!( - select_backend(AeadPrimitive::AsconAead128, Arch::X86_64, x86::AVX2 | x86::VAES_READY), - AeadBackend::Portable - ); - assert_eq!( - select_backend(AeadPrimitive::AsconAead128, Arch::Wasm32, Caps::NONE), - AeadBackend::WasmPortable - ); - } } diff --git a/src/aead/xchacha20poly1305.rs b/src/aead/xchacha20poly1305.rs index 8e1cd870..8922a552 100644 --- a/src/aead/xchacha20poly1305.rs +++ b/src/aead/xchacha20poly1305.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] - //! XChaCha20-Poly1305 public AEAD surface. use core::fmt; @@ -220,8 +218,12 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"hello xchacha"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); - cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &tag).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("XChaCha20-Poly1305 encryption must succeed"); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buf, &tag) + .expect("XChaCha20-Poly1305 decryption must succeed"); assert_eq!(&buf, b"hello xchacha"); } @@ -232,11 +234,15 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"nonce test"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("XChaCha20-Poly1305 test setup encryption must succeed"); let wrong_nonce = Nonce192::from_bytes([0x08u8; 24]); - let result = cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&wrong_nonce, b"aad", &mut buf, &tag), + Err(OpenError::verification()) + ); } #[test] @@ -246,14 +252,18 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"zero me on failure"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buf) + .expect("XChaCha20-Poly1305 test setup encryption must succeed"); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0xFF; let bad_tag = XChaCha20Poly1305Tag::from_bytes(bad_tag); - let result = cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"aad", &mut buf, &bad_tag), + Err(OpenError::verification()) + ); assert!(buf.iter().all(|&b| b == 0), "buffer not zeroed on auth failure"); } @@ -264,9 +274,13 @@ mod tests { let cipher = XChaCha20Poly1305::new(&key); let mut buf = *b"aad test"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buf).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buf) + .expect("XChaCha20-Poly1305 test setup encryption must succeed"); - let result = cipher.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag); - assert!(result.is_err()); + assert_eq!( + cipher.decrypt_in_place(&nonce, b"wrong", &mut buf, &tag), + Err(OpenError::verification()) + ); } } diff --git a/src/auth/argon2/aarch64.rs b/src/auth/argon2/aarch64.rs index d0ad0a7a..442c5d15 100644 --- a/src/auth/argon2/aarch64.rs +++ b/src/auth/argon2/aarch64.rs @@ -43,6 +43,17 @@ use core::arch::aarch64::{ use super::BLOCK_WORDS; +struct NeonState { + a_lo: uint64x2_t, + a_hi: uint64x2_t, + b_lo: uint64x2_t, + b_hi: uint64x2_t, + c_lo: uint64x2_t, + c_hi: uint64x2_t, + d_lo: uint64x2_t, + d_hi: uint64x2_t, +} + /// NEON BlaMka compression kernel. /// /// # Safety @@ -63,41 +74,41 @@ pub(super) unsafe fn compress_neon( // value; uint64x2_t has no invalid bit patterns. let mut r = unsafe { [core::mem::zeroed::(); 64] }; let mut q = r; - for i in 0..64 { + for i in 0usize..64 { + let offset = i.strict_mul(2); // SAFETY: x and y are [u64; BLOCK_WORDS] with BLOCK_WORDS == 128; the // loop reads 2 u64 per iteration at offset `2 * i` for i ∈ 0..64, so // the last read is at words 126..=127. NEON inherited from the fn // target_feature. - let (xv, yv) = unsafe { (vld1q_u64(x.as_ptr().add(2 * i)), vld1q_u64(y.as_ptr().add(2 * i))) }; + let (xv, yv) = unsafe { (vld1q_u64(x.as_ptr().add(offset)), vld1q_u64(y.as_ptr().add(offset))) }; r[i] = veorq_u64(xv, yv); q[i] = r[i]; } // Row pass: 8 rows × 16 u64 per row = 8 P-rounds on contiguous blocks // of 8 uint64x2_t. - for row in 0..8 { - let base = row * 8; - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = ( - q[base], - q[base + 1], - q[base + 2], - q[base + 3], - q[base + 4], - q[base + 5], - q[base + 6], - q[base + 7], - ); + for row in 0usize..8 { + let base = row.strict_mul(8); + let mut state = NeonState { + a_lo: q[base], + a_hi: q[base.strict_add(1)], + b_lo: q[base.strict_add(2)], + b_hi: q[base.strict_add(3)], + c_lo: q[base.strict_add(4)], + c_hi: q[base.strict_add(5)], + d_lo: q[base.strict_add(6)], + d_hi: q[base.strict_add(7)], + }; // SAFETY: NEON inherited from outer target_feature. - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = - unsafe { p_round_neon(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) }; - q[base] = a_lo; - q[base + 1] = a_hi; - q[base + 2] = b_lo; - q[base + 3] = b_hi; - q[base + 4] = c_lo; - q[base + 5] = c_hi; - q[base + 6] = d_lo; - q[base + 7] = d_hi; + unsafe { p_round_neon(&mut state) }; + q[base] = state.a_lo; + q[base.strict_add(1)] = state.a_hi; + q[base.strict_add(2)] = state.b_lo; + q[base.strict_add(3)] = state.b_hi; + q[base.strict_add(4)] = state.c_lo; + q[base.strict_add(5)] = state.c_hi; + q[base.strict_add(6)] = state.d_lo; + q[base.strict_add(7)] = state.d_hi; } // Column pass: each P-round reads 16 u64s at stride-16 positions @@ -105,41 +116,41 @@ pub(super) unsafe fn compress_neon( // q[col*2+113]`). In uint64x2_t indexing each "col*2" pair corresponds // to a single uint64x2_t at index `col`, and the stride-16 becomes // stride 8 in uint64x2_t space. - for col in 0..8 { - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = ( - q[col], - q[col + 8], - q[col + 16], - q[col + 24], - q[col + 32], - q[col + 40], - q[col + 48], - q[col + 56], - ); + for col in 0usize..8 { + let mut state = NeonState { + a_lo: q[col], + a_hi: q[col.strict_add(8)], + b_lo: q[col.strict_add(16)], + b_hi: q[col.strict_add(24)], + c_lo: q[col.strict_add(32)], + c_hi: q[col.strict_add(40)], + d_lo: q[col.strict_add(48)], + d_hi: q[col.strict_add(56)], + }; // SAFETY: NEON inherited from outer target_feature. - let (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) = - unsafe { p_round_neon(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) }; - q[col] = a_lo; - q[col + 8] = a_hi; - q[col + 16] = b_lo; - q[col + 24] = b_hi; - q[col + 32] = c_lo; - q[col + 40] = c_hi; - q[col + 48] = d_lo; - q[col + 56] = d_hi; + unsafe { p_round_neon(&mut state) }; + q[col] = state.a_lo; + q[col.strict_add(8)] = state.a_hi; + q[col.strict_add(16)] = state.b_lo; + q[col.strict_add(24)] = state.b_hi; + q[col.strict_add(32)] = state.c_lo; + q[col.strict_add(40)] = state.c_hi; + q[col.strict_add(48)] = state.d_lo; + q[col.strict_add(56)] = state.d_hi; } // Final XOR with R, fused with the dst store. - for i in 0..64 { + for i in 0usize..64 { let final_v = veorq_u64(q[i], r[i]); // SAFETY: dst[2*i..2*i+2] is within BLOCK_WORDS for i ∈ 0..64. NEON // inherited from the fn target_feature. unsafe { + let offset = i.strict_mul(2); if xor_into { - let cur = vld1q_u64(dst.as_ptr().add(2 * i)); - vst1q_u64(dst.as_mut_ptr().add(2 * i), veorq_u64(cur, final_v)); + let cur = vld1q_u64(dst.as_ptr().add(offset)); + vst1q_u64(dst.as_mut_ptr().add(offset), veorq_u64(cur, final_v)); } else { - vst1q_u64(dst.as_mut_ptr().add(2 * i), final_v); + vst1q_u64(dst.as_mut_ptr().add(offset), final_v); } } } @@ -157,83 +168,48 @@ pub(super) unsafe fn compress_neon( /// aarch64 so any aarch64 build satisfies this, but the `unsafe fn` /// signature preserves the contract for completeness. #[inline(always)] -#[allow(clippy::too_many_arguments)] -unsafe fn p_round_neon( - mut a_lo: uint64x2_t, - mut a_hi: uint64x2_t, - mut b_lo: uint64x2_t, - mut b_hi: uint64x2_t, - mut c_lo: uint64x2_t, - mut c_hi: uint64x2_t, - mut d_lo: uint64x2_t, - mut d_hi: uint64x2_t, -) -> ( - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, - uint64x2_t, -) { +unsafe fn p_round_neon(state: &mut NeonState) { // Column step — 4 parallel GBs. // SAFETY: NEON precondition inherited from caller. - unsafe { - gb_neon( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - } + unsafe { gb_neon(state) }; // Diagonalise: rotate b by 1, c by 2, d by 3 across the 4-lane row. // SAFETY: vextq_u64 only operates on register values, no memory access. unsafe { - let b_lo2 = vextq_u64::<1>(b_lo, b_hi); - let b_hi2 = vextq_u64::<1>(b_hi, b_lo); - b_lo = b_lo2; - b_hi = b_hi2; + let b_lo = vextq_u64::<1>(state.b_lo, state.b_hi); + let b_hi = vextq_u64::<1>(state.b_hi, state.b_lo); + state.b_lo = b_lo; + state.b_hi = b_hi; // c: rotate by 2 == swap lo/hi. - let c_lo2 = c_hi; - let c_hi2 = c_lo; - c_lo = c_lo2; - c_hi = c_hi2; + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let d_lo2 = vextq_u64::<1>(d_hi, d_lo); - let d_hi2 = vextq_u64::<1>(d_lo, d_hi); - d_lo = d_lo2; - d_hi = d_hi2; + let d_lo = vextq_u64::<1>(state.d_hi, state.d_lo); + let d_hi = vextq_u64::<1>(state.d_lo, state.d_hi); + state.d_lo = d_lo; + state.d_hi = d_hi; } // Diagonal step — 4 parallel GBs on the rotated state. // SAFETY: NEON precondition inherited from caller. - unsafe { - gb_neon( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - } + unsafe { gb_neon(state) }; // Undo diagonalisation: rotate b by -1, c by -2, d by -3. // SAFETY: vextq_u64 only operates on register values. unsafe { - let b_lo2 = vextq_u64::<1>(b_hi, b_lo); - let b_hi2 = vextq_u64::<1>(b_lo, b_hi); - b_lo = b_lo2; - b_hi = b_hi2; + let b_lo = vextq_u64::<1>(state.b_hi, state.b_lo); + let b_hi = vextq_u64::<1>(state.b_lo, state.b_hi); + state.b_lo = b_lo; + state.b_hi = b_hi; // c: undo swap. - let c_lo2 = c_hi; - let c_hi2 = c_lo; - c_lo = c_lo2; - c_hi = c_hi2; + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let d_lo2 = vextq_u64::<1>(d_lo, d_hi); - let d_hi2 = vextq_u64::<1>(d_hi, d_lo); - d_lo = d_lo2; - d_hi = d_hi2; + let d_lo = vextq_u64::<1>(state.d_lo, state.d_hi); + let d_hi = vextq_u64::<1>(state.d_hi, state.d_lo); + state.d_lo = d_lo; + state.d_hi = d_hi; } - - (a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi) } // ─── 4-way parallel BlaMka G ─────────────────────────────────────────────── @@ -246,54 +222,44 @@ unsafe fn p_round_neon( /// Must be called from a context with NEON enabled (inherited from the /// outer `#[target_feature(enable = "neon")]` entry point). #[inline(always)] -#[allow(clippy::too_many_arguments)] -unsafe fn gb_neon( - a_lo: &mut uint64x2_t, - a_hi: &mut uint64x2_t, - b_lo: &mut uint64x2_t, - b_hi: &mut uint64x2_t, - c_lo: &mut uint64x2_t, - c_hi: &mut uint64x2_t, - d_lo: &mut uint64x2_t, - d_hi: &mut uint64x2_t, -) { +unsafe fn gb_neon(state: &mut NeonState) { // SAFETY: NEON precondition inherited; all ops below are register-only. unsafe { // Step 1: a ← a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vaddq_u64(vaddq_u64(*a_lo, *b_lo), p_lo); - *a_hi = vaddq_u64(vaddq_u64(*a_hi, *b_hi), p_hi); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vaddq_u64(vaddq_u64(state.a_lo, state.b_lo), p_lo); + state.a_hi = vaddq_u64(vaddq_u64(state.a_hi, state.b_hi), p_hi); // d ← (d ^ a) ROR 32 - *d_lo = ror32(veorq_u64(*d_lo, *a_lo)); - *d_hi = ror32(veorq_u64(*d_hi, *a_hi)); + state.d_lo = ror32(veorq_u64(state.d_lo, state.a_lo)); + state.d_hi = ror32(veorq_u64(state.d_hi, state.a_hi)); // Step 2: c ← c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vaddq_u64(vaddq_u64(*c_lo, *d_lo), p_lo); - *c_hi = vaddq_u64(vaddq_u64(*c_hi, *d_hi), p_hi); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vaddq_u64(vaddq_u64(state.c_lo, state.d_lo), p_lo); + state.c_hi = vaddq_u64(vaddq_u64(state.c_hi, state.d_hi), p_hi); // b ← (b ^ c) ROR 24 - *b_lo = ror24(veorq_u64(*b_lo, *c_lo)); - *b_hi = ror24(veorq_u64(*b_hi, *c_hi)); + state.b_lo = ror24(veorq_u64(state.b_lo, state.c_lo)); + state.b_hi = ror24(veorq_u64(state.b_hi, state.c_hi)); // Step 3: a ← a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vaddq_u64(vaddq_u64(*a_lo, *b_lo), p_lo); - *a_hi = vaddq_u64(vaddq_u64(*a_hi, *b_hi), p_hi); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vaddq_u64(vaddq_u64(state.a_lo, state.b_lo), p_lo); + state.a_hi = vaddq_u64(vaddq_u64(state.a_hi, state.b_hi), p_hi); // d ← (d ^ a) ROR 16 - *d_lo = ror16(veorq_u64(*d_lo, *a_lo)); - *d_hi = ror16(veorq_u64(*d_hi, *a_hi)); + state.d_lo = ror16(veorq_u64(state.d_lo, state.a_lo)); + state.d_hi = ror16(veorq_u64(state.d_hi, state.a_hi)); // Step 4: c ← c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vaddq_u64(vaddq_u64(*c_lo, *d_lo), p_lo); - *c_hi = vaddq_u64(vaddq_u64(*c_hi, *d_hi), p_hi); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vaddq_u64(vaddq_u64(state.c_lo, state.d_lo), p_lo); + state.c_hi = vaddq_u64(vaddq_u64(state.c_hi, state.d_hi), p_hi); // b ← (b ^ c) ROR 63 ≡ ROL 1 - *b_lo = ror63(veorq_u64(*b_lo, *c_lo)); - *b_hi = ror63(veorq_u64(*b_hi, *c_hi)); + state.b_lo = ror63(veorq_u64(state.b_lo, state.c_lo)); + state.b_hi = ror63(veorq_u64(state.b_hi, state.c_hi)); } } diff --git a/src/auth/argon2/dispatch.rs b/src/auth/argon2/dispatch.rs index 1df34ae5..8f5cf6bc 100644 --- a/src/auth/argon2/dispatch.rs +++ b/src/auth/argon2/dispatch.rs @@ -221,7 +221,12 @@ mod tests { #[test] fn all_kernels_terminate_on_portable() { - assert_eq!(*ALL_KERNELS.last().unwrap(), KernelId::Portable); + assert_eq!( + *ALL_KERNELS + .last() + .expect("Argon2 kernel table must contain the portable fallback"), + KernelId::Portable + ); } #[test] diff --git a/src/auth/argon2/kernels.rs b/src/auth/argon2/kernels.rs index e2a90811..b4d19a71 100644 --- a/src/auth/argon2/kernels.rs +++ b/src/auth/argon2/kernels.rs @@ -9,7 +9,6 @@ //! in sibling files (`aarch64.rs`, `x86_64.rs`, …) and are gated by //! `#[cfg(target_arch = ...)]`. The runtime dispatcher is [`super::dispatch`]. -#![allow(clippy::indexing_slicing)] // The portable kernel body is generic over the block word count (128) — // every index is bounds-proven by the fixed `[u64; BLOCK_WORDS]` shape. @@ -97,7 +96,7 @@ pub(super) unsafe fn compress_portable( let mut q = r; // Row pass: apply P to each 16-word row (8 rows of 16 u64s). - for chunk in q.chunks_exact_mut(P_LANE_WORDS) { + for chunk in q.as_chunks_mut::().0 { p_direct!( chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], chunk[12], chunk[13], chunk[14], chunk[15], @@ -107,24 +106,24 @@ pub(super) unsafe fn compress_portable( // Column pass: each 16-word column is two u64s per row at the same // register index. for col in 0usize..8 { - let base = col * 2; + let base = col.strict_mul(2); p_direct!( q[base], - q[base + 1], - q[base + 16], - q[base + 17], - q[base + 32], - q[base + 33], - q[base + 48], - q[base + 49], - q[base + 64], - q[base + 65], - q[base + 80], - q[base + 81], - q[base + 96], - q[base + 97], - q[base + 112], - q[base + 113], + q[base.strict_add(1)], + q[base.strict_add(16)], + q[base.strict_add(17)], + q[base.strict_add(32)], + q[base.strict_add(33)], + q[base.strict_add(48)], + q[base.strict_add(49)], + q[base.strict_add(64)], + q[base.strict_add(65)], + q[base.strict_add(80)], + q[base.strict_add(81)], + q[base.strict_add(96)], + q[base.strict_add(97)], + q[base.strict_add(112)], + q[base.strict_add(113)], ); } diff --git a/src/auth/argon2/mod.rs b/src/auth/argon2/mod.rs index d7a79da5..bcc65eea 100644 --- a/src/auth/argon2/mod.rs +++ b/src/auth/argon2/mod.rs @@ -56,8 +56,6 @@ //! //! [owasp-passwords]: https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html -#![allow(clippy::indexing_slicing)] -#![allow(clippy::unwrap_used)] // unwraps here are on slice→array conversions whose lengths are fixed by construction. use alloc::vec::Vec; @@ -141,15 +139,9 @@ impl MemoryBlock { /// repeated byte-slice transmutes. #[inline] fn zeroize_u64_slice_no_fence(words: &mut [u64]) { - let mut chunks = words.chunks_exact_mut(8); - for chunk in &mut chunks { - // SAFETY: chunk has exactly 8 initialized u64 values and [u64; 8] has - // the same alignment requirement as u64. - unsafe { core::ptr::write_volatile(chunk.as_mut_ptr().cast::<[u64; 8]>(), [0u64; 8]) }; - } - for w in chunks.into_remainder() { - // SAFETY: w is a valid, aligned, dereferenceable pointer to initialized u64. - unsafe { core::ptr::write_volatile(w, 0) }; + for word in words { + // SAFETY: `word` is a valid, aligned pointer to an initialized `u64` and remains exclusively borrowed. + unsafe { core::ptr::write_volatile(word, 0) }; } } @@ -171,8 +163,11 @@ fn zeroize_u64_slice(words: &mut [u64]) { /// (RFC 9106 §3.4.1). #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Argon2Variant { + /// Argon2d data-dependent addressing. Argon2d, + /// Argon2i data-independent addressing. Argon2i, + /// Argon2id hybrid addressing. Argon2id, } @@ -223,6 +218,9 @@ pub enum Argon2Error { ResourceOverflow, /// The allocator refused to provide the memory matrix. AllocationFailed, + /// A forced diagnostic backend is unavailable on the current host. + #[cfg(feature = "diag")] + BackendUnavailable, /// Password generation parameters exceed the verifier's resource limits. #[cfg(feature = "phc-strings")] VerificationLimitTooLow, @@ -242,6 +240,8 @@ impl fmt::Display for Argon2Error { Self::AssociatedDataTooLong => "Argon2 associated data exceeds 2^32-1 bytes", Self::ResourceOverflow => "Argon2 memory matrix exceeds the target's address space", Self::AllocationFailed => "Argon2 memory-matrix allocation failed", + #[cfg(feature = "diag")] + Self::BackendUnavailable => "requested Argon2 diagnostic backend is unavailable", #[cfg(all(feature = "phc-strings", feature = "getrandom"))] Self::EntropyUnavailable => "Argon2 entropy source unavailable", #[cfg(feature = "phc-strings")] @@ -392,7 +392,11 @@ struct Argon2Shape { const fn argon2_shape(params: Argon2Params) -> Argon2Shape { let lane_group = params.parallelism.strict_mul(SYNC_POINTS); - let blocks = (params.memory_cost_kib / lane_group).strict_mul(lane_group); + let blocks = params + .memory_cost_kib + .checked_div(lane_group) + .expect("validated Argon2 parallelism makes the lane group nonzero") + .strict_mul(lane_group); Argon2Shape { blocks, memory_bytes: (blocks as u64).strict_mul(BLOCK_SIZE as u64), @@ -488,6 +492,24 @@ pub fn diag_hash_portable( ) } +#[cfg(all( + feature = "diag", + any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64", + target_arch = "wasm32", + ) +))] +fn diag_compress_for(kernel: KernelId) -> Result { + if crate::platform::caps().has(dispatch::required_caps(kernel)) { + Ok(dispatch::compress_fn_for(kernel)) + } else { + Err(Argon2Error::BackendUnavailable) + } +} + /// Hash via the aarch64 NEON kernel. /// /// # Errors @@ -515,13 +537,7 @@ pub fn diag_hash_aarch64_neon( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support AVX2. The per-kernel tests -/// gate this call on `crate::platform::caps()` having the kernel's -/// required caps before invoking it. +/// Returns [`Argon2Error`] for invalid parameters or when AVX2 is unavailable. #[cfg(all(feature = "diag", target_arch = "x86_64"))] pub fn diag_hash_x86_avx2( params: &Argon2Params, @@ -530,17 +546,13 @@ pub fn diag_hash_x86_avx2( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::X86Avx2)), - "AVX2 not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::X86Avx2), + diag_compress_for(KernelId::X86Avx2)?, ) } @@ -548,11 +560,7 @@ pub fn diag_hash_x86_avx2( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support AVX-512F + AVX-512VL. +/// Returns [`Argon2Error`] for invalid parameters or when AVX-512F plus AVX-512VL is unavailable. #[cfg(all(feature = "diag", target_arch = "x86_64"))] pub fn diag_hash_x86_avx512( params: &Argon2Params, @@ -561,17 +569,13 @@ pub fn diag_hash_x86_avx512( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::X86Avx512)), - "AVX-512F + AVX-512VL not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::X86Avx512), + diag_compress_for(KernelId::X86Avx512)?, ) } @@ -579,11 +583,7 @@ pub fn diag_hash_x86_avx512( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support VSX. +/// Returns [`Argon2Error`] for invalid parameters or when VSX is unavailable. #[cfg(all(feature = "diag", target_arch = "powerpc64"))] pub fn diag_hash_power_vsx( params: &Argon2Params, @@ -592,17 +592,13 @@ pub fn diag_hash_power_vsx( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::PowerVsx)), - "POWER VSX not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::PowerVsx), + diag_compress_for(KernelId::PowerVsx)?, ) } @@ -610,11 +606,7 @@ pub fn diag_hash_power_vsx( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support the z13+ vector facility. +/// Returns [`Argon2Error`] for invalid parameters or when the z13+ vector facility is unavailable. #[cfg(all(feature = "diag", target_arch = "s390x"))] pub fn diag_hash_s390x_vector( params: &Argon2Params, @@ -623,17 +615,13 @@ pub fn diag_hash_s390x_vector( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::S390xVector)), - "s390x vector facility not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::S390xVector), + diag_compress_for(KernelId::S390xVector)?, ) } @@ -641,11 +629,7 @@ pub fn diag_hash_s390x_vector( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support the RISC-V V extension. +/// Returns [`Argon2Error`] for invalid parameters or when the RISC-V V extension is unavailable. #[cfg(all(feature = "diag", target_arch = "riscv64"))] pub fn diag_hash_riscv64_v( params: &Argon2Params, @@ -654,17 +638,13 @@ pub fn diag_hash_riscv64_v( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::Riscv64V)), - "RISC-V V extension not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::Riscv64V), + diag_compress_for(KernelId::Riscv64V)?, ) } @@ -672,11 +652,7 @@ pub fn diag_hash_riscv64_v( /// /// # Errors /// -/// Returns [`Argon2Error`] for invalid parameters. -/// -/// # Panics -/// -/// Panics if the host does not support wasm SIMD128. +/// Returns [`Argon2Error`] for invalid parameters or when WASM SIMD128 is unavailable. #[cfg(all(feature = "diag", target_arch = "wasm32"))] pub fn diag_hash_wasm_simd128( params: &Argon2Params, @@ -685,17 +661,13 @@ pub fn diag_hash_wasm_simd128( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - assert!( - crate::platform::caps().has(dispatch::required_caps(KernelId::WasmSimd128)), - "wasm simd128 not available on host" - ); argon2_hash_with_kernel( params, password, salt, variant, out, - dispatch::compress_fn_for(KernelId::WasmSimd128), + diag_compress_for(KernelId::WasmSimd128)?, ) } @@ -866,7 +838,7 @@ fn h_prime(input_parts: &[&[u8]], out: &mut [u8]) { // Feed LE32(out_len) then the input parts into Blake2b. For out_len <= 64 // the single-block output is the answer directly. let len_le = u32::try_from(out_len) - .unwrap_or_else(|_| unreachable!("Argon2 H' output length was checked before expansion")) + .expect("Argon2 output length is validated before H' expansion") .to_le_bytes(); if out_len <= 64 { @@ -916,13 +888,17 @@ fn h_prime_diag_blake2b_portable(input_parts: &[&[u8]], out: &mut [u8]) { let out_len = out.len(); assert!(out_len > 0, "H' output length must be positive"); let len_le = u32::try_from(out_len) - .unwrap_or_else(|_| unreachable!("Argon2 H' output length was checked before expansion")) + .expect("Argon2 output length is validated before H' expansion") .to_le_bytes(); if out_len <= 64 { let parts = [&len_le[..], input_parts[0]]; if input_parts.len() == 1 { - crate::hashes::crypto::blake2b::diag_hash_parts_portable(out_len as u8, &parts, out); + crate::hashes::crypto::blake2b::diag_hash_parts_portable( + u8::try_from(out_len).expect("the single-digest branch limits output to 64 bytes"), + &parts, + out, + ); } else { let mut data = [0u8; BLOCK_SIZE + 16]; let mut pos = 0usize; @@ -932,7 +908,11 @@ fn h_prime_diag_blake2b_portable(input_parts: &[&[u8]], out: &mut [u8]) { data[pos..pos.strict_add(part.len())].copy_from_slice(part); pos = pos.strict_add(part.len()); } - crate::hashes::crypto::blake2b::diag_hash_parts_portable(out_len as u8, &[&data[..pos]], out); + crate::hashes::crypto::blake2b::diag_hash_parts_portable( + u8::try_from(out_len).expect("the single-digest branch limits output to 64 bytes"), + &[&data[..pos]], + out, + ); } return; } @@ -960,7 +940,11 @@ fn h_prime_diag_blake2b_portable(input_parts: &[&[u8]], out: &mut [u8]) { let tail_off = r.strict_mul(32); let tail_len = out_len.strict_sub(tail_off); - crate::hashes::crypto::blake2b::diag_hash_parts_portable(tail_len as u8, &[&v_prev], &mut out[tail_off..]); + crate::hashes::crypto::blake2b::diag_hash_parts_portable( + u8::try_from(tail_len).expect("the H' tail is at most 64 bytes"), + &[&v_prev], + &mut out[tail_off..], + ); ct::zeroize(&mut v_prev); } @@ -979,26 +963,26 @@ fn compute_h0( // `check_inputs` before reaching here, so the `try_from` calls below are // infallible. We use `try_from + expect` rather than `as u32` to keep the // invariant readable at the call site. - let len_u32 = |label: &'static str, len: usize| -> [u8; 4] { + let len_u32 = |len: usize| -> [u8; 4] { u32::try_from(len) - .unwrap_or_else(|_| panic!("Argon2 H0: {label} length exceeded MAX_VAR_BYTES; check_inputs should have rejected")) + .expect("Argon2 input lengths are validated before H0 construction") .to_le_bytes() }; let mut hasher = Blake2b512::new(); hasher.update(¶ms.parallelism.to_le_bytes()); - hasher.update(&len_u32("output", output_len)); + hasher.update(&len_u32(output_len)); hasher.update(¶ms.memory_cost_kib.to_le_bytes()); hasher.update(¶ms.time_cost.to_le_bytes()); hasher.update(&ARGON2_VERSION.to_le_bytes()); hasher.update(&variant.y().to_le_bytes()); - hasher.update(&len_u32("password", password.len())); + hasher.update(&len_u32(password.len())); hasher.update(password); - hasher.update(&len_u32("salt", salt.len())); + hasher.update(&len_u32(salt.len())); hasher.update(salt); - hasher.update(&len_u32("secret", context.secret.len())); + hasher.update(&len_u32(context.secret.len())); hasher.update(context.secret); - hasher.update(&len_u32("associated_data", context.associated_data.len())); + hasher.update(&len_u32(context.associated_data.len())); hasher.update(context.associated_data); hasher.finalize() } @@ -1012,22 +996,22 @@ fn compute_h0_diag_blake2b_portable( variant: Argon2Variant, output_len: usize, ) -> [u8; 64] { - let len_u32 = |label: &'static str, len: usize| -> [u8; 4] { + let len_u32 = |len: usize| -> [u8; 4] { u32::try_from(len) - .unwrap_or_else(|_| panic!("Argon2 H0: {label} length exceeded MAX_VAR_BYTES; check_inputs should have rejected")) + .expect("Argon2 input lengths are validated before H0 construction") .to_le_bytes() }; let parallelism = params.parallelism.to_le_bytes(); - let output_len = len_u32("output", output_len); + let output_len = len_u32(output_len); let memory_cost = params.memory_cost_kib.to_le_bytes(); let time_cost = params.time_cost.to_le_bytes(); let version = ARGON2_VERSION.to_le_bytes(); let variant = variant.y().to_le_bytes(); - let password_len = len_u32("password", password.len()); - let salt_len = len_u32("salt", salt.len()); - let secret_len = len_u32("secret", context.secret.len()); - let associated_data_len = len_u32("associated_data", context.associated_data.len()); + let password_len = len_u32(password.len()); + let salt_len = len_u32(salt.len()); + let secret_len = len_u32(context.secret.len()); + let associated_data_len = len_u32(context.associated_data.len()); let mut out = [0u8; 64]; crate::hashes::crypto::blake2b::diag_hash_parts_portable( 64, @@ -1057,12 +1041,8 @@ fn compute_h0_diag_blake2b_portable( #[inline(always)] fn block_from_bytes(bytes: &[u8; BLOCK_SIZE]) -> MemoryBlock { let mut out = MemoryBlock::zero(); - for i in 0..BLOCK_WORDS { - out.0[i] = u64::from_le_bytes( - bytes[i.strict_mul(8)..i.strict_mul(8).strict_add(8)] - .try_into() - .unwrap(), - ); + for (word, bytes) in out.0.iter_mut().zip(bytes.as_chunks::<8>().0) { + *word = u64::from_le_bytes(*bytes); } out } @@ -1078,6 +1058,19 @@ fn block_to_bytes(block: &[u64; BLOCK_WORDS]) -> [u8; BLOCK_SIZE] { // ─── Argon2i pseudo-random address stream (RFC 9106 §3.4.2) ──────────────── +#[derive(Clone, Copy)] +struct SegmentConfig { + compress: CompressFn, + pass: u32, + slice: u32, + lanes: u32, + segment_len: u32, + lane_len: u32, + total_blocks: u32, + variant: Argon2Variant, + time_cost: u32, +} + /// Buffer of 128 `J1||J2` word pairs used for a single segment of Argon2i /// (or Argon2id's data-independent slices). #[derive(Clone)] @@ -1094,25 +1087,14 @@ impl AddressBlock { /// Generate a fresh address block keyed by /// `(pass, lane, slice, blocks, total_passes, variant_y, counter)`. - #[allow(clippy::too_many_arguments)] // RFC 9106 §3.4.2 fixes this list; wrapping it in a struct is empty ceremony. - fn refresh( - &mut self, - compress: CompressFn, - pass: u32, - lane: u32, - slice: u32, - blocks: u32, - total_passes: u32, - variant_y: u32, - counter: u64, - ) { + fn refresh(&mut self, config: SegmentConfig, lane: u32, counter: u64) { let mut input = MemoryBlock::zero(); - input.0[0] = pass as u64; - input.0[1] = lane as u64; - input.0[2] = slice as u64; - input.0[3] = blocks as u64; - input.0[4] = total_passes as u64; - input.0[5] = variant_y as u64; + input.0[0] = u64::from(config.pass); + input.0[1] = u64::from(lane); + input.0[2] = u64::from(config.slice); + input.0[3] = u64::from(config.total_blocks); + input.0[4] = u64::from(config.time_cost); + input.0[5] = u64::from(config.variant.y()); input.0[6] = counter; let zero = MemoryBlock::zero(); @@ -1121,8 +1103,8 @@ impl AddressBlock { // `compress_fn_for` in per-kernel tests), which only returns a // kernel whose `required_caps` are a subset of the host's caps. unsafe { - compress(&mut intermediate.0, &zero.0, &input.0, /* xor_into = */ false); - compress(&mut self.words.0, &zero.0, &intermediate.0, /* xor_into = */ false); + (config.compress)(&mut intermediate.0, &zero.0, &input.0, /* xor_into = */ false); + (config.compress)(&mut self.words.0, &zero.0, &intermediate.0, /* xor_into = */ false); } } } @@ -1143,12 +1125,16 @@ impl Matrix { let shape = argon2_shape(params); let lanes = params.parallelism; let m_prime = shape.blocks; - let lane_len = m_prime / lanes; - let segment_len = lane_len / SYNC_POINTS; + let lane_len = m_prime + .checked_div(lanes) + .expect("validated Argon2 parallelism is nonzero"); + let segment_len = lane_len + .checked_div(SYNC_POINTS) + .expect("the Argon2 synchronization-point count is nonzero"); if shape.memory_bytes > isize::MAX as u64 { return Err(Argon2Error::ResourceOverflow); } - let total = m_prime as usize; + let total = usize::try_from(m_prime).map_err(|_| Argon2Error::ResourceOverflow)?; let mut blocks = Vec::new(); blocks .try_reserve_exact(total) @@ -1286,27 +1272,21 @@ impl MatrixView { /// Compute the reference block `(ref_lane, ref_index)` for a position /// `(lane, col)` in pass `pass`, using `j1`/`j2` pseudo-random words. -#[allow(clippy::too_many_arguments)] // RFC 9106 §3.4 ties eight fields together; a ctx struct would just forward them all. #[inline(always)] -fn reference_index( - pass: u32, - lane: u32, - slice: u32, - col: u32, - j1: u32, - j2: u32, - lanes: u32, - segment_len: u32, - lane_len: u32, -) -> (u32, u32) { +fn reference_index(config: SegmentConfig, lane: u32, col: u32, j1: u32, j2: u32) -> (u32, u32) { // Reference lane - let ref_lane = if pass == 0 && slice == 0 { lane } else { j2 % lanes }; + let ref_lane = if config.pass == 0 && config.slice == 0 { + lane + } else { + j2.checked_rem(config.lanes) + .expect("validated Argon2 parallelism is nonzero") + }; // Reference area size let same_lane = ref_lane == lane; - let position_in_segment = col.wrapping_sub(slice.wrapping_mul(segment_len)); + let position_in_segment = col.wrapping_sub(config.slice.wrapping_mul(config.segment_len)); // We know position_in_segment < segment_len (by construction). - let area_size: u32 = if pass == 0 { + let area_size: u32 = if config.pass == 0 { // First pass: previous slices in the current lane are available. if same_lane { // Blocks 0..col − 1 available (col excludes position itself). @@ -1316,7 +1296,7 @@ fn reference_index( } else { // Other lane: blocks in slices 0..slice completed, minus 1 if position // in current segment == 0 (prevents self-reference in racing lane). - let completed_slices = slice.wrapping_mul(segment_len); + let completed_slices = config.slice.wrapping_mul(config.segment_len); if position_in_segment == 0 { completed_slices.wrapping_sub(1) } else { @@ -1328,13 +1308,14 @@ fn reference_index( // already-computed portion for same-lane references. if same_lane { // lane_len − segment_len + position_in_segment − 1 - lane_len - .wrapping_sub(segment_len) + config + .lane_len + .wrapping_sub(config.segment_len) .wrapping_add(position_in_segment) .wrapping_sub(1) } else { // lane_len − segment_len, minus 1 if position_in_segment == 0 - let base = lane_len.wrapping_sub(segment_len); + let base = config.lane_len.wrapping_sub(config.segment_len); if position_in_segment == 0 { base.wrapping_sub(1) } else { @@ -1347,18 +1328,23 @@ fn reference_index( let j1_u64 = j1 as u64; let relative_position = { let x = (j1_u64.wrapping_mul(j1_u64)) >> 32; - let y = (area_size as u64).wrapping_mul(x) >> 32; - (area_size as u64).wrapping_sub(1).wrapping_sub(y) as u32 + let y = u64::from(area_size).wrapping_mul(x) >> 32; + let relative = u64::from(area_size).wrapping_sub(1).wrapping_sub(y); + let [b0, b1, b2, b3, _, _, _, _] = relative.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) }; // Absolute start position of the reference area (wraps across the lane). - let start_position = if pass == 0 || slice == (SYNC_POINTS - 1) { + let start_position = if config.pass == 0 || config.slice == (SYNC_POINTS - 1) { 0 } else { - (slice.wrapping_add(1)).wrapping_mul(segment_len) + (config.slice.wrapping_add(1)).wrapping_mul(config.segment_len) }; - let ref_index = (start_position.wrapping_add(relative_position)) % lane_len; + let ref_index = start_position + .wrapping_add(relative_position) + .checked_rem(config.lane_len) + .expect("validated Argon2 geometry has a nonzero lane length"); (ref_lane, ref_index) } @@ -1369,7 +1355,6 @@ fn reference_index( /// [`MatrixView`] from the underlying block storage, and dispatches into /// [`fill_segment_inner`]. The exclusive borrow makes the inner kernel's /// safety contract trivially satisfied for single-threaded callers. -#[allow(clippy::too_many_arguments)] // RFC 9106 §3.4 fixes this list; struct wrapper is empty ceremony. fn fill_segment( matrix: &mut Matrix, compress: CompressFn, @@ -1379,29 +1364,24 @@ fn fill_segment( variant: Argon2Variant, time_cost: u32, ) { - let lanes = matrix.lanes; - let segment_len = matrix.segment_len; - let lane_len = matrix.lane_len; - let total_blocks = matrix.len() as u32; + let config = SegmentConfig { + compress, + pass, + slice, + lanes: matrix.lanes, + segment_len: matrix.segment_len, + lane_len: matrix.lane_len, + total_blocks: u32::try_from(matrix.len()).expect("matrix length originates from the validated u32 block count"), + variant, + time_cost, + }; let view = MatrixView::from_blocks(&mut matrix.blocks); // SAFETY: `&mut matrix` is held exclusively for the duration of the // call. The view is the only handle to the matrix's storage during // `fill_segment_inner`; aliasing and concurrency contracts are // trivially upheld. unsafe { - fill_segment_inner( - view, - compress, - pass, - lane, - slice, - lanes, - segment_len, - lane_len, - total_blocks, - variant, - time_cost, - ); + fill_segment_inner(view, lane, config); } } @@ -1416,31 +1396,26 @@ fn fill_segment( /// /// Callers must guarantee that, for the indices touched by this call: /// -/// - The current segment range `[lane * lane_len + slice * segment_len .. lane * lane_len + (slice -/// + 1) * segment_len]` is exclusively writeable by this task. Same-task reads may target -/// already-written positions in that range. -/// - Every immutable input remains stable for the duration of `compress`. The previous and -/// reference inputs may alias each other, but neither may alias the current mutable output. +/// - The task has exclusive write access to the current lane and slice's segment range; same-task reads may target positions it has already written there. +/// - Every immutable input remains stable for `compress`; previous and reference inputs may alias each other, but neither may alias the current mutable output. /// /// Both conditions are upheld by: +/// /// - The sequential path's exclusive `&mut Matrix` borrow, OR /// - The Argon2 reference-index function (RFC 9106 §3.4) when called from `fill_slice_parallel` — /// see the doc-comment on [`MatrixView`] for the disjointness argument. -#[allow(clippy::too_many_arguments, clippy::doc_lazy_continuation)] -unsafe fn fill_segment_inner( - view: MatrixView, - compress: CompressFn, - pass: u32, - lane: u32, - slice: u32, - lanes: u32, - segment_len: u32, - lane_len: u32, - total_blocks: u32, - variant: Argon2Variant, - time_cost: u32, -) { - let variant_y = variant.y(); +unsafe fn fill_segment_inner(view: MatrixView, lane: u32, config: SegmentConfig) { + let SegmentConfig { + compress, + pass, + slice, + lanes: _, + segment_len, + lane_len, + total_blocks: _, + variant, + time_cost: _, + } = config; let lane_len_usize = lane_len as usize; let lane_base = (lane as usize).strict_mul(lane_len_usize); @@ -1458,16 +1433,7 @@ unsafe fn fill_segment_inner( let mut address_counter: u64 = 0; if is_independent { address_counter = 1; // RFC 9106: counter starts at 1 for first block. - address_block.refresh( - compress, - pass, - lane, - slice, - total_blocks, - time_cost, - variant_y, - address_counter, - ); + address_block.refresh(config, lane, address_counter); } // Starting column for this segment. Skip first two blocks of lane 0 on @@ -1491,16 +1457,7 @@ unsafe fn fill_segment_inner( let addr_pos = (seg_col as usize) % BLOCK_WORDS; if addr_pos == 0 && seg_col != 0 { address_counter = address_counter.strict_add(1); - address_block.refresh( - compress, - pass, - lane, - slice, - total_blocks, - time_cost, - variant_y, - address_counter, - ); + address_block.refresh(config, lane, address_counter); } let word = address_block.words.0[addr_pos]; ((word & 0xFFFF_FFFFu64) as u32, (word >> 32) as u32) @@ -1515,7 +1472,7 @@ unsafe fn fill_segment_inner( ((word & 0xFFFF_FFFFu64) as u32, (word >> 32) as u32) }; - let (ref_lane, ref_index) = reference_index(pass, lane, slice, col, j1, j2, lanes, segment_len, lane_len); + let (ref_lane, ref_index) = reference_index(config, lane, col, j1, j2); // Compute new block: G(B[lane][prev], B[ref_lane][ref_index]), optionally // XOR-accumulated into existing block (v1.3, pass > 0). @@ -1591,8 +1548,19 @@ fn fill_slice_parallel( let lanes = matrix.lanes; let segment_len = matrix.segment_len; let lane_len = matrix.lane_len; - let total_blocks = matrix.len() as u32; + let total_blocks = u32::try_from(matrix.len()).expect("matrix length originates from the validated u32 block count"); let view = MatrixView::from_blocks(&mut matrix.blocks); + let config = SegmentConfig { + compress, + pass, + slice, + lanes, + segment_len, + lane_len, + total_blocks, + variant, + time_cost, + }; // Spawn lanes 1..lanes onto rayon workers; run lane 0 inline on the // calling thread. This is the standard Blake3 / parallel-tree-reduction @@ -1617,19 +1585,7 @@ fn fill_slice_parallel( // a writer. The Send/Sync impls on `MatrixView` are sound under // this discipline; see the `MatrixView` doc-comment. unsafe { - fill_segment_inner( - view, - compress, - pass, - lane, - slice, - lanes, - segment_len, - lane_len, - total_blocks, - variant, - time_cost, - ); + fill_segment_inner(view, lane, config); } }); } @@ -1638,19 +1594,7 @@ fn fill_slice_parallel( // call (no other spawned task touches lane 0). The same disjointness // argument used for the spawned tasks applies — see above. unsafe { - fill_segment_inner( - view, - compress, - pass, - 0, - slice, - lanes, - segment_len, - lane_len, - total_blocks, - variant, - time_cost, - ); + fill_segment_inner(view, 0, config); } }); } @@ -1678,6 +1622,13 @@ fn fill_slice( // ─── Full Argon2 hash function ───────────────────────────────────────────── +#[derive(Clone, Copy)] +struct HashBackend { + compress: CompressFn, + #[cfg(feature = "diag")] + diag_blake2b: bool, +} + fn argon2_hash( params: &Argon2Params, password: &[u8], @@ -1696,7 +1647,19 @@ fn argon2_hash_with_context( variant: Argon2Variant, out: &mut [u8], ) -> Result<(), Argon2Error> { - argon2_hash_with_kernel_inner(params, context, password, salt, variant, out, active_compress(), false) + argon2_hash_with_kernel_inner( + params, + context, + password, + salt, + variant, + out, + HashBackend { + compress: active_compress(), + #[cfg(feature = "diag")] + diag_blake2b: false, + }, + ) } #[cfg(feature = "diag")] @@ -1715,8 +1678,10 @@ fn argon2_hash_with_kernel( salt, variant, out, - compress, - false, + HashBackend { + compress, + diag_blake2b: false, + }, ) } @@ -1736,12 +1701,13 @@ fn argon2_hash_with_kernel_diag_blake2b( salt, variant, out, - compress, - true, + HashBackend { + compress, + diag_blake2b: true, + }, ) } -#[allow(clippy::too_many_arguments)] // Params, context, inputs, variant, output, and kernel are the real operation boundary. fn argon2_hash_with_kernel_inner( params: &Argon2Params, context: Argon2Context<'_>, @@ -1749,8 +1715,7 @@ fn argon2_hash_with_kernel_inner( salt: &[u8], variant: Argon2Variant, out: &mut [u8], - compress: CompressFn, - #[cfg_attr(not(feature = "diag"), allow(unused_variables))] diag_blake2b: bool, + backend: HashBackend, ) -> Result<(), Argon2Error> { Argon2Params::check_inputs(password, salt, context)?; if out.len() < MIN_OUTPUT_LEN || out.len() as u64 > MAX_VAR_BYTES { @@ -1767,7 +1732,7 @@ fn argon2_hash_with_kernel_inner( let mut h0 = { #[cfg(feature = "diag")] { - if diag_blake2b { + if backend.diag_blake2b { compute_h0_diag_blake2b_portable(params, context, password, salt, variant, out.len()) } else { compute_h0(params, context, password, salt, variant, out.len()) @@ -1785,7 +1750,7 @@ fn argon2_hash_with_kernel_inner( // B[lane][0] = H'(H0 || LE32(0) || LE32(lane), BLOCK_SIZE) let lane_le = lane.to_le_bytes(); #[cfg(feature = "diag")] - if diag_blake2b { + if backend.diag_blake2b { h_prime_diag_blake2b_portable(&[&h0, &0u32.to_le_bytes(), &lane_le], &mut buf); } else { h_prime(&[&h0, &0u32.to_le_bytes(), &lane_le], &mut buf); @@ -1796,7 +1761,7 @@ fn argon2_hash_with_kernel_inner( // B[lane][1] = H'(H0 || LE32(1) || LE32(lane), BLOCK_SIZE) #[cfg(feature = "diag")] - if diag_blake2b { + if backend.diag_blake2b { h_prime_diag_blake2b_portable(&[&h0, &1u32.to_le_bytes(), &lane_le], &mut buf); } else { h_prime(&[&h0, &1u32.to_le_bytes(), &lane_le], &mut buf); @@ -1812,21 +1777,22 @@ fn argon2_hash_with_kernel_inner( // skipped when `parallelism == 1` to avoid rayon overhead. for pass in 0..params.time_cost { for slice in 0..SYNC_POINTS { - fill_slice(&mut matrix, compress, pass, slice, variant, params.time_cost); + fill_slice(&mut matrix, backend.compress, pass, slice, variant, params.time_cost); } } // Finalisation: C = XOR of last block of each lane; output = H'(C, T). - let mut acc = *matrix.get(0, lane_len - 1); + let last_block = lane_len.strict_sub(1); + let mut acc = *matrix.get(0, last_block); for lane in 1..lanes { - let blk = matrix.get(lane, lane_len - 1); + let blk = matrix.get(lane, last_block); for i in 0..BLOCK_WORDS { acc[i] ^= blk[i]; } } let mut acc_bytes = block_to_bytes(&acc); #[cfg(feature = "diag")] - if diag_blake2b { + if backend.diag_blake2b { h_prime_diag_blake2b_portable(&[&acc_bytes], out); } else { h_prime(&[&acc_bytes], out); @@ -2148,7 +2114,7 @@ mod password_phc { } impl ApprovedPhc { - pub fn salt(&self) -> &[u8] { + pub(super) fn salt(&self) -> &[u8] { &self.salt[..self.salt_len as usize] } } @@ -2209,7 +2175,7 @@ mod password_phc { Ok(ApprovedPhc { params, salt, - salt_len: decoded_salt_len as u8, + salt_len: u8::try_from(decoded_salt_len).map_err(|_| PhcError::InvalidLength)?, expected, }) } @@ -2252,7 +2218,7 @@ mod tests { #[cfg(not(miri))] fn canon_params() -> Argon2Params { - Argon2Params::new(32, 3, 4).unwrap() + Argon2Params::new(32, 3, 4).expect("RFC 9106 test parameters must be valid") } #[cfg(not(miri))] @@ -2277,11 +2243,14 @@ mod tests { ]; let mut actual = [0u8; 32]; - Argon2d::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual).unwrap(); + Argon2d::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual) + .expect("RFC 9106 Argon2d vector inputs must derive"); assert_eq!(actual, expected_d); - Argon2i::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual).unwrap(); + Argon2i::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual) + .expect("RFC 9106 Argon2i vector inputs must derive"); assert_eq!(actual, expected_i); - Argon2id::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive_with_context(&canon_params(), canon_context(), PASSWORD, SALT, &mut actual) + .expect("RFC 9106 Argon2id vector inputs must derive"); assert_eq!(actual, expected_id); } @@ -2290,11 +2259,17 @@ mod tests { fn raw_verify_accepts_only_the_exact_inputs() { let params = canon_params(); let mut expected = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut expected).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut expected).expect("canonical inputs must derive"); - assert!(Argon2id::verify(¶ms, PASSWORD, SALT, &expected).is_ok()); - assert!(Argon2id::verify(¶ms, b"wrong", SALT, &expected).is_err()); - assert!(Argon2id::verify(¶ms, PASSWORD, &[0xff; 16], &expected).is_err()); + assert_eq!(Argon2id::verify(¶ms, PASSWORD, SALT, &expected), Ok(())); + assert_eq!( + Argon2id::verify(¶ms, b"wrong", SALT, &expected), + Err(VerificationError::new()) + ); + assert_eq!( + Argon2id::verify(¶ms, PASSWORD, &[0xff; 16], &expected), + Err(VerificationError::new()) + ); } #[test] @@ -2302,12 +2277,15 @@ mod tests { assert_eq!(Argon2Params::new(8, 0, 1), Err(Argon2Error::InvalidTimeCost)); assert_eq!(Argon2Params::new(8, 1, 0), Err(Argon2Error::InvalidParallelism)); assert_eq!(Argon2Params::new(16, 1, 4), Err(Argon2Error::InvalidMemoryCost)); - assert!(Argon2Params::new(32, 1, 4).is_ok()); + let valid = Argon2Params::new(32, 1, 4).expect("boundary-valid parameters must construct"); + assert_eq!(valid.get_memory_cost_kib(), 32); + assert_eq!(valid.get_time_cost(), 1); + assert_eq!(valid.get_parallelism(), 4); } #[test] fn derive_rejects_invalid_operation_lengths() { - let params = Argon2Params::new(32, 1, 4).unwrap(); + let params = Argon2Params::new(32, 1, 4).expect("test parameters must be valid"); let mut out = [0u8; 32]; assert_eq!( Argon2id::derive(¶ms, b"pw", &[0u8; 7], &mut out), @@ -2347,10 +2325,13 @@ mod tests { parallelism: u32, output_len: usize, ) -> vec::Vec { - let params = argon2::Params::new(memory_kib, time, parallelism, Some(output_len)).unwrap(); + let params = + argon2::Params::new(memory_kib, time, parallelism, Some(output_len)).expect("oracle parameters must be valid"); let oracle = argon2::Argon2::new(algorithm, argon2::Version::V0x13, params); let mut output = alloc::vec![0u8; output_len]; - oracle.hash_password_into(password, salt, &mut output).unwrap(); + oracle + .hash_password_into(password, salt, &mut output) + .expect("oracle inputs must derive"); output } @@ -2359,10 +2340,10 @@ mod tests { fn all_variants_match_the_oracle() { let cases: &[(u32, u32, u32, usize)] = &[(8, 1, 1, 16), (16, 2, 1, 32), (32, 3, 2, 64)]; for &(memory, time, parallelism, output_len) in cases { - let params = Argon2Params::new(memory, time, parallelism).unwrap(); + let params = Argon2Params::new(memory, time, parallelism).expect("oracle-case parameters must be valid"); let mut actual = alloc::vec![0u8; output_len]; - Argon2d::derive(¶ms, b"password", &[0u8; 16], &mut actual).unwrap(); + Argon2d::derive(¶ms, b"password", &[0u8; 16], &mut actual).expect("Argon2d oracle case must derive"); assert_eq!( actual, oracle_hash( @@ -2376,7 +2357,7 @@ mod tests { ) ); - Argon2i::derive(¶ms, b"password", &[0u8; 16], &mut actual).unwrap(); + Argon2i::derive(¶ms, b"password", &[0u8; 16], &mut actual).expect("Argon2i oracle case must derive"); assert_eq!( actual, oracle_hash( @@ -2390,7 +2371,7 @@ mod tests { ) ); - Argon2id::derive(¶ms, b"password", &[0u8; 16], &mut actual).unwrap(); + Argon2id::derive(¶ms, b"password", &[0u8; 16], &mut actual).expect("Argon2id oracle case must derive"); assert_eq!( actual, oracle_hash( @@ -2421,19 +2402,20 @@ mod tests { use crate::auth::{PasswordStatus, phc::PhcError}; fn small_params() -> Argon2Params { - Argon2Params::new(32, 2, 1).unwrap() + Argon2Params::new(32, 2, 1).expect("small test profile must be valid") } fn encode(params: Argon2Params, password: &[u8], salt: &[u8], context: Argon2Context<'_>) -> alloc::string::String { let mut verifier = [0u8; PASSWORD_OUTPUT_LEN]; - Argon2id::derive_with_context(¶ms, context, password, salt, &mut verifier).unwrap(); + Argon2id::derive_with_context(¶ms, context, password, salt, &mut verifier) + .expect("PHC test inputs must derive"); password_phc::encode(params, salt, &verifier) } #[test] fn canonical_password_record_round_trips() { let params = small_params(); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("test verification profile must be admissible"); let encoded = encode(params, b"password", &[0xaa; 16], Argon2Context::default()); assert!(encoded.starts_with("$argon2id$v=19$m=32,t=2,p=1$")); @@ -2441,13 +2423,16 @@ mod tests { password.verify_password(b"password", &encoded), Ok(PasswordStatus::Current) ); - assert!(password.verify_password(b"wrong", &encoded).is_err()); + assert_eq!( + password.verify_password(b"wrong", &encoded), + Err(VerificationError::new()) + ); } #[test] fn accepted_older_profile_requests_rehash() { - let generation = Argon2Params::new(40, 2, 1).unwrap(); - let password = Argon2idPassword::new(generation).unwrap(); + let generation = Argon2Params::new(40, 2, 1).expect("generation profile must be valid"); + let password = Argon2idPassword::new(generation).expect("generation profile must be admissible"); let encoded = encode(small_params(), b"password", &[0xbb; 16], Argon2Context::default()); assert_eq!( @@ -2459,7 +2444,7 @@ mod tests { #[test] fn accepted_noncurrent_salt_length_requests_rehash() { let params = small_params(); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("test verification profile must be admissible"); let encoded = encode(params, b"password", &[0xbb; 8], Argon2Context::default()); assert_eq!( @@ -2471,19 +2456,21 @@ mod tests { #[test] fn borrowed_context_is_required_for_context_bound_records() { let params = small_params(); - let password = Argon2idPassword::new(params).unwrap(); + let password = Argon2idPassword::new(params).expect("test verification profile must be admissible"); let context = Argon2Context::new(b"pepper", b"tenant"); let encoded = encode(params, b"password", &[0xcc; 16], context); - assert!(password.verify_password(b"password", &encoded).is_err()); + assert_eq!( + password.verify_password(b"password", &encoded), + Err(VerificationError::new()) + ); assert_eq!( password.verify_password_with_context(b"password", &encoded, context), Ok(PasswordStatus::Current) ); - assert!( - password - .verify_password_with_context(b"password", &encoded, Argon2Context::new(b"wrong", b"tenant"),) - .is_err() + assert_eq!( + password.verify_password_with_context(b"password", &encoded, Argon2Context::new(b"wrong", b"tenant"),), + Err(VerificationError::new()) ); } @@ -2506,19 +2493,19 @@ mod tests { #[test] fn actual_argon2_shape_defines_the_limit() { let limits = Argon2VerificationLimits::for_profile(small_params()); - let rounded_equivalent = Argon2Params::new(35, 2, 1).unwrap(); + let rounded_equivalent = Argon2Params::new(35, 2, 1).expect("rounded test profile must be valid"); assert!(limits.allows(rounded_equivalent)); - let next_matrix = Argon2Params::new(36, 2, 1).unwrap(); + let next_matrix = Argon2Params::new(36, 2, 1).expect("next-matrix test profile must be valid"); assert!(!limits.allows(next_matrix)); } #[test] fn generator_and_parser_share_the_full_argon2_parallelism_domain() { - let params = Argon2Params::new(2_048, 1, 256).unwrap(); + let params = Argon2Params::new(2_048, 1, 256).expect("maximum-domain test profile must be valid"); let encoded = password_phc::encode(params, &[0x44; 16], &[0u8; PASSWORD_OUTPUT_LEN]); let limits = Argon2VerificationLimits::for_profile(params); - assert!(password_phc::approve(&encoded, limits).is_ok()); + password_phc::approve(&encoded, limits).expect("generator output must be accepted by the parser"); } #[test] @@ -2535,16 +2522,22 @@ mod tests { format!("$argon2id$v=19$m=32,t=2,x=1$${salt}$${hash}"), ]; for encoded in cases { - assert!(password_phc::approve(&encoded, limits).is_err(), "{encoded}"); + password_phc::approve(&encoded, limits) + .err() + .expect("noncanonical record must be rejected"); } } #[cfg(feature = "getrandom")] #[test] fn generated_records_use_fresh_salts() { - let password = Argon2idPassword::new(small_params()).unwrap(); - let first = password.hash_password(b"password").unwrap(); - let second = password.hash_password(b"password").unwrap(); + let password = Argon2idPassword::new(small_params()).expect("test verification profile must be admissible"); + let first = password + .hash_password(b"password") + .expect("first salt generation must succeed"); + let second = password + .hash_password(b"password") + .expect("second salt generation must succeed"); assert_ne!(first, second); assert_eq!( diff --git a/src/auth/argon2/power.rs b/src/auth/argon2/power.rs index 8ff495f0..4e96ea7d 100644 --- a/src/auth/argon2/power.rs +++ b/src/auth/argon2/power.rs @@ -26,12 +26,11 @@ //! //! # Rotations //! -//! Lane-wise u64 rotate is expressed as a shift-right + shift-left + OR -//! vector sequence. Whether the compiler selects a native rotate instruction -//! is a generated-code property, not a source contract. +//! Lane-wise u64 rotation uses `rotate_right`. Whether the compiler selects a +//! native vector rotate instruction is a generated-code property, not a source +//! contract. #![cfg(target_arch = "powerpc64")] -#![allow(clippy::cast_possible_truncation)] use core::simd::u64x2; @@ -53,94 +52,93 @@ pub(super) unsafe fn compress_vsx( y: &[u64; BLOCK_WORDS], xor_into: bool, ) { - // SAFETY: VSX is enabled by this function's `#[target_feature]`. - // `core::simd::u64x2` arithmetic is always wrapping per the SIMD - // contract, so no further unsafe is needed for the algorithm itself. let mut r = [0u64; BLOCK_WORDS]; let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { - let xv = u64x2::from_array([x[i], x[i + 1]]); - let yv = u64x2::from_array([y[i], y[i + 1]]); + let next = i.strict_add(1); + let xv = u64x2::from_array([x[i], x[next]]); + let yv = u64x2::from_array([y[i], y[next]]); let rv = (xv ^ yv).to_array(); r[i] = rv[0]; - r[i + 1] = rv[1]; + r[next] = rv[1]; q[i] = rv[0]; - q[i + 1] = rv[1]; - i += 2; + q[next] = rv[1]; + i = i.strict_add(2); } // Row pass. let mut row = 0usize; while row < 8 { - let base = row * 16; - let mut a: Pair = [load_pair(&q, base), load_pair(&q, base + 2)]; - let mut b: Pair = [load_pair(&q, base + 4), load_pair(&q, base + 6)]; - let mut c: Pair = [load_pair(&q, base + 8), load_pair(&q, base + 10)]; - let mut d: Pair = [load_pair(&q, base + 12), load_pair(&q, base + 14)]; + let base = row.strict_mul(16); + let mut a: Pair = [load_pair(&q, base), load_pair(&q, base.strict_add(2))]; + let mut b: Pair = [load_pair(&q, base.strict_add(4)), load_pair(&q, base.strict_add(6))]; + let mut c: Pair = [load_pair(&q, base.strict_add(8)), load_pair(&q, base.strict_add(10))]; + let mut d: Pair = [load_pair(&q, base.strict_add(12)), load_pair(&q, base.strict_add(14))]; p_round(&mut a, &mut b, &mut c, &mut d); store_pair(&mut q, base, a[0]); - store_pair(&mut q, base + 2, a[1]); - store_pair(&mut q, base + 4, b[0]); - store_pair(&mut q, base + 6, b[1]); - store_pair(&mut q, base + 8, c[0]); - store_pair(&mut q, base + 10, c[1]); - store_pair(&mut q, base + 12, d[0]); - store_pair(&mut q, base + 14, d[1]); - row += 1; + store_pair(&mut q, base.strict_add(2), a[1]); + store_pair(&mut q, base.strict_add(4), b[0]); + store_pair(&mut q, base.strict_add(6), b[1]); + store_pair(&mut q, base.strict_add(8), c[0]); + store_pair(&mut q, base.strict_add(10), c[1]); + store_pair(&mut q, base.strict_add(12), d[0]); + store_pair(&mut q, base.strict_add(14), d[1]); + row = row.strict_add(1); } // Column pass. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a: Pair = [load_pair(&q, base), load_pair(&q, base + 16)]; - let mut b: Pair = [load_pair(&q, base + 32), load_pair(&q, base + 48)]; - let mut c: Pair = [load_pair(&q, base + 64), load_pair(&q, base + 80)]; - let mut d: Pair = [load_pair(&q, base + 96), load_pair(&q, base + 112)]; + let base = col.strict_mul(2); + let mut a: Pair = [load_pair(&q, base), load_pair(&q, base.strict_add(16))]; + let mut b: Pair = [load_pair(&q, base.strict_add(32)), load_pair(&q, base.strict_add(48))]; + let mut c: Pair = [load_pair(&q, base.strict_add(64)), load_pair(&q, base.strict_add(80))]; + let mut d: Pair = [load_pair(&q, base.strict_add(96)), load_pair(&q, base.strict_add(112))]; p_round(&mut a, &mut b, &mut c, &mut d); store_pair(&mut q, base, a[0]); - store_pair(&mut q, base + 16, a[1]); - store_pair(&mut q, base + 32, b[0]); - store_pair(&mut q, base + 48, b[1]); - store_pair(&mut q, base + 64, c[0]); - store_pair(&mut q, base + 80, c[1]); - store_pair(&mut q, base + 96, d[0]); - store_pair(&mut q, base + 112, d[1]); - col += 1; + store_pair(&mut q, base.strict_add(16), a[1]); + store_pair(&mut q, base.strict_add(32), b[0]); + store_pair(&mut q, base.strict_add(48), b[1]); + store_pair(&mut q, base.strict_add(64), c[0]); + store_pair(&mut q, base.strict_add(80), c[1]); + store_pair(&mut q, base.strict_add(96), d[0]); + store_pair(&mut q, base.strict_add(112), d[1]); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor. - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { + let next = i.strict_add(1); let qv = load_pair(&q, i); let rv = load_pair(&r, i); let f = (qv ^ rv).to_array(); if xor_into { dst[i] ^= f[0]; - dst[i + 1] ^= f[1]; + dst[next] ^= f[1]; } else { dst[i] = f[0]; - dst[i + 1] = f[1]; + dst[next] = f[1]; } - i += 2; + i = i.strict_add(2); } } #[inline(always)] fn load_pair(buf: &[u64; BLOCK_WORDS], idx: usize) -> u64x2 { - u64x2::from_array([buf[idx], buf[idx + 1]]) + u64x2::from_array([buf[idx], buf[idx.strict_add(1)]]) } #[inline(always)] fn store_pair(buf: &mut [u64; BLOCK_WORDS], idx: usize, v: u64x2) { let a = v.to_array(); buf[idx] = a[0]; - buf[idx + 1] = a[1]; + buf[idx.strict_add(1)] = a[1]; } // ─── 4-way P-round ───────────────────────────────────────────────────────── @@ -201,32 +199,32 @@ fn gb(a: &mut Pair, b: &mut Pair, c: &mut Pair, d: &mut Pair) { // Step 1: a = a + b + 2·lsb(a)·lsb(b) let p0 = bla_mul(a[0], b[0]); let p1 = bla_mul(a[1], b[1]); - a[0] = a[0] + b[0] + p0; - a[1] = a[1] + b[1] + p1; + a[0] = bla_add(a[0], b[0], p0); + a[1] = bla_add(a[1], b[1], p1); d[0] = ror::<32>(d[0] ^ a[0]); d[1] = ror::<32>(d[1] ^ a[1]); // Step 2: c = c + d + 2·lsb(c)·lsb(d) let p0 = bla_mul(c[0], d[0]); let p1 = bla_mul(c[1], d[1]); - c[0] = c[0] + d[0] + p0; - c[1] = c[1] + d[1] + p1; + c[0] = bla_add(c[0], d[0], p0); + c[1] = bla_add(c[1], d[1], p1); b[0] = ror::<24>(b[0] ^ c[0]); b[1] = ror::<24>(b[1] ^ c[1]); // Step 3: a = a + b + 2·lsb(a)·lsb(b) let p0 = bla_mul(a[0], b[0]); let p1 = bla_mul(a[1], b[1]); - a[0] = a[0] + b[0] + p0; - a[1] = a[1] + b[1] + p1; + a[0] = bla_add(a[0], b[0], p0); + a[1] = bla_add(a[1], b[1], p1); d[0] = ror::<16>(d[0] ^ a[0]); d[1] = ror::<16>(d[1] ^ a[1]); // Step 4: c = c + d + 2·lsb(c)·lsb(d) let p0 = bla_mul(c[0], d[0]); let p1 = bla_mul(c[1], d[1]); - c[0] = c[0] + d[0] + p0; - c[1] = c[1] + d[1] + p1; + c[0] = bla_add(c[0], d[0], p0); + c[1] = bla_add(c[1], d[1], p1); b[0] = ror::<63>(b[0] ^ c[0]); b[1] = ror::<63>(b[1] ^ c[1]); } @@ -237,9 +235,14 @@ fn gb(a: &mut Pair, b: &mut Pair, c: &mut Pair, d: &mut Pair) { #[inline(always)] fn ror(v: u64x2) -> u64x2 { const { assert!(N > 0 && N < 64) } - let s = u64x2::splat(N as u64); - let r = u64x2::splat((64 - N) as u64); - (v >> s) | (v << r) + let right = core::ops::Shr::shr(v, u64x2::splat(u64::from(N))); + let left = core::ops::Shl::shl(v, u64x2::splat(u64::from(64u32.strict_sub(N)))); + core::ops::BitOr::bitor(right, left) +} + +#[inline(always)] +fn bla_add(a: u64x2, b: u64x2, product: u64x2) -> u64x2 { + simd_wrapping_add(simd_wrapping_add(a, b), product) } /// `2 · lsb(a) · lsb(b)` lane-wise via masked `u64x2 *`. The product @@ -248,7 +251,16 @@ fn ror(v: u64x2) -> u64x2 { #[inline(always)] fn bla_mul(a: u64x2, b: u64x2) -> u64x2 { let mask = u64x2::splat(0xffff_ffff); - let al = a & mask; - let bl = b & mask; - (al * bl) << u64x2::splat(1) + let product = simd_wrapping_mul(a & mask, b & mask); + core::ops::Shl::shl(product, u64x2::splat(1)) +} + +#[inline(always)] +fn simd_wrapping_add(a: u64x2, b: u64x2) -> u64x2 { + core::ops::Add::add(a, b) +} + +#[inline(always)] +fn simd_wrapping_mul(a: u64x2, b: u64x2) -> u64x2 { + core::ops::Mul::mul(a, b) } diff --git a/src/auth/argon2/riscv64.rs b/src/auth/argon2/riscv64.rs index 8e2255e6..c0205634 100644 --- a/src/auth/argon2/riscv64.rs +++ b/src/auth/argon2/riscv64.rs @@ -1,9 +1,8 @@ //! RISC-V Vector (RVV) BlaMka compression kernel for Argon2. //! -//! Uses two-u64 pairs to expose independent operations to the optimizer while -//! preserving ordinary scalar Rust semantics. Dispatch requires the RISC-V V -//! extension for this backend, but exact RVV lowering is target- and -//! toolchain-specific generated-code evidence. +//! Uses `core::simd::u64x2` pairs to expose two independent lanes per +//! operation. Dispatch requires the RISC-V V extension for this backend; +//! exact RVV lowering remains target- and toolchain-specific evidence. //! //! # Vectorisation topology //! @@ -14,29 +13,41 @@ //! ``` //! //! Column step: `GB(a_i, b_i, c_i, d_i)` for `i ∈ 0..4` runs at 4-way -//! parallelism with each `[u64; 2]` pair holding two of the four lanes +//! parallelism with each `u64x2` pair holding two of the four lanes //! per row (a/b/c/d × {lo, hi}). //! //! Diagonal step: rotate `b` by 1, `c` by 2, `d` by 3 within each -//! 4-lane row. Implemented as plain index swaps. +//! 4-lane row using cross-pair swizzles. //! //! # BlaMka multiply //! -//! `2 · lsb(a) · lsb(b)` lane-wise via masked `wrapping_mul`. The -//! product of two 32-bit values fits in u64, so masking with -//! `0xffffffff` and one `wrapping_mul(2)` is exact. +//! `2 · lsb(a) · lsb(b)` lane-wise via a masked SIMD multiply. The product of +//! two 32-bit values fits in u64, so the operation is exact modulo the lane +//! representation. //! //! # Rotations //! -//! All four (32, 24, 16, 63) use scalar `u64::rotate_right`; instruction -//! selection is left to the target compiler. +//! All four rotations use lane-wise SIMD shifts and OR. Instruction selection +//! is left to the target compiler. #![cfg(target_arch = "riscv64")] -#![allow(clippy::cast_possible_truncation)] + +use core::simd::u64x2; use super::BLOCK_WORDS; -type Pair = [u64; 2]; +type Pair = u64x2; + +struct RvvState { + a_lo: Pair, + a_hi: Pair, + b_lo: Pair, + b_hi: Pair, + c_lo: Pair, + c_hi: Pair, + d_lo: Pair, + d_hi: Pair, +} /// RISC-V V extension BlaMka compression kernel. /// @@ -53,225 +64,217 @@ pub(super) unsafe fn compress_rvv( y: &[u64; BLOCK_WORDS], xor_into: bool, ) { - // SAFETY: V extension is enabled by this function's `#[target_feature]`. - // Memory accesses are explicit indices into fixed-size arrays. let mut r = [0u64; BLOCK_WORDS]; let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { - r[i] = x[i] ^ y[i]; - r[i + 1] = x[i + 1] ^ y[i + 1]; - q[i] = r[i]; - q[i + 1] = r[i + 1]; - i += 2; + let rv = vxor(load_pair(x, i), load_pair(y, i)); + store_pair(&mut r, i, rv); + store_pair(&mut q, i, rv); + i = i.strict_add(2); } // Row pass. let mut row = 0usize; while row < 8 { - let base = row * 16; - let mut a_lo: Pair = [q[base], q[base + 1]]; - let mut a_hi: Pair = [q[base + 2], q[base + 3]]; - let mut b_lo: Pair = [q[base + 4], q[base + 5]]; - let mut b_hi: Pair = [q[base + 6], q[base + 7]]; - let mut c_lo: Pair = [q[base + 8], q[base + 9]]; - let mut c_hi: Pair = [q[base + 10], q[base + 11]]; - let mut d_lo: Pair = [q[base + 12], q[base + 13]]; - let mut d_hi: Pair = [q[base + 14], q[base + 15]]; - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - q[base] = a_lo[0]; - q[base + 1] = a_lo[1]; - q[base + 2] = a_hi[0]; - q[base + 3] = a_hi[1]; - q[base + 4] = b_lo[0]; - q[base + 5] = b_lo[1]; - q[base + 6] = b_hi[0]; - q[base + 7] = b_hi[1]; - q[base + 8] = c_lo[0]; - q[base + 9] = c_lo[1]; - q[base + 10] = c_hi[0]; - q[base + 11] = c_hi[1]; - q[base + 12] = d_lo[0]; - q[base + 13] = d_lo[1]; - q[base + 14] = d_hi[0]; - q[base + 15] = d_hi[1]; - row += 1; + let base = row.strict_mul(16); + let mut state = RvvState { + a_lo: load_pair(&q, base), + a_hi: load_pair(&q, base.strict_add(2)), + b_lo: load_pair(&q, base.strict_add(4)), + b_hi: load_pair(&q, base.strict_add(6)), + c_lo: load_pair(&q, base.strict_add(8)), + c_hi: load_pair(&q, base.strict_add(10)), + d_lo: load_pair(&q, base.strict_add(12)), + d_hi: load_pair(&q, base.strict_add(14)), + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a_lo); + store_pair(&mut q, base.strict_add(2), state.a_hi); + store_pair(&mut q, base.strict_add(4), state.b_lo); + store_pair(&mut q, base.strict_add(6), state.b_hi); + store_pair(&mut q, base.strict_add(8), state.c_lo); + store_pair(&mut q, base.strict_add(10), state.c_hi); + store_pair(&mut q, base.strict_add(12), state.d_lo); + store_pair(&mut q, base.strict_add(14), state.d_hi); + row = row.strict_add(1); } // Column pass. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a_lo: Pair = [q[base], q[base + 1]]; - let mut a_hi: Pair = [q[base + 16], q[base + 17]]; - let mut b_lo: Pair = [q[base + 32], q[base + 33]]; - let mut b_hi: Pair = [q[base + 48], q[base + 49]]; - let mut c_lo: Pair = [q[base + 64], q[base + 65]]; - let mut c_hi: Pair = [q[base + 80], q[base + 81]]; - let mut d_lo: Pair = [q[base + 96], q[base + 97]]; - let mut d_hi: Pair = [q[base + 112], q[base + 113]]; - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - q[base] = a_lo[0]; - q[base + 1] = a_lo[1]; - q[base + 16] = a_hi[0]; - q[base + 17] = a_hi[1]; - q[base + 32] = b_lo[0]; - q[base + 33] = b_lo[1]; - q[base + 48] = b_hi[0]; - q[base + 49] = b_hi[1]; - q[base + 64] = c_lo[0]; - q[base + 65] = c_lo[1]; - q[base + 80] = c_hi[0]; - q[base + 81] = c_hi[1]; - q[base + 96] = d_lo[0]; - q[base + 97] = d_lo[1]; - q[base + 112] = d_hi[0]; - q[base + 113] = d_hi[1]; - col += 1; + let base = col.strict_mul(2); + let mut state = RvvState { + a_lo: load_pair(&q, base), + a_hi: load_pair(&q, base.strict_add(16)), + b_lo: load_pair(&q, base.strict_add(32)), + b_hi: load_pair(&q, base.strict_add(48)), + c_lo: load_pair(&q, base.strict_add(64)), + c_hi: load_pair(&q, base.strict_add(80)), + d_lo: load_pair(&q, base.strict_add(96)), + d_hi: load_pair(&q, base.strict_add(112)), + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a_lo); + store_pair(&mut q, base.strict_add(16), state.a_hi); + store_pair(&mut q, base.strict_add(32), state.b_lo); + store_pair(&mut q, base.strict_add(48), state.b_hi); + store_pair(&mut q, base.strict_add(64), state.c_lo); + store_pair(&mut q, base.strict_add(80), state.c_hi); + store_pair(&mut q, base.strict_add(96), state.d_lo); + store_pair(&mut q, base.strict_add(112), state.d_hi); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor. - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { - let f0 = q[i] ^ r[i]; - let f1 = q[i + 1] ^ r[i + 1]; - if xor_into { - dst[i] ^= f0; - dst[i + 1] ^= f1; + let result = vxor(load_pair(&q, i), load_pair(&r, i)); + let result = if xor_into { + vxor(load_pair(dst, i), result) } else { - dst[i] = f0; - dst[i + 1] = f1; - } - i += 2; + result + }; + store_pair(dst, i, result); + i = i.strict_add(2); } } +#[inline(always)] +fn load_pair(buf: &[u64; BLOCK_WORDS], idx: usize) -> Pair { + u64x2::from_array([buf[idx], buf[idx.strict_add(1)]]) +} + +#[inline(always)] +fn store_pair(buf: &mut [u64; BLOCK_WORDS], idx: usize, pair: Pair) { + let lanes = pair.to_array(); + buf[idx] = lanes[0]; + buf[idx.strict_add(1)] = lanes[1]; +} + // ─── 4-way P-round ───────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn p_round( - a_lo: &mut Pair, - a_hi: &mut Pair, - b_lo: &mut Pair, - b_hi: &mut Pair, - c_lo: &mut Pair, - c_hi: &mut Pair, - d_lo: &mut Pair, - d_hi: &mut Pair, -) { +fn p_round(state: &mut RvvState) { // Column step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Diagonalise: rotate b by 1, c by 2, d by 3 across the 4-lane row. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = [tb_lo[1], tb_hi[0]]; - *b_hi = [tb_hi[1], tb_lo[0]]; + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_a1_b0(b_lo, b_hi); + state.b_hi = pair_b1_a0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = [td_hi[1], td_lo[0]]; - *d_hi = [td_lo[1], td_hi[0]]; + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_b1_a0(d_lo, d_hi); + state.d_hi = pair_a1_b0(d_lo, d_hi); // Diagonal step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Undo diagonalisation. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = [tb_hi[1], tb_lo[0]]; - *b_hi = [tb_lo[1], tb_hi[0]]; + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_b1_a0(b_lo, b_hi); + state.b_hi = pair_a1_b0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = [td_lo[1], td_hi[0]]; - *d_hi = [td_hi[1], td_lo[0]]; + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_a1_b0(d_lo, d_hi); + state.d_hi = pair_b1_a0(d_lo, d_hi); +} + +#[inline(always)] +fn pair_a1_b0(a: Pair, b: Pair) -> Pair { + core::simd::simd_swizzle!(a, b, [1, 2]) +} + +#[inline(always)] +fn pair_b1_a0(a: Pair, b: Pair) -> Pair { + core::simd::simd_swizzle!(a, b, [3, 0]) } // ─── 4-way BlaMka G ──────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn gb( - a_lo: &mut Pair, - a_hi: &mut Pair, - b_lo: &mut Pair, - b_hi: &mut Pair, - c_lo: &mut Pair, - c_hi: &mut Pair, - d_lo: &mut Pair, - d_hi: &mut Pair, -) { +fn gb(state: &mut RvvState) { // Step 1: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vadd(vadd(*a_lo, *b_lo), p_lo); - *a_hi = vadd(vadd(*a_hi, *b_hi), p_hi); - *d_lo = ror(vxor(*d_lo, *a_lo), 32); - *d_hi = ror(vxor(*d_hi, *a_hi), 32); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = bla_add(state.a_lo, state.b_lo, p_lo); + state.a_hi = bla_add(state.a_hi, state.b_hi, p_hi); + state.d_lo = ror::<32>(vxor(state.d_lo, state.a_lo)); + state.d_hi = ror::<32>(vxor(state.d_hi, state.a_hi)); // Step 2: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vadd(vadd(*c_lo, *d_lo), p_lo); - *c_hi = vadd(vadd(*c_hi, *d_hi), p_hi); - *b_lo = ror(vxor(*b_lo, *c_lo), 24); - *b_hi = ror(vxor(*b_hi, *c_hi), 24); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = bla_add(state.c_lo, state.d_lo, p_lo); + state.c_hi = bla_add(state.c_hi, state.d_hi, p_hi); + state.b_lo = ror::<24>(vxor(state.b_lo, state.c_lo)); + state.b_hi = ror::<24>(vxor(state.b_hi, state.c_hi)); // Step 3: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vadd(vadd(*a_lo, *b_lo), p_lo); - *a_hi = vadd(vadd(*a_hi, *b_hi), p_hi); - *d_lo = ror(vxor(*d_lo, *a_lo), 16); - *d_hi = ror(vxor(*d_hi, *a_hi), 16); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = bla_add(state.a_lo, state.b_lo, p_lo); + state.a_hi = bla_add(state.a_hi, state.b_hi, p_hi); + state.d_lo = ror::<16>(vxor(state.d_lo, state.a_lo)); + state.d_hi = ror::<16>(vxor(state.d_hi, state.a_hi)); // Step 4: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vadd(vadd(*c_lo, *d_lo), p_lo); - *c_hi = vadd(vadd(*c_hi, *d_hi), p_hi); - *b_lo = ror(vxor(*b_lo, *c_lo), 63); - *b_hi = ror(vxor(*b_hi, *c_hi), 63); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = bla_add(state.c_lo, state.d_lo, p_lo); + state.c_hi = bla_add(state.c_hi, state.d_hi, p_hi); + state.b_lo = ror::<63>(vxor(state.b_lo, state.c_lo)); + state.b_hi = ror::<63>(vxor(state.b_hi, state.c_hi)); } -// ─── Micro-ops (compiler-vectorised at VL=2/SEW=64 under -C target-feature=+v) ─ +// ─── Micro-ops ───────────────────────────────────────────────────────────── #[inline(always)] -fn vadd(a: Pair, b: Pair) -> Pair { - [a[0].wrapping_add(b[0]), a[1].wrapping_add(b[1])] +fn vxor(a: Pair, b: Pair) -> Pair { + core::ops::BitXor::bitxor(a, b) } #[inline(always)] -fn vxor(a: Pair, b: Pair) -> Pair { - [a[0] ^ b[0], a[1] ^ b[1]] +fn ror(value: Pair) -> Pair { + const { assert!(N > 0 && N < 64) } + let right = core::ops::Shr::shr(value, u64x2::splat(u64::from(N))); + let left = core::ops::Shl::shl(value, u64x2::splat(u64::from(64u32.strict_sub(N)))); + core::ops::BitOr::bitor(right, left) } #[inline(always)] -fn ror(x: Pair, n: u32) -> Pair { - [x[0].rotate_right(n), x[1].rotate_right(n)] +fn bla_add(a: Pair, b: Pair, product: Pair) -> Pair { + simd_wrapping_add(simd_wrapping_add(a, b), product) } /// `2 · lsb(a) · lsb(b)` lane-wise. Masked u32 multiply fits in u64 so /// the result is exact without a 128-bit-wide product. #[inline(always)] fn bla_mul(a: Pair, b: Pair) -> Pair { - const MASK: u64 = 0xffff_ffff; - [ - (a[0] & MASK).wrapping_mul(b[0] & MASK).wrapping_shl(1), - (a[1] & MASK).wrapping_mul(b[1] & MASK).wrapping_shl(1), - ] + let mask = u64x2::splat(0xffff_ffff); + let product = simd_wrapping_mul(a & mask, b & mask); + core::ops::Shl::shl(product, u64x2::splat(1)) +} + +#[inline(always)] +fn simd_wrapping_add(a: Pair, b: Pair) -> Pair { + core::ops::Add::add(a, b) +} + +#[inline(always)] +fn simd_wrapping_mul(a: Pair, b: Pair) -> Pair { + core::ops::Mul::mul(a, b) } diff --git a/src/auth/argon2/s390x.rs b/src/auth/argon2/s390x.rs index 310023cb..7749bb98 100644 --- a/src/auth/argon2/s390x.rs +++ b/src/auth/argon2/s390x.rs @@ -27,16 +27,28 @@ //! `s390x::VECTOR`. #![cfg(target_arch = "s390x")] -#![allow(unsafe_code)] -#![allow(clippy::cast_possible_truncation)] use core::simd::i64x2; use super::BLOCK_WORDS; +struct VectorState { + a_lo: i64x2, + a_hi: i64x2, + b_lo: i64x2, + b_hi: i64x2, + c_lo: i64x2, + c_hi: i64x2, + d_lo: i64x2, + d_hi: i64x2, +} + // ─── Inline-asm primitives (z13+ vector facility) ────────────────────────── #[target_feature(enable = "vector")] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; // SAFETY: z13+ vector facility via target_feature. @@ -53,6 +65,9 @@ unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { } #[target_feature(enable = "vector")] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { let out: i64x2; // SAFETY: z13+ vector facility via target_feature. @@ -69,6 +84,10 @@ unsafe fn vx(a: i64x2, b: i64x2) -> i64x2 { } /// `verllg` ROL by 32 = ROR 32. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_32(x: i64x2) -> i64x2 { let out: i64x2; @@ -85,6 +104,10 @@ unsafe fn verllg_32(x: i64x2) -> i64x2 { } /// `verllg` ROL by 40 = ROR 24. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_40(x: i64x2) -> i64x2 { let out: i64x2; @@ -101,6 +124,10 @@ unsafe fn verllg_40(x: i64x2) -> i64x2 { } /// `verllg` ROL by 48 = ROR 16. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_48(x: i64x2) -> i64x2 { let out: i64x2; @@ -117,6 +144,10 @@ unsafe fn verllg_48(x: i64x2) -> i64x2 { } /// `verllg` ROL by 1 = ROR 63. +/// +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] unsafe fn verllg_1(x: i64x2) -> i64x2 { let out: i64x2; @@ -143,15 +174,22 @@ fn pair_b1_a0(a: i64x2, b: i64x2) -> i64x2 { } #[inline(always)] +/// # Safety +/// +/// `p` must remain valid to read two initialized `u64` values from one +/// allocation. unsafe fn vload_pair(p: *const u64) -> i64x2 { // SAFETY: caller ensures p is valid for 2 × u64. - unsafe { core::ptr::read_unaligned(p as *const i64x2) } + unsafe { core::ptr::read_unaligned(p.cast()) } } #[inline(always)] +/// # Safety +/// +/// `p` must remain valid to write two `u64` values into one allocation. unsafe fn vstore_pair(p: *mut u64, v: i64x2) { // SAFETY: caller ensures p is valid for 2 × u64. - unsafe { core::ptr::write_unaligned(p as *mut i64x2, v) } + unsafe { core::ptr::write_unaligned(p.cast(), v) } } /// `2 · lsb(a) · lsb(b)` lane-wise — scalar fallback (z13 has no native @@ -162,108 +200,94 @@ fn bla_mul(a: i64x2, b: i64x2) -> i64x2 { let aa = a.to_array(); let bb = b.to_array(); const MASK: u64 = 0xffff_ffff; - let r0 = ((aa[0] as u64) & MASK) - .wrapping_mul((bb[0] as u64) & MASK) + let r0 = (aa[0].cast_unsigned() & MASK) + .wrapping_mul(bb[0].cast_unsigned() & MASK) .wrapping_shl(1); - let r1 = ((aa[1] as u64) & MASK) - .wrapping_mul((bb[1] as u64) & MASK) + let r1 = (aa[1].cast_unsigned() & MASK) + .wrapping_mul(bb[1].cast_unsigned() & MASK) .wrapping_shl(1); - i64x2::from_array([r0 as i64, r1 as i64]) + i64x2::from_array([r0.cast_signed(), r1.cast_signed()]) } // ─── 4-way P-round ───────────────────────────────────────────────────────── -#[allow(clippy::too_many_arguments)] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] -unsafe fn p_round( - a_lo: &mut i64x2, - a_hi: &mut i64x2, - b_lo: &mut i64x2, - b_hi: &mut i64x2, - c_lo: &mut i64x2, - c_hi: &mut i64x2, - d_lo: &mut i64x2, - d_hi: &mut i64x2, -) { +unsafe fn p_round(state: &mut VectorState) { // SAFETY: vector facility inherited. unsafe { - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = pair_a1_b0(tb_lo, tb_hi); - *b_hi = pair_b1_a0(tb_lo, tb_hi); + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_a1_b0(b_lo, b_hi); + state.b_hi = pair_b1_a0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = pair_b1_a0(td_lo, td_hi); - *d_hi = pair_a1_b0(td_lo, td_hi); + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_b1_a0(d_lo, d_hi); + state.d_hi = pair_a1_b0(d_lo, d_hi); - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = pair_b1_a0(tb_lo, tb_hi); - *b_hi = pair_a1_b0(tb_lo, tb_hi); + let b_lo = state.b_lo; + let b_hi = state.b_hi; + state.b_lo = pair_b1_a0(b_lo, b_hi); + state.b_hi = pair_a1_b0(b_lo, b_hi); - core::mem::swap(c_lo, c_hi); + core::mem::swap(&mut state.c_lo, &mut state.c_hi); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = pair_a1_b0(td_lo, td_hi); - *d_hi = pair_b1_a0(td_lo, td_hi); + let d_lo = state.d_lo; + let d_hi = state.d_hi; + state.d_lo = pair_a1_b0(d_lo, d_hi); + state.d_hi = pair_b1_a0(d_lo, d_hi); } } // ─── 4-way BlaMka G ──────────────────────────────────────────────────────── -#[allow(clippy::too_many_arguments)] +/// # Safety +/// +/// The current CPU must support the z13+ vector facility. #[target_feature(enable = "vector")] -unsafe fn gb( - a_lo: &mut i64x2, - a_hi: &mut i64x2, - b_lo: &mut i64x2, - b_hi: &mut i64x2, - c_lo: &mut i64x2, - c_hi: &mut i64x2, - d_lo: &mut i64x2, - d_hi: &mut i64x2, -) { +unsafe fn gb(state: &mut VectorState) { // SAFETY: vector facility inherited. unsafe { // Step 1 - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vag(vag(*a_lo, *b_lo), p_lo); - *a_hi = vag(vag(*a_hi, *b_hi), p_hi); - *d_lo = verllg_32(vx(*d_lo, *a_lo)); - *d_hi = verllg_32(vx(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vag(vag(state.a_lo, state.b_lo), p_lo); + state.a_hi = vag(vag(state.a_hi, state.b_hi), p_hi); + state.d_lo = verllg_32(vx(state.d_lo, state.a_lo)); + state.d_hi = verllg_32(vx(state.d_hi, state.a_hi)); // Step 2 - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vag(vag(*c_lo, *d_lo), p_lo); - *c_hi = vag(vag(*c_hi, *d_hi), p_hi); - *b_lo = verllg_40(vx(*b_lo, *c_lo)); - *b_hi = verllg_40(vx(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vag(vag(state.c_lo, state.d_lo), p_lo); + state.c_hi = vag(vag(state.c_hi, state.d_hi), p_hi); + state.b_lo = verllg_40(vx(state.b_lo, state.c_lo)); + state.b_hi = verllg_40(vx(state.b_hi, state.c_hi)); // Step 3 - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = vag(vag(*a_lo, *b_lo), p_lo); - *a_hi = vag(vag(*a_hi, *b_hi), p_hi); - *d_lo = verllg_48(vx(*d_lo, *a_lo)); - *d_hi = verllg_48(vx(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a_lo, state.b_lo); + let p_hi = bla_mul(state.a_hi, state.b_hi); + state.a_lo = vag(vag(state.a_lo, state.b_lo), p_lo); + state.a_hi = vag(vag(state.a_hi, state.b_hi), p_hi); + state.d_lo = verllg_48(vx(state.d_lo, state.a_lo)); + state.d_hi = verllg_48(vx(state.d_hi, state.a_hi)); // Step 4 - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = vag(vag(*c_lo, *d_lo), p_lo); - *c_hi = vag(vag(*c_hi, *d_hi), p_hi); - *b_lo = verllg_1(vx(*b_lo, *c_lo)); - *b_hi = verllg_1(vx(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c_lo, state.d_lo); + let p_hi = bla_mul(state.c_hi, state.d_hi); + state.c_lo = vag(vag(state.c_lo, state.d_lo), p_lo); + state.c_hi = vag(vag(state.c_hi, state.d_hi), p_hi); + state.b_lo = verllg_1(vx(state.b_lo, state.c_lo)); + state.b_hi = verllg_1(vx(state.b_hi, state.c_hi)); } } @@ -289,74 +313,74 @@ pub(super) unsafe fn compress_vector( unsafe { let mut r = [0u64; BLOCK_WORDS]; let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { let xv = vload_pair(x.as_ptr().add(i)); let yv = vload_pair(y.as_ptr().add(i)); let rv = vx(xv, yv); vstore_pair(r.as_mut_ptr().add(i), rv); vstore_pair(q.as_mut_ptr().add(i), rv); - i += 2; + i = i.strict_add(2); } // Row pass. let mut row = 0usize; while row < 8 { - let base = row * 16; - let mut a_lo = vload_pair(q.as_ptr().add(base)); - let mut a_hi = vload_pair(q.as_ptr().add(base + 2)); - let mut b_lo = vload_pair(q.as_ptr().add(base + 4)); - let mut b_hi = vload_pair(q.as_ptr().add(base + 6)); - let mut c_lo = vload_pair(q.as_ptr().add(base + 8)); - let mut c_hi = vload_pair(q.as_ptr().add(base + 10)); - let mut d_lo = vload_pair(q.as_ptr().add(base + 12)); - let mut d_hi = vload_pair(q.as_ptr().add(base + 14)); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - vstore_pair(q.as_mut_ptr().add(base), a_lo); - vstore_pair(q.as_mut_ptr().add(base + 2), a_hi); - vstore_pair(q.as_mut_ptr().add(base + 4), b_lo); - vstore_pair(q.as_mut_ptr().add(base + 6), b_hi); - vstore_pair(q.as_mut_ptr().add(base + 8), c_lo); - vstore_pair(q.as_mut_ptr().add(base + 10), c_hi); - vstore_pair(q.as_mut_ptr().add(base + 12), d_lo); - vstore_pair(q.as_mut_ptr().add(base + 14), d_hi); - row += 1; + let base = row.strict_mul(16); + let mut state = VectorState { + a_lo: vload_pair(q.as_ptr().add(base)), + a_hi: vload_pair(q.as_ptr().add(base.strict_add(2))), + b_lo: vload_pair(q.as_ptr().add(base.strict_add(4))), + b_hi: vload_pair(q.as_ptr().add(base.strict_add(6))), + c_lo: vload_pair(q.as_ptr().add(base.strict_add(8))), + c_hi: vload_pair(q.as_ptr().add(base.strict_add(10))), + d_lo: vload_pair(q.as_ptr().add(base.strict_add(12))), + d_hi: vload_pair(q.as_ptr().add(base.strict_add(14))), + }; + + p_round(&mut state); + + vstore_pair(q.as_mut_ptr().add(base), state.a_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(2)), state.a_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(4)), state.b_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(6)), state.b_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(8)), state.c_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(10)), state.c_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(12)), state.d_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(14)), state.d_hi); + row = row.strict_add(1); } // Column pass. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a_lo = vload_pair(q.as_ptr().add(base)); - let mut a_hi = vload_pair(q.as_ptr().add(base + 16)); - let mut b_lo = vload_pair(q.as_ptr().add(base + 32)); - let mut b_hi = vload_pair(q.as_ptr().add(base + 48)); - let mut c_lo = vload_pair(q.as_ptr().add(base + 64)); - let mut c_hi = vload_pair(q.as_ptr().add(base + 80)); - let mut d_lo = vload_pair(q.as_ptr().add(base + 96)); - let mut d_hi = vload_pair(q.as_ptr().add(base + 112)); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - vstore_pair(q.as_mut_ptr().add(base), a_lo); - vstore_pair(q.as_mut_ptr().add(base + 16), a_hi); - vstore_pair(q.as_mut_ptr().add(base + 32), b_lo); - vstore_pair(q.as_mut_ptr().add(base + 48), b_hi); - vstore_pair(q.as_mut_ptr().add(base + 64), c_lo); - vstore_pair(q.as_mut_ptr().add(base + 80), c_hi); - vstore_pair(q.as_mut_ptr().add(base + 96), d_lo); - vstore_pair(q.as_mut_ptr().add(base + 112), d_hi); - col += 1; + let base = col.strict_mul(2); + let mut state = VectorState { + a_lo: vload_pair(q.as_ptr().add(base)), + a_hi: vload_pair(q.as_ptr().add(base.strict_add(16))), + b_lo: vload_pair(q.as_ptr().add(base.strict_add(32))), + b_hi: vload_pair(q.as_ptr().add(base.strict_add(48))), + c_lo: vload_pair(q.as_ptr().add(base.strict_add(64))), + c_hi: vload_pair(q.as_ptr().add(base.strict_add(80))), + d_lo: vload_pair(q.as_ptr().add(base.strict_add(96))), + d_hi: vload_pair(q.as_ptr().add(base.strict_add(112))), + }; + + p_round(&mut state); + + vstore_pair(q.as_mut_ptr().add(base), state.a_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(16)), state.a_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(32)), state.b_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(48)), state.b_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(64)), state.c_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(80)), state.c_hi); + vstore_pair(q.as_mut_ptr().add(base.strict_add(96)), state.d_lo); + vstore_pair(q.as_mut_ptr().add(base.strict_add(112)), state.d_hi); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor. - let mut i = 0; + let mut i = 0usize; while i < BLOCK_WORDS { let qv = vload_pair(q.as_ptr().add(i)); let rv = vload_pair(r.as_ptr().add(i)); @@ -367,7 +391,7 @@ pub(super) unsafe fn compress_vector( } else { vstore_pair(dst.as_mut_ptr().add(i), f); } - i += 2; + i = i.strict_add(2); } } } diff --git a/src/auth/argon2/wasm.rs b/src/auth/argon2/wasm.rs index 49c314ea..5c61e2c3 100644 --- a/src/auth/argon2/wasm.rs +++ b/src/auth/argon2/wasm.rs @@ -36,15 +36,32 @@ //! - ROR 63 ≡ ROL 1: shift-right + shift-left + OR. #![cfg(target_arch = "wasm32")] -#![allow(clippy::cast_possible_truncation)] use core::arch::wasm32::{ - i8x16_shuffle, i64x2_add, i64x2_mul, i64x2_shuffle, u64x2_shl, u64x2_shr, u64x2_splat, v128, v128_and, v128_load, - v128_or, v128_store, v128_xor, + i8x16_shuffle, i64x2_add, i64x2_mul, i64x2_shuffle, u64x2, u64x2_extract_lane, u64x2_shl, u64x2_shr, u64x2_splat, + v128, v128_and, v128_or, v128_xor, }; use super::BLOCK_WORDS; +struct RoundState { + a: [v128; 2], + b: [v128; 2], + c: [v128; 2], + d: [v128; 2], +} + +#[inline(always)] +fn load_pair(words: &[u64; N], offset: usize) -> v128 { + u64x2(words[offset], words[offset.strict_add(1)]) +} + +#[inline(always)] +fn store_pair(words: &mut [u64; N], offset: usize, value: v128) { + words[offset] = u64x2_extract_lane::<0>(value); + words[offset.strict_add(1)] = u64x2_extract_lane::<1>(value); +} + /// WebAssembly SIMD128 BlaMka compression kernel. /// /// # Safety @@ -60,190 +77,150 @@ pub(super) unsafe fn compress_simd128( y: &[u64; BLOCK_WORDS], xor_into: bool, ) { - // SAFETY: simd128 is enabled by this function's `#[target_feature]` - // attribute, so all `v128`/`u64x2_*` ops below are valid to call. - unsafe { - // R = X XOR Y, materialised to scratch for re-reads during the - // row + column passes plus the final XOR. - let mut r = [0u64; BLOCK_WORDS]; - let mut q = [0u64; BLOCK_WORDS]; - let mut i = 0; - while i < BLOCK_WORDS { - let xv = v128_load(x.as_ptr().add(i).cast()); - let yv = v128_load(y.as_ptr().add(i).cast()); - let rv = v128_xor(xv, yv); - v128_store(r.as_mut_ptr().add(i).cast(), rv); - v128_store(q.as_mut_ptr().add(i).cast(), rv); - i += 2; - } - - // Row pass: 8 P-rounds on contiguous 16-u64 chunks of q[]. - let mut row = 0usize; - while row < 8 { - let base = row * 16; - let mut a_lo = v128_load(q.as_ptr().add(base).cast()); - let mut a_hi = v128_load(q.as_ptr().add(base + 2).cast()); - let mut b_lo = v128_load(q.as_ptr().add(base + 4).cast()); - let mut b_hi = v128_load(q.as_ptr().add(base + 6).cast()); - let mut c_lo = v128_load(q.as_ptr().add(base + 8).cast()); - let mut c_hi = v128_load(q.as_ptr().add(base + 10).cast()); - let mut d_lo = v128_load(q.as_ptr().add(base + 12).cast()); - let mut d_hi = v128_load(q.as_ptr().add(base + 14).cast()); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - v128_store(q.as_mut_ptr().add(base).cast(), a_lo); - v128_store(q.as_mut_ptr().add(base + 2).cast(), a_hi); - v128_store(q.as_mut_ptr().add(base + 4).cast(), b_lo); - v128_store(q.as_mut_ptr().add(base + 6).cast(), b_hi); - v128_store(q.as_mut_ptr().add(base + 8).cast(), c_lo); - v128_store(q.as_mut_ptr().add(base + 10).cast(), c_hi); - v128_store(q.as_mut_ptr().add(base + 12).cast(), d_lo); - v128_store(q.as_mut_ptr().add(base + 14).cast(), d_hi); - row += 1; - } - - // Column pass: 8 P-rounds on stride-16 u64 sequences. Each lane of - // 2 u64 is loaded directly from the natural row-major positions — - // see RFC 9106 §3.6 column-step indexing. - let mut col = 0usize; - while col < 8 { - let base = col * 2; - let mut a_lo = v128_load(q.as_ptr().add(base).cast()); - let mut a_hi = v128_load(q.as_ptr().add(base + 16).cast()); - let mut b_lo = v128_load(q.as_ptr().add(base + 32).cast()); - let mut b_hi = v128_load(q.as_ptr().add(base + 48).cast()); - let mut c_lo = v128_load(q.as_ptr().add(base + 64).cast()); - let mut c_hi = v128_load(q.as_ptr().add(base + 80).cast()); - let mut d_lo = v128_load(q.as_ptr().add(base + 96).cast()); - let mut d_hi = v128_load(q.as_ptr().add(base + 112).cast()); - - p_round( - &mut a_lo, &mut a_hi, &mut b_lo, &mut b_hi, &mut c_lo, &mut c_hi, &mut d_lo, &mut d_hi, - ); - - v128_store(q.as_mut_ptr().add(base).cast(), a_lo); - v128_store(q.as_mut_ptr().add(base + 16).cast(), a_hi); - v128_store(q.as_mut_ptr().add(base + 32).cast(), b_lo); - v128_store(q.as_mut_ptr().add(base + 48).cast(), b_hi); - v128_store(q.as_mut_ptr().add(base + 64).cast(), c_lo); - v128_store(q.as_mut_ptr().add(base + 80).cast(), c_hi); - v128_store(q.as_mut_ptr().add(base + 96).cast(), d_lo); - v128_store(q.as_mut_ptr().add(base + 112).cast(), d_hi); - col += 1; - } - - // Final XOR with R, fused with dst store/xor. - let mut i = 0; - while i < BLOCK_WORDS { - let qv = v128_load(q.as_ptr().add(i).cast()); - let rv = v128_load(r.as_ptr().add(i).cast()); - let f = v128_xor(qv, rv); - if xor_into { - let cur = v128_load(dst.as_ptr().add(i).cast()); - v128_store(dst.as_mut_ptr().add(i).cast(), v128_xor(cur, f)); - } else { - v128_store(dst.as_mut_ptr().add(i).cast(), f); - } - i += 2; - } + // R = X XOR Y, materialised to scratch for re-reads during the + // row + column passes plus the final XOR. + let mut r = [0u64; BLOCK_WORDS]; + let mut q = [0u64; BLOCK_WORDS]; + let mut i = 0usize; + while i < BLOCK_WORDS { + let rv = v128_xor(load_pair(x, i), load_pair(y, i)); + store_pair(&mut r, i, rv); + store_pair(&mut q, i, rv); + i = i.strict_add(2); + } + + // Row pass: 8 P-rounds on contiguous 16-u64 chunks of q[]. + let mut row = 0usize; + while row < 8 { + let base = row.strict_mul(16); + let mut state = RoundState { + a: [load_pair(&q, base), load_pair(&q, base.strict_add(2))], + b: [load_pair(&q, base.strict_add(4)), load_pair(&q, base.strict_add(6))], + c: [load_pair(&q, base.strict_add(8)), load_pair(&q, base.strict_add(10))], + d: [load_pair(&q, base.strict_add(12)), load_pair(&q, base.strict_add(14))], + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a[0]); + store_pair(&mut q, base.strict_add(2), state.a[1]); + store_pair(&mut q, base.strict_add(4), state.b[0]); + store_pair(&mut q, base.strict_add(6), state.b[1]); + store_pair(&mut q, base.strict_add(8), state.c[0]); + store_pair(&mut q, base.strict_add(10), state.c[1]); + store_pair(&mut q, base.strict_add(12), state.d[0]); + store_pair(&mut q, base.strict_add(14), state.d[1]); + row = row.strict_add(1); + } + + // Column pass: 8 P-rounds on stride-16 u64 sequences. Each lane of + // 2 u64 is loaded directly from the natural row-major positions — + // see RFC 9106 §3.6 column-step indexing. + let mut col = 0usize; + while col < 8 { + let base = col.strict_mul(2); + let mut state = RoundState { + a: [load_pair(&q, base), load_pair(&q, base.strict_add(16))], + b: [load_pair(&q, base.strict_add(32)), load_pair(&q, base.strict_add(48))], + c: [load_pair(&q, base.strict_add(64)), load_pair(&q, base.strict_add(80))], + d: [load_pair(&q, base.strict_add(96)), load_pair(&q, base.strict_add(112))], + }; + + p_round(&mut state); + + store_pair(&mut q, base, state.a[0]); + store_pair(&mut q, base.strict_add(16), state.a[1]); + store_pair(&mut q, base.strict_add(32), state.b[0]); + store_pair(&mut q, base.strict_add(48), state.b[1]); + store_pair(&mut q, base.strict_add(64), state.c[0]); + store_pair(&mut q, base.strict_add(80), state.c[1]); + store_pair(&mut q, base.strict_add(96), state.d[0]); + store_pair(&mut q, base.strict_add(112), state.d[1]); + col = col.strict_add(1); + } + + // Final XOR with R, fused with dst store/xor. + let mut i = 0usize; + while i < BLOCK_WORDS { + let f = v128_xor(load_pair(&q, i), load_pair(&r, i)); + let output = if xor_into { v128_xor(load_pair(dst, i), f) } else { f }; + store_pair(dst, i, output); + i = i.strict_add(2); } } // ─── 4-way P-round ───────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn p_round( - a_lo: &mut v128, - a_hi: &mut v128, - b_lo: &mut v128, - b_hi: &mut v128, - c_lo: &mut v128, - c_hi: &mut v128, - d_lo: &mut v128, - d_hi: &mut v128, -) { +fn p_round(state: &mut RoundState) { // Column step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Diagonalise: rotate b by 1, c by 2, d by 3 across the 4-lane row. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); - *b_hi = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); + let tb_lo = state.b[0]; + let tb_hi = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); + state.b[1] = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); - core::mem::swap(c_lo, c_hi); + state.c.swap(0, 1); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = i64x2_shuffle::<1, 2>(td_hi, td_lo); - *d_hi = i64x2_shuffle::<1, 2>(td_lo, td_hi); + let td_lo = state.d[0]; + let td_hi = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td_hi, td_lo); + state.d[1] = i64x2_shuffle::<1, 2>(td_lo, td_hi); // Diagonal step. - gb(a_lo, a_hi, b_lo, b_hi, c_lo, c_hi, d_lo, d_hi); + gb(state); // Undo diagonalisation. - let tb_lo = *b_lo; - let tb_hi = *b_hi; - *b_lo = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); - *b_hi = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); + let tb_lo = state.b[0]; + let tb_hi = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb_hi, tb_lo); + state.b[1] = i64x2_shuffle::<1, 2>(tb_lo, tb_hi); - core::mem::swap(c_lo, c_hi); + state.c.swap(0, 1); - let td_lo = *d_lo; - let td_hi = *d_hi; - *d_lo = i64x2_shuffle::<1, 2>(td_lo, td_hi); - *d_hi = i64x2_shuffle::<1, 2>(td_hi, td_lo); + let td_lo = state.d[0]; + let td_hi = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td_lo, td_hi); + state.d[1] = i64x2_shuffle::<1, 2>(td_hi, td_lo); } // ─── 4-way BlaMka G ──────────────────────────────────────────────────────── #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn gb( - a_lo: &mut v128, - a_hi: &mut v128, - b_lo: &mut v128, - b_hi: &mut v128, - c_lo: &mut v128, - c_hi: &mut v128, - d_lo: &mut v128, - d_hi: &mut v128, -) { +fn gb(state: &mut RoundState) { // Step 1: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = i64x2_add(i64x2_add(*a_lo, *b_lo), p_lo); - *a_hi = i64x2_add(i64x2_add(*a_hi, *b_hi), p_hi); - *d_lo = ror32(v128_xor(*d_lo, *a_lo)); - *d_hi = ror32(v128_xor(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a[0], state.b[0]); + let p_hi = bla_mul(state.a[1], state.b[1]); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), p_lo); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), p_hi); + state.d[0] = ror32(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror32(v128_xor(state.d[1], state.a[1])); // Step 2: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = i64x2_add(i64x2_add(*c_lo, *d_lo), p_lo); - *c_hi = i64x2_add(i64x2_add(*c_hi, *d_hi), p_hi); - *b_lo = ror24(v128_xor(*b_lo, *c_lo)); - *b_hi = ror24(v128_xor(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c[0], state.d[0]); + let p_hi = bla_mul(state.c[1], state.d[1]); + state.c[0] = i64x2_add(i64x2_add(state.c[0], state.d[0]), p_lo); + state.c[1] = i64x2_add(i64x2_add(state.c[1], state.d[1]), p_hi); + state.b[0] = ror24(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror24(v128_xor(state.b[1], state.c[1])); // Step 3: a = a + b + 2·lsb(a)·lsb(b) - let p_lo = bla_mul(*a_lo, *b_lo); - let p_hi = bla_mul(*a_hi, *b_hi); - *a_lo = i64x2_add(i64x2_add(*a_lo, *b_lo), p_lo); - *a_hi = i64x2_add(i64x2_add(*a_hi, *b_hi), p_hi); - *d_lo = ror16(v128_xor(*d_lo, *a_lo)); - *d_hi = ror16(v128_xor(*d_hi, *a_hi)); + let p_lo = bla_mul(state.a[0], state.b[0]); + let p_hi = bla_mul(state.a[1], state.b[1]); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), p_lo); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), p_hi); + state.d[0] = ror16(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror16(v128_xor(state.d[1], state.a[1])); // Step 4: c = c + d + 2·lsb(c)·lsb(d) - let p_lo = bla_mul(*c_lo, *d_lo); - let p_hi = bla_mul(*c_hi, *d_hi); - *c_lo = i64x2_add(i64x2_add(*c_lo, *d_lo), p_lo); - *c_hi = i64x2_add(i64x2_add(*c_hi, *d_hi), p_hi); - *b_lo = ror63(v128_xor(*b_lo, *c_lo)); - *b_hi = ror63(v128_xor(*b_hi, *c_hi)); + let p_lo = bla_mul(state.c[0], state.d[0]); + let p_hi = bla_mul(state.c[1], state.d[1]); + state.c[0] = i64x2_add(i64x2_add(state.c[0], state.d[0]), p_lo); + state.c[1] = i64x2_add(i64x2_add(state.c[1], state.d[1]), p_hi); + state.b[0] = ror63(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror63(v128_xor(state.b[1], state.c[1])); } // ─── Micro-ops ───────────────────────────────────────────────────────────── diff --git a/src/auth/argon2/x86_64.rs b/src/auth/argon2/x86_64.rs index 10cb9b9f..f24180bd 100644 --- a/src/auth/argon2/x86_64.rs +++ b/src/auth/argon2/x86_64.rs @@ -36,7 +36,6 @@ //! left shift by 1 supplies the `2 · …` factor. #![cfg(target_arch = "x86_64")] -#![allow(clippy::cast_possible_truncation)] use core::arch::x86_64::{ __m256i, __m512i, _mm_loadu_si128, _mm_storeu_si128, _mm256_add_epi64, _mm256_castsi128_si256, @@ -100,25 +99,25 @@ pub(super) unsafe fn compress_avx2( let rv = _mm256_xor_si256(xv, yv); _mm256_storeu_si256(r.as_mut_ptr().add(i).cast(), rv); _mm256_storeu_si256(q.as_mut_ptr().add(i).cast(), rv); - i += 4; + i = i.strict_add(4); } // Row pass: 8 P-rounds on contiguous 16-u64 chunks of q[]. let mut row = 0usize; while row < 8 { - let base = row * 16; + let base = row.strict_mul(16); let mut a = _mm256_loadu_si256(q.as_ptr().add(base).cast()); - let mut b = _mm256_loadu_si256(q.as_ptr().add(base + 4).cast()); - let mut c = _mm256_loadu_si256(q.as_ptr().add(base + 8).cast()); - let mut d = _mm256_loadu_si256(q.as_ptr().add(base + 12).cast()); + let mut b = _mm256_loadu_si256(q.as_ptr().add(base.strict_add(4)).cast()); + let mut c = _mm256_loadu_si256(q.as_ptr().add(base.strict_add(8)).cast()); + let mut d = _mm256_loadu_si256(q.as_ptr().add(base.strict_add(12)).cast()); p_round_avx2(&mut a, &mut b, &mut c, &mut d); _mm256_storeu_si256(q.as_mut_ptr().add(base).cast(), a); - _mm256_storeu_si256(q.as_mut_ptr().add(base + 4).cast(), b); - _mm256_storeu_si256(q.as_mut_ptr().add(base + 8).cast(), c); - _mm256_storeu_si256(q.as_mut_ptr().add(base + 12).cast(), d); - row += 1; + _mm256_storeu_si256(q.as_mut_ptr().add(base.strict_add(4)).cast(), b); + _mm256_storeu_si256(q.as_mut_ptr().add(base.strict_add(8)).cast(), c); + _mm256_storeu_si256(q.as_mut_ptr().add(base.strict_add(12)).cast(), d); + row = row.strict_add(1); } // Column pass: 8 P-rounds on stride-16 u64 sequences. Each YMM holds @@ -126,19 +125,19 @@ pub(super) unsafe fn compress_avx2( // high half = row 2k+1) — see RFC 9106 §3.6 column-step indexing. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a = load_col_pair_avx2(&q, base, base + 16); - let mut b = load_col_pair_avx2(&q, base + 32, base + 48); - let mut c = load_col_pair_avx2(&q, base + 64, base + 80); - let mut d = load_col_pair_avx2(&q, base + 96, base + 112); + let base = col.strict_mul(2); + let mut a = load_col_pair_avx2(&q, base, base.strict_add(16)); + let mut b = load_col_pair_avx2(&q, base.strict_add(32), base.strict_add(48)); + let mut c = load_col_pair_avx2(&q, base.strict_add(64), base.strict_add(80)); + let mut d = load_col_pair_avx2(&q, base.strict_add(96), base.strict_add(112)); p_round_avx2(&mut a, &mut b, &mut c, &mut d); - store_col_pair_avx2(&mut q, base, base + 16, a); - store_col_pair_avx2(&mut q, base + 32, base + 48, b); - store_col_pair_avx2(&mut q, base + 64, base + 80, c); - store_col_pair_avx2(&mut q, base + 96, base + 112, d); - col += 1; + store_col_pair_avx2(&mut q, base, base.strict_add(16), a); + store_col_pair_avx2(&mut q, base.strict_add(32), base.strict_add(48), b); + store_col_pair_avx2(&mut q, base.strict_add(64), base.strict_add(80), c); + store_col_pair_avx2(&mut q, base.strict_add(96), base.strict_add(112), d); + col = col.strict_add(1); } // Final XOR with R, fused with the dst store/xor. @@ -153,7 +152,7 @@ pub(super) unsafe fn compress_avx2( } else { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast(), f); } - i += 4; + i = i.strict_add(4); } } } @@ -251,12 +250,18 @@ unsafe fn gb_avx2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror32_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; shuffle imm 0xB1 swaps adjacent u32 halves. unsafe { _mm256_shuffle_epi32(x, 0xB1) } } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror24_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; ROT24_MASK is 32-byte aligned static data. unsafe { @@ -266,6 +271,9 @@ unsafe fn ror24_avx2(x: __m256i) -> __m256i { } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror16_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; ROT16_MASK is 32-byte aligned static data. unsafe { @@ -275,6 +283,9 @@ unsafe fn ror16_avx2(x: __m256i) -> __m256i { } #[inline(always)] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn ror63_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 inherited; (x << 1) | (x >> 63). unsafe { _mm256_or_si256(_mm256_add_epi64(x, x), _mm256_srli_epi64(x, 63)) } @@ -319,7 +330,7 @@ pub(super) unsafe fn compress_avx512( let rv = _mm512_xor_si512(xv, yv); _mm512_storeu_si512(r.as_mut_ptr().add(i).cast(), rv); _mm512_storeu_si512(q.as_mut_ptr().add(i).cast(), rv); - i += 8; + i = i.strict_add(8); } // Row pass: 4 iterations × 2 rows per iter = 8 P-rounds done at @@ -339,14 +350,14 @@ pub(super) unsafe fn compress_avx512( // Each `VSHUFI64X2` picks 4 of 8 contiguous 128-bit lanes (4 from // src1, 4 from src2) using a single imm8. The 0x44 imm picks lanes // 0,1 from each source; 0xEE picks lanes 2,3. - let mut iter = 0; + let mut iter = 0usize; while iter < 4 { - let off = iter * 32; + let off = iter.strict_mul(32); let r0_lo = _mm512_loadu_si512(q.as_ptr().add(off).cast()); - let r0_hi = _mm512_loadu_si512(q.as_ptr().add(off + 8).cast()); - let r1_lo = _mm512_loadu_si512(q.as_ptr().add(off + 16).cast()); - let r1_hi = _mm512_loadu_si512(q.as_ptr().add(off + 24).cast()); + let r0_hi = _mm512_loadu_si512(q.as_ptr().add(off.strict_add(8)).cast()); + let r1_lo = _mm512_loadu_si512(q.as_ptr().add(off.strict_add(16)).cast()); + let r1_hi = _mm512_loadu_si512(q.as_ptr().add(off.strict_add(24)).cast()); let mut a = _mm512_shuffle_i64x2(r0_lo, r1_lo, 0x44); let mut b = _mm512_shuffle_i64x2(r0_lo, r1_lo, 0xEE); @@ -363,10 +374,10 @@ pub(super) unsafe fn compress_avx512( let r1_hi_out = _mm512_shuffle_i64x2(c, d, 0xEE); _mm512_storeu_si512(q.as_mut_ptr().add(off).cast(), r0_lo_out); - _mm512_storeu_si512(q.as_mut_ptr().add(off + 8).cast(), r0_hi_out); - _mm512_storeu_si512(q.as_mut_ptr().add(off + 16).cast(), r1_lo_out); - _mm512_storeu_si512(q.as_mut_ptr().add(off + 24).cast(), r1_hi_out); - iter += 1; + _mm512_storeu_si512(q.as_mut_ptr().add(off.strict_add(8)).cast(), r0_hi_out); + _mm512_storeu_si512(q.as_mut_ptr().add(off.strict_add(16)).cast(), r1_lo_out); + _mm512_storeu_si512(q.as_mut_ptr().add(off.strict_add(24)).cast(), r1_hi_out); + iter = iter.strict_add(1); } // Column pass: 4-way YMM with native `VPRORQ` rotations. @@ -379,19 +390,19 @@ pub(super) unsafe fn compress_avx512( // 2 × insert per GB-lane. let mut col = 0usize; while col < 8 { - let base = col * 2; - let mut a = load_col_pair_avx2(&q, base, base + 16); - let mut b = load_col_pair_avx2(&q, base + 32, base + 48); - let mut c = load_col_pair_avx2(&q, base + 64, base + 80); - let mut d = load_col_pair_avx2(&q, base + 96, base + 112); + let base = col.strict_mul(2); + let mut a = load_col_pair_avx2(&q, base, base.strict_add(16)); + let mut b = load_col_pair_avx2(&q, base.strict_add(32), base.strict_add(48)); + let mut c = load_col_pair_avx2(&q, base.strict_add(64), base.strict_add(80)); + let mut d = load_col_pair_avx2(&q, base.strict_add(96), base.strict_add(112)); p_round_avx512vl(&mut a, &mut b, &mut c, &mut d); - store_col_pair_avx2(&mut q, base, base + 16, a); - store_col_pair_avx2(&mut q, base + 32, base + 48, b); - store_col_pair_avx2(&mut q, base + 64, base + 80, c); - store_col_pair_avx2(&mut q, base + 96, base + 112, d); - col += 1; + store_col_pair_avx2(&mut q, base, base.strict_add(16), a); + store_col_pair_avx2(&mut q, base.strict_add(32), base.strict_add(48), b); + store_col_pair_avx2(&mut q, base.strict_add(64), base.strict_add(80), c); + store_col_pair_avx2(&mut q, base.strict_add(96), base.strict_add(112), d); + col = col.strict_add(1); } // Final XOR with R, fused with dst store/xor at ZMM width. @@ -406,7 +417,7 @@ pub(super) unsafe fn compress_avx512( } else { _mm512_storeu_si512(dst.as_mut_ptr().add(i).cast(), f); } - i += 8; + i = i.strict_add(8); } } } diff --git a/src/auth/curve25519_edwards.rs b/src/auth/curve25519_edwards.rs index 48b65cc4..85e13729 100644 --- a/src/auth/curve25519_edwards.rs +++ b/src/auth/curve25519_edwards.rs @@ -4,28 +4,23 @@ //! same Curve25519 basepoint. Reuse the Ed25519 basepoint machinery instead of //! maintaining a second precompute stack for the identical scalar-mul problem. -#[allow(dead_code)] +#[cfg(feature = "ed25519")] #[path = "ed25519/constants.rs"] pub(crate) mod constants; -#[allow(dead_code)] #[path = "ed25519/field.rs"] pub(crate) mod field; #[cfg(target_arch = "x86_64")] -#[allow(dead_code)] #[path = "ed25519/field_avx2.rs"] pub(crate) mod field_avx2; #[cfg(target_arch = "x86_64")] -#[allow(dead_code)] #[path = "ed25519/field_ifma.rs"] pub(crate) mod field_ifma; -#[allow(dead_code)] #[path = "ed25519/point.rs"] pub(crate) mod point; #[cfg(target_arch = "x86_64")] -#[allow(dead_code)] #[path = "ed25519/point_avx2.rs"] pub(crate) mod point_avx2; -#[allow(dead_code)] +#[cfg(feature = "ed25519")] #[path = "ed25519/scalar.rs"] pub(crate) mod scalar; @@ -38,8 +33,18 @@ pub use point_avx2::{ }; /// Dispatch `[s]B` (fixed-base scalar mul) to the fastest validated CT path. +#[cfg_attr( + all( + target_arch = "x86_64", + target_os = "linux", + not(any(test, miri, feature = "portable-only")) + ), + expect( + dead_code, + reason = "x86_64 Linux library builds use the assembly fixed-base entry points" + ) +)] #[must_use] -#[allow(dead_code)] pub(crate) fn basepoint_mul_dispatch(scalar_bytes: &[u8; 32]) -> point::ExtendedPoint { #[cfg(target_arch = "x86_64")] { @@ -59,3 +64,70 @@ pub(crate) fn basepoint_mul_dispatch(scalar_bytes: &[u8; 32]) -> point::Extended point::ExtendedPoint::scalar_mul_basepoint(scalar_bytes) } + +/// Decompose a scalar encoding into signed radix-16 digits in `[-8, 8]`. +#[must_use] +fn scalar_radix_16(bytes: &[u8; 32]) -> [i8; 64] { + debug_assert!(bytes[31] <= 127); + + let mut digits = [0i8; 64]; + for (index, byte) in bytes.iter().copied().enumerate() { + let low = index.strict_mul(2); + let high = low.strict_add(1); + digits[low] = i8::from_ne_bytes([byte & 0x0F]); + digits[high] = i8::from_ne_bytes([(byte >> 4) & 0x0F]); + } + + for index in 0usize..63 { + let next = index.strict_add(1); + let carry = digits[index].strict_add(8) >> 4; + digits[index] = digits[index].strict_sub(carry << 4); + digits[next] = digits[next].strict_add(carry); + } + + digits +} + +#[cfg(test)] +mod tests { + use super::*; + + // Keeps `basepoint_mul_dispatch` reachable under every feature + // combination: the x86_64/aarch64 assembly entry points route around it + // in ed25519's and x25519's own callers, so without a direct test caller + // it goes dead under feature sets that enable only one of the two curves. + // Encoding is curve-specific (`to_bytes` needs ed25519, `to_montgomery_u` + // needs x25519), so each active curve gets its own comparison. + + #[cfg(feature = "ed25519")] + #[test] + fn basepoint_mul_dispatch_matches_portable_edwards_encoding() { + let mut scalar = [0u8; 32]; + scalar[0] = 9; + + let dispatched = basepoint_mul_dispatch(&scalar); + let portable = point::ExtendedPoint::scalar_mul_basepoint(&scalar); + + assert_eq!( + dispatched.to_bytes(), + portable.to_bytes(), + "dispatched basepoint mul should match the portable reference" + ); + } + + #[cfg(feature = "x25519")] + #[test] + fn basepoint_mul_dispatch_matches_portable_montgomery_encoding() { + let mut scalar = [0u8; 32]; + scalar[0] = 9; + + let dispatched = basepoint_mul_dispatch(&scalar); + let portable = point::ExtendedPoint::scalar_mul_basepoint(&scalar); + + assert_eq!( + dispatched.to_montgomery_u().normalize().to_bytes(), + portable.to_montgomery_u().normalize().to_bytes(), + "dispatched basepoint mul should match the portable reference" + ); + } +} diff --git a/src/auth/ecdsa.rs b/src/auth/ecdsa.rs index 5ebffd1d..0c701e0e 100644 --- a/src/auth/ecdsa.rs +++ b/src/auth/ecdsa.rs @@ -21,6 +21,13 @@ use crate::{ mod ecdsa_aarch64_asm; #[path = "ecdsa_generator_tables.rs"] mod ecdsa_generator_tables; +#[cfg(any( + test, + not(any( + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + )) +))] #[path = "ecdsa_p384_field.rs"] mod ecdsa_p384_field; #[cfg(all(target_arch = "x86_64", target_os = "linux"))] @@ -1609,23 +1616,20 @@ fn ct_mul_u64_wide(lhs: u64, rhs: u64) -> (u64, u64) { let mut multiplicand_lo = lhs; let mut multiplicand_hi = 0u64; let mut multiplier = rhs; - let mut bit = 0u32; - - while bit < u64::BITS { + for _ in 0..u64::BITS { // Keep LLVM from recognizing the bit-serial product and lowering it back to a target multiply. // The CT artifact gate independently rejects scalar multiply in these ECDSA closures. let selected_bit = core::hint::black_box(multiplier & 1); let mask = 0u64.wrapping_sub(selected_bit); let (next_lo, carry) = product_lo.overflowing_add(multiplicand_lo & mask); let (next_hi, _) = product_hi.overflowing_add(multiplicand_hi & mask); - let (next_hi, _) = next_hi.overflowing_add(carry as u64); + let (next_hi, _) = next_hi.overflowing_add(u64::from(carry)); product_lo = next_lo; product_hi = next_hi; multiplicand_hi = (multiplicand_hi << 1) | (multiplicand_lo >> 63); multiplicand_lo <<= 1; multiplier >>= 1; - bit += 1; } (product_lo, product_hi) @@ -1633,26 +1637,25 @@ fn ct_mul_u64_wide(lhs: u64, rhs: u64) -> (u64, u64) { #[inline(always)] fn mul_u64_wide(lhs: u64, rhs: u64) -> (u64, u64) { - #[cfg(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] { ct_mul_u64_wide(lhs, rhs) } - #[cfg(not(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] + #[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] { - let product = (lhs as u128) * (rhs as u128); - (product as u64, (product >> 64) as u64) + split_u128(u128::from(lhs).strict_mul(u128::from(rhs))) } } #[inline(always)] fn mul_u64_low(lhs: u64, rhs: u64) -> u64 { - #[cfg(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] { ct_mul_u64_wide(lhs, rhs).0 } - #[cfg(not(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] + #[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] { lhs.wrapping_mul(rhs) } @@ -1714,17 +1717,15 @@ impl Uint { } fn from_be_slice(bytes: &[u8]) -> Result { - if bytes.len() > L * 8 { + if bytes.len() > L.strict_mul(8) { return Err(EcdsaError::InvalidSignature); } let mut out = Self::ZERO; - for (index, byte) in bytes.iter().rev().copied().enumerate() { - let limb = index / 8; - let shift = (index % 8) * 8; - let Some(out_limb) = out.0.get_mut(limb) else { - return Err(EcdsaError::InvalidSignature); - }; - *out_limb |= u64::from(byte) << shift; + for (out_limb, chunk) in out.0.iter_mut().zip(bytes.rchunks(8)) { + let mut encoded = [0u8; 8]; + let offset = encoded.len().strict_sub(chunk.len()); + encoded[offset..].copy_from_slice(chunk); + *out_limb = u64::from_be_bytes(encoded); } Ok(out) } @@ -1739,12 +1740,10 @@ impl Uint { fn write_be(self, out: &mut [u8]) { out.fill(0); - for (index, byte) in out.iter_mut().rev().enumerate() { - let limb = index / 8; - let shift = (index % 8) * 8; - if let Some(limb_value) = self.0.get(limb) { - *byte = (*limb_value >> shift) as u8; - } + for (chunk, limb) in out.rchunks_mut(8).zip(self.0) { + let encoded = limb.to_be_bytes(); + let offset = encoded.len().strict_sub(chunk.len()); + chunk.copy_from_slice(&encoded[offset..]); } } @@ -1948,24 +1947,18 @@ impl Uint { } const WINDOW_BITS: usize = 4; - const WINDOW_SIZE: usize = 1 << WINDOW_BITS; + const WINDOW_SIZE: usize = 16; let mut base = montgomery_mul(*self, modulus.r2, modulus); let mut acc = montgomery_mul(Self::ONE, modulus.r2, modulus); let mut powers = [acc; WINDOW_SIZE]; - let mut next_power = base; - let mut remaining = WINDOW_SIZE - 1; + let mut next_power = acc; for power in powers.iter_mut().skip(1) { + next_power = montgomery_mul(next_power, base, modulus); *power = next_power; - remaining -= 1; - if remaining != 0 { - next_power = montgomery_mul(next_power, base, modulus); - } } - let mut bit = L * 64; - while bit > 0 { - bit -= WINDOW_BITS; + for bit in (0..L.strict_mul(64)).step_by(WINDOW_BITS).rev() { for _ in 0..WINDOW_BITS { acc = montgomery_square(acc, modulus); } @@ -1999,15 +1992,14 @@ impl SecretScalar { } fn from_be_bytes(bytes: &[u8; N]) -> Self { - debug_assert!(N <= L * 8); + debug_assert!(N <= L.strict_mul(8)); let mut out = Uint::ZERO; - for (index, byte) in bytes.iter().rev().copied().enumerate() { - let limb = index / 8; - let shift = (index % 8) * 8; - if let Some(out_limb) = out.0.get_mut(limb) { - *out_limb |= u64::from(byte) << shift; - } + for (out_limb, chunk) in out.0.iter_mut().zip(bytes.rchunks(8)) { + let mut encoded = [0u8; 8]; + let offset = encoded.len().strict_sub(chunk.len()); + encoded[offset..].copy_from_slice(chunk); + *out_limb = u64::from_be_bytes(encoded); } Self::new(out) } @@ -2068,7 +2060,6 @@ impl FieldElement { Self::from_uint(Uint::ONE, modulus) } - #[allow(clippy::indexing_slicing)] fn add(self, rhs: Self) -> Self { if is_p384_field_modulus(self.modulus) { let reduced = add_p384_field( @@ -2096,7 +2087,6 @@ impl FieldElement { Self::from_montgomery(self.value.add_mod_ct(&rhs.value, self.modulus.value), self.modulus) } - #[allow(clippy::indexing_slicing)] fn sub(self, rhs: Self) -> Self { if is_p384_field_modulus(self.modulus) { let reduced = sub_p384_field( @@ -2360,6 +2350,11 @@ impl Jacobian { } } + #[cfg(any( + feature = "diag", + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + ))] fn to_affine_x_ct(self, exponent: Uint) -> FieldElement { let inv_z = self.z.inv_ct(exponent); let z2 = inv_z.square(); @@ -2492,39 +2487,47 @@ fn sign_digest_p384_blinded( } fn hmac_expand_p256(secret: &[u8; 32], digest: &[u8; 32], out: &mut [u8; 64]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha256::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha256::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha256::new(secret); mac.update(P256_NONCE_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); mac.update(digest); block.copy_from_slice(mac.finalize().as_bytes()); } } fn hmac_expand_p256_public_blind(secret: &[u8; 32], out: &mut [u8; 64]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha256::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha256::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha256::new(secret); mac.update(P256_PUBKEY_BLIND_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); block.copy_from_slice(mac.finalize().as_bytes()); } } fn hmac_expand_p384(secret: &[u8; 48], digest: &[u8; 48], out: &mut [u8; 96]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha384::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha384::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha384::new(secret); mac.update(P384_NONCE_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); mac.update(digest); block.copy_from_slice(mac.finalize().as_bytes()); } } fn hmac_expand_p384_public_blind(secret: &[u8; 48], out: &mut [u8; 96]) { - for (block_index, block) in out.chunks_exact_mut(HmacSha384::TAG_SIZE).enumerate() { + let (blocks, remainder) = out.as_chunks_mut::<{ HmacSha384::TAG_SIZE }>(); + debug_assert!(remainder.is_empty()); + for (block_index, block) in [0u8, 1].into_iter().zip(blocks) { let mut mac = HmacSha384::new(secret); mac.update(P384_PUBKEY_BLIND_DOMAIN); - mac.update(&[block_index as u8]); + mac.update(&[block_index]); block.copy_from_slice(mac.finalize().as_bytes()); } } @@ -2755,7 +2758,9 @@ fn reduce_wide_order_nonzero_owned( } let mut acc = Uint::ZERO; - for chunk in bytes.chunks_exact(8) { + let (chunks, remainder) = bytes.as_chunks::<8>(); + debug_assert!(remainder.is_empty()); + for chunk in chunks { acc = mul_mod_montgomery_ct(acc, radix, modulus); let word = chunk .iter() @@ -2946,12 +2951,12 @@ fn is_p384_curve(curve: &Curve) -> bool { all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), all(target_arch = "x86_64", target_os = "linux") ))] -#[allow(clippy::indexing_slicing)] fn affine_from_words(modulus: &'static Modulus, words: &[u64]) -> Affine { let mut x = [0u64; L]; let mut y = [0u64; L]; - x.copy_from_slice(&words[..L]); - y.copy_from_slice(&words[L..L * 2]); + let (x_words, y_words) = words.split_at(L); + x.copy_from_slice(x_words); + y.copy_from_slice(y_words); let x = Uint(x); let y = Uint(y); Affine { @@ -3276,14 +3281,13 @@ fn p384_jacobian_from_words(words: &[u64; 18]) -> Jacobian<6> { } #[cfg(all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")))] -#[allow(clippy::indexing_slicing)] fn jacobian_from_p384_words(modulus: &'static Modulus, words: &[u64; 18]) -> Jacobian { let mut x = [0u64; L]; let mut y = [0u64; L]; let mut z = [0u64; L]; x.copy_from_slice(&words[..L]); - y.copy_from_slice(&words[6..6 + L]); - z.copy_from_slice(&words[12..12 + L]); + y.copy_from_slice(&words[6..6usize.strict_add(L)]); + z.copy_from_slice(&words[12..12usize.strict_add(L)]); Jacobian { x: FieldElement::from_montgomery(Uint(x), modulus), y: FieldElement::from_montgomery(Uint(y), modulus), @@ -3293,7 +3297,6 @@ fn jacobian_from_p384_words(modulus: &'static Modulus, words: } #[cfg(test)] -#[allow(clippy::indexing_slicing)] fn scalar_mul_basepoint_comb_ct(curve: &Curve, scalar: Uint) -> Jacobian { let rows = curve.signing_comb_rows; let mut acc = Jacobian::infinity(curve.field_modulus); @@ -3308,7 +3311,6 @@ fn scalar_mul_basepoint_comb_ct(curve: &Curve, scalar: Uint( curve: &Curve, scalar: &SecretScalar, @@ -3327,7 +3329,6 @@ fn scalar_mul_basepoint_comb_ct_secret( acc } -#[allow(clippy::indexing_slicing)] fn scalar_mul_basepoint_comb_ct_secret_blinded( curve: &Curve, scalar: &SecretScalar, @@ -3373,7 +3374,6 @@ fn signing_comb_digit_ct(scalar: Uint, row: usize, rows: usiz digit } -#[allow(clippy::indexing_slicing)] fn select_signing_generator_affine_ct(curve: &Curve, digit: usize) -> Affine { let mut x = curve.signing_generator_comb_x[0]; let mut y = curve.signing_generator_comb_y[0]; @@ -3393,6 +3393,7 @@ fn select_signing_generator_affine_ct(curve: &Curve, digit: u } } +/// Return the P-256 signing-comb coordinates selected by `digit` as Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_select_signing_generator_affine_limb_digest(digit: u8) -> [u64; 8] { let selected = select_signing_generator_affine_ct(&P256, usize::from(digit)); @@ -3424,6 +3425,7 @@ pub(crate) fn diag_zeroize_ecdsa_p256_platform_scratch(wide: [u8; 64]) -> u64 { core::hint::black_box(inverse.value().0[0]) } +/// Derive the deterministic P-256 nonce for `message` and return its scalar limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_nonce_reduce_limb_digest(secret: [u8; 32], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3434,6 +3436,7 @@ pub fn diag_ecdsa_p256_nonce_reduce_limb_digest(secret: [u8; 32], message: &[u8] nonce.value().0 } +/// Reduce a wide P-256 nonce candidate to a nonzero scalar and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_reduce_wide_order_limb_digest(wide: [u8; 64]) -> [u64; 4] { let wide = ZeroizingBytes::new(wide); @@ -3441,6 +3444,7 @@ pub fn diag_ecdsa_p256_reduce_wide_order_limb_digest(wide: [u8; 64]) -> [u64; 4] nonce.value().0 } +/// Return the affine limbs produced by blinded P-256 basepoint multiplication. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_basepoint_blinded_limb_digest(secret: [u8; 32], blind: [u8; 64], message: &[u8]) -> [u64; 8] { let secret = ZeroizingBytes::new(secret); @@ -3457,6 +3461,7 @@ pub fn diag_ecdsa_p256_basepoint_blinded_limb_digest(secret: [u8; 32], blind: [u out } +/// Run P-256 scalar signing finalization with supplied nonce material and return `r || s` limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_scalar_finish_limb_digest(secret: [u8; 32], nonce_wide: [u8; 64], message: &[u8]) -> [u64; 8] { let secret = ZeroizingBytes::new(secret); @@ -3472,6 +3477,7 @@ pub fn diag_ecdsa_p256_scalar_finish_limb_digest(secret: [u8; 32], nonce_wide: [ out } +/// Multiply the P-256 secret scalar by a fixed public `r` and return the order-field limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret: [u8; 32]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3481,6 +3487,7 @@ pub fn diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret: [u8; 32]) -> [u64; rd.value().0 } +/// Run the blinded P-256 order multiplication stage for a fixed public `r` and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret: [u8; 32], blind: [u8; 64]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3497,6 +3504,7 @@ pub fn diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret: [u8; 32], b rd.value().0 } +/// Derive and invert the deterministic P-256 nonce and return its Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_nonce_inverse_limb_digest(secret: [u8; 32], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3512,6 +3520,7 @@ pub fn diag_ecdsa_p256_nonce_inverse_limb_digest(secret: [u8; 32], message: &[u8 inverse.value().0 } +/// Run the final P-256 signing multiplication with supplied nonce material and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p256"))] pub fn diag_ecdsa_p256_final_multiply_limb_digest(secret: [u8; 32], nonce_wide: [u8; 64], message: &[u8]) -> [u64; 4] { let secret = ZeroizingBytes::new(secret); @@ -3536,6 +3545,7 @@ pub fn diag_ecdsa_p256_final_multiply_limb_digest(secret: [u8; 32], nonce_wide: product.value().0 } +/// Return the P-384 signing-comb coordinates selected by `digit` as Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_select_signing_generator_affine_limb_digest(digit: u8) -> [u64; 12] { let selected = select_signing_generator_affine_ct(&P384, usize::from(digit)); @@ -3565,6 +3575,7 @@ pub(crate) fn diag_zeroize_ecdsa_p384_platform_scratch(wide: [u8; 96]) -> u64 { core::hint::black_box(inverse.value().0[0]) } +/// Derive the deterministic P-384 nonce for `message` and return its scalar limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_nonce_reduce_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3575,6 +3586,7 @@ pub fn diag_ecdsa_p384_nonce_reduce_limb_digest(secret: [u8; 48], message: &[u8] nonce.value().0 } +/// Reduce a wide P-384 nonce candidate to a nonzero scalar and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_reduce_wide_order_limb_digest(wide: [u8; 96]) -> [u64; 6] { let wide = ZeroizingBytes::new(wide); @@ -3582,6 +3594,7 @@ pub fn diag_ecdsa_p384_reduce_wide_order_limb_digest(wide: [u8; 96]) -> [u64; 6] nonce.value().0 } +/// Return the affine limbs produced by blinded P-384 basepoint multiplication. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_basepoint_blinded_limb_digest(secret: [u8; 48], blind: [u8; 96], message: &[u8]) -> [u64; 12] { let secret = ZeroizingBytes::new(secret); @@ -3598,6 +3611,7 @@ pub fn diag_ecdsa_p384_basepoint_blinded_limb_digest(secret: [u8; 48], blind: [u out } +/// Derive the P-384 nonce point and return its reduced affine x-coordinate limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_basepoint_r_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3614,6 +3628,7 @@ pub fn diag_ecdsa_p384_basepoint_r_limb_digest(secret: [u8; 48], message: &[u8]) .0 } +/// Run P-384 scalar signing finalization with supplied nonce material and return `r || s` limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_scalar_finish_limb_digest(secret: [u8; 48], nonce_wide: [u8; 96], message: &[u8]) -> [u64; 12] { let secret = ZeroizingBytes::new(secret); @@ -3629,6 +3644,7 @@ pub fn diag_ecdsa_p384_scalar_finish_limb_digest(secret: [u8; 48], nonce_wide: [ out } +/// Multiply the P-384 secret scalar by a fixed public `r` and return the order-field limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret: [u8; 48]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3638,6 +3654,7 @@ pub fn diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret: [u8; 48]) -> [u64; rd.value().0 } +/// Derive and invert the deterministic P-384 nonce and return its Montgomery limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_nonce_inverse_limb_digest(secret: [u8; 48], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3653,6 +3670,7 @@ pub fn diag_ecdsa_p384_nonce_inverse_limb_digest(secret: [u8; 48], message: &[u8 inverse.value().0 } +/// Run the final P-384 signing multiplication with supplied nonce material and return its limbs. #[cfg(all(feature = "diag", feature = "ecdsa-p384"))] pub fn diag_ecdsa_p384_final_multiply_limb_digest(secret: [u8; 48], nonce_wide: [u8; 96], message: &[u8]) -> [u64; 6] { let secret = ZeroizingBytes::new(secret); @@ -3697,7 +3715,6 @@ fn projective_x_matches_scalar(point: Jacobian, scalar: Uint< } } -#[allow(clippy::indexing_slicing)] fn scalar_mul_two( curve: &Curve, lhs_scalar: Uint, @@ -3723,13 +3740,13 @@ fn scalar_mul_two( } const fn comb_rows() -> usize { - (L * 64).div_ceil(COMB_WIDTH) + L.strict_mul(64).div_ceil(COMB_WIDTH) } fn comb_digit(scalar: Uint, row: usize, rows: usize) -> usize { let mut digit = 0usize; for column in 0..COMB_WIDTH { - let bit = row + column * rows; + let bit = row.strict_add(column.strict_mul(rows)); if scalar.bit(bit) { digit |= 1usize << column; } @@ -3737,7 +3754,6 @@ fn comb_digit(scalar: Uint, row: usize, rows: usize) -> usize digit } -#[allow(clippy::indexing_slicing)] fn precompute_comb_table(point: Affine) -> [Affine; COMB_TABLE_SIZE] { let rows = comb_rows::(); let mut column_points = [Jacobian::from_affine(point); COMB_WIDTH]; @@ -3763,14 +3779,16 @@ fn precompute_comb_table(point: Affine) -> [Affine; COMB_T normalize_jacobian_table(table) } -#[allow(clippy::indexing_slicing)] fn normalize_jacobian_table(table: [Jacobian; N]) -> [Affine; N] { - let modulus = table[0].x.modulus; + let modulus = table + .first() + .map(|point| point.x.modulus) + .expect("ECDSA comb tables are nonempty"); let mut prefixes = [FieldElement::one(modulus); N]; let mut acc = FieldElement::one(modulus); - for i in 0..N { - prefixes[i] = acc; - acc = acc.mul(table[i].z); + for (prefix, point) in prefixes.iter_mut().zip(table.iter()) { + *prefix = acc; + acc = acc.mul(point.z); } let mut acc_inv = acc.inv(); @@ -3778,30 +3796,29 @@ fn normalize_jacobian_table(table: [Jacobian; x: FieldElement::zero(modulus), y: FieldElement::zero(modulus), }; N]; - for i in (0..N).rev() { - let z_inv = acc_inv.mul(prefixes[i]); - acc_inv = acc_inv.mul(table[i].z); + for ((point, prefix), output) in table.iter().zip(prefixes).zip(out.iter_mut()).rev() { + let z_inv = acc_inv.mul(prefix); + acc_inv = acc_inv.mul(point.z); let z2 = z_inv.square(); let z3 = z2.mul(z_inv); - out[i] = Affine { - x: table[i].x.mul(z2), - y: table[i].y.mul(z3), + *output = Affine { + x: point.x.mul(z2), + y: point.y.mul(z3), }; } out } -#[allow(clippy::indexing_slicing)] fn parse_public_key(bytes: &[u8], curve: &Curve) -> Result, EcdsaError> { - let field_len = L * 8; + let field_len = L.strict_mul(8); if bytes.len() != field_len.strict_mul(2).strict_add(1) || bytes.first().copied() != Some(0x04) { return Err(EcdsaError::InvalidPublicKey); } - let x = Uint::from_be_slice(bytes.get(1..1 + field_len).ok_or(EcdsaError::InvalidPublicKey)?) - .map_err(|_| EcdsaError::InvalidPublicKey)?; - let y = Uint::from_be_slice(bytes.get(1 + field_len..).ok_or(EcdsaError::InvalidPublicKey)?) - .map_err(|_| EcdsaError::InvalidPublicKey)?; + let coordinates = bytes.get(1..).ok_or(EcdsaError::InvalidPublicKey)?; + let (x_bytes, y_bytes) = coordinates.split_at(field_len); + let x = Uint::from_be_slice(x_bytes).map_err(|_| EcdsaError::InvalidPublicKey)?; + let y = Uint::from_be_slice(y_bytes).map_err(|_| EcdsaError::InvalidPublicKey)?; if x.cmp(&curve.field_modulus.value).is_ge() || y.cmp(&curve.field_modulus.value).is_ge() { return Err(EcdsaError::InvalidPublicKey); } @@ -3827,10 +3844,11 @@ fn is_on_curve(point: Affine, curve: &Curve) -> bool { fn encode_sec1(point: &Affine) -> [u8; N] { let mut out = [0u8; N]; - out[0] = 0x04; - let field_len = L * 8; - point.x.to_uint().write_be(&mut out[1..1 + field_len]); - point.y.to_uint().write_be(&mut out[1 + field_len..]); + let (tag, coordinates) = out.split_first_mut().expect("SEC1 output includes a tag byte"); + *tag = 0x04; + let (x_bytes, y_bytes) = coordinates.split_at_mut(L.strict_mul(8)); + point.x.to_uint().write_be(x_bytes); + point.y.to_uint().write_be(y_bytes); out } @@ -3838,9 +3856,9 @@ fn parse_signature_scalars( bytes: &[u8; BYTES], scalar_modulus: Uint, ) -> Result<(Uint, Uint), EcdsaError> { - let field_len = LIMBS * 8; - let r = Uint::from_be_slice(&bytes[..field_len])?; - let s = Uint::from_be_slice(&bytes[field_len..])?; + let (r_bytes, s_bytes) = bytes.split_at(LIMBS.strict_mul(8)); + let r = Uint::from_be_slice(r_bytes)?; + let s = Uint::from_be_slice(s_bytes)?; if !r.is_in_range(&scalar_modulus) || !s.is_in_range(&scalar_modulus) { return Err(EcdsaError::InvalidSignature); } @@ -3848,7 +3866,7 @@ fn parse_signature_scalars( } fn parse_signature_der_bytes(der: &[u8]) -> Result<[u8; BYTES], EcdsaError> { - let field_len = LIMBS * 8; + let field_len = LIMBS.strict_mul(8); let mut root = DerReader::new(der); let sig = root.read_constructed(TAG_SEQUENCE)?; root.finish()?; @@ -3859,8 +3877,9 @@ fn parse_signature_der_bytes(der: &[u8]) sig.finish()?; let mut bytes = [0u8; BYTES]; - parse_der_integer_into(r_value, &mut bytes[..field_len])?; - parse_der_integer_into(s_value, &mut bytes[field_len..])?; + let (r_bytes, s_bytes) = bytes.split_at_mut(field_len); + parse_der_integer_into(r_value, r_bytes)?; + parse_der_integer_into(s_value, s_bytes)?; Ok(bytes) } @@ -3959,7 +3978,6 @@ fn mul_mod_montgomery_blinded_ct( product_share.value().add_mod_ct(&product_blind.value(), modulus.value) } -#[allow(clippy::indexing_slicing)] fn montgomery_mul(lhs: Uint, rhs: Uint, modulus: &'static Modulus) -> Uint { debug_assert!(L <= 6); @@ -4015,27 +4033,32 @@ fn montgomery_mul(lhs: Uint, rhs: Uint, modulus: &'static for i in 0..L { let mut carry = 0u64; for j in 0..L { - let k = i + j; + let k = i.strict_add(j); (limbs[k], carry) = mac_limb(limbs[k], lhs.0[i], rhs.0[j], carry); } - add_limb(&mut limbs, i + L, carry); + add_limb(&mut limbs, i.strict_add(L), carry); } for i in 0..L { let factor = mul_u64_low(limbs[i], modulus.n0_inv); let mut carry = 0u64; for j in 0..L { - let k = i + j; + let k = i.strict_add(j); (limbs[k], carry) = mac_limb(limbs[k], factor, modulus.value.0[j], carry); } - add_limb(&mut limbs, i + L, carry); + add_limb(&mut limbs, i.strict_add(L), carry); } + let double_limbs = L.strict_mul(2); let mut out = [0u64; L]; - out.copy_from_slice(&limbs[L..L + L]); + out.copy_from_slice(&limbs[L..double_limbs]); let out = Uint(out); let (reduced, borrow) = out.sub_raw(&modulus.value); - Uint::select(out, reduced, mask_nonzero_u64(limbs[L + L]) | mask_zero_u64(borrow)) + Uint::select( + out, + reduced, + mask_nonzero_u64(limbs[double_limbs]) | mask_zero_u64(borrow), + ) } fn montgomery_square(value: Uint, modulus: &'static Modulus) -> Uint { @@ -4095,7 +4118,7 @@ fn montgomery_mul_p256_order(lhs: [u64; 4], rhs: [u64; 4]) -> Uint<4> { t1 = v1; t2 = v2; t3 = v3; - t4 = carry_extra + carry; + t4 = carry_extra.strict_add(carry); }}; } @@ -4159,7 +4182,7 @@ fn montgomery_mul_p384_order(lhs: [u64; 6], rhs: [u64; 6]) -> Uint<6> { t3 = v3; t4 = v4; t5 = v5; - t6 = carry_extra + carry; + t6 = carry_extra.strict_add(carry); }}; } @@ -4260,7 +4283,6 @@ fn montgomery_mul_p256_field(lhs: [u64; 4], rhs: [u64; 4]) -> Uint<4> { montgomery_reduce_p256_field([w0, w1, w2, w3, w4, w5, w6, w7]) } -#[allow(clippy::indexing_slicing)] fn montgomery_reduce_p256_field(limbs: [u64; 8]) -> Uint<4> { let [r0, r1, r2, r3, r4, r5, r6, r7] = limbs; let p = P256_FIELD.0; @@ -4306,8 +4328,9 @@ fn sub_p256_field_once(limbs: [u64; 5]) -> Uint<4> { #[inline(always)] fn adc_limb(lhs: u64, rhs: u64, carry: u64) -> (u64, u64) { - let result = u128::from(lhs) + u128::from(rhs) + u128::from(carry); - (result as u64, (result >> 64) as u64) + let (result, overflow0) = lhs.overflowing_add(rhs); + let (result, overflow1) = result.overflowing_add(carry); + (result, u64::from(overflow0 | overflow1)) } #[inline(always)] @@ -4319,29 +4342,32 @@ fn sbb_limb(lhs: u64, rhs: u64, borrow: u64) -> (u64, u64) { #[inline(always)] fn mac_limb(acc: u64, lhs: u64, rhs: u64, carry: u64) -> (u64, u64) { - #[cfg(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x"))] - { - let (product_lo, product_hi) = mul_u64_wide(lhs, rhs); - let (result, carry0) = product_lo.overflowing_add(acc); - let (result, carry1) = result.overflowing_add(carry); - let (high, overflow0) = product_hi.overflowing_add(u64::from(carry0)); - let (high, overflow1) = high.overflowing_add(u64::from(carry1)); + let (product_lo, product_hi) = mul_u64_wide(lhs, rhs); + let (result, carry0) = product_lo.overflowing_add(acc); + let (result, carry1) = result.overflowing_add(carry); + let (high, overflow0) = product_hi.overflowing_add(u64::from(carry0)); + let (high, overflow1) = high.overflowing_add(u64::from(carry1)); - // lhs*rhs + acc + carry is at most 2^128 - 1, so the high limb cannot overflow. - // Keep that invariant checked in debug builds without emitting secret-fed panic branches - // in release ECDSA arithmetic on s390x and RISC-V. - debug_assert!(!overflow0 && !overflow1); - (result, high) - } + // lhs*rhs + acc + carry is at most 2^128 - 1, so the high limb cannot overflow. + // Keep that invariant checked in debug builds without emitting secret-fed panic branches + // in release ECDSA arithmetic. + debug_assert!(!overflow0 && !overflow1); + (result, high) +} - #[cfg(not(any(test, target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")))] - { - let result = u128::from(acc) + (u128::from(lhs) * u128::from(rhs)) + u128::from(carry); - (result as u64, (result >> 64) as u64) - } +#[inline(always)] +#[cfg(any( + test, + not(any(target_arch = "riscv32", target_arch = "riscv64", target_arch = "s390x")) +))] +fn split_u128(value: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = value.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) } -#[allow(clippy::indexing_slicing)] fn add_limb(limbs: &mut [u64; 13], mut index: usize, mut value: u64) { while index < limbs.len() { let (sum, carry) = limbs[index].overflowing_add(value); @@ -4461,7 +4487,8 @@ mod tests { assert_eq!(reader.read_len(), Err(EcdsaError::MalformedDer)); } - let oversized_len_len = [0x80 | (core::mem::size_of::() as u8 + 1)]; + let length_bytes = u8::try_from(core::mem::size_of::()).expect("usize width fits in one DER length byte"); + let oversized_len_len = [0x80 | length_bytes.strict_add(1)]; let mut reader = DerReader::new(&oversized_len_len); assert_eq!(reader.read_len(), Err(EcdsaError::MalformedDer)); } @@ -4471,7 +4498,7 @@ mod tests { sec1[0] = 0x04; P256_GX.write_be(&mut sec1[1..33]); P256_GY.write_be(&mut sec1[33..]); - EcdsaP256PublicKey::from_sec1_bytes(&sec1).unwrap() + EcdsaP256PublicKey::from_sec1_bytes(&sec1).expect("the P-256 generator is a valid public key") } fn p384_public_key() -> EcdsaP384PublicKey { @@ -4479,7 +4506,7 @@ mod tests { sec1[0] = 0x04; P384_GX.write_be(&mut sec1[1..49]); P384_GY.write_be(&mut sec1[49..]); - EcdsaP384PublicKey::from_sec1_bytes(&sec1).unwrap() + EcdsaP384PublicKey::from_sec1_bytes(&sec1).expect("the P-384 generator is a valid public key") } fn p384_sparse_scalar(bits: &[usize]) -> Uint<6> { @@ -4504,12 +4531,12 @@ mod tests { #[test] fn ct_mul_u64_wide_matches_u128_for_edges_and_generated_inputs() { - const EDGES: [u64; 8] = [0, 1, 2, u32::MAX as u64, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; + const EDGES: [u64; 8] = [0, 1, 2, 0xffff_ffff, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; for lhs in EDGES { for rhs in EDGES { - let product = u128::from(lhs) * u128::from(rhs); - assert_eq!(ct_mul_u64_wide(lhs, rhs), (product as u64, (product >> 64) as u64)); + let product = u128::from(lhs).strict_mul(u128::from(rhs)); + assert_eq!(ct_mul_u64_wide(lhs, rhs), split_u128(product)); } } @@ -4518,24 +4545,24 @@ mod tests { for _ in 0..1024 { lhs = lhs.wrapping_mul(0x9e37_79b9_7f4a_7c15).wrapping_add(1); rhs = rhs.wrapping_mul(0xd134_2543_de82_ef95).wrapping_add(1); - let product = u128::from(lhs) * u128::from(rhs); - assert_eq!(ct_mul_u64_wide(lhs, rhs), (product as u64, (product >> 64) as u64)); + let product = u128::from(lhs).strict_mul(u128::from(rhs)); + assert_eq!(ct_mul_u64_wide(lhs, rhs), split_u128(product)); } } #[test] fn mac_limb_matches_u128_for_edges_and_generated_inputs() { - const EDGES: [u64; 8] = [0, 1, 2, u32::MAX as u64, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; + const EDGES: [u64; 8] = [0, 1, 2, 0xffff_ffff, 1u64 << 32, 1u64 << 63, u64::MAX - 1, u64::MAX]; for acc in EDGES { for lhs in EDGES { for rhs in EDGES { for carry in EDGES { - let expected = u128::from(lhs) * u128::from(rhs) + u128::from(acc) + u128::from(carry); - assert_eq!( - mac_limb(acc, lhs, rhs, carry), - (expected as u64, (expected >> 64) as u64) - ); + let expected = u128::from(lhs) + .strict_mul(u128::from(rhs)) + .strict_add(u128::from(acc)) + .strict_add(u128::from(carry)); + assert_eq!(mac_limb(acc, lhs, rhs, carry), split_u128(expected)); } } } @@ -4550,11 +4577,11 @@ mod tests { let rhs = state; state = state.wrapping_mul(0xa409_3822_299f_31d0).wrapping_add(1); let carry = state; - let expected = u128::from(lhs) * u128::from(rhs) + u128::from(acc) + u128::from(carry); - assert_eq!( - mac_limb(acc, lhs, rhs, carry), - (expected as u64, (expected >> 64) as u64) - ); + let expected = u128::from(lhs) + .strict_mul(u128::from(rhs)) + .strict_add(u128::from(acc)) + .strict_add(u128::from(carry)); + assert_eq!(mac_limb(acc, lhs, rhs, carry), split_u128(expected)); } } @@ -4612,7 +4639,7 @@ mod tests { ); bytes[63] = 1; - assert!(EcdsaP256Signature::from_bytes(bytes).is_ok()); + EcdsaP256Signature::from_bytes(bytes).expect("nonzero in-range P-256 scalars form a valid signature encoding"); let mut out_of_range = bytes; P256_ORDER.write_be(&mut out_of_range[..32]); @@ -4632,7 +4659,7 @@ mod tests { ); bytes[95] = 1; - assert!(EcdsaP384Signature::from_bytes(bytes).is_ok()); + EcdsaP384Signature::from_bytes(bytes).expect("nonzero in-range P-384 scalars form a valid signature encoding"); let mut out_of_range = bytes; P384_ORDER.write_be(&mut out_of_range[..48]); @@ -4645,7 +4672,7 @@ mod tests { #[test] fn der_signature_parser_requires_canonical_unsigned_integers() { let good = [0x30, 0x06, 0x02, 0x01, 0x01, 0x02, 0x01, 0x01]; - assert!(EcdsaP256Signature::from_der(&good).is_ok()); + EcdsaP256Signature::from_der(&good).expect("canonical unsigned DER integers must parse"); let negative = [0x30, 0x07, 0x02, 0x02, 0x80, 0x01, 0x02, 0x01, 0x01]; assert_eq!( @@ -4693,8 +4720,8 @@ mod tests { #[test] fn p256_secret_key_one_derives_generator_public_key() { let mut one = [0u8; EcdsaP256SecretKey::LENGTH]; - one[EcdsaP256SecretKey::LENGTH - 1] = 1; - let secret = EcdsaP256SecretKey::from_bytes(one).unwrap(); + *one.last_mut().expect("P-256 scalar encoding is nonempty") = 1; + let secret = EcdsaP256SecretKey::from_bytes(one).expect("one is a valid P-256 secret scalar"); assert_eq!(secret.public_key(), p256_public_key()); assert_eq!(secret.public_key_blinded(|blind| blind.fill(0xa5)), p256_public_key()); @@ -4703,8 +4730,8 @@ mod tests { #[test] fn p384_secret_key_one_derives_generator_public_key() { let mut one = [0u8; EcdsaP384SecretKey::LENGTH]; - one[EcdsaP384SecretKey::LENGTH - 1] = 1; - let secret = EcdsaP384SecretKey::from_bytes(one).unwrap(); + *one.last_mut().expect("P-384 scalar encoding is nonempty") = 1; + let secret = EcdsaP384SecretKey::from_bytes(one).expect("one is a valid P-384 secret scalar"); assert_eq!(secret.public_key(), p384_public_key()); assert_eq!(secret.public_key_blinded(|blind| blind.fill(0x5a)), p384_public_key()); @@ -4712,21 +4739,28 @@ mod tests { #[test] fn p256_try_sign_is_deterministic_low_s_and_verifies() { - let secret = EcdsaP256SecretKey::from_bytes([0x42; EcdsaP256SecretKey::LENGTH]).unwrap(); + let secret = EcdsaP256SecretKey::from_bytes([0x42; EcdsaP256SecretKey::LENGTH]) + .expect("fixture is a valid P-256 secret scalar"); let public = secret.public_key(); let message = b"rscrypto ecdsa p256 signing"; - let first = secret.try_sign(message).unwrap(); - let second = secret.try_sign(message).unwrap(); - let blinded = secret.try_sign_blinded(message, |blind| blind.fill(0x7b)).unwrap(); + let first = secret.try_sign(message).expect("valid P-256 signing fixture must sign"); + let second = secret.try_sign(message).expect("valid P-256 signing fixture must sign"); + let blinded = secret + .try_sign_blinded(message, |blind| blind.fill(0x7b)) + .expect("valid P-256 blinded signing fixture must sign"); assert_eq!(first, second); - assert!(public.verify(message, &first).is_ok()); - assert!(public.verify(message, &blinded).is_ok()); + public + .verify(message, &first) + .expect("fresh P-256 signature must verify"); + public + .verify(message, &blinded) + .expect("fresh blinded P-256 signature must verify"); assert_eq!(first, blinded); assert!( Uint::from_be_slice(&first.as_bytes()[32..]) - .unwrap() + .expect("P-256 signature scalar has the fixed field width") .cmp(&P256_ORDER_HALF) .is_le() ); @@ -4734,21 +4768,28 @@ mod tests { #[test] fn p384_try_sign_is_deterministic_low_s_and_verifies() { - let secret = EcdsaP384SecretKey::from_bytes([0x24; EcdsaP384SecretKey::LENGTH]).unwrap(); + let secret = EcdsaP384SecretKey::from_bytes([0x24; EcdsaP384SecretKey::LENGTH]) + .expect("fixture is a valid P-384 secret scalar"); let public = secret.public_key(); let message = b"rscrypto ecdsa p384 signing"; - let first = secret.try_sign(message).unwrap(); - let second = secret.try_sign(message).unwrap(); - let blinded = secret.try_sign_blinded(message, |blind| blind.fill(0xb7)).unwrap(); + let first = secret.try_sign(message).expect("valid P-384 signing fixture must sign"); + let second = secret.try_sign(message).expect("valid P-384 signing fixture must sign"); + let blinded = secret + .try_sign_blinded(message, |blind| blind.fill(0xb7)) + .expect("valid P-384 blinded signing fixture must sign"); assert_eq!(first, second); - assert!(public.verify(message, &first).is_ok()); - assert!(public.verify(message, &blinded).is_ok()); + public + .verify(message, &first) + .expect("fresh P-384 signature must verify"); + public + .verify(message, &blinded) + .expect("fresh blinded P-384 signature must verify"); assert_eq!(first, blinded); assert!( Uint::from_be_slice(&first.as_bytes()[48..]) - .unwrap() + .expect("P-384 signature scalar has the fixed field width") .cmp(&P384_ORDER_HALF) .is_le() ); @@ -4809,6 +4850,36 @@ mod tests { } } + #[cfg(any( + all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), + all(target_arch = "x86_64", target_os = "linux") + ))] + #[test] + fn p384_portable_field_arithmetic_matches_platform_backend() { + let values = [ + Uint::ZERO, + P384_FIELD_MONTGOMERY_ONE, + P384_GX, + P384_GY, + P384_B, + P384_FIELD.sub_raw(&Uint::ONE).0, + ]; + + for lhs in values { + assert_eq!( + ecdsa_p384_field::square(lhs.0), + ecdsa_platform_asm::p384_field_square(&lhs.0) + ); + + for rhs in values { + assert_eq!( + ecdsa_p384_field::mul(lhs.0, rhs.0), + ecdsa_platform_asm::p384_field_mul(&lhs.0, &rhs.0) + ); + } + } + } + #[cfg(any( all(target_arch = "aarch64", any(target_os = "macos", target_os = "linux")), all(target_arch = "x86_64", target_os = "linux") @@ -4852,8 +4923,8 @@ mod tests { for row in [0usize, 1, 2, 23, 46] { for column in [0usize, 1, 3, 7] { - let bit = row + column * P384_SIGNING_COMB_ROWS; - let next_bit = bit + 1; + let bit = row.strict_add(column.strict_mul(P384_SIGNING_COMB_ROWS)); + let next_bit = bit.strict_add(1); assert_p384_nonexceptional_comb_matches_complete(p384_sparse_scalar(&[bit, next_bit])); } } @@ -4888,8 +4959,10 @@ mod tests { fn p384_owned_wide_order_reduction_matches_platform_reduction() { for bytes in [[0u8; 96], [0xffu8; 96], { let mut bytes = [0u8; 96]; - for (index, byte) in bytes.iter_mut().enumerate() { - *byte = (index as u8).wrapping_mul(17).wrapping_add(0xa5); + let mut value = 0xa5u8; + for byte in &mut bytes { + *byte = value; + value = value.wrapping_add(17); } bytes }] { @@ -4902,8 +4975,10 @@ mod tests { #[test] fn ecdsa_secret_key_debug_is_redacted() { - let p256 = EcdsaP256SecretKey::from_bytes([0x11; EcdsaP256SecretKey::LENGTH]).unwrap(); - let p384 = EcdsaP384SecretKey::from_bytes([0x22; EcdsaP384SecretKey::LENGTH]).unwrap(); + let p256 = EcdsaP256SecretKey::from_bytes([0x11; EcdsaP256SecretKey::LENGTH]) + .expect("fixture is a valid P-256 secret scalar"); + let p384 = EcdsaP384SecretKey::from_bytes([0x22; EcdsaP384SecretKey::LENGTH]) + .expect("fixture is a valid P-384 secret scalar"); assert_eq!(format!("{p256:?}"), "EcdsaP256SecretKey(****)"); assert_eq!(format!("{p384:?}"), "EcdsaP384SecretKey(****)"); diff --git a/src/auth/ecdsa_aarch64_asm.rs b/src/auth/ecdsa_aarch64_asm.rs index 5d515d6e..cce0b20d 100644 --- a/src/auth/ecdsa_aarch64_asm.rs +++ b/src/auth/ecdsa_aarch64_asm.rs @@ -5,8 +5,6 @@ //! ABI boundary; `ecdsa.rs` owns scalar validation, blinding, affine //! conversion, and public API semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::ZeroizingWords; diff --git a/src/auth/ecdsa_p384_field.rs b/src/auth/ecdsa_p384_field.rs index 64c91d30..65f544c0 100644 --- a/src/auth/ecdsa_p384_field.rs +++ b/src/auth/ecdsa_p384_field.rs @@ -1,49 +1,23 @@ -//! Fiat-Crypto P-384 field arithmetic used by ECDSA verification. +//! Fiat-Crypto P-384 field arithmetic used by portable ECDSA paths. //! -//! This file contains a narrow extraction of generated Fiat-Crypto code +//! This file contains a narrow, adapted extraction of generated Fiat-Crypto code //! from the RustCrypto `p384` crate field backend. The generated source //! is distributed under MIT, Apache-2.0, or BSD-1-Clause terms by the //! Fiat-Crypto authors. It is kept private to this module and wrapped by //! small `mul`/`square` helpers below. -#![allow( - non_camel_case_types, - clippy::identity_op, - clippy::too_many_arguments, - clippy::unnecessary_cast, - clippy::unused_unit, - unused_assignments, - unused_variables, - dead_code, - unused_parens -)] - -type fiat_p384_u1 = u8; -type fiat_p384_i1 = i8; -type fiat_p384_u2 = u8; -type fiat_p384_i2 = i8; -type fiat_p384_montgomery_domain_field_element = [u64; 6]; - -const fn fiat_p384_addcarryx_u64(arg1: fiat_p384_u1, arg2: u64, arg3: u64) -> (u64, fiat_p384_u1) { - let mut out1: u64 = 0; - let mut out2: fiat_p384_u1 = 0; - let x1: u128 = (((arg1 as u128) + (arg2 as u128)) + (arg3 as u128)); - let x2: u64 = ((x1 & (0xffffffffffffffff as u128)) as u64); - let x3: fiat_p384_u1 = ((x1 >> 64) as fiat_p384_u1); - out1 = x2; - out2 = x3; - (out1, out2) +#[inline(always)] +fn fiat_p384_addcarryx_u64(carry: u8, lhs: u64, rhs: u64) -> (u64, u8) { + let (sum, overflow0) = lhs.overflowing_add(rhs); + let (sum, overflow1) = sum.overflowing_add(u64::from(carry)); + (sum, u8::from(overflow0 | overflow1)) } -const fn fiat_p384_subborrowx_u64(arg1: fiat_p384_u1, arg2: u64, arg3: u64) -> (u64, fiat_p384_u1) { - let mut out1: u64 = 0; - let mut out2: fiat_p384_u1 = 0; - let x1: i128 = (((arg2 as i128) - (arg1 as i128)) - (arg3 as i128)); - let x2: fiat_p384_i1 = ((x1 >> 64) as fiat_p384_i1); - let x3: u64 = ((x1 & (0xffffffffffffffff as i128)) as u64); - out1 = x3; - out2 = (((0x0 as fiat_p384_i2) - (x2 as fiat_p384_i2)) as fiat_p384_u1); - (out1, out2) +#[inline(always)] +fn fiat_p384_subborrowx_u64(borrow: u8, lhs: u64, rhs: u64) -> (u64, u8) { + let (difference, underflow0) = lhs.overflowing_sub(rhs); + let (difference, underflow1) = difference.overflowing_sub(u64::from(borrow)); + (difference, u8::from(underflow0 | underflow1)) } #[inline(always)] @@ -51,40 +25,33 @@ fn fiat_p384_mulx_u64(arg1: u64, arg2: u64) -> (u64, u64) { super::mul_u64_wide(arg1, arg2) } -const fn fiat_p384_cmovznz_u64(arg1: fiat_p384_u1, arg2: u64, arg3: u64) -> u64 { - let mut out1: u64 = 0; - let x1: fiat_p384_u1 = (!(!arg1)); - let x2: u64 = ((((((0x0 as fiat_p384_i2) - (x1 as fiat_p384_i2)) as fiat_p384_i1) as i128) - & (0xffffffffffffffff as i128)) as u64); - let x3: u64 = ((x2 & arg3) | ((!x2) & arg2)); - out1 = x3; - out1 +#[inline(always)] +fn fiat_p384_cmovznz_u64(choice: u8, zero: u64, nonzero: u64) -> u64 { + let mask = u64::from(choice).wrapping_neg(); + (mask & nonzero) | (!mask & zero) } -fn fiat_p384_mul( - arg1: &fiat_p384_montgomery_domain_field_element, - arg2: &fiat_p384_montgomery_domain_field_element, -) -> fiat_p384_montgomery_domain_field_element { - let mut out1: fiat_p384_montgomery_domain_field_element = [0; 6]; - let x1: u64 = (arg1[1]); - let x2: u64 = (arg1[2]); - let x3: u64 = (arg1[3]); - let x4: u64 = (arg1[4]); - let x5: u64 = (arg1[5]); - let x6: u64 = (arg1[0]); - let (x7, x8) = fiat_p384_mulx_u64(x6, (arg2[5])); - let (x9, x10) = fiat_p384_mulx_u64(x6, (arg2[4])); - let (x11, x12) = fiat_p384_mulx_u64(x6, (arg2[3])); - let (x13, x14) = fiat_p384_mulx_u64(x6, (arg2[2])); - let (x15, x16) = fiat_p384_mulx_u64(x6, (arg2[1])); - let (x17, x18) = fiat_p384_mulx_u64(x6, (arg2[0])); +fn fiat_p384_mul(arg1: &[u64; 6], arg2: &[u64; 6]) -> [u64; 6] { + let mut out1 = [0; 6]; + let x1: u64 = arg1[1]; + let x2: u64 = arg1[2]; + let x3: u64 = arg1[3]; + let x4: u64 = arg1[4]; + let x5: u64 = arg1[5]; + let x6: u64 = arg1[0]; + let (x7, x8) = fiat_p384_mulx_u64(x6, arg2[5]); + let (x9, x10) = fiat_p384_mulx_u64(x6, arg2[4]); + let (x11, x12) = fiat_p384_mulx_u64(x6, arg2[3]); + let (x13, x14) = fiat_p384_mulx_u64(x6, arg2[2]); + let (x15, x16) = fiat_p384_mulx_u64(x6, arg2[1]); + let (x17, x18) = fiat_p384_mulx_u64(x6, arg2[0]); let (x19, x20) = fiat_p384_addcarryx_u64(0x0, x18, x15); let (x21, x22) = fiat_p384_addcarryx_u64(x20, x16, x13); let (x23, x24) = fiat_p384_addcarryx_u64(x22, x14, x11); let (x25, x26) = fiat_p384_addcarryx_u64(x24, x12, x9); let (x27, x28) = fiat_p384_addcarryx_u64(x26, x10, x7); - let x29: u64 = ((x28 as u64) + x8); - let (x30, x31) = fiat_p384_mulx_u64(x17, 0x100000001); + let x29: u64 = u64::from(x28).wrapping_add(x8); + let (x30, _x31) = fiat_p384_mulx_u64(x17, 0x100000001); let (x32, x33) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x34, x35) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x36, x37) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); @@ -96,34 +63,34 @@ fn fiat_p384_mul( let (x48, x49) = fiat_p384_addcarryx_u64(x47, x39, x36); let (x50, x51) = fiat_p384_addcarryx_u64(x49, x37, x34); let (x52, x53) = fiat_p384_addcarryx_u64(x51, x35, x32); - let x54: u64 = ((x53 as u64) + x33); - let (x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); + let x54: u64 = u64::from(x53).wrapping_add(x33); + let (_x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); let (x57, x58) = fiat_p384_addcarryx_u64(x56, x19, x44); let (x59, x60) = fiat_p384_addcarryx_u64(x58, x21, x46); let (x61, x62) = fiat_p384_addcarryx_u64(x60, x23, x48); let (x63, x64) = fiat_p384_addcarryx_u64(x62, x25, x50); let (x65, x66) = fiat_p384_addcarryx_u64(x64, x27, x52); let (x67, x68) = fiat_p384_addcarryx_u64(x66, x29, x54); - let (x69, x70) = fiat_p384_mulx_u64(x1, (arg2[5])); - let (x71, x72) = fiat_p384_mulx_u64(x1, (arg2[4])); - let (x73, x74) = fiat_p384_mulx_u64(x1, (arg2[3])); - let (x75, x76) = fiat_p384_mulx_u64(x1, (arg2[2])); - let (x77, x78) = fiat_p384_mulx_u64(x1, (arg2[1])); - let (x79, x80) = fiat_p384_mulx_u64(x1, (arg2[0])); + let (x69, x70) = fiat_p384_mulx_u64(x1, arg2[5]); + let (x71, x72) = fiat_p384_mulx_u64(x1, arg2[4]); + let (x73, x74) = fiat_p384_mulx_u64(x1, arg2[3]); + let (x75, x76) = fiat_p384_mulx_u64(x1, arg2[2]); + let (x77, x78) = fiat_p384_mulx_u64(x1, arg2[1]); + let (x79, x80) = fiat_p384_mulx_u64(x1, arg2[0]); let (x81, x82) = fiat_p384_addcarryx_u64(0x0, x80, x77); let (x83, x84) = fiat_p384_addcarryx_u64(x82, x78, x75); let (x85, x86) = fiat_p384_addcarryx_u64(x84, x76, x73); let (x87, x88) = fiat_p384_addcarryx_u64(x86, x74, x71); let (x89, x90) = fiat_p384_addcarryx_u64(x88, x72, x69); - let x91: u64 = ((x90 as u64) + x70); + let x91: u64 = u64::from(x90).wrapping_add(x70); let (x92, x93) = fiat_p384_addcarryx_u64(0x0, x57, x79); let (x94, x95) = fiat_p384_addcarryx_u64(x93, x59, x81); let (x96, x97) = fiat_p384_addcarryx_u64(x95, x61, x83); let (x98, x99) = fiat_p384_addcarryx_u64(x97, x63, x85); let (x100, x101) = fiat_p384_addcarryx_u64(x99, x65, x87); let (x102, x103) = fiat_p384_addcarryx_u64(x101, x67, x89); - let (x104, x105) = fiat_p384_addcarryx_u64(x103, (x68 as u64), x91); - let (x106, x107) = fiat_p384_mulx_u64(x92, 0x100000001); + let (x104, x105) = fiat_p384_addcarryx_u64(x103, u64::from(x68), x91); + let (x106, _x107) = fiat_p384_mulx_u64(x92, 0x100000001); let (x108, x109) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x110, x111) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x112, x113) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); @@ -135,27 +102,27 @@ fn fiat_p384_mul( let (x124, x125) = fiat_p384_addcarryx_u64(x123, x115, x112); let (x126, x127) = fiat_p384_addcarryx_u64(x125, x113, x110); let (x128, x129) = fiat_p384_addcarryx_u64(x127, x111, x108); - let x130: u64 = ((x129 as u64) + x109); - let (x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); + let x130: u64 = u64::from(x129).wrapping_add(x109); + let (_x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); let (x133, x134) = fiat_p384_addcarryx_u64(x132, x94, x120); let (x135, x136) = fiat_p384_addcarryx_u64(x134, x96, x122); let (x137, x138) = fiat_p384_addcarryx_u64(x136, x98, x124); let (x139, x140) = fiat_p384_addcarryx_u64(x138, x100, x126); let (x141, x142) = fiat_p384_addcarryx_u64(x140, x102, x128); let (x143, x144) = fiat_p384_addcarryx_u64(x142, x104, x130); - let x145: u64 = ((x144 as u64) + (x105 as u64)); - let (x146, x147) = fiat_p384_mulx_u64(x2, (arg2[5])); - let (x148, x149) = fiat_p384_mulx_u64(x2, (arg2[4])); - let (x150, x151) = fiat_p384_mulx_u64(x2, (arg2[3])); - let (x152, x153) = fiat_p384_mulx_u64(x2, (arg2[2])); - let (x154, x155) = fiat_p384_mulx_u64(x2, (arg2[1])); - let (x156, x157) = fiat_p384_mulx_u64(x2, (arg2[0])); + let x145: u64 = u64::from(x144).wrapping_add(u64::from(x105)); + let (x146, x147) = fiat_p384_mulx_u64(x2, arg2[5]); + let (x148, x149) = fiat_p384_mulx_u64(x2, arg2[4]); + let (x150, x151) = fiat_p384_mulx_u64(x2, arg2[3]); + let (x152, x153) = fiat_p384_mulx_u64(x2, arg2[2]); + let (x154, x155) = fiat_p384_mulx_u64(x2, arg2[1]); + let (x156, x157) = fiat_p384_mulx_u64(x2, arg2[0]); let (x158, x159) = fiat_p384_addcarryx_u64(0x0, x157, x154); let (x160, x161) = fiat_p384_addcarryx_u64(x159, x155, x152); let (x162, x163) = fiat_p384_addcarryx_u64(x161, x153, x150); let (x164, x165) = fiat_p384_addcarryx_u64(x163, x151, x148); let (x166, x167) = fiat_p384_addcarryx_u64(x165, x149, x146); - let x168: u64 = ((x167 as u64) + x147); + let x168: u64 = u64::from(x167).wrapping_add(x147); let (x169, x170) = fiat_p384_addcarryx_u64(0x0, x133, x156); let (x171, x172) = fiat_p384_addcarryx_u64(x170, x135, x158); let (x173, x174) = fiat_p384_addcarryx_u64(x172, x137, x160); @@ -163,7 +130,7 @@ fn fiat_p384_mul( let (x177, x178) = fiat_p384_addcarryx_u64(x176, x141, x164); let (x179, x180) = fiat_p384_addcarryx_u64(x178, x143, x166); let (x181, x182) = fiat_p384_addcarryx_u64(x180, x145, x168); - let (x183, x184) = fiat_p384_mulx_u64(x169, 0x100000001); + let (x183, _x184) = fiat_p384_mulx_u64(x169, 0x100000001); let (x185, x186) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x187, x188) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x189, x190) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); @@ -175,27 +142,27 @@ fn fiat_p384_mul( let (x201, x202) = fiat_p384_addcarryx_u64(x200, x192, x189); let (x203, x204) = fiat_p384_addcarryx_u64(x202, x190, x187); let (x205, x206) = fiat_p384_addcarryx_u64(x204, x188, x185); - let x207: u64 = ((x206 as u64) + x186); - let (x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); + let x207: u64 = u64::from(x206).wrapping_add(x186); + let (_x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); let (x210, x211) = fiat_p384_addcarryx_u64(x209, x171, x197); let (x212, x213) = fiat_p384_addcarryx_u64(x211, x173, x199); let (x214, x215) = fiat_p384_addcarryx_u64(x213, x175, x201); let (x216, x217) = fiat_p384_addcarryx_u64(x215, x177, x203); let (x218, x219) = fiat_p384_addcarryx_u64(x217, x179, x205); let (x220, x221) = fiat_p384_addcarryx_u64(x219, x181, x207); - let x222: u64 = ((x221 as u64) + (x182 as u64)); - let (x223, x224) = fiat_p384_mulx_u64(x3, (arg2[5])); - let (x225, x226) = fiat_p384_mulx_u64(x3, (arg2[4])); - let (x227, x228) = fiat_p384_mulx_u64(x3, (arg2[3])); - let (x229, x230) = fiat_p384_mulx_u64(x3, (arg2[2])); - let (x231, x232) = fiat_p384_mulx_u64(x3, (arg2[1])); - let (x233, x234) = fiat_p384_mulx_u64(x3, (arg2[0])); + let x222: u64 = u64::from(x221).wrapping_add(u64::from(x182)); + let (x223, x224) = fiat_p384_mulx_u64(x3, arg2[5]); + let (x225, x226) = fiat_p384_mulx_u64(x3, arg2[4]); + let (x227, x228) = fiat_p384_mulx_u64(x3, arg2[3]); + let (x229, x230) = fiat_p384_mulx_u64(x3, arg2[2]); + let (x231, x232) = fiat_p384_mulx_u64(x3, arg2[1]); + let (x233, x234) = fiat_p384_mulx_u64(x3, arg2[0]); let (x235, x236) = fiat_p384_addcarryx_u64(0x0, x234, x231); let (x237, x238) = fiat_p384_addcarryx_u64(x236, x232, x229); let (x239, x240) = fiat_p384_addcarryx_u64(x238, x230, x227); let (x241, x242) = fiat_p384_addcarryx_u64(x240, x228, x225); let (x243, x244) = fiat_p384_addcarryx_u64(x242, x226, x223); - let x245: u64 = ((x244 as u64) + x224); + let x245: u64 = u64::from(x244).wrapping_add(x224); let (x246, x247) = fiat_p384_addcarryx_u64(0x0, x210, x233); let (x248, x249) = fiat_p384_addcarryx_u64(x247, x212, x235); let (x250, x251) = fiat_p384_addcarryx_u64(x249, x214, x237); @@ -203,7 +170,7 @@ fn fiat_p384_mul( let (x254, x255) = fiat_p384_addcarryx_u64(x253, x218, x241); let (x256, x257) = fiat_p384_addcarryx_u64(x255, x220, x243); let (x258, x259) = fiat_p384_addcarryx_u64(x257, x222, x245); - let (x260, x261) = fiat_p384_mulx_u64(x246, 0x100000001); + let (x260, _x261) = fiat_p384_mulx_u64(x246, 0x100000001); let (x262, x263) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x264, x265) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x266, x267) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); @@ -215,27 +182,27 @@ fn fiat_p384_mul( let (x278, x279) = fiat_p384_addcarryx_u64(x277, x269, x266); let (x280, x281) = fiat_p384_addcarryx_u64(x279, x267, x264); let (x282, x283) = fiat_p384_addcarryx_u64(x281, x265, x262); - let x284: u64 = ((x283 as u64) + x263); - let (x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); + let x284: u64 = u64::from(x283).wrapping_add(x263); + let (_x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); let (x287, x288) = fiat_p384_addcarryx_u64(x286, x248, x274); let (x289, x290) = fiat_p384_addcarryx_u64(x288, x250, x276); let (x291, x292) = fiat_p384_addcarryx_u64(x290, x252, x278); let (x293, x294) = fiat_p384_addcarryx_u64(x292, x254, x280); let (x295, x296) = fiat_p384_addcarryx_u64(x294, x256, x282); let (x297, x298) = fiat_p384_addcarryx_u64(x296, x258, x284); - let x299: u64 = ((x298 as u64) + (x259 as u64)); - let (x300, x301) = fiat_p384_mulx_u64(x4, (arg2[5])); - let (x302, x303) = fiat_p384_mulx_u64(x4, (arg2[4])); - let (x304, x305) = fiat_p384_mulx_u64(x4, (arg2[3])); - let (x306, x307) = fiat_p384_mulx_u64(x4, (arg2[2])); - let (x308, x309) = fiat_p384_mulx_u64(x4, (arg2[1])); - let (x310, x311) = fiat_p384_mulx_u64(x4, (arg2[0])); + let x299: u64 = u64::from(x298).wrapping_add(u64::from(x259)); + let (x300, x301) = fiat_p384_mulx_u64(x4, arg2[5]); + let (x302, x303) = fiat_p384_mulx_u64(x4, arg2[4]); + let (x304, x305) = fiat_p384_mulx_u64(x4, arg2[3]); + let (x306, x307) = fiat_p384_mulx_u64(x4, arg2[2]); + let (x308, x309) = fiat_p384_mulx_u64(x4, arg2[1]); + let (x310, x311) = fiat_p384_mulx_u64(x4, arg2[0]); let (x312, x313) = fiat_p384_addcarryx_u64(0x0, x311, x308); let (x314, x315) = fiat_p384_addcarryx_u64(x313, x309, x306); let (x316, x317) = fiat_p384_addcarryx_u64(x315, x307, x304); let (x318, x319) = fiat_p384_addcarryx_u64(x317, x305, x302); let (x320, x321) = fiat_p384_addcarryx_u64(x319, x303, x300); - let x322: u64 = ((x321 as u64) + x301); + let x322: u64 = u64::from(x321).wrapping_add(x301); let (x323, x324) = fiat_p384_addcarryx_u64(0x0, x287, x310); let (x325, x326) = fiat_p384_addcarryx_u64(x324, x289, x312); let (x327, x328) = fiat_p384_addcarryx_u64(x326, x291, x314); @@ -243,7 +210,7 @@ fn fiat_p384_mul( let (x331, x332) = fiat_p384_addcarryx_u64(x330, x295, x318); let (x333, x334) = fiat_p384_addcarryx_u64(x332, x297, x320); let (x335, x336) = fiat_p384_addcarryx_u64(x334, x299, x322); - let (x337, x338) = fiat_p384_mulx_u64(x323, 0x100000001); + let (x337, _x338) = fiat_p384_mulx_u64(x323, 0x100000001); let (x339, x340) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x341, x342) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x343, x344) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); @@ -255,27 +222,27 @@ fn fiat_p384_mul( let (x355, x356) = fiat_p384_addcarryx_u64(x354, x346, x343); let (x357, x358) = fiat_p384_addcarryx_u64(x356, x344, x341); let (x359, x360) = fiat_p384_addcarryx_u64(x358, x342, x339); - let x361: u64 = ((x360 as u64) + x340); - let (x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); + let x361: u64 = u64::from(x360).wrapping_add(x340); + let (_x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); let (x364, x365) = fiat_p384_addcarryx_u64(x363, x325, x351); let (x366, x367) = fiat_p384_addcarryx_u64(x365, x327, x353); let (x368, x369) = fiat_p384_addcarryx_u64(x367, x329, x355); let (x370, x371) = fiat_p384_addcarryx_u64(x369, x331, x357); let (x372, x373) = fiat_p384_addcarryx_u64(x371, x333, x359); let (x374, x375) = fiat_p384_addcarryx_u64(x373, x335, x361); - let x376: u64 = ((x375 as u64) + (x336 as u64)); - let (x377, x378) = fiat_p384_mulx_u64(x5, (arg2[5])); - let (x379, x380) = fiat_p384_mulx_u64(x5, (arg2[4])); - let (x381, x382) = fiat_p384_mulx_u64(x5, (arg2[3])); - let (x383, x384) = fiat_p384_mulx_u64(x5, (arg2[2])); - let (x385, x386) = fiat_p384_mulx_u64(x5, (arg2[1])); - let (x387, x388) = fiat_p384_mulx_u64(x5, (arg2[0])); + let x376: u64 = u64::from(x375).wrapping_add(u64::from(x336)); + let (x377, x378) = fiat_p384_mulx_u64(x5, arg2[5]); + let (x379, x380) = fiat_p384_mulx_u64(x5, arg2[4]); + let (x381, x382) = fiat_p384_mulx_u64(x5, arg2[3]); + let (x383, x384) = fiat_p384_mulx_u64(x5, arg2[2]); + let (x385, x386) = fiat_p384_mulx_u64(x5, arg2[1]); + let (x387, x388) = fiat_p384_mulx_u64(x5, arg2[0]); let (x389, x390) = fiat_p384_addcarryx_u64(0x0, x388, x385); let (x391, x392) = fiat_p384_addcarryx_u64(x390, x386, x383); let (x393, x394) = fiat_p384_addcarryx_u64(x392, x384, x381); let (x395, x396) = fiat_p384_addcarryx_u64(x394, x382, x379); let (x397, x398) = fiat_p384_addcarryx_u64(x396, x380, x377); - let x399: u64 = ((x398 as u64) + x378); + let x399: u64 = u64::from(x398).wrapping_add(x378); let (x400, x401) = fiat_p384_addcarryx_u64(0x0, x364, x387); let (x402, x403) = fiat_p384_addcarryx_u64(x401, x366, x389); let (x404, x405) = fiat_p384_addcarryx_u64(x403, x368, x391); @@ -283,7 +250,7 @@ fn fiat_p384_mul( let (x408, x409) = fiat_p384_addcarryx_u64(x407, x372, x395); let (x410, x411) = fiat_p384_addcarryx_u64(x409, x374, x397); let (x412, x413) = fiat_p384_addcarryx_u64(x411, x376, x399); - let (x414, x415) = fiat_p384_mulx_u64(x400, 0x100000001); + let (x414, _x415) = fiat_p384_mulx_u64(x400, 0x100000001); let (x416, x417) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x418, x419) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x420, x421) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); @@ -295,28 +262,28 @@ fn fiat_p384_mul( let (x432, x433) = fiat_p384_addcarryx_u64(x431, x423, x420); let (x434, x435) = fiat_p384_addcarryx_u64(x433, x421, x418); let (x436, x437) = fiat_p384_addcarryx_u64(x435, x419, x416); - let x438: u64 = ((x437 as u64) + x417); - let (x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); + let x438: u64 = u64::from(x437).wrapping_add(x417); + let (_x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); let (x441, x442) = fiat_p384_addcarryx_u64(x440, x402, x428); let (x443, x444) = fiat_p384_addcarryx_u64(x442, x404, x430); let (x445, x446) = fiat_p384_addcarryx_u64(x444, x406, x432); let (x447, x448) = fiat_p384_addcarryx_u64(x446, x408, x434); let (x449, x450) = fiat_p384_addcarryx_u64(x448, x410, x436); let (x451, x452) = fiat_p384_addcarryx_u64(x450, x412, x438); - let x453: u64 = ((x452 as u64) + (x413 as u64)); + let x453: u64 = u64::from(x452).wrapping_add(u64::from(x413)); let (x454, x455) = fiat_p384_subborrowx_u64(0x0, x441, 0xffffffff); let (x456, x457) = fiat_p384_subborrowx_u64(x455, x443, 0xffffffff00000000); let (x458, x459) = fiat_p384_subborrowx_u64(x457, x445, 0xfffffffffffffffe); let (x460, x461) = fiat_p384_subborrowx_u64(x459, x447, 0xffffffffffffffff); let (x462, x463) = fiat_p384_subborrowx_u64(x461, x449, 0xffffffffffffffff); let (x464, x465) = fiat_p384_subborrowx_u64(x463, x451, 0xffffffffffffffff); - let (x466, x467) = fiat_p384_subborrowx_u64(x465, x453, (0x0 as u64)); - let (x468) = fiat_p384_cmovznz_u64(x467, x454, x441); - let (x469) = fiat_p384_cmovznz_u64(x467, x456, x443); - let (x470) = fiat_p384_cmovznz_u64(x467, x458, x445); - let (x471) = fiat_p384_cmovznz_u64(x467, x460, x447); - let (x472) = fiat_p384_cmovznz_u64(x467, x462, x449); - let (x473) = fiat_p384_cmovznz_u64(x467, x464, x451); + let (_x466, x467) = fiat_p384_subborrowx_u64(x465, x453, 0x0_u64); + let x468 = fiat_p384_cmovznz_u64(x467, x454, x441); + let x469 = fiat_p384_cmovznz_u64(x467, x456, x443); + let x470 = fiat_p384_cmovznz_u64(x467, x458, x445); + let x471 = fiat_p384_cmovznz_u64(x467, x460, x447); + let x472 = fiat_p384_cmovznz_u64(x467, x462, x449); + let x473 = fiat_p384_cmovznz_u64(x467, x464, x451); out1[0] = x468; out1[1] = x469; out1[2] = x470; @@ -326,27 +293,27 @@ fn fiat_p384_mul( out1 } -fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p384_montgomery_domain_field_element { - let mut out1: fiat_p384_montgomery_domain_field_element = [0; 6]; - let x1: u64 = (arg1[1]); - let x2: u64 = (arg1[2]); - let x3: u64 = (arg1[3]); - let x4: u64 = (arg1[4]); - let x5: u64 = (arg1[5]); - let x6: u64 = (arg1[0]); - let (x7, x8) = fiat_p384_mulx_u64(x6, (arg1[5])); - let (x9, x10) = fiat_p384_mulx_u64(x6, (arg1[4])); - let (x11, x12) = fiat_p384_mulx_u64(x6, (arg1[3])); - let (x13, x14) = fiat_p384_mulx_u64(x6, (arg1[2])); - let (x15, x16) = fiat_p384_mulx_u64(x6, (arg1[1])); - let (x17, x18) = fiat_p384_mulx_u64(x6, (arg1[0])); +fn fiat_p384_square(arg1: &[u64; 6]) -> [u64; 6] { + let mut out1 = [0; 6]; + let x1: u64 = arg1[1]; + let x2: u64 = arg1[2]; + let x3: u64 = arg1[3]; + let x4: u64 = arg1[4]; + let x5: u64 = arg1[5]; + let x6: u64 = arg1[0]; + let (x7, x8) = fiat_p384_mulx_u64(x6, arg1[5]); + let (x9, x10) = fiat_p384_mulx_u64(x6, arg1[4]); + let (x11, x12) = fiat_p384_mulx_u64(x6, arg1[3]); + let (x13, x14) = fiat_p384_mulx_u64(x6, arg1[2]); + let (x15, x16) = fiat_p384_mulx_u64(x6, arg1[1]); + let (x17, x18) = fiat_p384_mulx_u64(x6, arg1[0]); let (x19, x20) = fiat_p384_addcarryx_u64(0x0, x18, x15); let (x21, x22) = fiat_p384_addcarryx_u64(x20, x16, x13); let (x23, x24) = fiat_p384_addcarryx_u64(x22, x14, x11); let (x25, x26) = fiat_p384_addcarryx_u64(x24, x12, x9); let (x27, x28) = fiat_p384_addcarryx_u64(x26, x10, x7); - let x29: u64 = ((x28 as u64) + x8); - let (x30, x31) = fiat_p384_mulx_u64(x17, 0x100000001); + let x29: u64 = u64::from(x28).wrapping_add(x8); + let (x30, _x31) = fiat_p384_mulx_u64(x17, 0x100000001); let (x32, x33) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x34, x35) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); let (x36, x37) = fiat_p384_mulx_u64(x30, 0xffffffffffffffff); @@ -358,34 +325,34 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x48, x49) = fiat_p384_addcarryx_u64(x47, x39, x36); let (x50, x51) = fiat_p384_addcarryx_u64(x49, x37, x34); let (x52, x53) = fiat_p384_addcarryx_u64(x51, x35, x32); - let x54: u64 = ((x53 as u64) + x33); - let (x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); + let x54: u64 = u64::from(x53).wrapping_add(x33); + let (_x55, x56) = fiat_p384_addcarryx_u64(0x0, x17, x42); let (x57, x58) = fiat_p384_addcarryx_u64(x56, x19, x44); let (x59, x60) = fiat_p384_addcarryx_u64(x58, x21, x46); let (x61, x62) = fiat_p384_addcarryx_u64(x60, x23, x48); let (x63, x64) = fiat_p384_addcarryx_u64(x62, x25, x50); let (x65, x66) = fiat_p384_addcarryx_u64(x64, x27, x52); let (x67, x68) = fiat_p384_addcarryx_u64(x66, x29, x54); - let (x69, x70) = fiat_p384_mulx_u64(x1, (arg1[5])); - let (x71, x72) = fiat_p384_mulx_u64(x1, (arg1[4])); - let (x73, x74) = fiat_p384_mulx_u64(x1, (arg1[3])); - let (x75, x76) = fiat_p384_mulx_u64(x1, (arg1[2])); - let (x77, x78) = fiat_p384_mulx_u64(x1, (arg1[1])); - let (x79, x80) = fiat_p384_mulx_u64(x1, (arg1[0])); + let (x69, x70) = fiat_p384_mulx_u64(x1, arg1[5]); + let (x71, x72) = fiat_p384_mulx_u64(x1, arg1[4]); + let (x73, x74) = fiat_p384_mulx_u64(x1, arg1[3]); + let (x75, x76) = fiat_p384_mulx_u64(x1, arg1[2]); + let (x77, x78) = fiat_p384_mulx_u64(x1, arg1[1]); + let (x79, x80) = fiat_p384_mulx_u64(x1, arg1[0]); let (x81, x82) = fiat_p384_addcarryx_u64(0x0, x80, x77); let (x83, x84) = fiat_p384_addcarryx_u64(x82, x78, x75); let (x85, x86) = fiat_p384_addcarryx_u64(x84, x76, x73); let (x87, x88) = fiat_p384_addcarryx_u64(x86, x74, x71); let (x89, x90) = fiat_p384_addcarryx_u64(x88, x72, x69); - let x91: u64 = ((x90 as u64) + x70); + let x91: u64 = u64::from(x90).wrapping_add(x70); let (x92, x93) = fiat_p384_addcarryx_u64(0x0, x57, x79); let (x94, x95) = fiat_p384_addcarryx_u64(x93, x59, x81); let (x96, x97) = fiat_p384_addcarryx_u64(x95, x61, x83); let (x98, x99) = fiat_p384_addcarryx_u64(x97, x63, x85); let (x100, x101) = fiat_p384_addcarryx_u64(x99, x65, x87); let (x102, x103) = fiat_p384_addcarryx_u64(x101, x67, x89); - let (x104, x105) = fiat_p384_addcarryx_u64(x103, (x68 as u64), x91); - let (x106, x107) = fiat_p384_mulx_u64(x92, 0x100000001); + let (x104, x105) = fiat_p384_addcarryx_u64(x103, u64::from(x68), x91); + let (x106, _x107) = fiat_p384_mulx_u64(x92, 0x100000001); let (x108, x109) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x110, x111) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); let (x112, x113) = fiat_p384_mulx_u64(x106, 0xffffffffffffffff); @@ -397,27 +364,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x124, x125) = fiat_p384_addcarryx_u64(x123, x115, x112); let (x126, x127) = fiat_p384_addcarryx_u64(x125, x113, x110); let (x128, x129) = fiat_p384_addcarryx_u64(x127, x111, x108); - let x130: u64 = ((x129 as u64) + x109); - let (x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); + let x130: u64 = u64::from(x129).wrapping_add(x109); + let (_x131, x132) = fiat_p384_addcarryx_u64(0x0, x92, x118); let (x133, x134) = fiat_p384_addcarryx_u64(x132, x94, x120); let (x135, x136) = fiat_p384_addcarryx_u64(x134, x96, x122); let (x137, x138) = fiat_p384_addcarryx_u64(x136, x98, x124); let (x139, x140) = fiat_p384_addcarryx_u64(x138, x100, x126); let (x141, x142) = fiat_p384_addcarryx_u64(x140, x102, x128); let (x143, x144) = fiat_p384_addcarryx_u64(x142, x104, x130); - let x145: u64 = ((x144 as u64) + (x105 as u64)); - let (x146, x147) = fiat_p384_mulx_u64(x2, (arg1[5])); - let (x148, x149) = fiat_p384_mulx_u64(x2, (arg1[4])); - let (x150, x151) = fiat_p384_mulx_u64(x2, (arg1[3])); - let (x152, x153) = fiat_p384_mulx_u64(x2, (arg1[2])); - let (x154, x155) = fiat_p384_mulx_u64(x2, (arg1[1])); - let (x156, x157) = fiat_p384_mulx_u64(x2, (arg1[0])); + let x145: u64 = u64::from(x144).wrapping_add(u64::from(x105)); + let (x146, x147) = fiat_p384_mulx_u64(x2, arg1[5]); + let (x148, x149) = fiat_p384_mulx_u64(x2, arg1[4]); + let (x150, x151) = fiat_p384_mulx_u64(x2, arg1[3]); + let (x152, x153) = fiat_p384_mulx_u64(x2, arg1[2]); + let (x154, x155) = fiat_p384_mulx_u64(x2, arg1[1]); + let (x156, x157) = fiat_p384_mulx_u64(x2, arg1[0]); let (x158, x159) = fiat_p384_addcarryx_u64(0x0, x157, x154); let (x160, x161) = fiat_p384_addcarryx_u64(x159, x155, x152); let (x162, x163) = fiat_p384_addcarryx_u64(x161, x153, x150); let (x164, x165) = fiat_p384_addcarryx_u64(x163, x151, x148); let (x166, x167) = fiat_p384_addcarryx_u64(x165, x149, x146); - let x168: u64 = ((x167 as u64) + x147); + let x168: u64 = u64::from(x167).wrapping_add(x147); let (x169, x170) = fiat_p384_addcarryx_u64(0x0, x133, x156); let (x171, x172) = fiat_p384_addcarryx_u64(x170, x135, x158); let (x173, x174) = fiat_p384_addcarryx_u64(x172, x137, x160); @@ -425,7 +392,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x177, x178) = fiat_p384_addcarryx_u64(x176, x141, x164); let (x179, x180) = fiat_p384_addcarryx_u64(x178, x143, x166); let (x181, x182) = fiat_p384_addcarryx_u64(x180, x145, x168); - let (x183, x184) = fiat_p384_mulx_u64(x169, 0x100000001); + let (x183, _x184) = fiat_p384_mulx_u64(x169, 0x100000001); let (x185, x186) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x187, x188) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); let (x189, x190) = fiat_p384_mulx_u64(x183, 0xffffffffffffffff); @@ -437,27 +404,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x201, x202) = fiat_p384_addcarryx_u64(x200, x192, x189); let (x203, x204) = fiat_p384_addcarryx_u64(x202, x190, x187); let (x205, x206) = fiat_p384_addcarryx_u64(x204, x188, x185); - let x207: u64 = ((x206 as u64) + x186); - let (x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); + let x207: u64 = u64::from(x206).wrapping_add(x186); + let (_x208, x209) = fiat_p384_addcarryx_u64(0x0, x169, x195); let (x210, x211) = fiat_p384_addcarryx_u64(x209, x171, x197); let (x212, x213) = fiat_p384_addcarryx_u64(x211, x173, x199); let (x214, x215) = fiat_p384_addcarryx_u64(x213, x175, x201); let (x216, x217) = fiat_p384_addcarryx_u64(x215, x177, x203); let (x218, x219) = fiat_p384_addcarryx_u64(x217, x179, x205); let (x220, x221) = fiat_p384_addcarryx_u64(x219, x181, x207); - let x222: u64 = ((x221 as u64) + (x182 as u64)); - let (x223, x224) = fiat_p384_mulx_u64(x3, (arg1[5])); - let (x225, x226) = fiat_p384_mulx_u64(x3, (arg1[4])); - let (x227, x228) = fiat_p384_mulx_u64(x3, (arg1[3])); - let (x229, x230) = fiat_p384_mulx_u64(x3, (arg1[2])); - let (x231, x232) = fiat_p384_mulx_u64(x3, (arg1[1])); - let (x233, x234) = fiat_p384_mulx_u64(x3, (arg1[0])); + let x222: u64 = u64::from(x221).wrapping_add(u64::from(x182)); + let (x223, x224) = fiat_p384_mulx_u64(x3, arg1[5]); + let (x225, x226) = fiat_p384_mulx_u64(x3, arg1[4]); + let (x227, x228) = fiat_p384_mulx_u64(x3, arg1[3]); + let (x229, x230) = fiat_p384_mulx_u64(x3, arg1[2]); + let (x231, x232) = fiat_p384_mulx_u64(x3, arg1[1]); + let (x233, x234) = fiat_p384_mulx_u64(x3, arg1[0]); let (x235, x236) = fiat_p384_addcarryx_u64(0x0, x234, x231); let (x237, x238) = fiat_p384_addcarryx_u64(x236, x232, x229); let (x239, x240) = fiat_p384_addcarryx_u64(x238, x230, x227); let (x241, x242) = fiat_p384_addcarryx_u64(x240, x228, x225); let (x243, x244) = fiat_p384_addcarryx_u64(x242, x226, x223); - let x245: u64 = ((x244 as u64) + x224); + let x245: u64 = u64::from(x244).wrapping_add(x224); let (x246, x247) = fiat_p384_addcarryx_u64(0x0, x210, x233); let (x248, x249) = fiat_p384_addcarryx_u64(x247, x212, x235); let (x250, x251) = fiat_p384_addcarryx_u64(x249, x214, x237); @@ -465,7 +432,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x254, x255) = fiat_p384_addcarryx_u64(x253, x218, x241); let (x256, x257) = fiat_p384_addcarryx_u64(x255, x220, x243); let (x258, x259) = fiat_p384_addcarryx_u64(x257, x222, x245); - let (x260, x261) = fiat_p384_mulx_u64(x246, 0x100000001); + let (x260, _x261) = fiat_p384_mulx_u64(x246, 0x100000001); let (x262, x263) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x264, x265) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); let (x266, x267) = fiat_p384_mulx_u64(x260, 0xffffffffffffffff); @@ -477,27 +444,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x278, x279) = fiat_p384_addcarryx_u64(x277, x269, x266); let (x280, x281) = fiat_p384_addcarryx_u64(x279, x267, x264); let (x282, x283) = fiat_p384_addcarryx_u64(x281, x265, x262); - let x284: u64 = ((x283 as u64) + x263); - let (x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); + let x284: u64 = u64::from(x283).wrapping_add(x263); + let (_x285, x286) = fiat_p384_addcarryx_u64(0x0, x246, x272); let (x287, x288) = fiat_p384_addcarryx_u64(x286, x248, x274); let (x289, x290) = fiat_p384_addcarryx_u64(x288, x250, x276); let (x291, x292) = fiat_p384_addcarryx_u64(x290, x252, x278); let (x293, x294) = fiat_p384_addcarryx_u64(x292, x254, x280); let (x295, x296) = fiat_p384_addcarryx_u64(x294, x256, x282); let (x297, x298) = fiat_p384_addcarryx_u64(x296, x258, x284); - let x299: u64 = ((x298 as u64) + (x259 as u64)); - let (x300, x301) = fiat_p384_mulx_u64(x4, (arg1[5])); - let (x302, x303) = fiat_p384_mulx_u64(x4, (arg1[4])); - let (x304, x305) = fiat_p384_mulx_u64(x4, (arg1[3])); - let (x306, x307) = fiat_p384_mulx_u64(x4, (arg1[2])); - let (x308, x309) = fiat_p384_mulx_u64(x4, (arg1[1])); - let (x310, x311) = fiat_p384_mulx_u64(x4, (arg1[0])); + let x299: u64 = u64::from(x298).wrapping_add(u64::from(x259)); + let (x300, x301) = fiat_p384_mulx_u64(x4, arg1[5]); + let (x302, x303) = fiat_p384_mulx_u64(x4, arg1[4]); + let (x304, x305) = fiat_p384_mulx_u64(x4, arg1[3]); + let (x306, x307) = fiat_p384_mulx_u64(x4, arg1[2]); + let (x308, x309) = fiat_p384_mulx_u64(x4, arg1[1]); + let (x310, x311) = fiat_p384_mulx_u64(x4, arg1[0]); let (x312, x313) = fiat_p384_addcarryx_u64(0x0, x311, x308); let (x314, x315) = fiat_p384_addcarryx_u64(x313, x309, x306); let (x316, x317) = fiat_p384_addcarryx_u64(x315, x307, x304); let (x318, x319) = fiat_p384_addcarryx_u64(x317, x305, x302); let (x320, x321) = fiat_p384_addcarryx_u64(x319, x303, x300); - let x322: u64 = ((x321 as u64) + x301); + let x322: u64 = u64::from(x321).wrapping_add(x301); let (x323, x324) = fiat_p384_addcarryx_u64(0x0, x287, x310); let (x325, x326) = fiat_p384_addcarryx_u64(x324, x289, x312); let (x327, x328) = fiat_p384_addcarryx_u64(x326, x291, x314); @@ -505,7 +472,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x331, x332) = fiat_p384_addcarryx_u64(x330, x295, x318); let (x333, x334) = fiat_p384_addcarryx_u64(x332, x297, x320); let (x335, x336) = fiat_p384_addcarryx_u64(x334, x299, x322); - let (x337, x338) = fiat_p384_mulx_u64(x323, 0x100000001); + let (x337, _x338) = fiat_p384_mulx_u64(x323, 0x100000001); let (x339, x340) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x341, x342) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); let (x343, x344) = fiat_p384_mulx_u64(x337, 0xffffffffffffffff); @@ -517,27 +484,27 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x355, x356) = fiat_p384_addcarryx_u64(x354, x346, x343); let (x357, x358) = fiat_p384_addcarryx_u64(x356, x344, x341); let (x359, x360) = fiat_p384_addcarryx_u64(x358, x342, x339); - let x361: u64 = ((x360 as u64) + x340); - let (x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); + let x361: u64 = u64::from(x360).wrapping_add(x340); + let (_x362, x363) = fiat_p384_addcarryx_u64(0x0, x323, x349); let (x364, x365) = fiat_p384_addcarryx_u64(x363, x325, x351); let (x366, x367) = fiat_p384_addcarryx_u64(x365, x327, x353); let (x368, x369) = fiat_p384_addcarryx_u64(x367, x329, x355); let (x370, x371) = fiat_p384_addcarryx_u64(x369, x331, x357); let (x372, x373) = fiat_p384_addcarryx_u64(x371, x333, x359); let (x374, x375) = fiat_p384_addcarryx_u64(x373, x335, x361); - let x376: u64 = ((x375 as u64) + (x336 as u64)); - let (x377, x378) = fiat_p384_mulx_u64(x5, (arg1[5])); - let (x379, x380) = fiat_p384_mulx_u64(x5, (arg1[4])); - let (x381, x382) = fiat_p384_mulx_u64(x5, (arg1[3])); - let (x383, x384) = fiat_p384_mulx_u64(x5, (arg1[2])); - let (x385, x386) = fiat_p384_mulx_u64(x5, (arg1[1])); - let (x387, x388) = fiat_p384_mulx_u64(x5, (arg1[0])); + let x376: u64 = u64::from(x375).wrapping_add(u64::from(x336)); + let (x377, x378) = fiat_p384_mulx_u64(x5, arg1[5]); + let (x379, x380) = fiat_p384_mulx_u64(x5, arg1[4]); + let (x381, x382) = fiat_p384_mulx_u64(x5, arg1[3]); + let (x383, x384) = fiat_p384_mulx_u64(x5, arg1[2]); + let (x385, x386) = fiat_p384_mulx_u64(x5, arg1[1]); + let (x387, x388) = fiat_p384_mulx_u64(x5, arg1[0]); let (x389, x390) = fiat_p384_addcarryx_u64(0x0, x388, x385); let (x391, x392) = fiat_p384_addcarryx_u64(x390, x386, x383); let (x393, x394) = fiat_p384_addcarryx_u64(x392, x384, x381); let (x395, x396) = fiat_p384_addcarryx_u64(x394, x382, x379); let (x397, x398) = fiat_p384_addcarryx_u64(x396, x380, x377); - let x399: u64 = ((x398 as u64) + x378); + let x399: u64 = u64::from(x398).wrapping_add(x378); let (x400, x401) = fiat_p384_addcarryx_u64(0x0, x364, x387); let (x402, x403) = fiat_p384_addcarryx_u64(x401, x366, x389); let (x404, x405) = fiat_p384_addcarryx_u64(x403, x368, x391); @@ -545,7 +512,7 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x408, x409) = fiat_p384_addcarryx_u64(x407, x372, x395); let (x410, x411) = fiat_p384_addcarryx_u64(x409, x374, x397); let (x412, x413) = fiat_p384_addcarryx_u64(x411, x376, x399); - let (x414, x415) = fiat_p384_mulx_u64(x400, 0x100000001); + let (x414, _x415) = fiat_p384_mulx_u64(x400, 0x100000001); let (x416, x417) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x418, x419) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); let (x420, x421) = fiat_p384_mulx_u64(x414, 0xffffffffffffffff); @@ -557,28 +524,28 @@ fn fiat_p384_square(arg1: &fiat_p384_montgomery_domain_field_element) -> fiat_p3 let (x432, x433) = fiat_p384_addcarryx_u64(x431, x423, x420); let (x434, x435) = fiat_p384_addcarryx_u64(x433, x421, x418); let (x436, x437) = fiat_p384_addcarryx_u64(x435, x419, x416); - let x438: u64 = ((x437 as u64) + x417); - let (x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); + let x438: u64 = u64::from(x437).wrapping_add(x417); + let (_x439, x440) = fiat_p384_addcarryx_u64(0x0, x400, x426); let (x441, x442) = fiat_p384_addcarryx_u64(x440, x402, x428); let (x443, x444) = fiat_p384_addcarryx_u64(x442, x404, x430); let (x445, x446) = fiat_p384_addcarryx_u64(x444, x406, x432); let (x447, x448) = fiat_p384_addcarryx_u64(x446, x408, x434); let (x449, x450) = fiat_p384_addcarryx_u64(x448, x410, x436); let (x451, x452) = fiat_p384_addcarryx_u64(x450, x412, x438); - let x453: u64 = ((x452 as u64) + (x413 as u64)); + let x453: u64 = u64::from(x452).wrapping_add(u64::from(x413)); let (x454, x455) = fiat_p384_subborrowx_u64(0x0, x441, 0xffffffff); let (x456, x457) = fiat_p384_subborrowx_u64(x455, x443, 0xffffffff00000000); let (x458, x459) = fiat_p384_subborrowx_u64(x457, x445, 0xfffffffffffffffe); let (x460, x461) = fiat_p384_subborrowx_u64(x459, x447, 0xffffffffffffffff); let (x462, x463) = fiat_p384_subborrowx_u64(x461, x449, 0xffffffffffffffff); let (x464, x465) = fiat_p384_subborrowx_u64(x463, x451, 0xffffffffffffffff); - let (x466, x467) = fiat_p384_subborrowx_u64(x465, x453, (0x0 as u64)); - let (x468) = fiat_p384_cmovznz_u64(x467, x454, x441); - let (x469) = fiat_p384_cmovznz_u64(x467, x456, x443); - let (x470) = fiat_p384_cmovznz_u64(x467, x458, x445); - let (x471) = fiat_p384_cmovznz_u64(x467, x460, x447); - let (x472) = fiat_p384_cmovznz_u64(x467, x462, x449); - let (x473) = fiat_p384_cmovznz_u64(x467, x464, x451); + let (_x466, x467) = fiat_p384_subborrowx_u64(x465, x453, 0x0_u64); + let x468 = fiat_p384_cmovznz_u64(x467, x454, x441); + let x469 = fiat_p384_cmovznz_u64(x467, x456, x443); + let x470 = fiat_p384_cmovznz_u64(x467, x458, x445); + let x471 = fiat_p384_cmovznz_u64(x467, x460, x447); + let x472 = fiat_p384_cmovznz_u64(x467, x462, x449); + let x473 = fiat_p384_cmovznz_u64(x467, x464, x451); out1[0] = x468; out1[1] = x469; out1[2] = x470; diff --git a/src/auth/ecdsa_x86_64_asm.rs b/src/auth/ecdsa_x86_64_asm.rs index 79e0dcbe..8349f44b 100644 --- a/src/auth/ecdsa_x86_64_asm.rs +++ b/src/auth/ecdsa_x86_64_asm.rs @@ -5,8 +5,6 @@ //! module owns the ABI boundary; `ecdsa.rs` owns scalar validation, blinding, //! affine conversion, and public API semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::ZeroizingWords; diff --git a/src/auth/ed25519.rs b/src/auth/ed25519.rs index 2733617c..980cc4a7 100644 --- a/src/auth/ed25519.rs +++ b/src/auth/ed25519.rs @@ -429,7 +429,7 @@ pub struct Ed25519Keypair { #[cfg(feature = "diag")] #[doc(hidden)] -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct DiagEd25519VerifyScalars { pub r_bytes: [u8; PUBLIC_KEY_LENGTH], pub s_canonical: [u8; SECRET_KEY_LENGTH], @@ -992,24 +992,8 @@ mod tests { } fn decode_hex(hex: &str) -> [u8; N] { - let bytes = hex.as_bytes(); let mut out = [0u8; N]; - - for (dst, chunk) in out.iter_mut().zip(bytes.chunks_exact(2)) { - *dst = match chunk[0] { - b'0'..=b'9' => chunk[0] - b'0', - b'a'..=b'f' => chunk[0] - b'a' + 10, - b'A'..=b'F' => chunk[0] - b'A' + 10, - _ => panic!("invalid hex"), - } << 4 - | match chunk[1] { - b'0'..=b'9' => chunk[1] - b'0', - b'a'..=b'f' => chunk[1] - b'a' + 10, - b'A'..=b'F' => chunk[1] - b'A' + 10, - _ => panic!("invalid hex"), - }; - } - + crate::hex::from_hex(hex, &mut out).expect("RFC 8032 vectors must contain valid fixed-width hexadecimal"); out } @@ -1033,8 +1017,12 @@ mod tests { let signature = keypair.sign(message); assert_eq!(secret.sign(message), signature); - assert!(keypair.public_key().verify(message, &signature).is_ok()); - assert!(verify(message, &keypair.public_key(), &signature).is_ok()); + keypair + .public_key() + .verify(message, &signature) + .expect("the keypair signature must verify with its public key"); + verify(message, &keypair.public_key(), &signature) + .expect("the keypair signature must verify through the free function"); } #[test] @@ -1189,7 +1177,7 @@ mod tests { for scalar_bytes in [[0u8; Ed25519SecretKey::LENGTH], one, reduced] { let portable = point::ExtendedPoint::scalar_mul_basepoint(&scalar_bytes) .to_bytes() - .unwrap(); + .expect("basepoint multiplication produces an encodable Edwards point"); let asm = super::basepoint_mul_encoded_dispatch(&scalar_bytes); assert_eq!(asm, portable); @@ -1203,7 +1191,12 @@ mod tests { { let (asm_public, asm_point) = super::x86_64_asm::basepoint_mul_public(&scalar_bytes); assert_eq!(asm_public, portable); - assert_eq!(asm_point.to_bytes().unwrap(), portable); + assert_eq!( + asm_point + .to_bytes() + .expect("assembly basepoint multiplication produces an encodable Edwards point"), + portable + ); } } } @@ -1225,8 +1218,11 @@ mod tests { let public = keypair.public_key(); let signature = keypair.sign(message); let (r_bytes, s_bytes) = super::split_signature(&signature); - let a_point = public.point().unwrap(); - let s = scalar::from_canonical_bytes(&s_bytes).unwrap(); + let a_point = public + .point() + .expect("a locally generated Ed25519 public key decodes to an Edwards point"); + let s = + scalar::from_canonical_bytes(&s_bytes).expect("a locally generated Ed25519 signature has a canonical scalar"); let challenge_digest = super::hash_challenge(&r_bytes, public.as_bytes(), message); let challenge = scalar::reduce_bytes_mod_order(&challenge_digest); @@ -1236,10 +1232,10 @@ mod tests { let portable = point::straus_wnaf_basepoint_vartime(&s_canonical, &neg_challenge_bytes, &a_point) .to_bytes() - .unwrap(); + .expect("double-scalar multiplication produces an encodable Edwards point"); let asm = super::aarch64_asm::double_scalar_basepoint_encoded(&s_canonical, &neg_challenge_bytes, public.as_bytes()) - .unwrap(); + .expect("the assembly double-scalar path accepts a locally generated public key"); assert_eq!(asm, portable); assert_eq!(asm, r_bytes); diff --git a/src/auth/ed25519/aarch64_asm.rs b/src/auth/ed25519/aarch64_asm.rs index ef8d5c08..e35a2378 100644 --- a/src/auth/ed25519/aarch64_asm.rs +++ b/src/auth/ed25519/aarch64_asm.rs @@ -5,8 +5,6 @@ //! backends. This module owns the ABI boundary; `ed25519.rs` owns //! public-key/signature validation semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::constants::{PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; diff --git a/src/auth/ed25519/basepoint_table_ifma.rs b/src/auth/ed25519/basepoint_table_ifma.rs index 316b1e8b..f92d98e0 100644 --- a/src/auth/ed25519/basepoint_table_ifma.rs +++ b/src/auth/ed25519/basepoint_table_ifma.rs @@ -6,8 +6,8 @@ //! Lanes: `(d2·(Y−X), d2·(Y+X), 2·d2·Z, −2·d1·T)`. //! //! Generated from the canonical Ed25519 basepoint using portable scalar math. -//! Validated by `gen_basepoint_wnaf8_ifma_table` test in `point.rs`. -#![allow(clippy::unreadable_literal)] +//! Validated by `ifma_wnaf8_table_matches_portable_odd_multiples` in +//! `point_avx2.rs`. pub(crate) static BASEPOINT_WNAF8_IFMA_RAW: [[[i64; 4]; 5]; 64] = [ // 1B diff --git a/src/auth/ed25519/field_avx2.rs b/src/auth/ed25519/field_avx2.rs index 5384e270..7c64d58a 100644 --- a/src/auth/ed25519/field_avx2.rs +++ b/src/auth/ed25519/field_avx2.rs @@ -31,9 +31,7 @@ use super::field::FieldElement; const LOW_25_BITS: i64 = (1 << 25) - 1; const LOW_26_BITS: i64 = (1 << 26) - 1; - -/// D-lane blend mask: positions 5 and 7 in each `__m256i` (u32 view). -const D_BLEND: i32 = 0b1010_0000u8 as i32; +const LOW_26_BITS_U64: u64 = (1 << 26) - 1; /// Four field elements packed for AVX2 parallel processing. /// @@ -49,30 +47,32 @@ pub(crate) struct FieldElement2625x4(pub(crate) [__m256i; 5]); /// Lane rearrangement patterns for `shuffle`. #[derive(Clone, Copy)] #[repr(u8)] -#[allow(clippy::upper_case_acronyms, dead_code)] // Lane labels, complete API. pub(crate) enum Shuffle { - /// Identity: (A, B, C, D) → (A, B, C, D) - ABCD, /// Swap pairs: (A, B, C, D) → (B, A, D, C) - BADC, + SwapPairs, /// Swap left pair only: (A, B, C, D) → (B, A, C, D) - BACD, + SwapAB, /// Swap right pair only: (A, B, C, D) → (A, B, D, C) - ABDC, + SwapCD, /// Broadcast A: (A, B, C, D) → (A, A, A, A) - AAAA, + #[cfg(feature = "ed25519")] + BroadcastA, /// Broadcast B: (A, B, C, D) → (B, B, B, B) - BBBB, + #[cfg(feature = "ed25519")] + BroadcastB, /// (A, B, C, D) → (C, A, C, A) - CACA, + #[cfg(feature = "ed25519")] + AlternateCA, /// (A, B, C, D) → (D, B, B, D) - DBBD, + #[cfg(feature = "ed25519")] + OuterDInnerB, /// (A, B, C, D) → (A, D, D, A) - ADDA, + OuterAInnerD, /// (A, B, C, D) → (C, B, C, B) - CBCB, + AlternateCB, /// (A, B, C, D) → (A, B, A, B) - ABAB, + #[cfg(feature = "ed25519")] + RepeatAB, } impl Shuffle { @@ -80,17 +80,21 @@ impl Shuffle { #[inline(always)] fn control(self) -> [i32; 8] { match self { - Self::ABCD => [0, 1, 2, 3, 4, 5, 6, 7], - Self::BADC => [1, 0, 3, 2, 5, 4, 7, 6], - Self::BACD => [1, 0, 3, 2, 4, 5, 6, 7], - Self::ABDC => [0, 1, 2, 3, 5, 4, 7, 6], - Self::AAAA => [0, 0, 2, 2, 0, 0, 2, 2], - Self::BBBB => [1, 1, 3, 3, 1, 1, 3, 3], - Self::CACA => [4, 0, 6, 2, 4, 0, 6, 2], - Self::DBBD => [5, 1, 7, 3, 1, 5, 3, 7], - Self::ADDA => [0, 5, 2, 7, 5, 0, 7, 2], - Self::CBCB => [4, 1, 6, 3, 4, 1, 6, 3], - Self::ABAB => [0, 1, 2, 3, 0, 1, 2, 3], + Self::SwapPairs => [1, 0, 3, 2, 5, 4, 7, 6], + Self::SwapAB => [1, 0, 3, 2, 4, 5, 6, 7], + Self::SwapCD => [0, 1, 2, 3, 5, 4, 7, 6], + #[cfg(feature = "ed25519")] + Self::BroadcastA => [0, 0, 2, 2, 0, 0, 2, 2], + #[cfg(feature = "ed25519")] + Self::BroadcastB => [1, 1, 3, 3, 1, 1, 3, 3], + #[cfg(feature = "ed25519")] + Self::AlternateCA => [4, 0, 6, 2, 4, 0, 6, 2], + #[cfg(feature = "ed25519")] + Self::OuterDInnerB => [5, 1, 7, 3, 1, 5, 3, 7], + Self::OuterAInnerD => [0, 5, 2, 7, 5, 0, 7, 2], + Self::AlternateCB => [4, 1, 6, 3, 4, 1, 6, 3], + #[cfg(feature = "ed25519")] + Self::RepeatAB => [0, 1, 2, 3, 0, 1, 2, 3], } } } @@ -103,13 +107,9 @@ impl Shuffle { /// Lane positions: `[a_even, b_even, a_odd, b_odd, c_even, d_even, c_odd, d_odd]`. #[derive(Clone, Copy)] #[repr(u8)] -#[allow(clippy::upper_case_acronyms, dead_code)] // Lane labels, complete API. pub(crate) enum Lanes { - /// Select A lanes: positions 0, 2 - A = 0b0000_0101, - /// Select B lanes: positions 1, 3 - B = 0b0000_1010, /// Select C lanes: positions 4, 6 + #[cfg(feature = "ed25519")] C = 0b0101_0000, /// Select D lanes: positions 5, 7 D = 0b1010_0000, @@ -118,15 +118,14 @@ pub(crate) enum Lanes { /// Select A and C lanes: positions 0, 2, 4, 6 AC = 0b0101_0101, /// Select A and D lanes: positions 0, 2, 5, 7 + #[cfg(feature = "ed25519")] AD = 0b1010_0101, /// Select B and C lanes: positions 1, 3, 4, 6 + #[cfg(feature = "ed25519")] BC = 0b0101_1010, - /// Select B, C, and D lanes: positions 1, 3, 4, 5, 6, 7 - BCD = 0b1111_1010, - /// Select C and D lanes: positions 4-7 - CD = 0b1111_0000, - /// Select all lanes - ABCD = 0b1111_1111, + /// Select every lane except A: positions 1, 3, 4, 5, 6, 7 + #[cfg(feature = "ed25519")] + ExceptA = 0b1111_1010, } /// Zero-extend packed u32 pairs into two u64x4 vectors suitable for `vpmuludq`. @@ -138,11 +137,10 @@ pub(crate) enum Lanes { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn unpack_pair(v: __m256i) -> (__m256i, __m256i) { +fn unpack_pair(v: __m256i) -> (__m256i, __m256i) { let zero = _mm256_setzero_si256(); let lo = _mm256_unpacklo_epi32(v, zero); let hi = _mm256_unpackhi_epi32(v, zero); @@ -155,11 +153,10 @@ unsafe fn unpack_pair(v: __m256i) -> (__m256i, __m256i) { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn repack_pair(lo: __m256i, hi: __m256i) -> __m256i { +fn repack_pair(lo: __m256i, hi: __m256i) -> __m256i { // Shuffle to pack the low u32 of each 64-bit lane into consecutive positions. // _mm256_shuffle_epi32 with imm [0, 2, 0, 2] = 0b10_00_10_00 packs positions // 0 and 2 within each 128-bit lane. @@ -173,11 +170,10 @@ unsafe fn repack_pair(lo: __m256i, hi: __m256i) -> __m256i { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn mul32(a: __m256i, b: __m256i) -> __m256i { +fn mul32(a: __m256i, b: __m256i) -> __m256i { _mm256_mul_epu32(a, b) } @@ -185,14 +181,19 @@ unsafe fn mul32(a: __m256i, b: __m256i) -> __m256i { /// /// # Safety /// -/// Caller must ensure AVX2 is available. +/// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add64(a: __m256i, b: __m256i) -> __m256i { +fn add64(a: __m256i, b: __m256i) -> __m256i { _mm256_add_epi64(a, b) } +#[inline(always)] +fn low_word_as_i32(value: u64) -> i32 { + let bytes = value.to_le_bytes(); + i32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + // FieldElement2625x4 implementation #[cfg(target_arch = "x86_64")] @@ -201,11 +202,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn zero() -> Self { + pub(crate) fn zero() -> Self { Self([_mm256_setzero_si256(); 5]) } @@ -216,42 +216,43 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { + pub(crate) fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { let al = a.limbs(); let bl = b.limbs(); let cl = c.limbs(); let dl = d.limbs(); - let mask = LOW_26_BITS as u64; let out = [ - Self::pack_limb_pair(al[0], bl[0], cl[0], dl[0], mask), - Self::pack_limb_pair(al[1], bl[1], cl[1], dl[1], mask), - Self::pack_limb_pair(al[2], bl[2], cl[2], dl[2], mask), - Self::pack_limb_pair(al[3], bl[3], cl[3], dl[3], mask), - Self::pack_limb_pair(al[4], bl[4], cl[4], dl[4], mask), + Self::pack_limb_pair(al[0], bl[0], cl[0], dl[0]), + Self::pack_limb_pair(al[1], bl[1], cl[1], dl[1]), + Self::pack_limb_pair(al[2], bl[2], cl[2], dl[2]), + Self::pack_limb_pair(al[3], bl[3], cl[3], dl[3]), + Self::pack_limb_pair(al[4], bl[4], cl[4], dl[4]), ]; // Odd limbs from a non-reduced FieldElement may exceed 25 bits. Self(out).reduce() } + /// # Safety + /// + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn pack_limb_pair(al: u64, bl: u64, cl: u64, dl: u64, mask: u64) -> __m256i { + fn pack_limb_pair(al: u64, bl: u64, cl: u64, dl: u64) -> __m256i { + // Valid radix-51 limbs stay below 2^55, so each shifted half fits in an unsigned 32-bit lane. _mm256_setr_epi32( - (al & mask) as i32, - (bl & mask) as i32, - (al >> 26) as i32, - (bl >> 26) as i32, - (cl & mask) as i32, - (dl & mask) as i32, - (cl >> 26) as i32, - (dl >> 26) as i32, + low_word_as_i32(al & LOW_26_BITS_U64), + low_word_as_i32(bl & LOW_26_BITS_U64), + low_word_as_i32(al >> 26), + low_word_as_i32(bl >> 26), + low_word_as_i32(cl & LOW_26_BITS_U64), + low_word_as_i32(dl & LOW_26_BITS_U64), + low_word_as_i32(cl >> 26), + low_word_as_i32(dl >> 26), ) } @@ -259,11 +260,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn split(&self) -> [FieldElement; 4] { + pub(crate) fn split(&self) -> [FieldElement; 4] { let mut al = [0u64; 5]; let mut bl = [0u64; 5]; let mut cl = [0u64; 5]; @@ -277,7 +277,8 @@ impl FieldElement2625x4 { .zip(self.0.iter()) { let mut tmp = [0u32; 8]; - _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec); + // SAFETY: AVX2 is active in this function, and `tmp` provides 32 writable bytes for the unaligned store. + unsafe { _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec) }; *a_out = u64::from(tmp[0]) | (u64::from(tmp[2]) << 26); *b_out = u64::from(tmp[1]) | (u64::from(tmp[3]) << 26); @@ -302,11 +303,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add(&self, rhs: &Self) -> Self { + pub(crate) fn add(&self, rhs: &Self) -> Self { Self([ _mm256_add_epi32(self.0[0], rhs.0[0]), _mm256_add_epi32(self.0[1], rhs.0[1]), @@ -323,35 +323,34 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn sub(&self, rhs: &Self) -> Self { + pub(crate) fn sub(&self, rhs: &Self) -> Self { // 2p in radix 10×(26/25): // p = 2^255 - 19 // Limb 0: 2 * ((1 << 26) - 19) = 2^27 - 38 // Even limbs 2,4,6,8: 2 * ((1 << 26) - 1) = 2^27 - 2 // Odd limbs 1,3,5,7,9: 2 * ((1 << 25) - 1) = 2^26 - 2 let bias_0 = _mm256_setr_epi32( - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 26) - 19)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, + (1 << 27) - 38, + (1 << 27) - 38, + (1 << 26) - 2, + (1 << 26) - 2, + (1 << 27) - 38, + (1 << 27) - 38, + (1 << 26) - 2, + (1 << 26) - 2, ); let bias_n = _mm256_setr_epi32( - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 26) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, - (2 * ((1i64 << 25) - 1)) as i32, + (1 << 27) - 2, + (1 << 27) - 2, + (1 << 26) - 2, + (1 << 26) - 2, + (1 << 27) - 2, + (1 << 27) - 2, + (1 << 26) - 2, + (1 << 26) - 2, ); Self([ @@ -370,11 +369,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn negate_lazy(&self) -> Self { + pub(crate) fn negate_lazy(&self) -> Self { Self::zero().sub(self) } @@ -387,11 +385,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn shuffle(&self, pattern: Shuffle) -> Self { + pub(crate) fn shuffle(&self, pattern: Shuffle) -> Self { let ctrl = pattern.control(); let c = _mm256_setr_epi32(ctrl[0], ctrl[1], ctrl[2], ctrl[3], ctrl[4], ctrl[5], ctrl[6], ctrl[7]); Self([ @@ -410,11 +407,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn blend(&self, other: &Self, lanes: Lanes) -> Self { + pub(crate) fn blend(&self, other: &Self, lanes: Lanes) -> Self { // _mm256_blend_epi32 requires a compile-time immediate, so dispatch // on the enum variant. macro_rules! do_blend { @@ -430,17 +426,17 @@ impl FieldElement2625x4 { } match lanes { - Lanes::A => do_blend!(0b0000_0101), - Lanes::B => do_blend!(0b0000_1010), + #[cfg(feature = "ed25519")] Lanes::C => do_blend!(0b0101_0000), Lanes::D => do_blend!(0b1010_0000), Lanes::AB => do_blend!(0b0000_1111), Lanes::AC => do_blend!(0b0101_0101), + #[cfg(feature = "ed25519")] Lanes::AD => do_blend!(0b1010_0101), + #[cfg(feature = "ed25519")] Lanes::BC => do_blend!(0b0101_1010), - Lanes::BCD => do_blend!(0b1111_1010), - Lanes::CD => do_blend!(0b1111_0000), - Lanes::ABCD => do_blend!(0b1111_1111), + #[cfg(feature = "ed25519")] + Lanes::ExceptA => do_blend!(0b1111_1010), } } @@ -448,12 +444,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn select_mask(&self, other: &Self, mask: u64) -> Self { - let mask = _mm256_set1_epi64x(mask as i64); + pub(crate) fn select_mask(&self, other: &Self, mask: u64) -> Self { + let mask = _mm256_set1_epi64x(i64::from_ne_bytes(mask.to_ne_bytes())); Self([ _mm256_xor_si256( self.0[0], @@ -485,12 +480,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn diff_sum(&self) -> Self { - let swapped = self.shuffle(Shuffle::BADC); // (B, A, D, C) + pub(crate) fn diff_sum(&self) -> Self { + let swapped = self.shuffle(Shuffle::SwapPairs); // (B, A, D, C) let negated = self.negate_lazy(); // (-A, -B, -C, -D) let neg_ac = self.blend(&negated, Lanes::AC); // (-A, B, -C, D) swapped.add(&neg_ac) // (B-A, A+B, D-C, C+D) @@ -506,11 +500,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn reduce(&self) -> Self { + pub(crate) fn reduce(&self) -> Self { // Unpack to 10 × u64x4 for carry propagation. let (z0, z1) = unpack_pair(self.0[0]); let (z2, z3) = unpack_pair(self.0[1]); @@ -528,11 +521,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn reduce64(z: &mut [__m256i; 10]) -> Self { + fn reduce64(z: &mut [__m256i; 10]) -> Self { let mask_26 = _mm256_set1_epi64x(LOW_26_BITS); let mask_25 = _mm256_set1_epi64x(LOW_25_BITS); let v19 = _mm256_set1_epi64x(19); @@ -608,11 +600,10 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul(&self, rhs: &Self) -> Self { + pub(crate) fn mul(&self, rhs: &Self) -> Self { let v19 = _mm256_set1_epi64x(19); // Unpack self into 10 �� u64x4 (zero-extended for vpmuludq) @@ -790,11 +781,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn square_accum(&self) -> [__m256i; 10] { + #[cfg(feature = "ed25519")] + fn square_accum(&self) -> [__m256i; 10] { let v19 = _mm256_set1_epi64x(19); let (x0, x1) = unpack_pair(self.0[0]); @@ -916,12 +907,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. - #[cfg(test)] + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. + #[cfg(all(test, feature = "ed25519"))] #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square(&self) -> Self { + pub(crate) fn square(&self) -> Self { let mut z = self.square_accum(); Self::reduce64(&mut z) } @@ -940,11 +930,11 @@ impl FieldElement2625x4 { /// /// # Safety /// - /// Caller must ensure AVX2 is available. + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square_and_negate_d(&self) -> Self { + #[cfg(feature = "ed25519")] + pub(crate) fn square_and_negate_d(&self) -> Self { let mut z = self.square_accum(); Self::negate_d_accum(&mut z); Self::reduce64(&mut z) @@ -957,10 +947,14 @@ impl FieldElement2625x4 { /// min bias ≈ 2^62) and small enough to fit in u64. The subsequent /// `reduce64` carry chain processes the biased values normally, producing /// a fully reduced negation with b < 0.007. + /// + /// # Safety + /// + /// Calls from outside an AVX2-enabled context require runtime AVX2 support. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn negate_d_accum(z: &mut [__m256i; 10]) { + #[cfg(feature = "ed25519")] + fn negate_d_accum(z: &mut [__m256i; 10]) { // p × 2^37 per limb (radix-26/25): let bias_even_0 = _mm256_set1_epi64x(((1i64 << 26) - 19) << 37); let bias_even = _mm256_set1_epi64x(((1i64 << 26) - 1) << 37); @@ -987,43 +981,13 @@ impl FieldElement2625x4 { neg_d!(8, bias_even); neg_d!(9, bias_odd); } - - /// Negate the D lane of each limb via `2p − D` in the packed u32 domain. - /// - /// **Not used by AVX2 `square_and_negate_d`** (which negates in u64 domain - /// for tighter bounds). Retained for potential external callers. - #[inline] - #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn, dead_code)] - unsafe fn negate_d_lane(fe: &mut Self) { - let p2_limb0_even = (2i64.wrapping_mul((1i64 << 26) - 19)) as i32; - let p2_limb_even = (2i64.wrapping_mul((1i64 << 26) - 1)) as i32; - let p2_limb_odd = (2i64.wrapping_mul((1i64 << 25) - 1)) as i32; - - let bias_0 = _mm256_setr_epi32(0, 0, 0, 0, 0, p2_limb0_even, 0, p2_limb_odd); - let bias_n = _mm256_setr_epi32(0, 0, 0, 0, 0, p2_limb_even, 0, p2_limb_odd); - - let neg0 = _mm256_sub_epi32(bias_0, fe.0[0]); - fe.0[0] = _mm256_blend_epi32::(fe.0[0], neg0); - - let neg1 = _mm256_sub_epi32(bias_n, fe.0[1]); - fe.0[1] = _mm256_blend_epi32::(fe.0[1], neg1); - - let neg2 = _mm256_sub_epi32(bias_n, fe.0[2]); - fe.0[2] = _mm256_blend_epi32::(fe.0[2], neg2); - - let neg3 = _mm256_sub_epi32(bias_n, fe.0[3]); - fe.0[3] = _mm256_blend_epi32::(fe.0[3], neg3); - - let neg4 = _mm256_sub_epi32(bias_n, fe.0[4]); - fe.0[4] = _mm256_blend_epi32::(fe.0[4], neg4); - } } // Tests #[cfg(test)] #[cfg(target_arch = "x86_64")] +#[cfg(feature = "ed25519")] mod tests { use super::{FieldElement, *}; @@ -1229,7 +1193,7 @@ mod tests { // SAFETY: AVX2 availability checked by the runtime guard above. unsafe { let packed = FieldElement2625x4::new(&a, &b, &c, &d); - let shuffled = packed.shuffle(Shuffle::BADC); + let shuffled = packed.shuffle(Shuffle::SwapPairs); let [ra, rb, rc, rd] = shuffled.split(); assert_eq!(ra.normalize(), b.normalize(), "BADC: A should be B"); diff --git a/src/auth/ed25519/field_ifma.rs b/src/auth/ed25519/field_ifma.rs index d27c7730..923cdf58 100644 --- a/src/auth/ed25519/field_ifma.rs +++ b/src/auth/ed25519/field_ifma.rs @@ -16,8 +16,8 @@ //! //! # Arithmetic convention //! -//! Field arithmetic is modular math (mod 2^255 - 19). Per CLAUDE.md rules, -//! `wrapping_*` is the correct choice for intentional modular arithmetic. +//! Field arithmetic is modular math (mod 2^255 - 19). Wrapping arithmetic is +//! reserved for operations whose field representation is intentionally modular. #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -30,7 +30,7 @@ use super::{ // Constants const MASK51: i64 = (1i64 << 51) - 1; -#[cfg(test)] +#[cfg(all(test, feature = "ed25519"))] const MASK52: i64 = (1i64 << 52) - 1; /// Subtraction bias: 2p in radix-51. Limb 0 accounts for the -19 term. @@ -60,8 +60,7 @@ pub(crate) struct FieldElement51x4(pub(crate) [__m256i; 5]); /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn mul19(x: __m256i) -> __m256i { +fn mul19(x: __m256i) -> __m256i { let x16 = _mm256_slli_epi64::<4>(x); let x2 = _mm256_slli_epi64::<1>(x); _mm256_add_epi64(_mm256_add_epi64(x16, x2), x) @@ -74,8 +73,7 @@ unsafe fn mul19(x: __m256i) -> __m256i { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn madd52lo(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { +fn madd52lo(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { _mm256_madd52lo_epu64(acc, a, b) } @@ -86,8 +84,7 @@ unsafe fn madd52lo(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { +fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { _mm256_madd52hi_epu64(acc, a, b) } @@ -99,16 +96,20 @@ unsafe fn madd52hi(acc: __m256i, a: __m256i, b: __m256i) -> __m256i { /// # Safety /// /// Caller must ensure AVX2 is available. -#[cfg(test)] +#[cfg(all(test, feature = "ed25519"))] #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_by_bit(bit: __m256i, val: __m256i) -> __m256i { +fn select_by_bit(bit: __m256i, val: __m256i) -> __m256i { // 0 → 0, 1 → 0xFFFF_FFFF_FFFF_FFFF let mask = _mm256_sub_epi64(_mm256_setzero_si256(), bit); _mm256_and_si256(mask, val) } +#[inline(always)] +fn u64_as_i64_bits(value: u64) -> i64 { + i64::from_ne_bytes(value.to_ne_bytes()) +} + // FieldElement51x4 implementation #[cfg(target_arch = "x86_64")] @@ -120,8 +121,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn zero() -> Self { + pub(crate) fn zero() -> Self { Self([_mm256_setzero_si256(); 5]) } @@ -132,19 +132,43 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { + pub(crate) fn new(a: &FieldElement, b: &FieldElement, c: &FieldElement, d: &FieldElement) -> Self { let al = a.limbs(); let bl = b.limbs(); let cl = c.limbs(); let dl = d.limbs(); Self([ - _mm256_set_epi64x(dl[0] as i64, cl[0] as i64, bl[0] as i64, al[0] as i64), - _mm256_set_epi64x(dl[1] as i64, cl[1] as i64, bl[1] as i64, al[1] as i64), - _mm256_set_epi64x(dl[2] as i64, cl[2] as i64, bl[2] as i64, al[2] as i64), - _mm256_set_epi64x(dl[3] as i64, cl[3] as i64, bl[3] as i64, al[3] as i64), - _mm256_set_epi64x(dl[4] as i64, cl[4] as i64, bl[4] as i64, al[4] as i64), + _mm256_set_epi64x( + u64_as_i64_bits(dl[0]), + u64_as_i64_bits(cl[0]), + u64_as_i64_bits(bl[0]), + u64_as_i64_bits(al[0]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[1]), + u64_as_i64_bits(cl[1]), + u64_as_i64_bits(bl[1]), + u64_as_i64_bits(al[1]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[2]), + u64_as_i64_bits(cl[2]), + u64_as_i64_bits(bl[2]), + u64_as_i64_bits(al[2]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[3]), + u64_as_i64_bits(cl[3]), + u64_as_i64_bits(bl[3]), + u64_as_i64_bits(al[3]), + ), + _mm256_set_epi64x( + u64_as_i64_bits(dl[4]), + u64_as_i64_bits(cl[4]), + u64_as_i64_bits(bl[4]), + u64_as_i64_bits(al[4]), + ), ]) } @@ -155,8 +179,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn split(&self) -> [FieldElement; 4] { + pub(crate) fn split(&self) -> [FieldElement; 4] { let mut al = [0u64; 5]; let mut bl = [0u64; 5]; let mut cl = [0u64; 5]; @@ -169,7 +192,8 @@ impl FieldElement51x4 { .zip(self.0.iter()) { let mut tmp = [0u64; 4]; - _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec); + // SAFETY: AVX2 is active in this function, and `tmp` provides 32 writable bytes for the unaligned store. + unsafe { _mm256_storeu_si256(tmp.as_mut_ptr().cast(), *vec) }; *a_out = tmp[0]; *b_out = tmp[1]; *c_out = tmp[2]; @@ -191,8 +215,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add(&self, rhs: &Self) -> Self { + pub(crate) fn add(&self, rhs: &Self) -> Self { Self([ _mm256_add_epi64(self.0[0], rhs.0[0]), _mm256_add_epi64(self.0[1], rhs.0[1]), @@ -209,8 +232,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn sub(&self, rhs: &Self) -> Self { + pub(crate) fn sub(&self, rhs: &Self) -> Self { let bias_0 = _mm256_set1_epi64x(BIAS_0); let bias_n = _mm256_set1_epi64x(BIAS_N); @@ -230,8 +252,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn negate_lazy(&self) -> Self { + pub(crate) fn negate_lazy(&self) -> Self { Self::zero().sub(self) } @@ -244,8 +265,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn shuffle(&self, pattern: Shuffle) -> Self { + pub(crate) fn shuffle(&self, pattern: Shuffle) -> Self { // _mm256_permute4x64_epi64 requires a compile-time immediate. // IMM8 = (d_src << 6) | (c_src << 4) | (b_src << 2) | a_src macro_rules! do_shuffle { @@ -261,17 +281,21 @@ impl FieldElement51x4 { } match pattern { - Shuffle::ABCD => do_shuffle!(0b11_10_01_00), - Shuffle::BADC => do_shuffle!(0b10_11_00_01), - Shuffle::BACD => do_shuffle!(0b11_10_00_01), - Shuffle::ABDC => do_shuffle!(0b10_11_01_00), - Shuffle::AAAA => do_shuffle!(0b00_00_00_00), - Shuffle::BBBB => do_shuffle!(0b01_01_01_01), - Shuffle::CACA => do_shuffle!(0b00_10_00_10), - Shuffle::DBBD => do_shuffle!(0b11_01_01_11), - Shuffle::ADDA => do_shuffle!(0b00_11_11_00), - Shuffle::CBCB => do_shuffle!(0b01_10_01_10), - Shuffle::ABAB => do_shuffle!(0b01_00_01_00), + Shuffle::SwapPairs => do_shuffle!(0b10_11_00_01), + Shuffle::SwapAB => do_shuffle!(0b11_10_00_01), + Shuffle::SwapCD => do_shuffle!(0b10_11_01_00), + #[cfg(feature = "ed25519")] + Shuffle::BroadcastA => do_shuffle!(0b00_00_00_00), + #[cfg(feature = "ed25519")] + Shuffle::BroadcastB => do_shuffle!(0b01_01_01_01), + #[cfg(feature = "ed25519")] + Shuffle::AlternateCA => do_shuffle!(0b00_10_00_10), + #[cfg(feature = "ed25519")] + Shuffle::OuterDInnerB => do_shuffle!(0b11_01_01_11), + Shuffle::OuterAInnerD => do_shuffle!(0b00_11_11_00), + Shuffle::AlternateCB => do_shuffle!(0b01_10_01_10), + #[cfg(feature = "ed25519")] + Shuffle::RepeatAB => do_shuffle!(0b01_00_01_00), } } @@ -285,8 +309,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn blend(&self, other: &Self, lanes: Lanes) -> Self { + pub(crate) fn blend(&self, other: &Self, lanes: Lanes) -> Self { // u64 lane → u32 pair: A={0,1}, B={2,3}, C={4,5}, D={6,7} macro_rules! do_blend { ($imm:expr) => { @@ -301,17 +324,17 @@ impl FieldElement51x4 { } match lanes { - Lanes::A => do_blend!(0b0000_0011), - Lanes::B => do_blend!(0b0000_1100), + #[cfg(feature = "ed25519")] Lanes::C => do_blend!(0b0011_0000), Lanes::D => do_blend!(0b1100_0000), Lanes::AB => do_blend!(0b0000_1111), Lanes::AC => do_blend!(0b0011_0011), + #[cfg(feature = "ed25519")] Lanes::AD => do_blend!(0b1100_0011), + #[cfg(feature = "ed25519")] Lanes::BC => do_blend!(0b0011_1100), - Lanes::BCD => do_blend!(0b1111_1100), - Lanes::CD => do_blend!(0b1111_0000), - Lanes::ABCD => do_blend!(0b1111_1111), + #[cfg(feature = "ed25519")] + Lanes::ExceptA => do_blend!(0b1111_1100), } } @@ -322,9 +345,8 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn select_mask(&self, other: &Self, mask: u64) -> Self { - let mask = _mm256_set1_epi64x(mask as i64); + pub(crate) fn select_mask(&self, other: &Self, mask: u64) -> Self { + let mask = _mm256_set1_epi64x(u64_as_i64_bits(mask)); Self([ _mm256_xor_si256( self.0[0], @@ -356,9 +378,8 @@ impl FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn diff_sum(&self) -> Self { - let swapped = self.shuffle(Shuffle::BADC); // (B, A, D, C) + pub(crate) fn diff_sum(&self) -> Self { + let swapped = self.shuffle(Shuffle::SwapPairs); // (B, A, D, C) let negated = self.negate_lazy(); // (-A, -B, -C, -D) let neg_ac = self.blend(&negated, Lanes::AC); // (-A, B, -C, D) swapped.add(&neg_ac) // (B-A, A+B, D-C, C+D) @@ -371,8 +392,7 @@ impl FieldElement51x4 { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn reduce(mut self) -> Self { + pub(crate) fn reduce(mut self) -> Self { let mask = _mm256_set1_epi64x(MASK51); let r19 = _mm256_set1_epi64x(19); @@ -423,8 +443,7 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul(&self, rhs: &Self) -> Self { + pub(crate) fn mul(&self, rhs: &Self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; let g = &rhs.0; @@ -577,10 +596,9 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul_unreduced(&self, rhs: &Self) -> Self { + pub(crate) fn mul_unreduced(&self, rhs: &Self) -> Self { let zero = _mm256_setzero_si256(); let mask52 = _mm256_set1_epi64x(MASK52); @@ -880,10 +898,9 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul_small_unreduced(&self, small: &Self) -> Self { + pub(crate) fn mul_small_unreduced(&self, small: &Self) -> Self { let zero = _mm256_setzero_si256(); let mask52 = _mm256_set1_epi64x(MASK52); let c = small.0[0]; // The only non-zero limb (≤18 bits, fits in 52 bits) @@ -950,8 +967,8 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square(&self) -> Self { + #[cfg(feature = "ed25519")] + pub(crate) fn square(&self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; @@ -1054,10 +1071,9 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - unsafe fn square_wide_fold(&self) -> [__m256i; 5] { + fn square_wide_fold(&self) -> [__m256i; 5] { let zero = _mm256_setzero_si256(); let f = &self.0; @@ -1176,11 +1192,10 @@ impl FieldElement51x4 { /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. - #[cfg(test)] + #[cfg(all(test, feature = "ed25519"))] #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn square_and_negate_d_wide(&self) -> Self { + pub(crate) fn square_and_negate_d_wide(&self) -> Self { let mut folded = self.square_wide_fold(); // Negate D lane in u64 domain using p × 2^10 bias. @@ -1229,8 +1244,7 @@ impl FieldElement51x4 { /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn mul_small(&self, small: &Self) -> Self { + pub(crate) fn mul_small(&self, small: &Self) -> Self { let zero = _mm256_setzero_si256(); let f = &self.0; let c = small.0[0]; // The only non-zero limb @@ -1269,6 +1283,7 @@ impl FieldElement51x4 { #[cfg(test)] #[cfg(target_arch = "x86_64")] +#[cfg(feature = "ed25519")] mod tests { use super::{FieldElement, *}; @@ -1468,7 +1483,7 @@ mod tests { // SAFETY: AVX2 checked above. unsafe { let packed = FieldElement51x4::new(&a, &b, &c, &d); - let shuffled = packed.shuffle(Shuffle::BADC); + let shuffled = packed.shuffle(Shuffle::SwapPairs); let [ra, rb, rc, rd] = shuffled.split(); assert_eq!(ra.limbs(), b.limbs(), "BADC lane 0 = B"); diff --git a/src/auth/ed25519/point.rs b/src/auth/ed25519/point.rs index ec0c014f..cad1bfe1 100644 --- a/src/auth/ed25519/point.rs +++ b/src/auth/ed25519/point.rs @@ -5,16 +5,23 @@ use core::fmt; -use super::{field::FieldElement, scalar}; +#[cfg(feature = "ed25519")] +use super::scalar; +use super::{field::FieldElement, scalar_radix_16}; +#[cfg(feature = "ed25519")] use crate::traits::ct; #[path = "basepoint_tables.rs"] mod basepoint_tables; +#[cfg(feature = "ed25519")] #[path = "basepoint_wnaf5_table.rs"] mod basepoint_wnaf5_table; -pub(crate) use self::{basepoint_tables::BASEPOINT_RADIX16_TABLE, basepoint_wnaf5_table::BASEPOINT_WNAF5_TABLE}; +pub(crate) use self::basepoint_tables::BASEPOINT_RADIX16_TABLE; +#[cfg(feature = "ed25519")] +pub(crate) use self::basepoint_wnaf5_table::BASEPOINT_WNAF5_TABLE; +#[cfg(feature = "ed25519")] const EDWARDS_D: FieldElement = FieldElement::from_limbs([ 929_955_233_495_203, 466_365_720_129_213, @@ -22,6 +29,7 @@ const EDWARDS_D: FieldElement = FieldElement::from_limbs([ 2_033_849_074_728_123, 1_442_794_654_840_575, ]); +#[cfg(feature = "ed25519")] const EDWARDS_D2: FieldElement = FieldElement::from_limbs([ 1_859_910_466_990_425, 932_731_440_258_426, @@ -29,6 +37,7 @@ const EDWARDS_D2: FieldElement = FieldElement::from_limbs([ 1_815_898_335_770_999, 633_789_495_995_903, ]); +#[cfg(feature = "ed25519")] const BASEPOINT_X: FieldElement = FieldElement::from_limbs([ 1_738_742_601_995_546, 1_146_398_526_822_698, @@ -36,6 +45,7 @@ const BASEPOINT_X: FieldElement = FieldElement::from_limbs([ 562_264_141_797_630, 587_772_402_128_613, ]); +#[cfg(feature = "ed25519")] const BASEPOINT_Y: FieldElement = FieldElement::from_limbs([ 1_801_439_850_948_184, 1_351_079_888_211_148, @@ -95,6 +105,7 @@ impl CachedPoint { /// multiply per addition than affine `CachedPoint`, but avoids the /// expensive field inversion required to convert runtime tables to affine. #[derive(Clone, Copy)] +#[cfg(feature = "ed25519")] struct ProjectiveCachedPoint { y_plus_x: FieldElement, y_minus_x: FieldElement, @@ -102,6 +113,7 @@ struct ProjectiveCachedPoint { t2d: FieldElement, } +#[cfg(feature = "ed25519")] impl ProjectiveCachedPoint { const IDENTITY: Self = Self { y_plus_x: FieldElement::ONE, @@ -144,6 +156,7 @@ impl ExtendedPoint { } /// Construct an extended point from affine coordinates. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn from_affine(x: FieldElement, y: FieldElement) -> Self { Self { @@ -155,6 +168,7 @@ impl ExtendedPoint { } /// Add two extended Edwards points. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn add(&self, rhs: &Self) -> Self { let a = self.y.sub(&self.x).mul(&rhs.y.sub(&rhs.x)); @@ -197,6 +211,7 @@ impl ExtendedPoint { /// Add a projective cached point (8M — 1 more than affine cached, but /// avoids the inversion needed to build affine runtime tables). + #[cfg(feature = "ed25519")] #[must_use] fn add_projective_cached(&self, rhs: &ProjectiveCachedPoint) -> Self { let a = self.y.sub(&self.x).mul(&rhs.y_minus_x); @@ -223,6 +238,7 @@ impl ExtendedPoint { /// multiplications, no curve-constant multiply. The general `add(self)` /// path costs 4 squarings + 5 multiplications and can't exploit squaring /// symmetry in the compiler. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn double(&self) -> Self { let a = self.x.square(); @@ -244,6 +260,7 @@ impl ExtendedPoint { } /// Compress the point into the standard Ed25519 encoding. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn to_bytes(self) -> Option<[u8; 32]> { let (x, y) = self.to_affine()?; @@ -253,6 +270,7 @@ impl ExtendedPoint { } /// Decode a compressed Ed25519 point. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn from_bytes(bytes: &[u8; 32]) -> Option { let sign = (bytes[31] >> 7) != 0; @@ -275,12 +293,14 @@ impl ExtendedPoint { } /// Standard Ed25519 basepoint. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn basepoint() -> Self { Self::from_affine(BASEPOINT_X, BASEPOINT_Y) } /// Scalar multiplication by a little-endian 32-byte scalar. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn scalar_mul(&self, scalar: &[u8; 32]) -> Self { let mut acc = Self::identity(); @@ -301,9 +321,10 @@ impl ExtendedPoint { /// Variable-base signed radix-16 multiplication using a projective runtime /// table (no field inversion). + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn scalar_mul_vartime(&self, scalar: &[u8; 32]) -> Self { - let digits = scalar::as_radix_16(scalar); + let digits = scalar_radix_16(scalar); let table = projective_cached_multiples(self); let mut acc = Self::identity(); @@ -324,7 +345,7 @@ impl ExtendedPoint { /// repeated-doubling work from the fixed-base path. #[must_use] pub(crate) fn scalar_mul_basepoint(scalar: &[u8; 32]) -> Self { - let digits = scalar::as_radix_16(scalar); + let digits = scalar_radix_16(scalar); let mut acc = Self::identity(); for (digit, table) in digits.iter().copied().zip(BASEPOINT_RADIX16_TABLE.iter()) { @@ -338,18 +359,21 @@ impl ExtendedPoint { // Radix-16 Straus removed — superseded by straus_wnaf_basepoint_vartime. /// Multiply by the Edwards cofactor. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn mul_by_cofactor(&self) -> Self { self.double().double().double() } /// Whether this point lies in the low-order torsion subgroup. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn is_small_order(&self) -> bool { self.mul_by_cofactor().equals_projective(&Self::identity()) } /// Convert the point to affine coordinates when `Z != 0`. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn to_affine(self) -> Option<(FieldElement, FieldElement)> { if self.z.is_zero() { @@ -365,8 +389,22 @@ impl ExtendedPoint { /// `u = (Z + Y) / (Z - Y)` is the standard birational map from /// Edwards25519 to Curve25519. The identity maps to the Montgomery /// 2-torsion point `u = 0`. + #[cfg(all( + feature = "x25519", + any( + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ))] #[must_use] - #[allow(dead_code)] pub(crate) fn to_montgomery_u(self) -> FieldElement { let numerator = self.z.add(&self.y); let denominator = self.z.sub(&self.y); @@ -378,6 +416,7 @@ impl ExtendedPoint { } /// Compare two extended points without converting to affine coordinates. + #[cfg(feature = "ed25519")] #[must_use] pub(crate) fn equals_projective(&self, rhs: &Self) -> bool { let zero = [0u8; 32]; @@ -402,6 +441,7 @@ impl ExtendedPoint { } /// Borrow the extended-coordinate components. + #[cfg(any(feature = "ed25519", target_arch = "x86_64"))] #[must_use] pub(crate) const fn components(&self) -> (&FieldElement, &FieldElement, &FieldElement, &FieldElement) { (&self.x, &self.y, &self.z, &self.t) @@ -419,16 +459,27 @@ fn ct_eq_mask_u8(lhs: u8, rhs: u8) -> u64 { #[inline(always)] #[must_use] fn ct_negative_mask_i8(value: i8) -> u64 { - let bit = ((i16::from(value) >> 15) & 1) as u64; + let bit = u64::from(value.to_ne_bytes()[0] >> 7); 0u64.wrapping_sub(bit) } #[inline(always)] #[must_use] fn ct_abs_i8(value: i8) -> u8 { - let value = i16::from(value); - let sign = value >> 15; - ((value ^ sign) - sign) as u8 + let value = value.to_ne_bytes()[0]; + let sign = value >> 7; + let mask = 0u8.wrapping_sub(sign); + (value ^ mask).wrapping_add(sign) +} + +#[inline] +fn volatile_copy_field(field: &FieldElement) -> FieldElement { + let mut limbs = [0u64; 5]; + for (output, input) in limbs.iter_mut().zip(field.limbs()) { + // SAFETY: `input` is an aligned reference to an initialized `u64` that remains live for this read. + *output = unsafe { core::ptr::read_volatile(input) }; + } + FieldElement::from_limbs(limbs) } #[inline(always)] @@ -456,15 +507,13 @@ fn select_cached(lhs: &CachedPoint, rhs: &CachedPoint, mask: u64) -> CachedPoint fn select_signed_cached(table: &[CachedPoint; 8], digit: i8) -> CachedPoint { let abs = core::hint::black_box(ct_abs_i8(digit)); let mut selected = CachedPoint::IDENTITY; - for (i, candidate) in table.iter().enumerate() { - // SAFETY: Volatile table read is used as a compiler barrier because: - // 1. `candidate` is a valid shared reference to one `CachedPoint` entry. - // 2. `CachedPoint` is `Copy`, so the volatile read does not create ownership aliasing. - // 3. Every table entry is read unconditionally; the selected digit affects only masks below. - // Without this barrier, LLVM can rewrite the masked selection into a secret-dependent branch - // ladder. - let candidate = unsafe { core::ptr::read_volatile(candidate) }; - let mask = core::hint::black_box(ct_eq_mask_u8(abs, (i as u8).wrapping_add(1))); + for (expected, candidate) in (1u8..=8).zip(table) { + let candidate = CachedPoint { + y_plus_x: volatile_copy_field(&candidate.y_plus_x), + y_minus_x: volatile_copy_field(&candidate.y_minus_x), + t2d: volatile_copy_field(&candidate.t2d), + }; + let mask = core::hint::black_box(ct_eq_mask_u8(abs, expected)); selected = select_cached(&selected, &candidate, mask); } @@ -472,6 +521,7 @@ fn select_signed_cached(table: &[CachedPoint; 8], digit: i8) -> CachedPoint { select_cached(&selected, &neg, core::hint::black_box(ct_negative_mask_i8(digit))) } +/// Select one signed digit from the first portable basepoint table and return its field limbs. #[cfg(feature = "diag")] #[inline(always)] pub fn diag_select_basepoint_cached_limb_digest(digit: i8) -> [u64; 15] { @@ -484,6 +534,7 @@ pub fn diag_select_basepoint_cached_limb_digest(digit: i8) -> [u64; 15] { } /// Add a signed digit from a projective cached table (runtime table). +#[cfg(feature = "ed25519")] #[inline] #[must_use] fn add_signed_projective_cached(acc: ExtendedPoint, table: &[ProjectiveCachedPoint; 8], digit: i8) -> ExtendedPoint { @@ -502,6 +553,7 @@ fn add_signed_projective_cached(acc: ExtendedPoint, table: &[ProjectiveCachedPoi /// Build a projective cached table of `[1P, 2P, ..., 8P]` without any /// field inversion. Each entry stores `(Y+X, Y-X, Z, 2dT)` in extended /// projective coordinates. +#[cfg(feature = "ed25519")] #[must_use] fn projective_cached_multiples(point: &ExtendedPoint) -> [ProjectiveCachedPoint; 8] { let mut out = [ProjectiveCachedPoint::IDENTITY; 8]; @@ -517,6 +569,7 @@ fn projective_cached_multiples(point: &ExtendedPoint) -> [ProjectiveCachedPoint; } /// Build odd multiples `[1P, 3P, 5P, ..., (2n-1)P]` in projective cached format. +#[cfg(feature = "ed25519")] #[must_use] fn odd_projective_cached_multiples(point: &ExtendedPoint) -> [ProjectiveCachedPoint; N] { let p2 = point.double(); @@ -533,6 +586,7 @@ fn odd_projective_cached_multiples(point: &ExtendedPoint) -> [Pr } /// Add a signed wNAF digit from an odd-multiples projective cached table. +#[cfg(feature = "ed25519")] #[inline] #[must_use] fn add_wnaf_digit_projective(acc: ExtendedPoint, table: &[ProjectiveCachedPoint], digit: i8) -> ExtendedPoint { @@ -549,6 +603,7 @@ fn add_wnaf_digit_projective(acc: ExtendedPoint, table: &[ProjectiveCachedPoint] } /// Add a signed wNAF digit from an affine cached table. +#[cfg(feature = "ed25519")] #[inline] #[must_use] fn add_wnaf_digit_cached(acc: ExtendedPoint, table: &[CachedPoint], digit: i8) -> ExtendedPoint { @@ -567,8 +622,8 @@ fn add_wnaf_digit_cached(acc: ExtendedPoint, table: &[CachedPoint], digit: i8) - /// wNAF-based portable Straus: `[s]B + [h]A`. /// /// Uses wNAF(5) for both scalars (8-entry odd-multiples tables). +#[cfg(feature = "ed25519")] #[must_use] -#[allow(clippy::indexing_slicing)] // i bounded by top < 256, naf arrays are [i8; 256] pub(crate) fn straus_wnaf_basepoint_vartime(s: &[u8; 32], h: &[u8; 32], a: &ExtendedPoint) -> ExtendedPoint { let s_naf = scalar::non_adjacent_form(s, 5); let h_naf = scalar::non_adjacent_form(h, 5); @@ -617,75 +672,18 @@ impl fmt::Debug for ExtendedPoint { } } -#[cfg(test)] +#[cfg(all(test, feature = "ed25519"))] mod tests { use super::{ExtendedPoint, FieldElement}; - /// Generates the wNAF(8) basepoint table for IFMA verify. - /// Run with `--nocapture` to print the table source. - #[cfg(feature = "std")] - #[test] - fn gen_basepoint_wnaf8_ifma_table() { - let bp = ExtendedPoint::basepoint(); - let bp2 = bp.double(); - - let d2_fe = FieldElement::from_small(121_666); - let d2_2_fe = FieldElement::from_small(121_666u64.wrapping_mul(2)); - let d1_2_fe = FieldElement::from_small(121_665u64.wrapping_mul(2)); - - let mut acc = bp; - eprintln!("pub(crate) static BASEPOINT_WNAF8_IFMA_RAW: [[[i64; 4]; 5]; 64] = ["); - for i in 0..64u32 { - if i > 0 { - acc = acc.add(&bp2); - } - let (x, y, z, t) = acc.components(); - - let a = d2_fe.mul(&y.sub(x)).normalize(); - let b = d2_fe.mul(&y.add(x)).normalize(); - let c = d2_2_fe.mul(z).normalize(); - let d = d1_2_fe.mul(t).neg().normalize(); - - let al = a.limbs(); - let bl = b.limbs(); - let cl = c.limbs(); - let dl = d.limbs(); - - eprintln!(" // {}B", 2 * i + 1); - eprintln!(" ["); - for k in 0..5 { - eprintln!( - " [{}, {}, {}, {}],", - al[k] as i64, bl[k] as i64, cl[k] as i64, dl[k] as i64 - ); - } - eprintln!(" ],"); - } - eprintln!("];"); - } - fn basepoint() -> ExtendedPoint { ExtendedPoint::basepoint() } - fn decode_hex_32(hex: &str) -> [u8; 32] { - let bytes = hex.as_bytes(); + fn decode_hex_32(hex: &str) -> Result<[u8; 32], crate::hex::InvalidHexError> { let mut out = [0u8; 32]; - - for (dst, chunk) in out.iter_mut().zip(bytes.chunks_exact(2)) { - *dst = hex_value(chunk[0]) << 4 | hex_value(chunk[1]); - } - - out - } - - fn hex_value(byte: u8) -> u8 { - match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex"), - } + crate::hex::from_hex(hex, &mut out)?; + Ok(out) } #[test] @@ -722,15 +720,15 @@ mod tests { #[test] fn basepoint_roundtrips_compressed_encoding() { - let expected = decode_hex_32("5866666666666666666666666666666666666666666666666666666666666666"); + let expected = decode_hex_32("5866666666666666666666666666666666666666666666666666666666666666").ok(); let encoded = basepoint().to_bytes(); - assert_eq!(encoded, Some(expected)); + assert_eq!(encoded, expected); assert_eq!( encoded .and_then(|bytes| ExtendedPoint::from_bytes(&bytes)) .and_then(|point| point.to_bytes()), - Some(expected) + expected ); } @@ -781,8 +779,7 @@ mod tests { #[test] fn basepoint_wnaf5_table_matches_odd_basepoint_multiples() { - for (i, cached) in super::BASEPOINT_WNAF5_TABLE.iter().enumerate() { - let scalar = (2 * i + 1) as u8; + for (scalar, cached) in (1u8..).step_by(2).zip(&super::BASEPOINT_WNAF5_TABLE) { let mut scalar_bytes = [0u8; 32]; scalar_bytes[0] = scalar; @@ -808,13 +805,13 @@ mod tests { fn rfc8032_public_key_derivation_matches_vector_1() { use crate::auth::ed25519::{Ed25519SecretKey, hash::ExpandedSecret}; - let secret = Ed25519SecretKey::from_bytes(decode_hex_32( - "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", - )); + let secret_bytes = decode_hex_32("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60").ok(); + assert!(secret_bytes.is_some()); + let secret = Ed25519SecretKey::from_bytes(secret_bytes.unwrap_or_default()); let expanded = ExpandedSecret::from_secret_key(&secret); let public = ExtendedPoint::scalar_mul_basepoint(expanded.scalar_bytes()).to_bytes(); - let expected = decode_hex_32("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a"); + let expected = decode_hex_32("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a").ok(); - assert_eq!(public, Some(expected)); + assert_eq!(public, expected); } } diff --git a/src/auth/ed25519/point_avx2.rs b/src/auth/ed25519/point_avx2.rs index 2eb34880..babe4055 100644 --- a/src/auth/ed25519/point_avx2.rs +++ b/src/auth/ed25519/point_avx2.rs @@ -12,19 +12,22 @@ //! scaling all output coordinates by `d2²` — which cancels in projective //! coordinates. -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ed25519"))] use core::arch::x86_64::_mm256_loadu_si256; +#[cfg(all(target_arch = "x86_64", feature = "ed25519"))] +use super::scalar; #[cfg(target_arch = "x86_64")] use super::{ field::FieldElement, field_avx2::{FieldElement2625x4, Lanes, Shuffle}, field_ifma::FieldElement51x4, point::{CachedPoint, ExtendedPoint}, - scalar, + scalar_radix_16, }; #[cfg(target_arch = "x86_64")] #[path = "basepoint_table_ifma.rs"] +#[cfg(feature = "ed25519")] mod basepoint_table_ifma; /// Hamburg constants for the curve `d = -d1/d2`. @@ -57,8 +60,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn from_extended(p: &ExtendedPoint) -> Self { + pub(crate) fn from_extended(p: &ExtendedPoint) -> Self { let (x, y, z, t) = p.components(); Self(FieldElement2625x4::new(x, y, z, t)) } @@ -70,8 +72,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_extended(self) -> ExtendedPoint { + pub(crate) fn to_extended(self) -> ExtendedPoint { let [x, y, z, t] = self.0.split(); ExtendedPoint::from_raw(x, y, z, t) } @@ -85,8 +86,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_cached(self) -> CachedPointAvx2 { + pub(crate) fn to_cached(self) -> CachedPointAvx2 { // Step 1: Compute (Y-X, Y+X) in lanes A,B; keep Z,T in C,D. let ds = self.0.diff_sum(); // (Y-X, Y+X, T-Z, Z+T) let prepared = self.0.blend(&ds, Lanes::AB); // (Y-X, Y+X, Z, T) @@ -109,8 +109,7 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add_cached(&self, other: &CachedPointAvx2) -> Self { + pub(crate) fn add_cached(&self, other: &CachedPointAvx2) -> Self { // Step 1: Prepare self as (Y-X, Y+X, Z, T) by blending diff_sum into A,B. let ds = self.0.diff_sum(); let tmp = self.0.blend(&ds, Lanes::AB); // (Y1-X1, Y1+X1, Z1, T1) @@ -121,14 +120,14 @@ impl ExtendedPointAvx2 { // Step 3: Swap C↔D to align for diff_sum. // After swap: lane C has the T-product, lane D has the Z-product. - let swapped = product.shuffle(Shuffle::ABDC); + let swapped = product.shuffle(Shuffle::SwapCD); // Step 4: diff_sum computes (e, h, f, g) (up to Hamburg scaling). let ehfg = swapped.diff_sum(); // Step 5: Shuffle into final multiply operands. - let t0 = ehfg.shuffle(Shuffle::ADDA); // (e, g, g, e) - let t1 = ehfg.shuffle(Shuffle::CBCB); // (f, h, f, h) + let t0 = ehfg.shuffle(Shuffle::OuterAInnerD); // (e, g, g, e) + let t1 = ehfg.shuffle(Shuffle::AlternateCB); // (f, h, f, h) // Step 6: Uniform multiply → (e·f, g·h, g·f, e·h) = (X3, Y3, Z3, T3). Self(t0.mul(&t1)) @@ -143,11 +142,11 @@ impl ExtendedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn double(&self) -> Self { + #[cfg(feature = "ed25519")] + pub(crate) fn double(&self) -> Self { // Step 1: Build (X, Y, Z, X+Y) for squaring. - let ab = self.0.shuffle(Shuffle::ABAB); // (X, Y, X, Y) - let ba = ab.shuffle(Shuffle::BADC); // (Y, X, Y, X) + let ab = self.0.shuffle(Shuffle::RepeatAB); // (X, Y, X, Y) + let ba = ab.shuffle(Shuffle::SwapPairs); // (Y, X, Y, X) let xy_sum = ab.add(&ba); // (X+Y, Y+X, X+Y, Y+X) let prepared = self.0.blend(&xy_sum, Lanes::D); // (X, Y, Z, X+Y) @@ -163,8 +162,8 @@ impl ExtendedPointAvx2 { // Double-negations cancel in the final multiply. let zero = FieldElement2625x4::zero(); - let s1 = sq.shuffle(Shuffle::AAAA); // (S1, S1, S1, S1) - let s2 = sq.shuffle(Shuffle::BBBB); // (S2, S2, S2, S2) + let s1 = sq.shuffle(Shuffle::BroadcastA); // (S1, S1, S1, S1) + let s2 = sq.shuffle(Shuffle::BroadcastB); // (S2, S2, S2, S2) // Build the target vector incrementally: let sq_doubled = sq.add(&sq); // (2S1, 2S2, 2S3, −2S4) @@ -186,8 +185,8 @@ impl ExtendedPointAvx2 { let tmp = tmp.add(&neg_s2_in_bc); // (S5, S6, S8, S9) // Step 4: Shuffle into final multiply operands. - let t0 = tmp.shuffle(Shuffle::CACA); // (S8, S5, S8, S5) - let t1 = tmp.shuffle(Shuffle::DBBD); // (S9, S6, S6, S9) + let t0 = tmp.shuffle(Shuffle::AlternateCA); // (S8, S5, S8, S5) + let t1 = tmp.shuffle(Shuffle::OuterDInnerB); // (S9, S6, S6, S9) // Step 5: Uniform multiply → (S8·S9, S5·S6, S8·S6, S5·S9) = (X3, Y3, Z3, T3). Self(t0.mul(&t1)) @@ -206,9 +205,8 @@ impl CachedPointAvx2 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn neg(&self) -> Self { - let swapped = self.0.shuffle(Shuffle::BACD); // swap A↔B, keep C,D + pub(crate) fn neg(&self) -> Self { + let swapped = self.0.shuffle(Shuffle::SwapAB); // swap A↔B, keep C,D let negated = swapped.negate_lazy(); Self(swapped.blend(&negated, Lanes::D)) // negate D only } @@ -225,16 +223,27 @@ fn ct_eq_mask_u8(lhs: u8, rhs: u8) -> u64 { #[inline(always)] #[must_use] fn ct_negative_mask_i8(value: i8) -> u64 { - let bit = ((i16::from(value) >> 15) & 1) as u64; + let bit = u64::from(value.to_ne_bytes()[0] >> 7); 0u64.wrapping_sub(bit) } #[inline(always)] #[must_use] fn ct_abs_i8(value: i8) -> u8 { - let value = i16::from(value); - let sign = value >> 15; - ((value ^ sign) - sign) as u8 + let value = value.to_ne_bytes()[0]; + let sign = value >> 7; + let mask = 0u8.wrapping_sub(sign); + (value ^ mask).wrapping_add(sign) +} + +#[inline] +fn volatile_copy_field(field: &FieldElement) -> FieldElement { + let mut limbs = [0u64; 5]; + for (output, input) in limbs.iter_mut().zip(field.limbs()) { + // SAFETY: `input` is an aligned reference to an initialized `u64` that remains live for this read. + *output = unsafe { core::ptr::read_volatile(input) }; + } + FieldElement::from_limbs(limbs) } /// Fixed-schedule cached-point selection for AVX2 fixed-base tables. @@ -244,8 +253,7 @@ fn ct_abs_i8(value: i8) -> u8 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_cached_avx2(lhs: &CachedPointAvx2, rhs: &CachedPointAvx2, mask: u64) -> CachedPointAvx2 { +fn select_cached_avx2(lhs: &CachedPointAvx2, rhs: &CachedPointAvx2, mask: u64) -> CachedPointAvx2 { CachedPointAvx2(lhs.0.select_mask(&rhs.0, mask)) } @@ -256,8 +264,7 @@ unsafe fn select_cached_avx2(lhs: &CachedPointAvx2, rhs: &CachedPointAvx2, mask: /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_constants() -> FieldElement2625x4 { +fn hamburg_constants() -> FieldElement2625x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); let d1_fe_2 = FieldElement::from_small(D1.wrapping_mul(2)); @@ -274,8 +281,7 @@ unsafe fn hamburg_constants() -> FieldElement2625x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_affine_constants() -> FieldElement2625x4 { +fn hamburg_affine_constants() -> FieldElement2625x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); FieldElement2625x4::new(&d2_fe, &d2_fe, &d2_fe_2, &d2_fe) @@ -298,8 +304,8 @@ unsafe fn hamburg_affine_constants() -> FieldElement2625x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) -> CachedPointAvx2 { +#[cfg(feature = "ed25519")] +fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) -> CachedPointAvx2 { let (y_plus_x, y_minus_x, t2d) = cp.components(); let packed = FieldElement2625x4::new(y_minus_x, y_plus_x, &FieldElement::ONE, t2d); CachedPointAvx2(packed.mul(constants)) @@ -312,8 +318,7 @@ unsafe fn cached_from_affine(cp: &CachedPoint, constants: &FieldElement2625x4) - /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_signed_cached_avx2( +fn select_signed_cached_avx2( table: &[CachedPoint; 8], digit: i8, affine_k: &FieldElement2625x4, @@ -321,10 +326,14 @@ unsafe fn select_signed_cached_avx2( ) -> CachedPointAvx2 { let abs = core::hint::black_box(ct_abs_i8(digit)); let mut selected = *identity; - for (i, candidate) in table.iter().enumerate() { - let candidate = core::ptr::read_volatile(candidate); - let cached = cached_from_affine(&candidate, affine_k); - let mask = core::hint::black_box(ct_eq_mask_u8(abs, (i as u8).wrapping_add(1))); + for (expected, candidate) in (1u8..=8).zip(table) { + let (y_plus_x, y_minus_x, t2d) = candidate.components(); + let y_plus_x = volatile_copy_field(y_plus_x); + let y_minus_x = volatile_copy_field(y_minus_x); + let t2d = volatile_copy_field(t2d); + let packed = FieldElement2625x4::new(&y_minus_x, &y_plus_x, &FieldElement::ONE, &t2d); + let cached = CachedPointAvx2(packed.mul(affine_k)); + let mask = core::hint::black_box(ct_eq_mask_u8(abs, expected)); selected = select_cached_avx2(&selected, &cached, mask); } @@ -339,8 +348,8 @@ unsafe fn select_signed_cached_avx2( /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_cached_avx2( +#[cfg(feature = "ed25519")] +fn add_wnaf_digit_cached_avx2( acc: ExtendedPointAvx2, table: &[CachedPoint; 8], digit: i8, @@ -366,8 +375,8 @@ unsafe fn add_wnaf_digit_cached_avx2( /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_signed_runtime_cached_avx2( +#[cfg(feature = "ed25519")] +fn add_signed_runtime_cached_avx2( acc: ExtendedPointAvx2, table: &[CachedPointAvx2; 8], digit: i8, @@ -390,8 +399,8 @@ unsafe fn add_signed_runtime_cached_avx2( /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; 8] { +#[cfg(feature = "ed25519")] +fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; 8] { let mut acc = *point; let point_cached = point.to_cached(); let first = acc.to_cached(); @@ -415,9 +424,9 @@ unsafe fn cached_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn scalar_mul_vartime_avx2(point: &ExtendedPoint, scalar_bytes: &[u8; 32]) -> ExtendedPoint { - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let avx_point = ExtendedPointAvx2::from_extended(point); let table = cached_multiples_avx2(&avx_point); @@ -441,12 +450,18 @@ pub(crate) unsafe fn scalar_mul_vartime_avx2(point: &ExtendedPoint, scalar_bytes /// # Safety /// /// Caller must ensure AVX2 is available. +#[cfg_attr( + all(target_os = "linux", not(any(test, miri, feature = "portable-only"))), + expect( + dead_code, + reason = "x86_64 Linux library builds route fixed-base multiplication through assembly" + ) +)] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] pub(crate) unsafe fn scalar_mul_basepoint_avx2(scalar_bytes: &[u8; 32]) -> ExtendedPoint { use super::point::BASEPOINT_RADIX16_TABLE; - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let affine_k = hamburg_affine_constants(); let mut acc = ExtendedPointAvx2::from_extended(&ExtendedPoint::identity()); let identity = acc.to_cached(); @@ -467,7 +482,6 @@ pub(crate) unsafe fn scalar_mul_basepoint_avx2(scalar_bytes: &[u8; 32]) -> Exten #[cfg(feature = "diag")] #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; 20] { use super::point::BASEPOINT_RADIX16_TABLE; @@ -476,7 +490,7 @@ pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; let selected = select_signed_cached_avx2(&BASEPOINT_RADIX16_TABLE[0], digit, &affine_k, &identity); let fields = selected.0.split(); let mut out = [0u64; 20]; - for (chunk, field) in out.chunks_exact_mut(5).zip(fields.iter()) { + for (chunk, field) in out.as_chunks_mut::<5>().0.iter_mut().zip(fields.iter()) { chunk.copy_from_slice(field.limbs()); } out @@ -491,8 +505,8 @@ pub unsafe fn diag_select_basepoint_cached_avx2_limb_digest(digit: i8) -> [u64; /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; N] { +#[cfg(feature = "ed25519")] +fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [CachedPointAvx2; N] { let p2 = point.double(); let p2_cached = p2.to_cached(); @@ -513,8 +527,8 @@ unsafe fn odd_multiples_avx2(point: &ExtendedPointAvx2) -> [Cach /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], digit: i8) -> ExtendedPointAvx2 { +#[cfg(feature = "ed25519")] +fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], digit: i8) -> ExtendedPointAvx2 { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(point) = table.get(index) else { return acc; @@ -540,8 +554,7 @@ unsafe fn add_wnaf_digit_avx2(acc: ExtendedPointAvx2, table: &[CachedPointAvx2], /// /// Caller must ensure AVX2 is available. #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -#[allow(clippy::indexing_slicing)] // i bounded by top < 256, naf arrays are [i8; 256] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn straus_wnaf_vartime_avx2(s: &[u8; 32], h: &[u8; 32], a: &ExtendedPoint) -> ExtendedPoint { use super::point::BASEPOINT_WNAF5_TABLE; @@ -608,8 +621,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn from_extended(p: &ExtendedPoint) -> Self { + pub(crate) fn from_extended(p: &ExtendedPoint) -> Self { let (x, y, z, t) = p.components(); Self(FieldElement51x4::new(x, y, z, t)) } @@ -621,8 +633,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_extended(self) -> ExtendedPoint { + pub(crate) fn to_extended(self) -> ExtendedPoint { let [x, y, z, t] = self.0.split(); ExtendedPoint::from_raw(x, y, z, t) } @@ -634,8 +645,7 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn to_cached(self) -> CachedPointIfma { + pub(crate) fn to_cached(self) -> CachedPointIfma { let ds = self.0.diff_sum(); let prepared = self.0.blend(&ds, Lanes::AB); let constants = hamburg_constants_ifma(); @@ -651,16 +661,15 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn add_cached(&self, other: &CachedPointIfma) -> Self { + pub(crate) fn add_cached(&self, other: &CachedPointIfma) -> Self { let ds = self.0.diff_sum(); let tmp = self.0.blend(&ds, Lanes::AB); let product = tmp.reduce().mul(&other.0); - let swapped = product.shuffle(Shuffle::ABDC); + let swapped = product.shuffle(Shuffle::SwapCD); let ehfg = swapped.diff_sum(); let reduced = ehfg.reduce(); - let t0 = reduced.shuffle(Shuffle::ADDA); - let t1 = reduced.shuffle(Shuffle::CBCB); + let t0 = reduced.shuffle(Shuffle::OuterAInnerD); + let t1 = reduced.shuffle(Shuffle::AlternateCB); Self(t0.mul(&t1)) } @@ -674,11 +683,11 @@ impl ExtendedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn double(&self) -> Self { + #[cfg(feature = "ed25519")] + pub(crate) fn double(&self) -> Self { // Prepare (X, Y, Z, X+Y) for squaring. - let tmp0 = self.0.shuffle(Shuffle::BADC); // (Y, X, _, _) - let tmp1 = self.0.add(&tmp0).shuffle(Shuffle::ABAB); // (X+Y, X+Y, X+Y, X+Y) + let tmp0 = self.0.shuffle(Shuffle::SwapPairs); // (Y, X, _, _) + let tmp1 = self.0.add(&tmp0).shuffle(Shuffle::RepeatAB); // (X+Y, X+Y, X+Y, X+Y) let prepared = self.0.blend(&tmp1, Lanes::D); // (X, Y, Z, X+Y) // Square reduced inputs → (S1, S2, S3, S4) = (X², Y², Z², (X+Y)²) @@ -688,8 +697,8 @@ impl ExtendedPointIfma { // Compute (S5, S6, S8, S9) where: // S5 = S1+S2, S6 = S1-S2, S8 = S1-S2+2S3, S9 = S1+S2-S4 let zero = FieldElement51x4::zero(); - let s1 = sq.shuffle(Shuffle::AAAA); - let s2 = sq.shuffle(Shuffle::BBBB); + let s1 = sq.shuffle(Shuffle::BroadcastA); + let s2 = sq.shuffle(Shuffle::BroadcastB); // (-S2, -S2, -S2, -S4): negate S2 in A,B,C and S4 in D. let s2_s2_s2_s4 = s2.blend(&sq, Lanes::D).negate_lazy(); @@ -698,13 +707,13 @@ impl ExtendedPointIfma { // = (S1, S1, S1+2S3, S1) tmp0 = tmp0.add(&zero.blend(&s2, Lanes::AD)); // = (S1+S2, S1, S1+2S3, S1+S2) - tmp0 = tmp0.add(&zero.blend(&s2_s2_s2_s4, Lanes::BCD)); + tmp0 = tmp0.add(&zero.blend(&s2_s2_s2_s4, Lanes::ExceptA)); // = (S1+S2, S1-S2, S1+2S3-S2, S1+S2-S4) = (S5, S6, S8, S9) // Reduce before final multiply. let reduced = tmp0.reduce(); - let t0 = reduced.shuffle(Shuffle::CACA); // (S8, S5, S8, S5) - let t1 = reduced.shuffle(Shuffle::DBBD); // (S9, S6, S6, S9) + let t0 = reduced.shuffle(Shuffle::AlternateCA); // (S8, S5, S8, S5) + let t1 = reduced.shuffle(Shuffle::OuterDInnerB); // (S9, S6, S6, S9) // (S8·S9, S5·S6, S8·S6, S5·S9) = (X3, Y3, Z3, T3) Self(t0.mul(&t1)) @@ -720,9 +729,8 @@ impl CachedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] - #[allow(unsafe_op_in_unsafe_fn)] - pub(crate) unsafe fn neg(&self) -> Self { - let swapped = self.0.shuffle(Shuffle::BACD); + pub(crate) fn neg(&self) -> Self { + let swapped = self.0.shuffle(Shuffle::SwapAB); let negated = swapped.negate_lazy(); Self(swapped.blend(&negated, Lanes::D)) } @@ -735,8 +743,7 @@ impl CachedPointIfma { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_cached_ifma(lhs: &CachedPointIfma, rhs: &CachedPointIfma, mask: u64) -> CachedPointIfma { +fn select_cached_ifma(lhs: &CachedPointIfma, rhs: &CachedPointIfma, mask: u64) -> CachedPointIfma { CachedPointIfma(lhs.0.select_mask(&rhs.0, mask)) } @@ -747,8 +754,7 @@ unsafe fn select_cached_ifma(lhs: &CachedPointIfma, rhs: &CachedPointIfma, mask: /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_constants_ifma() -> FieldElement51x4 { +fn hamburg_constants_ifma() -> FieldElement51x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); let d1_fe_2 = FieldElement::from_small(D1.wrapping_mul(2)); @@ -762,29 +768,12 @@ unsafe fn hamburg_constants_ifma() -> FieldElement51x4 { /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn hamburg_affine_constants_ifma() -> FieldElement51x4 { +fn hamburg_affine_constants_ifma() -> FieldElement51x4 { let d2_fe = FieldElement::from_small(D2); let d2_fe_2 = FieldElement::from_small(D2.wrapping_mul(2)); FieldElement51x4::new(&d2_fe, &d2_fe, &d2_fe_2, &d2_fe) } -/// Convert an affine `CachedPoint` to Hamburg-scaled IFMA cached format. -/// -/// # Safety -/// -/// Caller must ensure AVX-512 IFMA + VL are available. -#[inline] -#[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_from_affine_ifma(cp: &CachedPoint, constants: &FieldElement51x4) -> CachedPointIfma { - let (y_plus_x, y_minus_x, t2d) = cp.components(); - let packed = FieldElement51x4::new(y_minus_x, y_plus_x, &FieldElement::ONE, t2d); - // Affine table entries are already reduced (≤51-bit limbs from static data). - CachedPointIfma(packed.mul_small(constants).reduce()) -} - -/// Add a signed digit from an affine cached table (IFMA). /// Select a signed digit from an affine cached basepoint table. /// /// # Safety @@ -792,8 +781,7 @@ unsafe fn cached_from_affine_ifma(cp: &CachedPoint, constants: &FieldElement51x4 /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn select_signed_cached_ifma( +fn select_signed_cached_ifma( table: &[CachedPoint; 8], digit: i8, affine_k: &FieldElement51x4, @@ -801,10 +789,14 @@ unsafe fn select_signed_cached_ifma( ) -> CachedPointIfma { let abs = core::hint::black_box(ct_abs_i8(digit)); let mut selected = *identity; - for (i, candidate) in table.iter().enumerate() { - let candidate = core::ptr::read_volatile(candidate); - let cached = cached_from_affine_ifma(&candidate, affine_k); - let mask = core::hint::black_box(ct_eq_mask_u8(abs, (i as u8).wrapping_add(1))); + for (expected, candidate) in (1u8..=8).zip(table) { + let (y_plus_x, y_minus_x, t2d) = candidate.components(); + let y_plus_x = volatile_copy_field(y_plus_x); + let y_minus_x = volatile_copy_field(y_minus_x); + let t2d = volatile_copy_field(t2d); + let packed = FieldElement51x4::new(&y_minus_x, &y_plus_x, &FieldElement::ONE, &t2d); + let cached = CachedPointIfma(packed.mul_small(affine_k).reduce()); + let mask = core::hint::black_box(ct_eq_mask_u8(abs, expected)); selected = select_cached_ifma(&selected, &cached, mask); } @@ -819,8 +811,8 @@ unsafe fn select_signed_cached_ifma( /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_signed_runtime_cached_ifma( +#[cfg(feature = "ed25519")] +fn add_signed_runtime_cached_ifma( acc: ExtendedPointIfma, table: &[CachedPointIfma; 8], digit: i8, @@ -842,8 +834,8 @@ unsafe fn add_signed_runtime_cached_ifma( /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; 8] { +#[cfg(feature = "ed25519")] +fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; 8] { let mut acc = *point; let point_cached = point.to_cached(); let first = acc.to_cached(); @@ -862,9 +854,9 @@ unsafe fn cached_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn scalar_mul_vartime_ifma(point: &ExtendedPoint, scalar_bytes: &[u8; 32]) -> ExtendedPoint { - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let ifma_point = ExtendedPointIfma::from_extended(point); let table = cached_multiples_ifma(&ifma_point); let mut acc = ExtendedPointIfma::from_extended(&ExtendedPoint::identity()); @@ -882,12 +874,18 @@ pub(crate) unsafe fn scalar_mul_vartime_ifma(point: &ExtendedPoint, scalar_bytes /// # Safety /// /// Caller must ensure AVX-512 IFMA + VL are available. +#[cfg_attr( + all(target_os = "linux", not(any(test, miri, feature = "portable-only"))), + expect( + dead_code, + reason = "x86_64 Linux library builds route fixed-base multiplication through assembly" + ) +)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] pub(crate) unsafe fn scalar_mul_basepoint_ifma(scalar_bytes: &[u8; 32]) -> ExtendedPoint { use super::point::BASEPOINT_RADIX16_TABLE; - let digits = scalar::as_radix_16(scalar_bytes); + let digits = scalar_radix_16(scalar_bytes); let affine_k = hamburg_affine_constants_ifma(); let mut acc = ExtendedPointIfma::from_extended(&ExtendedPoint::identity()); let identity = acc.to_cached(); @@ -908,7 +906,6 @@ pub(crate) unsafe fn scalar_mul_basepoint_ifma(scalar_bytes: &[u8; 32]) -> Exten #[cfg(feature = "diag")] #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; 20] { use super::point::BASEPOINT_RADIX16_TABLE; @@ -917,7 +914,7 @@ pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; let selected = select_signed_cached_ifma(&BASEPOINT_RADIX16_TABLE[0], digit, &affine_k, &identity); let fields = selected.0.split(); let mut out = [0u64; 20]; - for (chunk, field) in out.chunks_exact_mut(5).zip(fields.iter()) { + for (chunk, field) in out.as_chunks_mut::<5>().0.iter_mut().zip(fields.iter()) { chunk.copy_from_slice(field.limbs()); } out @@ -939,8 +936,8 @@ pub unsafe fn diag_select_basepoint_cached_ifma_limb_digest(digit: i8) -> [u64; /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; N] { +#[cfg(feature = "ed25519")] +fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [CachedPointIfma; N] { let p2 = point.double(); let p2_cached = p2.to_cached(); @@ -964,8 +961,8 @@ unsafe fn odd_multiples_ifma(point: &ExtendedPointIfma) -> [Cach /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], digit: i8) -> ExtendedPointIfma { +#[cfg(feature = "ed25519")] +fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], digit: i8) -> ExtendedPointIfma { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(point) = table.get(index) else { return acc; @@ -985,15 +982,19 @@ unsafe fn add_wnaf_digit_ifma(acc: ExtendedPointIfma, table: &[CachedPointIfma], /// Caller must ensure AVX2 is available. #[inline] #[target_feature(enable = "avx2")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { - CachedPointIfma(FieldElement51x4([ - _mm256_loadu_si256(entry[0].as_ptr().cast()), - _mm256_loadu_si256(entry[1].as_ptr().cast()), - _mm256_loadu_si256(entry[2].as_ptr().cast()), - _mm256_loadu_si256(entry[3].as_ptr().cast()), - _mm256_loadu_si256(entry[4].as_ptr().cast()), - ])) +#[cfg(feature = "ed25519")] +fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { + // SAFETY: AVX2 is active in this function, and every inner array provides 32 initialized bytes for an unaligned load. + let limbs = unsafe { + [ + _mm256_loadu_si256(entry[0].as_ptr().cast()), + _mm256_loadu_si256(entry[1].as_ptr().cast()), + _mm256_loadu_si256(entry[2].as_ptr().cast()), + _mm256_loadu_si256(entry[3].as_ptr().cast()), + _mm256_loadu_si256(entry[4].as_ptr().cast()), + ] + }; + CachedPointIfma(FieldElement51x4(limbs)) } /// Add a signed wNAF digit from the static raw basepoint table. @@ -1003,8 +1004,8 @@ unsafe fn load_cached_ifma_raw(entry: &[[i64; 4]; 5]) -> CachedPointIfma { /// Caller must ensure AVX-512 IFMA + VL are available. #[inline] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -unsafe fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5]], digit: i8) -> ExtendedPointIfma { +#[cfg(feature = "ed25519")] +fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5]], digit: i8) -> ExtendedPointIfma { let index = usize::from((digit.unsigned_abs().wrapping_sub(1)) / 2); let Some(entry) = table.get(index) else { return acc; @@ -1033,8 +1034,7 @@ unsafe fn add_wnaf_digit_ifma_raw(acc: ExtendedPointIfma, table: &[[[i64; 4]; 5] /// /// Caller must ensure AVX-512 IFMA + VL are available. #[target_feature(enable = "avx2,avx512ifma,avx512vl")] -#[allow(unsafe_op_in_unsafe_fn)] -#[allow(clippy::indexing_slicing)] // i bounded by top < 256, naf arrays are [i8; 256] +#[cfg(feature = "ed25519")] pub(crate) unsafe fn straus_wnaf_vartime_ifma(s: &[u8; 32], h: &[u8; 32], a: &ExtendedPoint) -> ExtendedPoint { let s_naf = scalar::non_adjacent_form(s, 8); let h_naf = scalar::non_adjacent_form(h, 5); @@ -1085,6 +1085,7 @@ pub(crate) unsafe fn straus_wnaf_vartime_ifma(s: &[u8; 32], h: &[u8; 32], a: &Ex #[cfg(test)] #[cfg(target_arch = "x86_64")] +#[cfg(feature = "ed25519")] mod tests { use super::{ExtendedPoint, *}; @@ -1096,20 +1097,37 @@ mod tests { ExtendedPoint::basepoint() } - fn decode_hex_32(hex: &str) -> [u8; 32] { - let bytes = hex.as_bytes(); + fn decode_hex_32(hex: &str) -> Option<[u8; 32]> { let mut out = [0u8; 32]; - for (dst, chunk) in out.iter_mut().zip(bytes.chunks_exact(2)) { - *dst = hex_val(chunk[0]) << 4 | hex_val(chunk[1]); - } - out + crate::hex::from_hex(hex, &mut out).ok()?; + Some(out) } - fn hex_val(b: u8) -> u8 { - match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => b - b'a' + 10, - _ => panic!("invalid hex"), + #[test] + fn ifma_wnaf8_table_matches_portable_odd_multiples() { + let step = basepoint().double(); + let mut point = basepoint(); + let d2 = FieldElement::from_small(D2); + let d2_twice = FieldElement::from_small(D2.wrapping_mul(2)); + let d1_twice = FieldElement::from_small(D1.wrapping_mul(2)); + + for entry in &basepoint_table_ifma::BASEPOINT_WNAF8_IFMA_RAW { + let (x, y, z, t) = point.components(); + let a = d2.mul(&y.sub(x)).normalize(); + let b = d2.mul(&y.add(x)).normalize(); + let c = d2_twice.mul(z).normalize(); + let d = d1_twice.mul(t).neg().normalize(); + + for ((((raw, &a_limb), &b_limb), &c_limb), &d_limb) in + entry.iter().zip(a.limbs()).zip(b.limbs()).zip(c.limbs()).zip(d.limbs()) + { + assert_eq!( + (*raw).map(|value| u64::from_ne_bytes(value.to_ne_bytes())), + [a_limb, b_limb, c_limb, d_limb] + ); + } + + point = point.add(&step); } } @@ -1303,7 +1321,6 @@ mod tests { } } - #[cfg(feature = "ed25519")] #[test] fn scalar_mul_basepoint_rfc8032_vector1() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -1311,9 +1328,9 @@ mod tests { } use crate::auth::ed25519::{Ed25519SecretKey, hash::ExpandedSecret}; - let secret = Ed25519SecretKey::from_bytes(decode_hex_32( - "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", - )); + let secret_bytes = decode_hex_32("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"); + assert!(secret_bytes.is_some()); + let secret = Ed25519SecretKey::from_bytes(secret_bytes.unwrap_or_default()); let expanded = ExpandedSecret::from_secret_key(&secret); let expected = decode_hex_32("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a"); @@ -1322,13 +1339,12 @@ mod tests { let avx_pub = scalar_mul_basepoint_avx2(expanded.scalar_bytes()); assert_eq!( avx_pub.to_bytes(), - Some(expected), + expected, "AVX2 basepoint mul should match RFC 8032 vector 1" ); } } - #[cfg(feature = "ed25519")] #[test] fn straus_matches_scalar() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -1354,7 +1370,6 @@ mod tests { } } - #[cfg(feature = "ed25519")] #[test] fn straus_matches_scalar_large_scalars() { if !std::arch::is_x86_feature_detected!("avx2") { @@ -1379,9 +1394,15 @@ mod tests { r_bytes.copy_from_slice(&sig_bytes[..32]); s_bytes.copy_from_slice(&sig_bytes[32..]); - let r_point = ExtendedPoint::from_bytes(&r_bytes).unwrap(); - let a_point = ExtendedPoint::from_bytes(public.as_bytes()).unwrap(); - let s_scalar = super::super::scalar::from_canonical_bytes(&s_bytes).unwrap(); + let r_point = ExtendedPoint::from_bytes(&r_bytes); + let a_point = ExtendedPoint::from_bytes(public.as_bytes()); + let s_scalar = super::super::scalar::from_canonical_bytes(&s_bytes); + assert!(r_point.is_some()); + assert!(a_point.is_some()); + assert!(s_scalar.is_some()); + let r_point = r_point.unwrap_or_default(); + let a_point = a_point.unwrap_or_default(); + let s_scalar = s_scalar.unwrap_or_default(); let mut hasher = Sha512::new(); hasher.update(&r_bytes); diff --git a/src/auth/ed25519/scalar.rs b/src/auth/ed25519/scalar.rs index 9724c1d6..b0866ec9 100644 --- a/src/auth/ed25519/scalar.rs +++ b/src/auth/ed25519/scalar.rs @@ -1,5 +1,3 @@ -#![allow(clippy::identity_op, clippy::indexing_slicing)] - //! Internal Ed25519 scalar arithmetic mod the group order `L`. //! //! This is the correctness-first baseline for signing and verification. It @@ -53,7 +51,30 @@ struct Scalar52([u64; 5]); #[inline(always)] #[must_use] fn wide_mul(lhs: u64, rhs: u64) -> u128 { - u128::from(lhs) * u128::from(rhs) + u128::from(lhs).strict_mul(u128::from(rhs)) +} + +#[inline(always)] +#[must_use] +fn split_u128(value: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = value.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) +} + +#[inline(always)] +#[must_use] +fn low_u64(value: u128) -> u64 { + split_u128(value).0 +} + +#[inline(always)] +#[must_use] +fn low_u8(value: u64) -> u8 { + let [low, ..] = value.to_le_bytes(); + low } impl Scalar52 { @@ -148,44 +169,20 @@ impl Scalar52 { Self::add(&hi, &lo) } - #[rustfmt::skip] #[must_use] fn as_bytes(&self) -> [u8; 32] { let limbs = &self.0; - [ - ( limbs[0] >> 0) as u8, - ( limbs[0] >> 8) as u8, - ( limbs[0] >> 16) as u8, - ( limbs[0] >> 24) as u8, - ( limbs[0] >> 32) as u8, - ( limbs[0] >> 40) as u8, - ((limbs[0] >> 48) | (limbs[1] << 4)) as u8, - ( limbs[1] >> 4) as u8, - ( limbs[1] >> 12) as u8, - ( limbs[1] >> 20) as u8, - ( limbs[1] >> 28) as u8, - ( limbs[1] >> 36) as u8, - ( limbs[1] >> 44) as u8, - ( limbs[2] >> 0) as u8, - ( limbs[2] >> 8) as u8, - ( limbs[2] >> 16) as u8, - ( limbs[2] >> 24) as u8, - ( limbs[2] >> 32) as u8, - ( limbs[2] >> 40) as u8, - ((limbs[2] >> 48) | (limbs[3] << 4)) as u8, - ( limbs[3] >> 4) as u8, - ( limbs[3] >> 12) as u8, - ( limbs[3] >> 20) as u8, - ( limbs[3] >> 28) as u8, - ( limbs[3] >> 36) as u8, - ( limbs[3] >> 44) as u8, - ( limbs[4] >> 0) as u8, - ( limbs[4] >> 8) as u8, - ( limbs[4] >> 16) as u8, - ( limbs[4] >> 24) as u8, - ( limbs[4] >> 32) as u8, - ( limbs[4] >> 40) as u8, - ] + let words = [ + limbs[0] | (limbs[1] << 52), + (limbs[1] >> 12) | (limbs[2] << 40), + (limbs[2] >> 24) | (limbs[3] << 28), + (limbs[3] >> 36) | (limbs[4] << 16), + ]; + let mut bytes = [0u8; 32]; + for (chunk, word) in bytes.as_chunks_mut::<8>().0.iter_mut().zip(words) { + chunk.copy_from_slice(&word.to_le_bytes()); + } + bytes } #[must_use] @@ -207,7 +204,7 @@ impl Scalar52 { fn barrier(value: u64) -> u64 { // SAFETY: `value` is a local `u64`; reading it through a volatile pointer // preserves the arithmetic shape without violating aliasing or lifetime rules. - unsafe { core::ptr::read_volatile(&value) } + unsafe { core::ptr::read_volatile(core::ptr::from_ref(&value)) } } let mut out = [0u64; 5]; @@ -236,7 +233,7 @@ impl Scalar52 { fn barrier(value: u64) -> u64 { // SAFETY: `value` is a local `u64`; reading it through a volatile pointer // preserves the arithmetic shape without violating aliasing or lifetime rules. - unsafe { core::ptr::read_volatile(&value) } + unsafe { core::ptr::read_volatile(core::ptr::from_ref(&value)) } } let mut out = [0u64; 5]; @@ -261,18 +258,31 @@ impl Scalar52 { result } - #[rustfmt::skip] #[must_use] fn mul_internal(lhs: &Self, rhs: &Self) -> [u128; 9] { [ wide_mul(lhs.0[0], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[1]) + wide_mul(lhs.0[1], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[2]) + wide_mul(lhs.0[1], rhs.0[1]) + wide_mul(lhs.0[2], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[3]) + wide_mul(lhs.0[1], rhs.0[2]) + wide_mul(lhs.0[2], rhs.0[1]) + wide_mul(lhs.0[3], rhs.0[0]), - wide_mul(lhs.0[0], rhs.0[4]) + wide_mul(lhs.0[1], rhs.0[3]) + wide_mul(lhs.0[2], rhs.0[2]) + wide_mul(lhs.0[3], rhs.0[1]) + wide_mul(lhs.0[4], rhs.0[0]), - wide_mul(lhs.0[1], rhs.0[4]) + wide_mul(lhs.0[2], rhs.0[3]) + wide_mul(lhs.0[3], rhs.0[2]) + wide_mul(lhs.0[4], rhs.0[1]), - wide_mul(lhs.0[2], rhs.0[4]) + wide_mul(lhs.0[3], rhs.0[3]) + wide_mul(lhs.0[4], rhs.0[2]), - wide_mul(lhs.0[3], rhs.0[4]) + wide_mul(lhs.0[4], rhs.0[3]), + wide_mul(lhs.0[0], rhs.0[1]).strict_add(wide_mul(lhs.0[1], rhs.0[0])), + wide_mul(lhs.0[0], rhs.0[2]) + .strict_add(wide_mul(lhs.0[1], rhs.0[1])) + .strict_add(wide_mul(lhs.0[2], rhs.0[0])), + wide_mul(lhs.0[0], rhs.0[3]) + .strict_add(wide_mul(lhs.0[1], rhs.0[2])) + .strict_add(wide_mul(lhs.0[2], rhs.0[1])) + .strict_add(wide_mul(lhs.0[3], rhs.0[0])), + wide_mul(lhs.0[0], rhs.0[4]) + .strict_add(wide_mul(lhs.0[1], rhs.0[3])) + .strict_add(wide_mul(lhs.0[2], rhs.0[2])) + .strict_add(wide_mul(lhs.0[3], rhs.0[1])) + .strict_add(wide_mul(lhs.0[4], rhs.0[0])), + wide_mul(lhs.0[1], rhs.0[4]) + .strict_add(wide_mul(lhs.0[2], rhs.0[3])) + .strict_add(wide_mul(lhs.0[3], rhs.0[2])) + .strict_add(wide_mul(lhs.0[4], rhs.0[1])), + wide_mul(lhs.0[2], rhs.0[4]) + .strict_add(wide_mul(lhs.0[3], rhs.0[3])) + .strict_add(wide_mul(lhs.0[4], rhs.0[2])), + wide_mul(lhs.0[3], rhs.0[4]).strict_add(wide_mul(lhs.0[4], rhs.0[3])), wide_mul(lhs.0[4], rhs.0[4]), ] } @@ -298,13 +308,13 @@ impl Scalar52 { fn montgomery_reduce_unreduced(limbs: &[u128; 9]) -> Self { #[inline(always)] fn part1(sum: u128) -> (u128, u64) { - let p = (sum as u64).wrapping_mul(LFACTOR52) & RADIX52_MASK; + let p = low_u64(sum).wrapping_mul(LFACTOR52) & RADIX52_MASK; ((sum.strict_add(wide_mul(p, ORDER52.0[0]))) >> 52, p) } #[inline(always)] fn part2(sum: u128) -> (u128, u64) { - let word = (sum as u64) & RADIX52_MASK; + let word = low_u64(sum) & RADIX52_MASK; (sum >> 52, word) } @@ -345,7 +355,7 @@ impl Scalar52 { ); let (carry, r2) = part2(carry.strict_add(limbs[7]).strict_add(wide_mul(n3, ORDER52.0[4]))); let (carry, r3) = part2(carry.strict_add(limbs[8]).strict_add(wide_mul(n4, ORDER52.0[4]))); - let r4 = carry as u64; + let r4 = low_u64(carry); Self([r0, r1, r2, r3, r4]) } @@ -413,7 +423,7 @@ pub(crate) fn clamp_secret_scalar(bytes: &mut [u8; SECRET_KEY_LENGTH]) { #[must_use] pub(crate) fn decode_words_le(bytes: &[u8; SECRET_KEY_LENGTH]) -> Scalar { let mut limbs = [0u64; SCALAR_LIMBS]; - for (limb, chunk) in limbs.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (limb, chunk) in limbs.iter_mut().zip(bytes.as_chunks::<8>().0) { *limb = read_u64_le(chunk); } limbs @@ -434,15 +444,8 @@ pub(crate) fn from_canonical_bytes(bytes: &[u8; SECRET_KEY_LENGTH]) -> Option [u8; SECRET_KEY_LENGTH] { let mut out = [0u8; SECRET_KEY_LENGTH]; - for (chunk, limb) in out.as_mut_slice().chunks_exact_mut(8).zip(words.iter().copied()) { - chunk[0] = limb as u8; - chunk[1] = (limb >> 8) as u8; - chunk[2] = (limb >> 16) as u8; - chunk[3] = (limb >> 24) as u8; - chunk[4] = (limb >> 32) as u8; - chunk[5] = (limb >> 40) as u8; - chunk[6] = (limb >> 48) as u8; - chunk[7] = (limb >> 56) as u8; + for (chunk, limb) in out.as_chunks_mut::<8>().0.iter_mut().zip(words.iter().copied()) { + chunk.copy_from_slice(&limb.to_le_bytes()); } out } @@ -521,28 +524,6 @@ pub(crate) fn negate_mod(s: &Scalar) -> Scalar { } } -/// Decompose a scalar encoding into signed radix-16 digits in `[-8, 8]`. -#[must_use] -#[allow(clippy::indexing_slicing)] -pub(crate) fn as_radix_16(bytes: &[u8; SECRET_KEY_LENGTH]) -> [i8; 64] { - debug_assert!(bytes[31] <= 127); - - let mut digits = [0i8; 64]; - - for (i, byte) in bytes.iter().copied().enumerate() { - digits[2 * i] = (byte & 0x0F) as i8; - digits[2 * i + 1] = ((byte >> 4) & 0x0F) as i8; - } - - for i in 0..63 { - let carry = (digits[i] + 8) >> 4; - digits[i] -= carry << 4; - digits[i + 1] += carry; - } - - digits -} - /// Decompose a scalar into width-`w` non-adjacent form (wNAF). /// /// Returns a 256-element array of signed digits. Non-zero digits are odd @@ -556,7 +537,6 @@ pub(crate) fn as_radix_16(bytes: &[u8; SECRET_KEY_LENGTH]) -> [i8; 64] { /// /// Debug-panics if `w < 2` or `w > 8`. #[must_use] -#[allow(clippy::indexing_slicing)] pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { debug_assert!((2..=8).contains(&w)); @@ -565,7 +545,7 @@ pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { // Load scalar into mutable u64 words. The 5th word absorbs carry from // the 256th bit position. let mut x = [0u64; 5]; - for (dst, chunk) in x.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (dst, chunk) in x.iter_mut().zip(bytes.as_chunks::<8>().0) { *dst = read_u64_le(chunk); } @@ -597,10 +577,10 @@ pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { // Odd window: emit a signed digit. if window < width / 2 { - naf[pos] = window as i8; + naf[pos] = i8::from_ne_bytes([low_u8(window)]); carry = 0; } else { - naf[pos] = (window as i8).wrapping_sub(width as i8); + naf[pos] = i8::from_ne_bytes([low_u8(window).wrapping_sub(low_u8(width))]); carry = 1; } @@ -613,22 +593,15 @@ pub(crate) fn non_adjacent_form(bytes: &[u8; 32], w: usize) -> [i8; 256] { #[inline] #[must_use] -fn read_u64_le(chunk: &[u8]) -> u64 { - u64::from(chunk[0]) - | (u64::from(chunk[1]) << 8) - | (u64::from(chunk[2]) << 16) - | (u64::from(chunk[3]) << 24) - | (u64::from(chunk[4]) << 32) - | (u64::from(chunk[5]) << 40) - | (u64::from(chunk[6]) << 48) - | (u64::from(chunk[7]) << 56) +fn read_u64_le(chunk: &[u8; 8]) -> u64 { + u64::from_le_bytes(*chunk) } #[inline] #[must_use] fn read_words_le_32(bytes: &[u8; 32]) -> [u64; 4] { let mut out = [0u64; 4]; - for (dst, chunk) in out.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (dst, chunk) in out.iter_mut().zip(bytes.as_chunks::<8>().0) { *dst = read_u64_le(chunk); } out @@ -638,7 +611,7 @@ fn read_words_le_32(bytes: &[u8; 32]) -> [u64; 4] { #[must_use] fn read_words_le_64(bytes: &[u8; 64]) -> [u64; 8] { let mut out = [0u64; 8]; - for (dst, chunk) in out.iter_mut().zip(bytes.as_slice().chunks_exact(8)) { + for (dst, chunk) in out.iter_mut().zip(bytes.as_chunks::<8>().0) { *dst = read_u64_le(chunk); } out @@ -710,15 +683,16 @@ fn compare(lhs: &Scalar, rhs: &Scalar) -> Ordering { #[must_use] fn add_raw(lhs: &Scalar, rhs: &Scalar) -> (Scalar, u64) { let mut out = ZERO; - let mut carry = 0u128; + let mut carry = 0u64; for (dst, (&left, &right)) in out.iter_mut().zip(lhs.iter().zip(rhs.iter())) { - let sum = u128::from(left).strict_add(u128::from(right)).strict_add(carry); - *dst = sum as u64; - carry = sum >> 64; + let sum = u128::from(left) + .strict_add(u128::from(right)) + .strict_add(u128::from(carry)); + (*dst, carry) = split_u128(sum); } - (out, carry as u64) + (out, carry) } #[inline] @@ -751,8 +725,8 @@ fn maybe_sub_order(words: Scalar) -> Scalar { #[cfg(test)] mod tests { use super::{ - ORDER, Scalar, add_mod, as_radix_16, clamp_secret_scalar, decode_words_le, from_canonical_bytes, mul_add_mod, - mul_add_mod_secret, non_adjacent_form, reduce_64_bytes_mod_order_secret, reduce_bytes_mod_order, + ORDER, Scalar, add_mod, clamp_secret_scalar, decode_words_le, from_canonical_bytes, mul_add_mod, + mul_add_mod_secret, non_adjacent_form, read_u64_le, reduce_64_bytes_mod_order_secret, reduce_bytes_mod_order, reduce_bytes_mod_order_fallback, to_bytes, }; @@ -760,6 +734,14 @@ mod tests { [value, 0, 0, 0] } + fn patterned_bytes(mut value: u8, step: u8) -> [u8; 64] { + core::array::from_fn(|_| { + let current = value; + value = value.wrapping_add(step); + current + }) + } + #[test] fn clamp_secret_scalar_matches_rfc_8032_bit_rules() { let mut bytes = [0xFFu8; 32]; @@ -816,14 +798,14 @@ mod tests { #[test] fn wide_reduction_matches_fallback_for_fixed_input() { - let bytes = core::array::from_fn::<_, 64, _>(|i| (i as u8).wrapping_mul(17).wrapping_add(9)); + let bytes = patterned_bytes(9, 17); assert_eq!(reduce_bytes_mod_order(&bytes), reduce_bytes_mod_order_fallback(&bytes)); } #[test] fn secret_wide_reduction_matches_public_reduction() { - let bytes = core::array::from_fn::<_, 64, _>(|i| (i as u8).wrapping_mul(29).wrapping_add(13)); + let bytes = patterned_bytes(13, 29); assert_eq!(reduce_64_bytes_mod_order_secret(&bytes), reduce_bytes_mod_order(&bytes)); } @@ -832,7 +814,7 @@ mod tests { fn radix16_recenters_nibbles_into_signed_digits() { let mut bytes = [0u8; 32]; bytes[0] = 0x19; - let digits = as_radix_16(&bytes); + let digits = super::super::scalar_radix_16(&bytes); assert_eq!(digits[0], -7); assert_eq!(digits[1], 2); @@ -843,7 +825,7 @@ mod tests { #[test] fn modular_addition_wraps_order_boundary() { - let near_order = [ORDER[0] - 1, ORDER[1], ORDER[2], ORDER[3]]; + let near_order = [ORDER[0].strict_sub(1), ORDER[1], ORDER[2], ORDER[3]]; let wrapped = add_mod(&near_order, &from_u64(1)); assert_eq!(wrapped, [0, 0, 0, 0]); @@ -907,7 +889,7 @@ mod tests { let word = i / 64; let bit = i % 64; if digit > 0 { - let d = digit as u64; + let d = u64::from(digit.unsigned_abs()); let (new_val, carry) = reconstructed[word].overflowing_add(d << bit); reconstructed[word] = new_val; if carry { @@ -920,11 +902,12 @@ mod tests { } } // Handle cross-word shift - if bit > 0 && word + 1 < 5 { - reconstructed[word + 1] = reconstructed[word + 1].wrapping_add(d >> (64 - bit)); + let next_word = word.strict_add(1); + if bit > 0 && next_word < 5 { + reconstructed[next_word] = reconstructed[next_word].wrapping_add(d >> 64usize.strict_sub(bit)); } } else { - let d = (-digit) as u64; + let d = u64::from(digit.unsigned_abs()); let (new_val, borrow) = reconstructed[word].overflowing_sub(d << bit); reconstructed[word] = new_val; if borrow { @@ -936,16 +919,17 @@ mod tests { } } } - if bit > 0 && word + 1 < 5 { - reconstructed[word + 1] = reconstructed[word + 1].wrapping_sub(d >> (64 - bit)); + let next_word = word.strict_add(1); + if bit > 0 && next_word < 5 { + reconstructed[next_word] = reconstructed[next_word].wrapping_sub(d >> 64usize.strict_sub(bit)); } } } // Compare lower 4 words (256 bits) let mut expected = [0u64; 4]; - for (dst, chunk) in expected.iter_mut().zip(scalar_bytes.chunks_exact(8)) { - *dst = u64::from_le_bytes(chunk.try_into().unwrap()); + for (dst, chunk) in expected.iter_mut().zip(scalar_bytes.as_chunks::<8>().0) { + *dst = read_u64_le(chunk); } assert_eq!(&reconstructed[..4], &expected, "wNAF({w}) reconstruction failed"); } @@ -962,7 +946,7 @@ mod tests { for w in [5, 8] { let naf = non_adjacent_form(&s, w); - let max_abs = (1i16 << (w - 1)) - 1; // 2^(w-1) - 1 + let max_abs = (1i16 << w.strict_sub(1)).strict_sub(1); // 2^(w-1) - 1 for (i, &digit) in naf.iter().enumerate() { if digit != 0 { assert!(digit.abs() % 2 == 1, "wNAF({w}) digit[{i}] = {digit} is even"); diff --git a/src/auth/ed25519/x86_64_asm.rs b/src/auth/ed25519/x86_64_asm.rs index ed77a964..c0ac29d1 100644 --- a/src/auth/ed25519/x86_64_asm.rs +++ b/src/auth/ed25519/x86_64_asm.rs @@ -4,8 +4,6 @@ //! fixed-base multiplication backend. This module owns the ABI boundary; //! `ed25519.rs` owns signing semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::{ diff --git a/src/auth/hkdf.rs b/src/auth/hkdf.rs index 7c2c29be..f8a6864e 100644 --- a/src/auth/hkdf.rs +++ b/src/auth/hkdf.rs @@ -25,7 +25,6 @@ const SHA512_MAX_OUTPUT_SIZE: usize = 255 * SHA512_OUTPUT_SIZE; const SHA512_BLOCK_SIZE: usize = 128; #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u32x8_be(dst: &mut [u8; SHA256_OUTPUT_SIZE], words: &[u32; 8]) { dst[0..4].copy_from_slice(&words[0].to_be_bytes()); dst[4..8].copy_from_slice(&words[1].to_be_bytes()); @@ -38,7 +37,6 @@ fn write_u32x8_be(dst: &mut [u8; SHA256_OUTPUT_SIZE], words: &[u32; 8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u64x6_be(dst: &mut [u8], words: &[u64; 8]) { debug_assert!(dst.len() >= SHA384_OUTPUT_SIZE); dst[0..8].copy_from_slice(&words[0].to_be_bytes()); @@ -50,7 +48,6 @@ fn write_u64x6_be(dst: &mut [u8], words: &[u64; 8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u64x8_be(dst: &mut [u8], words: &[u64; 8]) { debug_assert!(dst.len() >= SHA512_OUTPUT_SIZE); dst[0..8].copy_from_slice(&words[0].to_be_bytes()); @@ -173,7 +170,6 @@ impl HkdfSha256 { /// bypassing all `Sha256` struct creation, `Drop` zeroization, and dispatch /// overhead in the inner loop. #[inline] - #[allow(clippy::indexing_slicing)] pub fn expand(&self, info: &[u8], okm: &mut [u8]) -> Result<(), HkdfOutputLengthError> { if okm.len() > SHA256_MAX_OUTPUT_SIZE { return Err(HkdfOutputLengthError::new()); @@ -326,6 +322,7 @@ impl HkdfSha256 { #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Derive a portable HKDF-SHA256 diagnostic output using fixed `b"salt"` and `b"info"` inputs. pub fn diag_hkdf_sha256_derive_portable(input_key_material: &[u8; SHA256_OUTPUT_SIZE]) -> [u8; SHA256_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha256::kernels::compress_blocks_fn( crate::hashes::crypto::sha256::kernels::Sha256KernelId::Portable, @@ -428,7 +425,6 @@ impl HkdfSha384 { /// bypassing all `Sha384` struct creation, `Drop` zeroization, and dispatch /// overhead in the inner loop. #[inline] - #[allow(clippy::indexing_slicing)] pub fn expand(&self, info: &[u8], okm: &mut [u8]) -> Result<(), HkdfOutputLengthError> { if okm.len() > SHA384_MAX_OUTPUT_SIZE { return Err(HkdfOutputLengthError::new()); @@ -565,6 +561,7 @@ impl HkdfSha384 { #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Derive a portable HKDF-SHA384 diagnostic output using fixed `b"salt"` and `b"info"` inputs. pub fn diag_hkdf_sha384_derive_portable(input_key_material: &[u8; SHA384_OUTPUT_SIZE]) -> [u8; SHA384_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha384::kernels::compress_blocks_fn( crate::hashes::crypto::sha384::kernels::Sha384KernelId::Portable, @@ -648,7 +645,6 @@ impl HkdfSha512 { /// Expand the stored pseudorandom key into `okm`. #[inline] - #[allow(clippy::indexing_slicing)] pub fn expand(&self, info: &[u8], okm: &mut [u8]) -> Result<(), HkdfOutputLengthError> { if okm.len() > SHA512_MAX_OUTPUT_SIZE { return Err(HkdfOutputLengthError::new()); @@ -785,6 +781,7 @@ impl HkdfSha512 { #[cfg(feature = "diag")] #[unsafe(no_mangle)] #[inline(never)] +/// Derive a portable HKDF-SHA512 diagnostic output using fixed `b"salt"` and `b"info"` inputs. pub fn diag_hkdf_sha512_derive_portable(input_key_material: &[u8; SHA512_OUTPUT_SIZE]) -> [u8; SHA512_OUTPUT_SIZE] { let compress = crate::hashes::crypto::sha512::kernels::compress_blocks_fn( crate::hashes::crypto::sha512::kernels::Sha512KernelId::Portable, @@ -806,7 +803,6 @@ impl Drop for HkdfSha512 { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha256_inner( compress: Sha256CompressBlocksFn, prev: Option<&[u8; SHA256_OUTPUT_SIZE]>, @@ -863,7 +859,6 @@ fn expand_hmac_sha256_inner( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha256_outer( compress: Sha256CompressBlocksFn, outer_init: &[u32; 8], @@ -880,7 +875,6 @@ fn expand_hmac_sha256_outer( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha384_inner( compress: Sha384CompressBlocksFn, prev: Option<&[u8; SHA384_OUTPUT_SIZE]>, @@ -937,7 +931,6 @@ fn expand_hmac_sha384_inner( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha384_outer( compress: Sha384CompressBlocksFn, outer_init: &[u64; 8], @@ -952,7 +945,6 @@ fn expand_hmac_sha384_outer( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha512_inner( compress: Sha512CompressBlocksFn, prev: Option<&[u8; SHA512_OUTPUT_SIZE]>, @@ -1009,7 +1001,6 @@ fn expand_hmac_sha512_inner( } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn expand_hmac_sha512_outer( compress: Sha512CompressBlocksFn, outer_init: &[u64; 8], @@ -1052,9 +1043,9 @@ mod tests { fn pattern(len: usize, mul: u8, add: u8) -> Vec { (0..len) .map(|i| { - (i as u8) + i.to_le_bytes()[0] .wrapping_mul(mul) - .wrapping_add(((i >> 2) as u8).wrapping_add(add)) + .wrapping_add((i >> 2).to_le_bytes()[0].wrapping_add(add)) }) .collect() } @@ -1079,13 +1070,15 @@ mod tests { let mut expected = vec![0u8; out_len]; RustCryptoHkdfSha256::new(Some(&salt), &ikm) .expand(&info, &mut expected) - .unwrap(); + .expect("test output length should be within the HKDF-SHA256 limit"); let public = HkdfSha256::new(&salt, &ikm); let forced = HkdfSha256::extract_with_compress_for_test(&salt, &ikm, compress); let mut public_out = vec![0u8; out_len]; - public.expand(&info, &mut public_out).unwrap(); + public + .expand(&info, &mut public_out) + .expect("test output length should be within the HKDF-SHA256 limit"); assert_eq!( public_out, expected, @@ -1098,7 +1091,9 @@ mod tests { ); let mut forced_out = vec![0u8; out_len]; - forced.expand(&info, &mut forced_out).unwrap(); + forced + .expand(&info, &mut forced_out) + .expect("test output length should be within the HKDF-SHA256 limit"); assert_eq!( forced_out, expected, @@ -1137,13 +1132,15 @@ mod tests { let mut expected = vec![0u8; out_len]; RustCryptoHkdfSha384::new(Some(&salt), &ikm) .expand(&info, &mut expected) - .unwrap(); + .expect("test output length should be within the HKDF-SHA384 limit"); let public = HkdfSha384::new(&salt, &ikm); let forced = HkdfSha384::extract_with_compress_for_test(&salt, &ikm, compress); let mut public_out = vec![0u8; out_len]; - public.expand(&info, &mut public_out).unwrap(); + public + .expand(&info, &mut public_out) + .expect("test output length should be within the HKDF-SHA384 limit"); assert_eq!( public_out, expected, @@ -1156,7 +1153,9 @@ mod tests { ); let mut forced_out = vec![0u8; out_len]; - forced.expand(&info, &mut forced_out).unwrap(); + forced + .expand(&info, &mut forced_out) + .expect("test output length should be within the HKDF-SHA384 limit"); assert_eq!( forced_out, expected, @@ -1195,13 +1194,15 @@ mod tests { let mut expected = vec![0u8; out_len]; RustCryptoHkdfSha512::new(Some(&salt), &ikm) .expand(&info, &mut expected) - .unwrap(); + .expect("test output length should be within the HKDF-SHA512 limit"); let public = HkdfSha512::new(&salt, &ikm); let forced = HkdfSha512::extract_with_compress_for_test(&salt, &ikm, compress); let mut public_out = vec![0u8; out_len]; - public.expand(&info, &mut public_out).unwrap(); + public + .expand(&info, &mut public_out) + .expect("test output length should be within the HKDF-SHA512 limit"); assert_eq!( public_out, expected, @@ -1214,7 +1215,9 @@ mod tests { ); let mut forced_out = vec![0u8; out_len]; - forced.expand(&info, &mut forced_out).unwrap(); + forced + .expand(&info, &mut forced_out) + .expect("test output length should be within the HKDF-SHA512 limit"); assert_eq!( forced_out, expected, diff --git a/src/auth/hmac.rs b/src/auth/hmac.rs index 2458d623..ef4a2b58 100644 --- a/src/auth/hmac.rs +++ b/src/auth/hmac.rs @@ -392,8 +392,7 @@ impl HmacSha256 { } } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(any(feature = "diag", all(test, feature = "hkdf")))] pub(crate) fn new_with_compress_for_test( key: &[u8], compress: crate::hashes::crypto::sha256::kernels::CompressBlocksFn, @@ -427,8 +426,7 @@ impl HmacSha256 { } } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(any(feature = "diag", all(test, feature = "hkdf")))] pub(crate) fn mac_with_compress_for_test( key: &[u8], data: &[u8], @@ -441,6 +439,7 @@ impl HmacSha256 { } #[cfg(feature = "diag")] +/// Compare the portable HMAC-SHA256 tag for the diagnostic message `b"binsec"` with `expected`. pub fn diag_hmac_sha256_verify_portable( key: &[u8; SHA256_TAG_SIZE], expected: &[u8; SHA256_TAG_SIZE], @@ -453,6 +452,7 @@ pub fn diag_hmac_sha256_verify_portable( } #[cfg(feature = "diag")] +/// Compare the leading 64 bits of the portable HMAC-SHA256 diagnostic tag with `expected`. pub fn diag_hmac_sha256_verify_truncated_64_portable( key: &[u8; SHA256_TAG_SIZE], expected: &[u8; 8], @@ -545,7 +545,6 @@ impl Mac for HmacSha256 { /// dispatch, state save/restore) that dominates on fast SHA2-CE cores. /// The outer hash is always merged into a single 128-byte (2-block) call. #[inline] - #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { if hmac_sha256_oneshot_should_stream() { // The Sapphire Rapids policy uses the public streaming path instead of @@ -598,7 +597,12 @@ impl Mac for HmacSha256 { 256 => compress_inline_inner!(256), 320 => compress_inline_inner!(320), 384 => compress_inline_inner!(384), - _ => unreachable!("HMAC-SHA256 inline inner padding is bounded to 128..=384 bytes"), + _ => { + ct::zeroize(&mut ipad); + let mut mac = Self::new(key); + mac.update(data); + return mac.finalize(); + } } } else { compress(&mut state, &ipad); @@ -634,7 +638,7 @@ impl Mac for HmacSha256 { compress(&mut state, &outer); let mut tag = [0u8; SHA256_TAG_SIZE]; - for (chunk, &word) in tag.chunks_exact_mut(4).zip(state.iter()) { + for (chunk, &word) in tag.as_chunks_mut::<4>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } @@ -768,6 +772,7 @@ impl HmacSha384 { } #[cfg(feature = "diag")] +/// Compare the portable HMAC-SHA384 tag for the diagnostic message `b"binsec"` with `expected`. pub fn diag_hmac_sha384_verify_portable( key: &[u8; SHA384_TAG_SIZE], expected: &[u8; SHA384_TAG_SIZE], @@ -841,7 +846,6 @@ impl Mac for HmacSha384 { } #[inline] - #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { const INLINE_DATA_MAX: usize = 256; @@ -884,7 +888,12 @@ impl Mac for HmacSha384 { 256 => compress_inline_inner!(256), 384 => compress_inline_inner!(384), 512 => compress_inline_inner!(512), - _ => unreachable!("HMAC-SHA384 inline inner padding is bounded to 256..=512 bytes"), + _ => { + ct::zeroize(&mut ipad); + let mut mac = Self::new(key); + mac.update(data); + return mac.finalize(); + } } } else { compress(&mut state, &ipad); @@ -920,7 +929,7 @@ impl Mac for HmacSha384 { compress(&mut state, &outer); let mut tag = [0u8; SHA384_TAG_SIZE]; - for (chunk, &word) in tag.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in tag.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } @@ -1054,6 +1063,7 @@ impl HmacSha512 { } #[cfg(feature = "diag")] +/// Compare the portable HMAC-SHA512 tag for the diagnostic message `b"binsec"` with `expected`. pub fn diag_hmac_sha512_verify_portable( key: &[u8; SHA512_TAG_SIZE], expected: &[u8; SHA512_TAG_SIZE], @@ -1127,7 +1137,6 @@ impl Mac for HmacSha512 { } #[inline] - #[allow(clippy::indexing_slicing)] // All indices bounded by prior length checks + fixed-size arrays. fn mac(key: &[u8], data: &[u8]) -> Self::Tag { const INLINE_DATA_MAX: usize = 256; @@ -1170,7 +1179,12 @@ impl Mac for HmacSha512 { 256 => compress_inline_inner!(256), 384 => compress_inline_inner!(384), 512 => compress_inline_inner!(512), - _ => unreachable!("HMAC-SHA512 inline inner padding is bounded to 256..=512 bytes"), + _ => { + ct::zeroize(&mut ipad); + let mut mac = Self::new(key); + mac.update(data); + return mac.finalize(); + } } } else { compress(&mut state, &ipad); @@ -1206,7 +1220,7 @@ impl Mac for HmacSha512 { compress(&mut state, &outer); let mut tag = [0u8; SHA512_TAG_SIZE]; - for (chunk, &word) in tag.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in tag.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } @@ -1263,15 +1277,15 @@ mod tests { fn pattern(len: usize, mul: u8, add: u8) -> Vec { (0..len) .map(|i| { - (i as u8) + i.to_le_bytes()[0] .wrapping_mul(mul) - .wrapping_add(((i >> 3) as u8).wrapping_add(add)) + .wrapping_add((i >> 3).to_le_bytes()[0].wrapping_add(add)) }) .collect() } fn oracle_hmac_sha384(key: &[u8], data: &[u8]) -> [u8; SHA384_TAG_SIZE] { - let mut mac = RustCryptoHmacSha384::new_from_slice(key).unwrap(); + let mut mac = RustCryptoHmacSha384::new_from_slice(key).expect("HMAC accepts keys of any length"); mac.update(data); let bytes = mac.finalize().into_bytes(); let mut tag = [0u8; SHA384_TAG_SIZE]; @@ -1280,7 +1294,7 @@ mod tests { } fn oracle_hmac_sha512(key: &[u8], data: &[u8]) -> [u8; SHA512_TAG_SIZE] { - let mut mac = RustCryptoHmacSha512::new_from_slice(key).unwrap(); + let mut mac = RustCryptoHmacSha512::new_from_slice(key).expect("HMAC accepts keys of any length"); mac.update(data); let bytes = mac.finalize().into_bytes(); let mut tag = [0u8; SHA512_TAG_SIZE]; diff --git a/src/auth/hmac_sha3.rs b/src/auth/hmac_sha3.rs index a88bd445..d2d1637d 100644 --- a/src/auth/hmac_sha3.rs +++ b/src/auth/hmac_sha3.rs @@ -1,7 +1,5 @@ //! HMAC-SHA3 family (RFC 2104, FIPS 198-1, FIPS 202). -#![allow(clippy::indexing_slicing)] // HMAC pads are fixed-size arrays bounded by rate constants. - use crate::{ hashes::crypto::keccak::KeccakCore, traits::{Mac, VerificationError, ct}, diff --git a/src/auth/kmac.rs b/src/auth/kmac.rs index c2ea51ad..b6639675 100644 --- a/src/auth/kmac.rs +++ b/src/auth/kmac.rs @@ -1,7 +1,5 @@ //! KMAC128 and KMAC256 (SP 800-185). -#![allow(clippy::indexing_slicing)] // Fixed-size scratch buffers and encoded suffix slices. - use core::fmt; use crate::{ @@ -190,6 +188,7 @@ define_kmac!(Kmac256, Cshake256, "256", 32); #[cfg(test)] mod tests { use super::{Kmac128, Kmac256}; + use crate::traits::VerificationError; #[test] fn reset_restores_keyed_state() { @@ -226,11 +225,16 @@ mod tests { ] { let mut tag = [0u8; Kmac128::MIN_AUTH_TAG_SIZE + 1]; Kmac128::mac_into(KEY, CUSTOMIZATION, MESSAGE, &mut tag[..len]); + let expected = if len >= Kmac128::MIN_AUTH_TAG_SIZE { + Ok(()) + } else { + Err(VerificationError::new()) + }; + assert_eq!(Kmac128::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), expected); assert_eq!( - Kmac128::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok(), - len >= Kmac128::MIN_AUTH_TAG_SIZE + Kmac128::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), + Ok(()) ); - assert!(Kmac128::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok()); } for len in [ @@ -240,11 +244,16 @@ mod tests { ] { let mut tag = [0u8; Kmac256::MIN_AUTH_TAG_SIZE + 1]; Kmac256::mac_into(KEY, CUSTOMIZATION, MESSAGE, &mut tag[..len]); + let expected = if len >= Kmac256::MIN_AUTH_TAG_SIZE { + Ok(()) + } else { + Err(VerificationError::new()) + }; + assert_eq!(Kmac256::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), expected); assert_eq!( - Kmac256::verify_tag(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok(), - len >= Kmac256::MIN_AUTH_TAG_SIZE + Kmac256::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]), + Ok(()) ); - assert!(Kmac256::verify_tag_primitive(KEY, CUSTOMIZATION, MESSAGE, &tag[..len]).is_ok()); } } } diff --git a/src/auth/mlkem.rs b/src/auth/mlkem.rs index 1f3ebc44..763bbecf 100644 --- a/src/auth/mlkem.rs +++ b/src/auth/mlkem.rs @@ -815,7 +815,7 @@ macro_rules! impl_mlkem_profile_ops { >( decapsulation_key.as_bytes(), ciphertext.as_bytes(), - ))) + )?)) } } }; diff --git a/src/auth/mlkem/portable.rs b/src/auth/mlkem/portable.rs index 50865676..76636042 100644 --- a/src/auth/mlkem/portable.rs +++ b/src/auth/mlkem/portable.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size FIPS buffers and public loop indices bound every access. - #[cfg(all( target_arch = "aarch64", any(target_os = "macos", target_os = "linux"), @@ -12,13 +10,6 @@ mod s390x; #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] mod x86_64; -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -use core::arch::aarch64::vgetq_lane_u16; #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] use core::arch::aarch64::{ int16x4_t, int16x8_t, int32x4_t, uint16x4_t, uint16x8_t, uint16x8x2_t, vaddq_s16, vaddq_s32, vaddq_u16, vaddq_u32, @@ -35,21 +26,13 @@ use core::arch::aarch64::{ vdup_n_u16, vget_low_u16, vreinterpret_s16_u16, vreinterpret_u16_s16, vreinterpret_u32_u16, vreinterpretq_u16_u32, vreinterpretq_u32_u16, vset_lane_s16, vshr_n_s16, vsub_u16, vuzp1q_u32, vuzp2q_u32, vzip1_u32, vzip2_u32, }; -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -use core::arch::aarch64::{vget_high_u8, vget_low_u8, vld3q_u8, vmovl_u8, vorrq_u16, vshlq_n_u16, vshrq_n_u16}; #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] use core::arch::x86_64::{ - __m128i, __m256i, _mm_add_epi16, _mm_and_si128, _mm_cmpgt_epi16, _mm_loadl_epi64, _mm_loadu_si128, _mm_mulhi_epi16, - _mm_mullo_epi16, _mm_set1_epi16, _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, - _mm_storel_epi64, _mm_storeu_si128, _mm_sub_epi16, _mm_unpacklo_epi16, _mm256_add_epi32, _mm256_and_si256, - _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_loadu_si256, _mm256_mulhi_epi16, _mm256_mullo_epi16, - _mm256_mullo_epi32, _mm256_or_si256, _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, _mm256_srai_epi32, - _mm256_srli_epi32, _mm256_storeu_si256, _mm256_sub_epi16, _mm256_sub_epi32, + __m128i, __m256i, _mm_add_epi16, _mm_and_si128, _mm_cmpgt_epi16, _mm_mulhi_epi16, _mm_mullo_epi16, _mm_set1_epi16, + _mm_setr_epi8, _mm_setzero_si128, _mm_shuffle_epi8, _mm_srli_epi16, _mm_sub_epi16, _mm_unpacklo_epi16, + _mm256_add_epi32, _mm256_and_si256, _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_mulhi_epi16, + _mm256_mullo_epi16, _mm256_mullo_epi32, _mm256_or_si256, _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, + _mm256_srai_epi32, _mm256_srli_epi32, _mm256_sub_epi16, _mm256_sub_epi32, }; #[cfg(test)] @@ -73,7 +56,7 @@ const Q_U32: u32 = Q as u32; all(target_arch = "aarch64", not(miri), not(feature = "portable-only")), all(target_arch = "x86_64", not(miri), not(feature = "portable-only")) ))] -const Q_I16: i16 = Q as i16; +const Q_I16: i16 = Q.cast_signed(); #[cfg(any( test, miri, @@ -90,7 +73,7 @@ const Q_DIV_RECIP: u64 = 20_642_679; #[cfg(any(test, not(target_arch = "s390x")))] const Q_COMPRESS_DIV_SHIFT: i32 = 33; #[cfg(any(test, not(target_arch = "s390x")))] -const Q_COMPRESS_DIV_RECIP: u64 = 2_580_335; +const Q_COMPRESS_DIV_RECIP: u32 = 2_580_335; const Q_MONT_INV_U16: u16 = 62_209; const MONT_R_SQUARED_MOD_Q: i16 = 1353; #[cfg(test)] @@ -113,6 +96,42 @@ const SAMPLE_NTT_ACC_CHUNK_COEFFS: usize = 16; type Poly = [u16; N]; type PolyVec = [Poly; K]; type PolyMatrix = [PolyVec; K]; + +#[inline(always)] +fn low_byte(value: u16) -> u8 { + value.to_le_bytes()[0] +} + +#[inline(always)] +fn low_u16(value: u32) -> u16 { + let [b0, b1, _, _] = value.to_le_bytes(); + u16::from_le_bytes([b0, b1]) +} + +#[cfg(any(test, not(target_arch = "s390x")))] +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, _, _, _, _] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + +#[cfg(test)] +#[inline] +fn test_low_byte(value: usize) -> u8 { + value.to_le_bytes()[0] +} + +#[cfg(test)] +#[inline] +fn test_u8(value: usize) -> u8 { + u8::try_from(value).expect("test value fits in u8") +} + +#[cfg(test)] +#[inline] +fn test_u16(value: usize) -> u16 { + u16::try_from(value).expect("test value fits in u16") +} #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] type PolyMulCache = [u16; N / 2]; #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] @@ -254,10 +273,7 @@ pub(super) fn validate_and_prepare_decapsulation_key< return Err(MlKemError::InvalidDecapsulationKey); } - let ek = match <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]) { - Ok(ek) => ek, - Err(_) => unreachable!("ML-KEM decapsulation key layout must include an encapsulation key"), - }; + let ek = <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]).map_err(|_| MlKemError::InvalidDecapsulationKey)?; Ok(PreparedDecapsulationArithmetic { s_hat: prepare_decapsulation_key::(dk), encapsulation: prepare_encapsulation_key::(ek), @@ -291,7 +307,9 @@ fn prepare_matrix_transpose(rho: &[u8; SEED_BYTES]) -> PolyMatri let mut a_transpose_hat = [[[0u16; N]; K]; K]; for (i, row) in a_transpose_hat.iter_mut().enumerate() { for (j, poly) in row.iter_mut().enumerate() { - sample_ntt_into(rho, i as u8, j as u8, poly); + let i = u8::try_from(i).expect("ML-KEM matrix row fits in u8"); + let j = u8::try_from(j).expect("ML-KEM matrix column fits in u8"); + sample_ntt_into(rho, i, j, poly); } } a_transpose_hat @@ -580,14 +598,11 @@ pub(super) fn decapsulate< >( dk: &[u8; DK_BYTES], c: &[u8; CT_BYTES], -) -> [u8; SHARED_SECRET_BYTES] { +) -> Result<[u8; SHARED_SECRET_BYTES], MlKemError> { let dk_pke = &dk[..DK_PKE_BYTES]; let ek_start = DK_PKE_BYTES; let ek_end = ek_start.strict_add(EK_BYTES); - let ek = match <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]) { - Ok(ek) => ek, - Err(_) => unreachable!("ML-KEM decapsulation key layout must include an encapsulation key"), - }; + let ek = <&[u8; EK_BYTES]>::try_from(&dk[ek_start..ek_end]).map_err(|_| MlKemError::InvalidDecapsulationKey)?; let h_start = ek_end; let h_stored = &dk[h_start..h_start.strict_add(HASH_BYTES)]; let z = &dk[h_start.strict_add(HASH_BYTES)..]; @@ -624,7 +639,7 @@ pub(super) fn decapsulate< ct::zeroize(&mut c_prime); ct::zeroize(core::slice::from_mut(&mut match_mask)); - shared + Ok(shared) } fn pke_keygen< @@ -686,22 +701,22 @@ fn keygen_sample_noise( let (s1, _) = s_tail.split_at_mut(1); let (e0, e_tail) = t_hat.split_at_mut(1); let (e1, _) = e_tail.split_at_mut(1); - sample_noise_quad::(sigma, 0, &mut s0[0], 1, &mut s1[0], 2, &mut e0[0], 3, &mut e1[0]); + sample_noise_quad::(sigma, [0, 1, 2, 3], [&mut s0[0], &mut s1[0], &mut e0[0], &mut e1[0]]); } else if ETA1_RANDOM_BYTES == ETA2_RANDOM_BYTES && K == 4 { let (s0, s_tail) = s_hat.split_at_mut(1); let (s1, s_tail) = s_tail.split_at_mut(1); let (s2, s3) = s_tail.split_at_mut(1); - sample_noise_quad::(sigma, 0, &mut s0[0], 1, &mut s1[0], 2, &mut s2[0], 3, &mut s3[0]); + sample_noise_quad::(sigma, [0, 1, 2, 3], [&mut s0[0], &mut s1[0], &mut s2[0], &mut s3[0]]); let (e0, e_tail) = t_hat.split_at_mut(1); let (e1, e_tail) = e_tail.split_at_mut(1); let (e2, e3) = e_tail.split_at_mut(1); - sample_noise_quad::(sigma, 4, &mut e0[0], 5, &mut e1[0], 6, &mut e2[0], 7, &mut e3[0]); + sample_noise_quad::(sigma, [4, 5, 6, 7], [&mut e0[0], &mut e1[0], &mut e2[0], &mut e3[0]]); } else if ETA1_RANDOM_BYTES == ETA2_RANDOM_BYTES && K == 3 { let (s0, s_tail) = s_hat.split_at_mut(1); let (s1, s2) = s_tail.split_at_mut(1); let (e0, e_tail) = t_hat.split_at_mut(1); - sample_noise_quad::(sigma, 0, &mut s0[0], 1, &mut s1[0], 2, &mut s2[0], 3, &mut e0[0]); + sample_noise_quad::(sigma, [0, 1, 2, 3], [&mut s0[0], &mut s1[0], &mut s2[0], &mut e0[0]]); let (e1, e2) = e_tail.split_at_mut(1); sample_noise_pair::(sigma, 4, &mut e1[0], 5, &mut e2[0]); @@ -740,14 +755,15 @@ fn keygen_matrix_accumulate(rho: &[u8; SEED_BYTES], s_hat: &Poly #[inline(always)] fn keygen_matrix_accumulate_fused(rho: &[u8; SEED_BYTES], s_hat: &PolyVec, t_hat: &mut PolyVec) { for (i, t_hat_i) in t_hat.iter_mut().enumerate() { + let i_u8 = u8::try_from(i).expect("ML-KEM matrix row fits in u8"); if K == 4 { sample_ntt_quad_mul_accumulate( rho, [ - (0, i as u8, &s_hat[0]), - (1, i as u8, &s_hat[1]), - (2, i as u8, &s_hat[2]), - (3, i as u8, &s_hat[3]), + (0, i_u8, &s_hat[0]), + (1, i_u8, &s_hat[1]), + (2, i_u8, &s_hat[2]), + (3, i_u8, &s_hat[3]), ], t_hat_i, ); @@ -755,17 +771,15 @@ fn keygen_matrix_accumulate_fused(rho: &[u8; SEED_BYTES], s_hat: let mut j = 0usize; while j.strict_add(1) < K { let next = j.strict_add(1); - sample_ntt_pair_mul_accumulate( - rho, - (j as u8, i as u8, &s_hat[j]), - (next as u8, i as u8, &s_hat[next]), - t_hat_i, - ); + let j_u8 = u8::try_from(j).expect("ML-KEM matrix column fits in u8"); + let next_u8 = u8::try_from(next).expect("ML-KEM matrix column fits in u8"); + sample_ntt_pair_mul_accumulate(rho, (j_u8, i_u8, &s_hat[j]), (next_u8, i_u8, &s_hat[next]), t_hat_i); j = j.strict_add(2); } if j < K { - sample_ntt_mul_accumulate(rho, j as u8, i as u8, &s_hat[j], t_hat_i); + let j_u8 = u8::try_from(j).expect("ML-KEM matrix column fits in u8"); + sample_ntt_mul_accumulate(rho, j_u8, i_u8, &s_hat[j], t_hat_i); } } } @@ -1149,7 +1163,8 @@ pub(super) unsafe fn diag_s390x_compress_decompress_values_digest(values: [u16; #[cfg(test)] fn fill_diag_seed(out: &mut [u8; SEED_BYTES], seed: u8) { for (i, byte) in out.iter_mut().enumerate() { - *byte = seed.wrapping_add(i as u8); + let i = u8::try_from(i).expect("ML-KEM diagnostic seed offset fits in u8"); + *byte = seed.wrapping_add(i); } } @@ -1158,7 +1173,8 @@ fn fill_diag_seed(out: &mut [u8; SEED_BYTES], seed: u8) { fn diag_fold_poly(poly: &Poly) -> u16 { let mut acc = 0u16; for (i, &coeff) in poly.iter().enumerate() { - acc ^= coeff.wrapping_mul((i as u16).wrapping_add(1)); + let i = u16::try_from(i).expect("ML-KEM polynomial index fits in u16"); + acc ^= coeff.wrapping_mul(i.wrapping_add(1)); } acc } @@ -1190,7 +1206,7 @@ fn pke_encrypt_prepared_768( let mut e1 = [[0u16; N]; 3]; let [y0, y1, y2] = &mut y_hat; let [e10, e11, e12] = &mut e1; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, e10); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, e10]); sample_noise_pair::(r, 4, e11, 5, e12); let mut e2 = [0u16; N]; sample_noise::(r, 6, &mut e2); @@ -1233,9 +1249,9 @@ fn pke_encrypt_prepared_1024( let mut y_hat = [[0u16; N]; 4]; let mut e1 = [[0u16; N]; 4]; let [y0, y1, y2, y3] = &mut y_hat; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, y3); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, y3]); let [e10, e11, e12, e13] = &mut e1; - sample_noise_quad::(r, 4, e10, 5, e11, 6, e12, 7, e13); + sample_noise_quad::(r, [4, 5, 6, 7], [e10, e11, e12, e13]); let mut e2 = [0u16; N]; sample_noise::(r, 8, &mut e2); @@ -1324,7 +1340,7 @@ fn pke_encrypt_prepared_768_compare( let mut e1 = [[0u16; N]; 3]; let [y0, y1, y2] = &mut y_hat; let [e10, e11, e12] = &mut e1; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, e10); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, e10]); sample_noise_pair::(r, 4, e11, 5, e12); let mut e2 = [0u16; N]; sample_noise::(r, 6, &mut e2); @@ -1368,9 +1384,9 @@ fn pke_encrypt_prepared_1024_compare( let mut y_hat = [[0u16; N]; 4]; let mut e1 = [[0u16; N]; 4]; let [y0, y1, y2, y3] = &mut y_hat; - sample_noise_quad::<128>(r, 0, y0, 1, y1, 2, y2, 3, y3); + sample_noise_quad::<128>(r, [0, 1, 2, 3], [y0, y1, y2, y3]); let [e10, e11, e12, e13] = &mut e1; - sample_noise_quad::(r, 4, e10, 5, e11, 6, e12, 7, e13); + sample_noise_quad::(r, [4, 5, 6, 7], [e10, e11, e12, e13]); let mut e2 = [0u16; N]; sample_noise::(r, 8, &mut e2); @@ -2162,13 +2178,11 @@ fn sample_matrix_ntt_mul_accumulate_materialized_k4_transpose( #[inline] fn matrix_accumulate_coord(entry: usize, transpose: bool) -> ((u8, u8), usize, usize) { - let dst = entry / K; - let rhs = entry % K; - let sample = if transpose { - (dst as u8, rhs as u8) - } else { - (rhs as u8, dst as u8) - }; + let dst = entry.checked_div(K).expect("ML-KEM dimension must be nonzero"); + let rhs = entry.checked_rem(K).expect("ML-KEM dimension must be nonzero"); + let dst_u8 = u8::try_from(dst).expect("ML-KEM matrix row fits in u8"); + let rhs_u8 = u8::try_from(rhs).expect("ML-KEM matrix column fits in u8"); + let sample = if transpose { (dst_u8, rhs_u8) } else { (rhs_u8, dst_u8) }; (sample, dst, rhs) } @@ -2397,7 +2411,7 @@ fn sample_ntt_initial_3blocks_then_tail_into(rho: &[u8; SEED_BYTES], j: u8, i: u let mut filled = 0usize; sample_ntt_initial_3blocks_public(&buf, out, &mut filled); - let mut blocks = SAMPLE_NTT_INITIAL_BLOCKS as u16; + let mut blocks = u16::try_from(SAMPLE_NTT_INITIAL_BLOCKS).expect("initial ML-KEM sample block count fits in u16"); let mut tail = [0u8; SHAKE128_RATE_BYTES]; while filled < N { reader.squeeze(&mut tail); @@ -2537,7 +2551,8 @@ fn sample_ntt_triple_initial_3blocks_then_tail_into( sample_ntt_initial_3blocks_public(&initial[1], out1, &mut filled[1]); sample_ntt_initial_3blocks_public(&initial[2], out2, &mut filled[2]); - let mut blocks = [SAMPLE_NTT_INITIAL_BLOCKS as u16; 3]; + let initial_blocks = u16::try_from(SAMPLE_NTT_INITIAL_BLOCKS).expect("initial ML-KEM sample block count fits in u16"); + let mut blocks = [initial_blocks; 3]; let mut tail = [[0u8; SHAKE128_RATE_BYTES]; 3]; while filled[0] < N && filled[1] < N && filled[2] < N { @@ -2895,125 +2910,6 @@ fn sample_ntt_pair_block_scalar( *filled1 = n1; } -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -macro_rules! sample_ntt_extract_16_candidate_vectors_neon { - ($input:expr) => {{ - let mask = vdupq_n_u16(0x0f); - // SAFETY: 48-byte deinterleaved load because: - // 1. The caller guarantees `$input..$input+48` is readable. - // 2. `vld3q_u8` reads exactly three 16-byte vectors from that public SampleNTT byte block. - // 3. The surrounding function is gated by `#[target_feature(enable = "neon")]`. - let triples = unsafe { vld3q_u8($input) }; - - let a0 = vmovl_u8(vget_low_u8(triples.0)); - let a1 = vmovl_u8(vget_low_u8(triples.1)); - let a2 = vmovl_u8(vget_low_u8(triples.2)); - let d0 = vorrq_u16(a0, vshlq_n_u16(vandq_u16(a1, mask), 8)); - let d1 = vorrq_u16(vshrq_n_u16(a1, 4), vshlq_n_u16(a2, 4)); - - let a0 = vmovl_u8(vget_high_u8(triples.0)); - let a1 = vmovl_u8(vget_high_u8(triples.1)); - let a2 = vmovl_u8(vget_high_u8(triples.2)); - let d2 = vorrq_u16(a0, vshlq_n_u16(vandq_u16(a1, mask), 8)); - let d3 = vorrq_u16(vshrq_n_u16(a1, 4), vshlq_n_u16(a2, 4)); - - (d0, d1, d2, d3) - }}; -} - -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -macro_rules! sample_ntt_store_public_candidate_unchecked { - ($out:expr, $n:ident, $candidate:expr) => {{ - let candidate = $candidate; - if candidate < Q { - // SAFETY: unchecked public-sample store because: - // 1. The caller reserved enough output capacity for every possible accepted candidate in this - // block, so the current accepted candidate is in bounds. - // 2. `$out` comes from a unique mutable polynomial borrow for the duration of parsing. - // 3. The branch depends only on public matrix-A sample bytes. - unsafe { - *$out.add($n) = candidate; - } - $n = $n.strict_add(1); - } - }}; -} - -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -macro_rules! sample_ntt_store_candidate_vectors_neon { - ($out:expr, $n:ident, $candidates:expr) => {{ - let (d0_lo, d1_lo, d0_hi, d1_hi) = $candidates; - macro_rules! store_pair_lane { - ($lane:literal, $d0:expr, $d1:expr) => {{ - sample_ntt_store_public_candidate_unchecked!($out, $n, vgetq_lane_u16::<$lane>($d0)); - sample_ntt_store_public_candidate_unchecked!($out, $n, vgetq_lane_u16::<$lane>($d1)); - }}; - } - - store_pair_lane!(0, d0_lo, d1_lo); - store_pair_lane!(1, d0_lo, d1_lo); - store_pair_lane!(2, d0_lo, d1_lo); - store_pair_lane!(3, d0_lo, d1_lo); - store_pair_lane!(4, d0_lo, d1_lo); - store_pair_lane!(5, d0_lo, d1_lo); - store_pair_lane!(6, d0_lo, d1_lo); - store_pair_lane!(7, d0_lo, d1_lo); - store_pair_lane!(0, d0_hi, d1_hi); - store_pair_lane!(1, d0_hi, d1_hi); - store_pair_lane!(2, d0_hi, d1_hi); - store_pair_lane!(3, d0_hi, d1_hi); - store_pair_lane!(4, d0_hi, d1_hi); - store_pair_lane!(5, d0_hi, d1_hi); - store_pair_lane!(6, d0_hi, d1_hi); - store_pair_lane!(7, d0_hi, d1_hi); - }}; -} - -#[cfg(all( - target_arch = "aarch64", - not(any(target_os = "macos", target_os = "linux")), - not(miri), - not(feature = "portable-only") -))] -#[target_feature(enable = "neon")] -/// # Safety -/// -/// `input` must point to 48 readable bytes. The caller must guarantee that the active CPU supports -/// NEON. `out` must be a unique 32-candidate destination. -unsafe fn sample_ntt_extract_16_candidates_neon(input: *const u8, out: &mut [u16; 32]) { - let (d0_lo, d1_lo, d0_hi, d1_hi) = sample_ntt_extract_16_candidate_vectors_neon!(input); - - // SAFETY: store the first 16 interleaved candidates because: - // 1. `out` is a unique `[u16; 32]` destination. - // 2. `vst2q_u16` writes exactly 16 u16 values from two eight-lane vectors. - // 3. The surrounding function is gated by `#[target_feature(enable = "neon")]`. - unsafe { - vst2q_u16(out.as_mut_ptr(), uint16x8x2_t(d0_lo, d1_lo)); - } - // SAFETY: store the last 16 interleaved candidates because: - // 1. `out.as_mut_ptr().add(16)..add(32)` is inside the 32-candidate destination. - // 2. `vst2q_u16` writes exactly 16 u16 values from two eight-lane vectors. - // 3. The surrounding function is gated by `#[target_feature(enable = "neon")]`. - unsafe { - vst2q_u16(out.as_mut_ptr().add(16), uint16x8x2_t(d0_hi, d1_hi)); - } -} - #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] /// # Safety @@ -3073,105 +2969,33 @@ fn sample_ntt_pair_block_neon( const MAX_CANDIDATES: usize = (SHAKE128_RATE_BYTES / 3) * 2; if N.strict_sub(*filled0) < MAX_CANDIDATES || N.strict_sub(*filled1) < MAX_CANDIDATES { - #[cfg(target_os = "linux")] - { - // SAFETY: bounded aarch64 SampleNTT tail parsing because: - // 1. `buf0` and `buf1` are fixed full SHAKE128 rate blocks. - // 2. The helper caps writes to each polynomial's remaining capacity. - // 3. `out0` and `out1` come from distinct mutable polynomial borrows. - // 4. Rejection branches and write positions depend only on public matrix-A XOF bytes. - unsafe { - sample_ntt_block_asm_bounded(buf0.as_ptr(), out0, filled0); - sample_ntt_block_asm_bounded(buf1.as_ptr(), out1, filled1); - } - } - #[cfg(not(target_os = "linux"))] - { - sample_ntt_block(buf0, out0, filled0); - sample_ntt_block(buf1, out1, filled1); - } - return; - } - - #[cfg(target_os = "linux")] - { - let n0 = *filled0; - let n1 = *filled1; - // SAFETY: aarch64 compact assembly SampleNTT block parsing because: - // 1. `buf0` and `buf1` are full 168-byte SHAKE128 rate blocks. - // 2. The preflight above reserves capacity for all 112 candidates each block can produce. + // SAFETY: bounded aarch64 SampleNTT tail parsing because: + // 1. `buf0` and `buf1` are fixed full SHAKE128 rate blocks. + // 2. The helper caps writes to each polynomial's remaining capacity. // 3. `out0` and `out1` come from distinct mutable polynomial borrows. // 4. Rejection branches and write positions depend only on public matrix-A XOF bytes. - let (count0, count1) = unsafe { - ( - aarch64::sample_ntt_rej_uniform_block_asm(out0.as_mut_ptr().add(n0), buf0.as_ptr()), - aarch64::sample_ntt_rej_uniform_block_asm(out1.as_mut_ptr().add(n1), buf1.as_ptr()), - ) - }; - *filled0 = n0.strict_add(count0); - *filled1 = n1.strict_add(count1); - } - - #[cfg(not(target_os = "linux"))] - { - const NEON_TRIPLES: usize = 16; - const NEON_BYTES: usize = NEON_TRIPLES * 3; - - let mut n0 = *filled0; - let mut n1 = *filled1; - let out0_ptr = out0.as_mut_ptr(); - let out1_ptr = out1.as_mut_ptr(); - let mut offset = 0usize; - - while offset.strict_add(NEON_BYTES) <= SHAKE128_RATE_BYTES { - // SAFETY: fixed 48-byte NEON extraction plus unchecked public-sample stores because: - // 1. `offset + NEON_BYTES <= SHAKE128_RATE_BYTES`, so both inputs name complete chunks. - // 2. The preflight above ensures each output has room for all candidates from the whole rate block. - // 3. Rejection branches and write counts depend only on public matrix-A samples. - let candidates0 = sample_ntt_extract_16_candidate_vectors_neon!(buf0.as_ptr().add(offset)); - let candidates1 = sample_ntt_extract_16_candidate_vectors_neon!(buf1.as_ptr().add(offset)); - sample_ntt_store_candidate_vectors_neon!(out0_ptr, n0, candidates0); - sample_ntt_store_candidate_vectors_neon!(out1_ptr, n1, candidates1); - - offset = offset.strict_add(NEON_BYTES); - } - - while offset.strict_add(2) < SHAKE128_RATE_BYTES { - let a0 = buf0[offset]; - let a1 = buf0[offset.strict_add(1)]; - let a2 = buf0[offset.strict_add(2)]; - let b0 = buf1[offset]; - let b1 = buf1[offset.strict_add(1)]; - let b2 = buf1[offset.strict_add(2)]; - - let d0 = u16::from(a0) | (u16::from(a1 & 0x0f) << 8); - let d1 = (u16::from(a1) >> 4) | (u16::from(a2) << 4); - let e0 = u16::from(b0) | (u16::from(b1 & 0x0f) << 8); - let e1 = (u16::from(b1) >> 4) | (u16::from(b2) << 4); - - if d0 < Q { - out0[n0] = d0; - n0 = n0.strict_add(1); - } - if d1 < Q { - out0[n0] = d1; - n0 = n0.strict_add(1); - } - if e0 < Q { - out1[n1] = e0; - n1 = n1.strict_add(1); - } - if e1 < Q { - out1[n1] = e1; - n1 = n1.strict_add(1); - } - - offset = offset.strict_add(3); + unsafe { + sample_ntt_block_asm_bounded(buf0.as_ptr(), out0, filled0); + sample_ntt_block_asm_bounded(buf1.as_ptr(), out1, filled1); } - - *filled0 = n0; - *filled1 = n1; + return; } + + let n0 = *filled0; + let n1 = *filled1; + // SAFETY: aarch64 compact assembly SampleNTT block parsing because: + // 1. `buf0` and `buf1` are full 168-byte SHAKE128 rate blocks. + // 2. The preflight above reserves capacity for all 112 candidates each block can produce. + // 3. `out0` and `out1` come from distinct mutable polynomial borrows. + // 4. Rejection branches and write positions depend only on public matrix-A XOF bytes. + let (count0, count1) = unsafe { + ( + aarch64::sample_ntt_rej_uniform_block_asm(out0.as_mut_ptr().add(n0), buf0.as_ptr()), + aarch64::sample_ntt_rej_uniform_block_asm(out1.as_mut_ptr().add(n1), buf1.as_ptr()), + ) + }; + *filled0 = n0.strict_add(count0); + *filled1 = n1.strict_add(count1); } fn sample_ntt_triple_from_xof_into(mut readers: [Shake128XofReader; 3], out: [&mut Poly; 3]) { @@ -3250,8 +3074,7 @@ fn sample_ntt_quad_from_xof_into(mut readers: [Shake128XofReader; 4], out: [&mut not(feature = "portable-only") )))] { - let [buf0, buf1, buf2, buf3] = &mut bufs; - Shake128XofReader::squeeze_quad(reader0, reader1, reader2, reader3, buf0, buf1, buf2, buf3); + reader0.squeeze_quad(reader1, reader2, reader3, &mut bufs); sample_ntt_quad_block(&bufs, [out0, out1, out2, out3], &mut filled); } } @@ -3384,7 +3207,8 @@ impl<'a> SampleNttProduct<'a> { #[cfg(not(all(target_arch = "aarch64", not(miri), not(feature = "portable-only"))))] #[inline(always)] fn absorb_block_scalar(&mut self, buf: &[u8; SHAKE128_RATE_BYTES], acc: &mut Poly) { - for chunk in buf.chunks_exact(3) { + let (chunks, remainder) = buf.as_chunks::<3>(); + for chunk in chunks { let d1 = u16::from(chunk[0]) | (u16::from(chunk[1] & 0x0f) << 8); let d2 = (u16::from(chunk[1]) >> 4) | (u16::from(chunk[2]) << 4); @@ -3394,6 +3218,7 @@ impl<'a> SampleNttProduct<'a> { break; } } + debug_assert!(remainder.is_empty()); } #[inline(always)] @@ -3493,6 +3318,9 @@ unsafe fn multiply_ntts_add_assign_chunk_neon_ptr(acc: &mut Poly, a_ptr: *const #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn sample_ntt_product_absorb_block_neon( product: &mut SampleNttProduct<'_>, buf: &[u8; SHAKE128_RATE_BYTES], @@ -3617,6 +3445,10 @@ unsafe fn sample_ntt_product_absorb_rate_ptr_neon( #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `buf` must be one complete SHAKE128 rate +/// block, and `product` and `acc` must retain their unique ML-KEM polynomial ownership. fn sample_ntt_product_absorb_block_avx2( product: &mut SampleNttProduct<'_>, buf: &[u8; SHAKE128_RATE_BYTES], @@ -3643,6 +3475,10 @@ fn sample_ntt_product_absorb_block_avx2( #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `product` and `acc` must retain their unique +/// ML-KEM polynomial ownership. fn sample_ntt_product_absorb_candidates_avx2( product: &mut SampleNttProduct<'_>, candidates: &[u16; 8], @@ -3755,8 +3591,7 @@ fn sample_ntt_quad_mul_accumulate_from_xof(mut readers: [Shake128XofReader; 4], { while !products[0].is_done() && !products[1].is_done() && !products[2].is_done() && !products[3].is_done() { let [reader0, reader1, reader2, reader3] = &mut readers; - let [buf0, buf1, buf2, buf3] = &mut bufs; - Shake128XofReader::squeeze_quad(reader0, reader1, reader2, reader3, buf0, buf1, buf2, buf3); + reader0.squeeze_quad(reader1, reader2, reader3, &mut bufs); products[0].absorb_block(&bufs[0], acc); products[1].absorb_block(&bufs[1], acc); products[2].absorb_block(&bufs[2], acc); @@ -3997,6 +3832,11 @@ unsafe fn sample_ntt_triple_block_neon_ptrs(rate_ptrs: [*const u8; 3], out: [&mu #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The active CPU must support AArch64 NEON. Every pointer in `rate_ptrs` must +/// reference a readable SHAKE128 rate block. The output polynomials must be +/// distinct, and every value in `filled` must be at most `N`. unsafe fn sample_ntt_quad_block_neon_ptrs(rate_ptrs: [*const u8; 4], out: [&mut Poly; 4], filled: &mut [usize; 4]) { const MAX_CANDIDATES: usize = (SHAKE128_RATE_BYTES / 3) * 2; @@ -4144,6 +3984,10 @@ unsafe fn sample_ntt_block_avx2(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `out` must have room for every accepted +/// candidate in `buf`, as established by the caller's remaining-capacity check. fn sample_ntt_block_avx2_full(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, filled: &mut usize) { let mut n = *filled; let out_ptr = out.as_mut_ptr(); @@ -4169,6 +4013,10 @@ fn sample_ntt_block_avx2_full(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, f #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3. `out` and `filled` must describe one unique +/// ML-KEM polynomial destination. fn sample_ntt_block_avx2_bounded(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, filled: &mut usize) { let mut n = *filled; let out_ptr = out.as_mut_ptr(); @@ -4241,6 +4089,11 @@ fn sample_ntt_store_candidates_bounded(out: *mut u16, n: &mut usize, candidates: #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// Decode eight SampleNTT candidates from a potentially unaligned 16-byte window. +/// +/// # Safety +/// +/// The active CPU must support AVX2, SSE4.1, and SSSE3, and `input` must be valid to read 16 bytes. unsafe fn sample_ntt_extract_8_candidates_avx2(input: *const u8, shifted_tail: bool) -> [u16; 8] { let pad = i8::MIN; let (even_mask, odd_mask) = if shifted_tail { @@ -4255,24 +4108,17 @@ unsafe fn sample_ntt_extract_8_candidates_avx2(input: *const u8, shifted_tail: b ) }; - // SAFETY: unaligned 16-byte input load because: - // 1. The caller guarantees `input..input + 16` is readable inside the SHAKE128 rate block. - // 2. `_mm_loadu_si128` accepts arbitrary alignment. - // 3. The surrounding function is gated by AVX2/SSE4.1/SSSE3 target features. - let block = unsafe { _mm_loadu_si128(input.cast::<__m128i>()) }; + // SAFETY: the caller guarantees `input..input + 16` is readable inside the SHAKE128 rate block. + let bytes = unsafe { input.cast::<[u8; 16]>().read_unaligned() }; + // SAFETY: `[u8; 16]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + let block = unsafe { core::mem::transmute::<[u8; 16], __m128i>(bytes) }; let mask_12 = _mm_set1_epi16(0x0fff); let even = _mm_and_si128(_mm_shuffle_epi8(block, even_mask), mask_12); let odd = _mm_srli_epi16::<4>(_mm_shuffle_epi8(block, odd_mask)); let interleaved = _mm_unpacklo_epi16(even, odd); - let mut candidates = [0u16; 8]; - // SAFETY: store exactly eight decoded candidates because: - // 1. `candidates` is a fully initialized local `[u16; 8]` destination. - // 2. `_mm_storeu_si128` writes exactly 16 bytes and accepts arbitrary alignment. - unsafe { - _mm_storeu_si128(candidates.as_mut_ptr().cast::<__m128i>(), interleaved); - } - candidates + // SAFETY: `__m128i` and `[u16; 8]` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<__m128i, [u16; 8]>(interleaved) } } #[cfg(not(all( @@ -4309,12 +4155,16 @@ fn sample_ntt_block(buf: &[u8; SHAKE128_RATE_BYTES], out: &mut Poly, filled: &mu } fn sample_noise(seed: &[u8; SEED_BYTES], nonce: u8, out: &mut Poly) { + assert!( + matches!(RANDOM_BYTES, ETA2_RANDOM_BYTES | ETA3_RANDOM_BYTES), + "unsupported ML-KEM noise width" + ); let mut buf = [0u8; RANDOM_BYTES]; prf_eta(seed, nonce, &mut buf); match RANDOM_BYTES { ETA2_RANDOM_BYTES => sample_poly_cbd_eta2(&buf, out), ETA3_RANDOM_BYTES => sample_poly_cbd_eta3(&buf, out), - _ => unreachable!("unsupported ML-KEM noise width"), + _ => {} } ct::zeroize(&mut buf); } @@ -4326,6 +4176,10 @@ fn sample_noise_pair( nonce1: u8, out1: &mut Poly, ) { + assert!( + matches!(RANDOM_BYTES, ETA2_RANDOM_BYTES | ETA3_RANDOM_BYTES), + "unsupported ML-KEM noise width" + ); let (mut reader0, mut reader1) = MlKemShake256XofReader::seeded_32_1_pair(seed, nonce0, nonce1); let mut buf0 = [0u8; RANDOM_BYTES]; let mut buf1 = [0u8; RANDOM_BYTES]; @@ -4339,59 +4193,43 @@ fn sample_noise_pair( sample_poly_cbd_eta3(&buf0, out0); sample_poly_cbd_eta3(&buf1, out1); } - _ => unreachable!("unsupported ML-KEM noise width"), + _ => {} } ct::zeroize(&mut buf0); ct::zeroize(&mut buf1); } -#[allow(clippy::too_many_arguments)] fn sample_noise_quad( seed: &[u8; SEED_BYTES], - nonce0: u8, - out0: &mut Poly, - nonce1: u8, - out1: &mut Poly, - nonce2: u8, - out2: &mut Poly, - nonce3: u8, - out3: &mut Poly, + [nonce0, nonce1, nonce2, nonce3]: [u8; 4], + [out0, out1, out2, out3]: [&mut Poly; 4], ) { + assert!( + matches!(RANDOM_BYTES, ETA2_RANDOM_BYTES | ETA3_RANDOM_BYTES), + "unsupported ML-KEM noise width" + ); let (mut reader0, mut reader1, mut reader2, mut reader3) = MlKemShake256XofReader::seeded_32_1_quad(seed, nonce0, nonce1, nonce2, nonce3); - let mut buf0 = [0u8; RANDOM_BYTES]; - let mut buf1 = [0u8; RANDOM_BYTES]; - let mut buf2 = [0u8; RANDOM_BYTES]; - let mut buf3 = [0u8; RANDOM_BYTES]; - MlKemShake256XofReader::squeeze_quad( - &mut reader0, - &mut reader1, - &mut reader2, - &mut reader3, - &mut buf0, - &mut buf1, - &mut buf2, - &mut buf3, - ); + let mut bufs = [[0u8; RANDOM_BYTES]; 4]; + reader0.squeeze_quad(&mut reader1, &mut reader2, &mut reader3, &mut bufs); match RANDOM_BYTES { ETA2_RANDOM_BYTES => { - sample_poly_cbd_eta2(&buf0, out0); - sample_poly_cbd_eta2(&buf1, out1); - sample_poly_cbd_eta2(&buf2, out2); - sample_poly_cbd_eta2(&buf3, out3); + sample_poly_cbd_eta2(&bufs[0], out0); + sample_poly_cbd_eta2(&bufs[1], out1); + sample_poly_cbd_eta2(&bufs[2], out2); + sample_poly_cbd_eta2(&bufs[3], out3); } ETA3_RANDOM_BYTES => { - sample_poly_cbd_eta3(&buf0, out0); - sample_poly_cbd_eta3(&buf1, out1); - sample_poly_cbd_eta3(&buf2, out2); - sample_poly_cbd_eta3(&buf3, out3); + sample_poly_cbd_eta3(&bufs[0], out0); + sample_poly_cbd_eta3(&bufs[1], out1); + sample_poly_cbd_eta3(&bufs[2], out2); + sample_poly_cbd_eta3(&bufs[3], out3); } - _ => unreachable!("unsupported ML-KEM noise width"), + _ => {} + } + for buf in &mut bufs { + ct::zeroize(buf); } - ct::zeroize(&mut buf0); - ct::zeroize(&mut buf1); - ct::zeroize(&mut buf2); - ct::zeroize(&mut buf3); } fn sample_poly_cbd_eta2(input: &[u8], out: &mut Poly) { @@ -4402,26 +4240,31 @@ fn sample_poly_cbd_eta2(input: &[u8], out: &mut Poly) { let y0 = ((byte >> 2) & 1).strict_add((byte >> 3) & 1); let x1 = ((byte >> 4) & 1).strict_add((byte >> 5) & 1); let y1 = ((byte >> 6) & 1).strict_add((byte >> 7) & 1); - out[i.strict_mul(2)] = small_signed_to_mod_q(i16::from(x0) - i16::from(y0)); - out[i.strict_mul(2).strict_add(1)] = small_signed_to_mod_q(i16::from(x1) - i16::from(y1)); + out[i.strict_mul(2)] = small_signed_to_mod_q(i16::from(x0).strict_sub(i16::from(y0))); + out[i.strict_mul(2).strict_add(1)] = small_signed_to_mod_q(i16::from(x1).strict_sub(i16::from(y1))); } } fn sample_poly_cbd_eta3(input: &[u8], out: &mut Poly) { debug_assert_eq!(input.len(), ETA3_RANDOM_BYTES); - for (i, bytes) in input.chunks_exact(3).enumerate() { + let (chunks, remainder) = input.as_chunks::<3>(); + for (i, bytes) in chunks.iter().enumerate() { let bits = u32::from(bytes[0]) | (u32::from(bytes[1]) << 8) | (u32::from(bytes[2]) << 16); - let counts = (bits & 0x0024_9249) + ((bits >> 1) & 0x0024_9249) + ((bits >> 2) & 0x0024_9249); + let counts = (bits & 0x0024_9249) + .strict_add((bits >> 1) & 0x0024_9249) + .strict_add((bits >> 2) & 0x0024_9249); let start = i.strict_mul(4); for j in 0usize..4 { let shift = j.strict_mul(6); let x = (counts >> shift) & 0x7; let y = (counts >> shift.strict_add(3)) & 0x7; - out[start.strict_add(j)] = small_signed_to_mod_q(x as i16 - y as i16); + out[start.strict_add(j)] = + small_signed_to_mod_q(i16::from(low_byte(low_u16(x))).strict_sub(i16::from(low_byte(low_u16(y))))); } } + debug_assert!(remainder.is_empty()); } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] @@ -5116,6 +4959,9 @@ fn multiply_ntts_add_assign_scalar(acc: &mut Poly, a: &Poly, b: &Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_neon(poly: &mut Poly) { ntt_neon_butterflies(poly); canonicalize_ntt_neon(poly); @@ -5123,6 +4969,9 @@ fn ntt_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_to_montgomery_product_domain_neon(poly: &mut Poly) { ntt_neon_butterflies(poly); canonicalize_ntt_product_domain_neon(poly); @@ -5130,6 +4979,9 @@ fn ntt_to_montgomery_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_neon_butterflies(poly: &mut Poly) { let mut zeta_index = 1usize; let mut len = 128usize; @@ -5179,6 +5031,9 @@ fn ntt_neon_butterflies(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5215,6 +5070,9 @@ fn ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5244,6 +5102,9 @@ fn ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn canonicalize_ntt_neon(poly: &mut Poly) { for i in (0..N).step_by(8) { // SAFETY: fixed-size NEON forward-NTT canonicalization because: @@ -5262,6 +5123,9 @@ fn canonicalize_ntt_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn canonicalize_ntt_product_domain_neon(poly: &mut Poly) { for i in (0..N).step_by(8) { // SAFETY: fixed-size NEON forward-NTT product-domain finalization because: @@ -5281,6 +5145,9 @@ fn canonicalize_ntt_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_neon(poly: &mut Poly, final_scale_mont: i16) { inverse_ntt_neon_butterflies(poly); @@ -5302,6 +5169,9 @@ fn inverse_ntt_neon(poly: &mut Poly, final_scale_mont: i16) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_neon_add_assign(poly: &mut Poly, addend: &Poly, final_scale_mont: i16) { inverse_ntt_neon_butterflies(poly); @@ -5323,6 +5193,9 @@ fn inverse_ntt_neon_add_assign(poly: &mut Poly, addend: &Poly, final_scale_mont: #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_neon_butterflies(poly: &mut Poly) { let mut zeta_index = 127usize; let mut len = 2usize; @@ -5371,6 +5244,9 @@ fn inverse_ntt_neon_butterflies(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5406,6 +5282,9 @@ fn inverse_ntt_len2_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn inverse_ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5432,6 +5311,9 @@ fn inverse_ntt_len4_neon(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn duplicate_i16_pair_lanes_neon(a: i16, b: i16) -> int16x4_t { let lanes = vdup_n_s16(a); let lanes = vset_lane_s16::<2>(b, lanes); @@ -5440,6 +5322,9 @@ fn duplicate_i16_pair_lanes_neon(a: i16, b: i16) -> int16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn zip_u16x4_pair_lanes_neon(lower: uint16x4_t, upper: uint16x4_t) -> uint16x8_t { let lower_pairs: uint32x2_t = vreinterpret_u32_u16(lower); let upper_pairs: uint32x2_t = vreinterpret_u32_u16(upper); @@ -5451,6 +5336,9 @@ fn zip_u16x4_pair_lanes_neon(lower: uint16x4_t, upper: uint16x4_t) -> uint16x8_t #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn poly_mulcache_compute_neon(poly: &Poly, cache: &mut PolyMulCache) { for i in (0..(N / 2)).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5477,6 +5365,9 @@ fn poly_mulcache_compute_neon(poly: &Poly, cache: &mut PolyMulCache) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn base_multiply_cached_8_neon( a: &Poly, b: &Poly, @@ -5524,6 +5415,9 @@ fn base_multiply_cached_8_neon( #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_accumulate_cached_neon( acc: &mut Poly, a: &PolyVec, @@ -5553,6 +5447,9 @@ fn multiply_ntts_accumulate_cached_neon( not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn base_multiply_8_neon(a: &Poly, b: &Poly, gamma: int16x8_t, coeff_offset: usize) -> (uint16x8_t, uint16x8_t) { debug_assert_eq!(coeff_offset % 16, 0); debug_assert!(coeff_offset.strict_add(16) <= N); @@ -5594,6 +5491,9 @@ fn base_multiply_8_neon(a: &Poly, b: &Poly, gamma: int16x8_t, coeff_offset: usiz #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn store_accumulated_8_neon(acc: &mut Poly, coeff_offset: usize, c0: uint16x8_t, c1: uint16x8_t) { debug_assert_eq!(coeff_offset % 16, 0); debug_assert!(coeff_offset.strict_add(16) <= N); @@ -5619,6 +5519,9 @@ fn store_accumulated_8_neon(acc: &mut Poly, coeff_offset: usize, c0: uint16x8_t, not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_add_assign_neon(acc: &mut Poly, a: &Poly, b: &Poly) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5643,6 +5546,9 @@ fn multiply_ntts_add_assign_neon(acc: &mut Poly, a: &Poly, b: &Poly) { not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_accumulate_k3_neon(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5673,6 +5579,9 @@ fn multiply_ntts_accumulate_k3_neon(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3] not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn multiply_ntts_accumulate_k4_neon(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -5701,6 +5610,9 @@ fn multiply_ntts_accumulate_k4_neon(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4] #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_i16x8_to_i32x4_neon(a: int16x8_t, b: int16x8_t) -> (int32x4_t, int32x4_t) { ( vmull_s16(vget_low_s16(a), vget_low_s16(b)), @@ -5710,38 +5622,56 @@ fn mul_i16x8_to_i32x4_neon(a: int16x8_t, b: int16x8_t) -> (int32x4_t, int32x4_t) #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_i16x8(a: int16x8_t, b_mont: i16) -> int16x8_t { montgomery_reduce_s16x8(vmulq_n_s16(a, b_mont), vshrq_n_s16::<1>(vqdmulhq_n_s16(a, b_mont))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_i16x4(a: int16x4_t, b_mont: i16) -> int16x4_t { montgomery_reduce_i32x4_neon(vmull_n_s16(a, b_mont)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_i16x4(a: int16x4_t, b_mont: int16x4_t) -> int16x4_t { montgomery_reduce_i32x4_neon(vmull_s16(a, b_mont)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn montgomery_reduce_i32x8_neon(lo: int32x4_t, hi: int32x4_t) -> int16x8_t { vcombine_s16(montgomery_reduce_i32x4_neon(lo), montgomery_reduce_i32x4_neon(hi)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn montgomery_reduce_i32x4_neon(value: int32x4_t) -> int16x4_t { - let k = vmul_n_s16(vmovn_s32(value), Q_MONT_INV_U16 as i16); + let k = vmul_n_s16(vmovn_s32(value), Q_MONT_INV_U16.cast_signed()); let c = vshrn_n_s32::<16>(vmull_n_s16(k, Q_I16)); vsub_s16(vshrn_n_s32::<16>(value), c) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_mod_u16x4(a: uint16x4_t, b_mont: i16) -> uint16x4_t { signed_to_mod_q_s16x4(montgomery_reduce_i32x4_neon(vmull_n_s16( vreinterpret_s16_u16(a), @@ -5751,12 +5681,18 @@ fn mul_mont_const_mod_u16x4(a: uint16x4_t, b_mont: i16) -> uint16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_mod_u16x4(a: uint16x4_t, b_mont: int16x4_t) -> uint16x4_t { signed_to_mod_q_s16x4(montgomery_reduce_i32x4_neon(vmull_s16(vreinterpret_s16_u16(a), b_mont))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn add_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { let sum = vadd_u16(a, b); let q = vdup_n_u16(Q); @@ -5766,6 +5702,9 @@ fn add_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn sub_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { let diff = vsub_u16(a, b); let q = vdup_n_u16(Q); @@ -5775,6 +5714,9 @@ fn sub_mod_u16x4(a: uint16x4_t, b: uint16x4_t) -> uint16x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn add_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { let sum = vaddq_u16(a, b); let q = vdupq_n_u16(Q); @@ -5784,6 +5726,9 @@ fn add_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn sub_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { let diff = vsubq_u16(a, b); let q = vdupq_n_u16(Q); @@ -5793,9 +5738,12 @@ fn sub_mod_u16x8(a: uint16x8_t, b: uint16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn canonicalize_lazy_i16x8(value: int16x8_t) -> uint16x8_t { let negative = vshrq_n_s16::<15>(value); - let shifted = vaddq_s16(value, vandq_s16(negative, vdupq_n_s16((Q as i16) * 8))); + let shifted = vaddq_s16(value, vandq_s16(negative, vdupq_n_s16(Q_I16.strict_mul(8)))); let shifted = vreinterpretq_u16_s16(shifted); let lo = reduce_lazy_u32x4(vmovl_u16(vget_low_u16(shifted))); let hi = reduce_lazy_u32x4(vmovl_u16(vget_high_u16(shifted))); @@ -5804,12 +5752,18 @@ fn canonicalize_lazy_i16x8(value: int16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn lazy_i16x8_to_product_domain(value: int16x8_t) -> uint16x8_t { signed_to_mod_q_s16x8(montgomery_reduce_s16x8(value, vshrq_n_s16::<15>(value))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn reduce_lazy_u32x4(value: uint32x4_t) -> uint32x4_t { let quotient = vshrq_n_u32::<26>(vaddq_u32(vmulq_n_u32(value, 20_159), vdupq_n_u32(1 << 25))); let product = vmulq_n_u32(quotient, Q_U32); @@ -5820,12 +5774,18 @@ fn reduce_lazy_u32x4(value: uint32x4_t) -> uint32x4_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn mul_mont_const_mod_u16x8(a: uint16x8_t, b_mont: i16) -> uint16x8_t { signed_to_mod_q_s16x8(mul_mont_const_i16x8(vreinterpretq_s16_u16(a), b_mont)) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn montgomery_reduce_s16x8(low: int16x8_t, high: int16x8_t) -> int16x8_t { let k = vreinterpretq_s16_u16(vmulq_n_u16(vreinterpretq_u16_s16(low), Q_MONT_INV_U16)); let c = vshrq_n_s16::<1>(vqdmulhq_n_s16(k, Q_I16)); @@ -5834,6 +5794,9 @@ fn montgomery_reduce_s16x8(low: int16x8_t, high: int16x8_t) -> int16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn signed_to_mod_q_s16x8(value: int16x8_t) -> uint16x8_t { let negative = vshrq_n_s16::<15>(value); vreinterpretq_u16_s16(vaddq_s16(value, vandq_s16(negative, vdupq_n_s16(Q_I16)))) @@ -5841,6 +5804,9 @@ fn signed_to_mod_q_s16x8(value: int16x8_t) -> uint16x8_t { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn signed_to_mod_q_s16x4(value: int16x4_t) -> uint16x4_t { let negative = vshr_n_s16::<15>(value); vreinterpret_u16_s16(vadd_s16(value, vand_s16(negative, vdup_n_s16(Q_I16)))) @@ -5853,6 +5819,9 @@ fn signed_to_mod_q_s16x4(value: int16x4_t) -> uint16x4_t { not(feature = "portable-only") ))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn to_montgomery_product_domain_neon(poly: &mut Poly) { let high = vdupq_n_s16(0); for i in (0..N).step_by(8) { @@ -5875,6 +5844,9 @@ fn to_montgomery_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn from_montgomery_product_domain_neon(poly: &mut Poly) { for i in (0..N).step_by(8) { // SAFETY: fixed-size NEON product-domain exit because: @@ -5897,6 +5869,9 @@ fn from_montgomery_product_domain_neon(poly: &mut Poly) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn ntt_avx2(poly: &mut Poly) { let mut zeta_index = 1usize; x86_64::ntt_len_ge16_avx2(poly, &mut zeta_index); @@ -5942,6 +5917,9 @@ fn ntt_avx2(poly: &mut Poly) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -5965,6 +5943,9 @@ fn ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn inverse_ntt_avx2(poly: &mut Poly, final_scale_mont: i16) { let mut zeta_index = 127usize; let mut len = 2usize; @@ -6029,6 +6010,9 @@ fn inverse_ntt_avx2(poly: &mut Poly, final_scale_mont: i16) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn inverse_ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -6055,6 +6039,9 @@ fn inverse_ntt_len4_avx2(poly: &mut Poly, zeta_index: &mut usize) { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn multiply_ntts_add_assign_avx2(acc: &mut Poly, a: &Poly, b: &Poly) { let mask = set1_u32x8_avx2(0xffff); for i in (0..GAMMAS_MONT.len()).step_by(8) { @@ -6069,9 +6056,9 @@ fn multiply_ntts_add_assign_avx2(acc: &mut Poly, a: &Poly, b: &Poly) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let a_pairs = _mm256_loadu_si256(a.as_ptr().add(coeff_offset).cast::<__m256i>()); - let b_pairs = _mm256_loadu_si256(b.as_ptr().add(coeff_offset).cast::<__m256i>()); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let a_pairs = load_u16x16_avx2(a.as_ptr().add(coeff_offset)); + let b_pairs = load_u16x16_avx2(b.as_ptr().add(coeff_offset)); + let acc_pairs = load_u16x16_avx2(acc.as_ptr().add(coeff_offset)); let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(i)); let a0 = _mm256_and_si256(a_pairs, mask); @@ -6093,13 +6080,17 @@ fn multiply_ntts_add_assign_avx2(acc: &mut Poly, a: &Poly, b: &Poly) { let out0 = add_mod_u32x8_avx2(acc0, c0); let out1 = add_mod_u32x8_avx2(acc1, c1); let packed = _mm256_or_si256(out0, _mm256_slli_epi32::<16>(out1)); - _mm256_storeu_si256(acc.as_mut_ptr().add(coeff_offset).cast::<__m256i>(), packed); + store_u16x16_avx2(acc.as_mut_ptr().add(coeff_offset), packed); } } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `coeff_offset..coeff_offset + 16` must be in +/// bounds for `b` and `acc`. fn multiply_ntts_add_assign_chunk_avx2( acc: &mut Poly, a: &[u16; SAMPLE_NTT_ACC_CHUNK_COEFFS], @@ -6118,9 +6109,9 @@ fn multiply_ntts_add_assign_chunk_avx2( // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let a_pairs = _mm256_loadu_si256(a.as_ptr().cast::<__m256i>()); - let b_pairs = _mm256_loadu_si256(b.as_ptr().add(coeff_offset).cast::<__m256i>()); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let a_pairs = load_u16x16_avx2(a.as_ptr()); + let b_pairs = load_u16x16_avx2(b.as_ptr().add(coeff_offset)); + let acc_pairs = load_u16x16_avx2(acc.as_ptr().add(coeff_offset)); let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(gamma_offset)); let a0 = _mm256_and_si256(a_pairs, mask); @@ -6142,60 +6133,109 @@ fn multiply_ntts_add_assign_chunk_avx2( let out0 = add_mod_u32x8_avx2(acc0, c0); let out1 = add_mod_u32x8_avx2(acc1, c1); let packed = _mm256_or_si256(out0, _mm256_slli_epi32::<16>(out1)); - _mm256_storeu_si256(acc.as_mut_ptr().add(coeff_offset).cast::<__m256i>(), packed); + store_u16x16_avx2(acc.as_mut_ptr().add(coeff_offset), packed); } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn set1_u32x8_avx2(value: u32) -> __m256i { - _mm256_set1_epi32(value as i32) + _mm256_set1_epi32(value.cast_signed()) +} + +#[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] +#[target_feature(enable = "avx2,sse4.1")] +/// Load 16 potentially unaligned coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read 16 initialized +/// `u16` values. +fn load_u16x16_avx2(ptr: *const u16) -> __m256i { + // SAFETY: the caller provides 16 readable coefficients; `[u16; 16]` retains two-byte alignment. + let lanes = unsafe { ptr.cast::<[u16; 16]>().read_unaligned() }; + // SAFETY: `[u16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 16], __m256i>(lanes) } +} + +#[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] +#[target_feature(enable = "avx2,sse4.1")] +/// Store 16 coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write 16 `u16` values. +fn store_u16x16_avx2(ptr: *mut u16, value: __m256i) { + // SAFETY: `__m256i` and `[u16; 16]` are 256-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m256i, [u16; 16]>(value) }; + // SAFETY: the caller provides 16 writable coefficients; `[u16; 16]` retains two-byte alignment. + unsafe { ptr.cast::<[u16; 16]>().write_unaligned(lanes) }; } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read eight initialized +/// `u16` values. fn load_u16x8_avx2(ptr: *const u16) -> __m128i { - // SAFETY: unaligned 8-coefficient input load because: - // 1. The caller proves `ptr..ptr + 8` is readable. - // 2. `_mm_loadu_si128` accepts arbitrary alignment. - // 3. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_loadu_si128(ptr.cast::<__m128i>()) } + // SAFETY: the caller proves `ptr..ptr + 8` is readable. + let lanes = unsafe { ptr.cast::<[u16; 8]>().read_unaligned() }; + // SAFETY: `[u16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 8], __m128i>(lanes) } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read four initialized +/// `u16` values. fn load_u16x4_avx2(ptr: *const u16) -> __m128i { - // SAFETY: unaligned 4-coefficient input load because: - // 1. The caller proves `ptr..ptr + 4` is readable. - // 2. `_mm_loadl_epi64` accepts arbitrary alignment. - // 3. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_loadl_epi64(ptr.cast::<__m128i>()) } + // SAFETY: the caller proves `ptr..ptr + 4` is readable. + let low = unsafe { ptr.cast::<[u16; 4]>().read_unaligned() }; + // Match `_mm_loadl_epi64` by zeroing the upper 64 bits. + let lanes = [low[0], low[1], low[2], low[3], 0, 0, 0, 0]; + // SAFETY: `[u16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 8], __m128i>(lanes) } } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write eight `u16` values. fn store_u16x8_avx2(ptr: *mut u16, values: __m128i) { - // SAFETY: unaligned 8-coefficient output store because: - // 1. The caller proves `ptr..ptr + 8` is writable. - // 2. Values are reduced modulo Q and fit in u16 before storing. - // 3. `_mm_storeu_si128` accepts arbitrary alignment. - // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_storeu_si128(ptr.cast::<__m128i>(), values) }; + // SAFETY: `__m128i` and `[u16; 8]` are 128-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m128i, [u16; 8]>(values) }; + // SAFETY: the caller proves `ptr..ptr + 8` is writable. + unsafe { ptr.cast::<[u16; 8]>().write_unaligned(lanes) }; } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write four `u16` values. fn store_u16x4_avx2(ptr: *mut u16, values: __m128i) { - // SAFETY: unaligned 4-coefficient output store because: - // 1. The caller proves `ptr..ptr + 4` is writable. - // 2. Values in the low four lanes are reduced modulo Q and fit in u16 before storing. - // 3. `_mm_storel_epi64` accepts arbitrary alignment. - // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - unsafe { _mm_storel_epi64(ptr.cast::<__m128i>(), values) }; + // SAFETY: `__m128i` and `[u16; 8]` are 128-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m128i, [u16; 8]>(values) }; + // SAFETY: the caller proves `ptr..ptr + 4` is writable. + unsafe { + ptr + .cast::<[u16; 4]>() + .write_unaligned([lanes[0], lanes[1], lanes[2], lanes[3]]) + }; } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn mul_mont_const_mod_u16x8_avx2(a: __m128i, b_mont: i16) -> __m128i { let b = _mm_set1_epi16(b_mont); signed_to_mod_q_s16x8_avx2(montgomery_reduce_s16x8_avx2( @@ -6206,14 +6246,20 @@ fn mul_mont_const_mod_u16x8_avx2(a: __m128i, b_mont: i16) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_s16x8_avx2(low: __m128i, high: __m128i) -> __m128i { - let k = _mm_mullo_epi16(low, _mm_set1_epi16(Q_MONT_INV_U16 as i16)); + let k = _mm_mullo_epi16(low, _mm_set1_epi16(Q_MONT_INV_U16.cast_signed())); let c = _mm_mulhi_epi16(k, _mm_set1_epi16(Q_I16)); _mm_sub_epi16(high, c) } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_s16x8_avx2(value: __m128i) -> __m128i { let negative = _mm_cmpgt_epi16(_mm_setzero_si128(), value); _mm_add_epi16(value, _mm_and_si128(negative, _mm_set1_epi16(Q_I16))) @@ -6221,6 +6267,9 @@ fn signed_to_mod_q_s16x8_avx2(value: __m128i) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { let sum = _mm_add_epi16(a, b); let ge_q = _mm_cmpgt_epi16(sum, _mm_set1_epi16(Q_I16 - 1)); @@ -6229,6 +6278,9 @@ fn add_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn sub_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { let diff = _mm_sub_epi16(a, b); let borrowed = _mm_cmpgt_epi16(b, a); @@ -6237,17 +6289,27 @@ fn sub_mod_u16x8_avx2(a: __m128i, b: __m128i) -> __m128i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read eight initialized +/// `i16` values. fn load_i16x8_as_i32x8_avx2(ptr: *const i16) -> __m256i { // SAFETY: unaligned 8-coefficient AVX2 input load because: // 1. The caller proves `ptr..ptr + 8` is readable. // 2. `_mm_loadu_si128` accepts arbitrary alignment. // 3. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`. - let packed = unsafe { _mm_loadu_si128(ptr.cast::<__m128i>()) }; + // SAFETY: the caller proves `ptr..ptr + 8` is readable. + let lanes = unsafe { ptr.cast::<[i16; 8]>().read_unaligned() }; + // SAFETY: `[i16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + let packed = unsafe { core::mem::transmute::<[i16; 8], __m128i>(lanes) }; _mm256_cvtepi16_epi32(packed) } #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { let k = _mm256_mullo_epi16(value, _mm256_set1_epi32(i32::from(Q_MONT_INV_U16))); let c = _mm256_mulhi_epi16(k, _mm256_set1_epi32(Q_I32)); @@ -6258,6 +6320,9 @@ fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_i32x8_avx2(value: __m256i) -> __m256i { let negative = _mm256_cmpgt_epi32(_mm256_setzero_si256(), value); _mm256_add_epi32(value, _mm256_and_si256(negative, set1_u32x8_avx2(Q_U32))) @@ -6265,6 +6330,9 @@ fn signed_to_mod_q_i32x8_avx2(value: __m256i) -> __m256i { #[cfg(all(target_arch = "x86_64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u32x8_avx2(a: __m256i, b: __m256i) -> __m256i { let sum = _mm256_add_epi32(a, b); let q = set1_u32x8_avx2(Q_U32); @@ -6284,10 +6352,10 @@ fn base_case_multiply(a0: u16, a1: u16, b0: u16, b1: u16, gamma_mont: i16) -> (u let a0b0 = mul_i32_secret(i32::from(a0), i32::from(b0)); let a1b1 = montgomery_reduce_i32(mul_i32_secret(i32::from(a1), i32::from(b1))); let c0 = signed_to_mod_q(montgomery_reduce_i32( - a0b0 + mul_i32_secret(i32::from(a1b1), i32::from(gamma_mont)), + a0b0.strict_add(mul_i32_secret(i32::from(a1b1), i32::from(gamma_mont))), )); let c1 = signed_to_mod_q(montgomery_reduce_i32( - mul_i32_secret(i32::from(a0), i32::from(b1)) + mul_i32_secret(i32::from(a1), i32::from(b0)), + mul_i32_secret(i32::from(a0), i32::from(b1)).strict_add(mul_i32_secret(i32::from(a1), i32::from(b0))), )); (c0, c1) } @@ -6439,8 +6507,8 @@ fn decompress_poly_add_assign(input: &Poly, out: &mut Poly) { #[inline] fn compress_value(value: u16) -> u16 { - let numerator = (u32::from(value) << D) + Q_HALF; - (div_q_compress_u32(numerator) & ((1u32 << D) - 1)) as u16 + let numerator = (u32::from(value) << D).strict_add(Q_HALF); + low_u16(div_q_compress_u32(numerator) & (1u32 << D).strict_sub(1)) } #[inline(always)] @@ -6493,6 +6561,9 @@ fn compress_values_4(values: [u16; 4]) -> [u16; 4] { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn compress_values_4_neon(values: [u16; 4]) -> [u16; 4] { debug_assert!(matches!(D, 1 | 4 | 5 | 10 | 11)); @@ -6514,18 +6585,25 @@ fn compress_values_4_neon(values: [u16; 4]) -> [u16; 4] { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn compress_lanes_4_neon(lanes: uint16x4_t) -> uint16x4_t { debug_assert!(matches!(D, 1 | 4 | 5 | 10 | 11)); let numerators = vaddq_u32(vshll_n_u16::(lanes), vdupq_n_u32(Q_HALF)); - let recip = vdup_n_u32(Q_COMPRESS_DIV_RECIP as u32); + let recip = vdup_n_u32(Q_COMPRESS_DIV_RECIP); let lo = vshrq_n_u64::(vmull_u32(vget_low_u32(numerators), recip)); let hi = vshrq_n_u64::(vmull_u32(vget_high_u32(numerators), recip)); let quotients = vcombine_u32(vmovn_u64(lo), vmovn_u64(hi)); - vmovn_u32(vandq_u32(quotients, vdupq_n_u32((1u32 << (D as u32)) - 1))) + let mask = 1u32.strict_shl(D.cast_unsigned()).strict_sub(1); + vmovn_u32(vandq_u32(quotients, vdupq_n_u32(mask))) } #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn load_compress_lanes_4_neon(input: &Poly, offset: usize) -> uint16x4_t { debug_assert!(offset.strict_add(4) <= N); @@ -6540,7 +6618,7 @@ fn load_compress_lanes_4_neon(input: &Poly, offset: usize) -> uint #[inline] fn decompress_value(value: u16) -> u16 { - ((mul_u32_secret(Q_U32, u32::from(value)) + (1u32 << (D - 1))) >> D) as u16 + low_u16(mul_u32_secret(Q_U32, u32::from(value)).strict_add(1u32 << D.strict_sub(1)) >> D) } #[inline(always)] @@ -6592,7 +6670,11 @@ fn decompress_message_add_assign(input: &[u8; SEED_BYTES], out: &mut Poly) { } fn compress_encode_poly(input: &Poly, out: &mut [u8]) { - debug_assert_eq!(out.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11), + "unsupported ML-KEM fused compress/encode width" + ); + debug_assert_eq!(out.len(), 32usize.strict_mul(D)); match D { 1 => compress_encode_1(input, out), @@ -6600,12 +6682,16 @@ fn compress_encode_poly(input: &Poly, out: &mut [u8]) { 5 => compress_encode_5(input, out), 10 => compress_encode_10(input, out), 11 => compress_encode_11(input, out), - _ => unreachable!("unsupported ML-KEM fused compress/encode width"), + _ => {} } } fn compress_encode_compare_poly(input: &Poly, expected: &[u8]) -> u8 { - debug_assert_eq!(BYTES, 32 * D); + assert!( + matches!(D, 4 | 5 | 10 | 11), + "unsupported ML-KEM fused compress/encode compare width" + ); + debug_assert_eq!(BYTES, 32usize.strict_mul(D)); debug_assert_eq!(expected.len(), BYTES); match D { @@ -6613,7 +6699,7 @@ fn compress_encode_compare_poly(input: &Poly 5 => compress_encode_compare_5(input, expected), 10 => compress_encode_compare_10(input, expected), 11 => compress_encode_compare_11(input, expected), - _ => unreachable!("unsupported ML-KEM fused compress/encode compare width"), + _ => 0, } } @@ -6625,7 +6711,7 @@ fn ct_zero_mask_u8(value: u8) -> u8 { #[inline] fn ct_zero_mask_u64(value: u64) -> u8 { - let nonzero = ((value | value.wrapping_neg()) >> 63) as u8; + let nonzero = ((value | value.wrapping_neg()) >> 63).to_le_bytes()[0]; 0u8.wrapping_sub(nonzero ^ 1) } @@ -6641,8 +6727,8 @@ fn compress_encode_compare_4(input: &Poly, expected: &[u8]) -> u8 { input[j.strict_add(2)], input[j.strict_add(3)], ]); - diff |= ((t[0] | (t[1] << 4)) as u8) ^ expected[k]; - diff |= ((t[2] | (t[3] << 4)) as u8) ^ expected[k.strict_add(1)]; + diff |= low_byte(t[0] | (t[1] << 4)) ^ expected[k]; + diff |= low_byte(t[2] | (t[3] << 4)) ^ expected[k.strict_add(1)]; } ct_zero_mask_u8(diff) } @@ -6668,11 +6754,11 @@ fn compress_encode_compare_5(input: &Poly, expected: &[u8]) -> u8 { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - diff |= ((t0 | (t1 << 5)) as u8) ^ expected[k]; - diff |= (((t1 >> 3) | (t2 << 2) | (t3 << 7)) as u8) ^ expected[k.strict_add(1)]; - diff |= (((t3 >> 1) | (t4 << 4)) as u8) ^ expected[k.strict_add(2)]; - diff |= (((t4 >> 4) | (t5 << 1) | (t6 << 6)) as u8) ^ expected[k.strict_add(3)]; - diff |= (((t6 >> 2) | (t7 << 3)) as u8) ^ expected[k.strict_add(4)]; + diff |= low_byte(t0 | (t1 << 5)) ^ expected[k]; + diff |= low_byte((t1 >> 3) | (t2 << 2) | (t3 << 7)) ^ expected[k.strict_add(1)]; + diff |= low_byte((t3 >> 1) | (t4 << 4)) ^ expected[k.strict_add(2)]; + diff |= low_byte((t4 >> 4) | (t5 << 1) | (t6 << 6)) ^ expected[k.strict_add(3)]; + diff |= low_byte((t6 >> 2) | (t7 << 3)) ^ expected[k.strict_add(4)]; } ct_zero_mask_u8(diff) } @@ -6703,11 +6789,11 @@ fn compress_encode_compare_10(input: &Poly, expected: &[u8]) -> u8 { input[j.strict_add(3)], ]); - diff |= (t0 as u8) ^ expected[k]; - diff |= (((t0 >> 8) | (t1 << 2)) as u8) ^ expected[k.strict_add(1)]; - diff |= (((t1 >> 6) | (t2 << 4)) as u8) ^ expected[k.strict_add(2)]; - diff |= (((t2 >> 4) | (t3 << 6)) as u8) ^ expected[k.strict_add(3)]; - diff |= ((t3 >> 2) as u8) ^ expected[k.strict_add(4)]; + diff |= low_byte(t0) ^ expected[k]; + diff |= low_byte((t0 >> 8) | (t1 << 2)) ^ expected[k.strict_add(1)]; + diff |= low_byte((t1 >> 6) | (t2 << 4)) ^ expected[k.strict_add(2)]; + diff |= low_byte((t2 >> 4) | (t3 << 6)) ^ expected[k.strict_add(3)]; + diff |= low_byte(t3 >> 2) ^ expected[k.strict_add(4)]; } ct_zero_mask_u8(diff) } @@ -6734,17 +6820,17 @@ fn compress_encode_compare_11(input: &Poly, expected: &[u8]) -> u8 { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - let encoded_lo = u64::from(t0 as u8) - | (u64::from(((t0 >> 8) | (t1 << 3)) as u8) << 8) - | (u64::from(((t1 >> 5) | (t2 << 6)) as u8) << 16) - | (u64::from((t2 >> 2) as u8) << 24) - | (u64::from(((t2 >> 10) | (t3 << 1)) as u8) << 32) - | (u64::from(((t3 >> 7) | (t4 << 4)) as u8) << 40) - | (u64::from(((t4 >> 4) | (t5 << 7)) as u8) << 48) - | (u64::from((t5 >> 1) as u8) << 56); - let encoded_hi = u32::from(((t5 >> 9) | (t6 << 2)) as u8) - | (u32::from(((t6 >> 6) | (t7 << 5)) as u8) << 8) - | (u32::from((t7 >> 3) as u8) << 16); + let encoded_lo = u64::from(low_byte(t0)) + | (u64::from(low_byte((t0 >> 8) | (t1 << 3))) << 8) + | (u64::from(low_byte((t1 >> 5) | (t2 << 6))) << 16) + | (u64::from(low_byte(t2 >> 2)) << 24) + | (u64::from(low_byte((t2 >> 10) | (t3 << 1))) << 32) + | (u64::from(low_byte((t3 >> 7) | (t4 << 4))) << 40) + | (u64::from(low_byte((t4 >> 4) | (t5 << 7))) << 48) + | (u64::from(low_byte(t5 >> 1)) << 56); + let encoded_hi = u32::from(low_byte((t5 >> 9) | (t6 << 2))) + | (u32::from(low_byte((t6 >> 6) | (t7 << 5))) << 8) + | (u32::from(low_byte(t7 >> 3)) << 16); let expected_lo = u64::from_le_bytes([ expected[k], @@ -6768,6 +6854,9 @@ fn compress_encode_compare_11(input: &Poly, expected: &[u8]) -> u8 { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[target_feature(enable = "neon")] +/// # Safety +/// +/// The caller must ensure the active CPU supports AArch64 NEON and preserve the ML-KEM coefficient-domain invariants of the calling transform. fn compress_encode_compare_10_neon(input: &Poly, expected: &[u8]) -> u8 { let mut diff = 0u8; for i in 0usize..64 { @@ -6779,17 +6868,21 @@ fn compress_encode_compare_10_neon(input: &Poly, expected: &[u8]) -> u8 { let t2 = vget_lane_u16::<2>(t); let t3 = vget_lane_u16::<3>(t); - diff |= (t0 as u8) ^ expected[k]; - diff |= (((t0 >> 8) | (t1 << 2)) as u8) ^ expected[k.strict_add(1)]; - diff |= (((t1 >> 6) | (t2 << 4)) as u8) ^ expected[k.strict_add(2)]; - diff |= (((t2 >> 4) | (t3 << 6)) as u8) ^ expected[k.strict_add(3)]; - diff |= ((t3 >> 2) as u8) ^ expected[k.strict_add(4)]; + diff |= low_byte(t0) ^ expected[k]; + diff |= low_byte((t0 >> 8) | (t1 << 2)) ^ expected[k.strict_add(1)]; + diff |= low_byte((t1 >> 6) | (t2 << 4)) ^ expected[k.strict_add(2)]; + diff |= low_byte((t2 >> 4) | (t3 << 6)) ^ expected[k.strict_add(3)]; + diff |= low_byte(t3 >> 2) ^ expected[k.strict_add(4)]; } ct_zero_mask_u8(diff) } fn decode_decompress_poly(input: &[u8], out: &mut Poly) { - debug_assert_eq!(input.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11), + "unsupported ML-KEM fused decode/decompress width" + ); + debug_assert_eq!(input.len(), 32usize.strict_mul(D)); match D { 1 => decode_decompress_1(input, out), @@ -6797,12 +6890,16 @@ fn decode_decompress_poly(input: &[u8], out: &mut Poly) { 5 => decode_decompress_5(input, out), 10 => decode_decompress_10(input, out), 11 => decode_decompress_11(input, out), - _ => unreachable!("unsupported ML-KEM fused decode/decompress width"), + _ => {} } } fn byte_encode(input: &Poly, out: &mut [u8]) { - debug_assert_eq!(out.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11 | 12), + "unsupported ML-KEM byte encoding width" + ); + debug_assert_eq!(out.len(), 32usize.strict_mul(D)); match D { 1 => byte_encode_1(input, out), @@ -6811,12 +6908,16 @@ fn byte_encode(input: &Poly, out: &mut [u8]) { 10 => byte_encode_10(input, out), 11 => byte_encode_11(input, out), 12 => byte_encode_12(input, out), - _ => unreachable!("unsupported ML-KEM byte encoding width"), + _ => {} } } fn byte_decode(input: &[u8], out: &mut Poly) { - debug_assert_eq!(input.len(), 32 * D); + assert!( + matches!(D, 1 | 4 | 5 | 10 | 11 | 12), + "unsupported ML-KEM byte decoding width" + ); + debug_assert_eq!(input.len(), 32usize.strict_mul(D)); match D { 1 => byte_decode_1(input, out), @@ -6825,7 +6926,7 @@ fn byte_decode(input: &[u8], out: &mut Poly) { 10 => byte_decode_10(input, out), 11 => byte_decode_11(input, out), 12 => byte_decode_12(input, out), - _ => unreachable!("unsupported ML-KEM byte decoding width"), + _ => {} } } @@ -6844,14 +6945,14 @@ fn compress_encode_1(input: &Poly, out: &mut [u8]) { input[start.strict_add(6)], input[start.strict_add(7)], ]); - *byte = (lo[0] as u8 & 1) - | ((lo[1] as u8 & 1) << 1) - | ((lo[2] as u8 & 1) << 2) - | ((lo[3] as u8 & 1) << 3) - | ((hi[0] as u8 & 1) << 4) - | ((hi[1] as u8 & 1) << 5) - | ((hi[2] as u8 & 1) << 6) - | ((hi[3] as u8 & 1) << 7); + *byte = (low_byte(lo[0]) & 1) + | ((low_byte(lo[1]) & 1) << 1) + | ((low_byte(lo[2]) & 1) << 2) + | ((low_byte(lo[3]) & 1) << 3) + | ((low_byte(hi[0]) & 1) << 4) + | ((low_byte(hi[1]) & 1) << 5) + | ((low_byte(hi[2]) & 1) << 6) + | ((low_byte(hi[3]) & 1) << 7); } } @@ -6870,14 +6971,14 @@ fn subtract_compress_encode_message(lhs: &Poly, rhs: &Poly, out: &mut [u8; SEED_ sub_mod(lhs[start.strict_add(6)], rhs[start.strict_add(6)]), sub_mod(lhs[start.strict_add(7)], rhs[start.strict_add(7)]), ]); - *byte = (lo[0] as u8 & 1) - | ((lo[1] as u8 & 1) << 1) - | ((lo[2] as u8 & 1) << 2) - | ((lo[3] as u8 & 1) << 3) - | ((hi[0] as u8 & 1) << 4) - | ((hi[1] as u8 & 1) << 5) - | ((hi[2] as u8 & 1) << 6) - | ((hi[3] as u8 & 1) << 7); + *byte = (low_byte(lo[0]) & 1) + | ((low_byte(lo[1]) & 1) << 1) + | ((low_byte(lo[2]) & 1) << 2) + | ((low_byte(lo[3]) & 1) << 3) + | ((low_byte(hi[0]) & 1) << 4) + | ((low_byte(hi[1]) & 1) << 5) + | ((low_byte(hi[2]) & 1) << 6) + | ((low_byte(hi[3]) & 1) << 7); } } @@ -6912,7 +7013,7 @@ fn byte_encode_1(input: &Poly, out: &mut [u8]) { let start = i.strict_mul(8); let mut packed = 0u8; for bit in 0..8 { - packed |= ((input[start.strict_add(bit)] & 1) as u8) << bit; + packed |= low_byte(input[start.strict_add(bit)] & 1) << bit; } *byte = packed; } @@ -6937,8 +7038,8 @@ fn compress_encode_4(input: &Poly, out: &mut [u8]) { input[j.strict_add(2)], input[j.strict_add(3)], ]); - out[k] = (t[0] | (t[1] << 4)) as u8; - out[k.strict_add(1)] = (t[2] | (t[3] << 4)) as u8; + out[k] = low_byte(t[0] | (t[1] << 4)); + out[k.strict_add(1)] = low_byte(t[2] | (t[3] << 4)); } } @@ -6964,7 +7065,7 @@ fn decode_decompress_4(input: &[u8], out: &mut Poly) { fn byte_encode_4(input: &Poly, out: &mut [u8]) { for (i, byte) in out.iter_mut().enumerate() { let j = i.strict_mul(2); - *byte = ((input[j] & 0x0f) | ((input[j.strict_add(1)] & 0x0f) << 4)) as u8; + *byte = low_byte((input[j] & 0x0f) | ((input[j.strict_add(1)] & 0x0f) << 4)); } } @@ -6995,11 +7096,11 @@ fn compress_encode_5(input: &Poly, out: &mut [u8]) { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - out[k] = (t0 | (t1 << 5)) as u8; - out[k.strict_add(1)] = ((t1 >> 3) | (t2 << 2) | (t3 << 7)) as u8; - out[k.strict_add(2)] = ((t3 >> 1) | (t4 << 4)) as u8; - out[k.strict_add(3)] = ((t4 >> 4) | (t5 << 1) | (t6 << 6)) as u8; - out[k.strict_add(4)] = ((t6 >> 2) | (t7 << 3)) as u8; + out[k] = low_byte(t0 | (t1 << 5)); + out[k.strict_add(1)] = low_byte((t1 >> 3) | (t2 << 2) | (t3 << 7)); + out[k.strict_add(2)] = low_byte((t3 >> 1) | (t4 << 4)); + out[k.strict_add(3)] = low_byte((t4 >> 4) | (t5 << 1) | (t6 << 6)); + out[k.strict_add(4)] = low_byte((t6 >> 2) | (t7 << 3)); } } @@ -7049,11 +7150,11 @@ fn byte_encode_5(input: &Poly, out: &mut [u8]) { let t6 = input[j.strict_add(6)] & 0x001f; let t7 = input[j.strict_add(7)] & 0x001f; - out[k] = (t0 | (t1 << 5)) as u8; - out[k.strict_add(1)] = ((t1 >> 3) | (t2 << 2) | (t3 << 7)) as u8; - out[k.strict_add(2)] = ((t3 >> 1) | (t4 << 4)) as u8; - out[k.strict_add(3)] = ((t4 >> 4) | (t5 << 1) | (t6 << 6)) as u8; - out[k.strict_add(4)] = ((t6 >> 2) | (t7 << 3)) as u8; + out[k] = low_byte(t0 | (t1 << 5)); + out[k.strict_add(1)] = low_byte((t1 >> 3) | (t2 << 2) | (t3 << 7)); + out[k.strict_add(2)] = low_byte((t3 >> 1) | (t4 << 4)); + out[k.strict_add(3)] = low_byte((t4 >> 4) | (t5 << 1) | (t6 << 6)); + out[k.strict_add(4)] = low_byte((t6 >> 2) | (t7 << 3)); } } @@ -7089,11 +7190,11 @@ fn compress_encode_10(input: &Poly, out: &mut [u8]) { input[j.strict_add(3)], ]); - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 2)) as u8; - out[k.strict_add(2)] = ((t1 >> 6) | (t2 << 4)) as u8; - out[k.strict_add(3)] = ((t2 >> 4) | (t3 << 6)) as u8; - out[k.strict_add(4)] = (t3 >> 2) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 2)); + out[k.strict_add(2)] = low_byte((t1 >> 6) | (t2 << 4)); + out[k.strict_add(3)] = low_byte((t2 >> 4) | (t3 << 6)); + out[k.strict_add(4)] = low_byte(t3 >> 2); } } @@ -7129,11 +7230,11 @@ fn byte_encode_10(input: &Poly, out: &mut [u8]) { let t2 = input[j.strict_add(2)] & 0x03ff; let t3 = input[j.strict_add(3)] & 0x03ff; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 2)) as u8; - out[k.strict_add(2)] = ((t1 >> 6) | (t2 << 4)) as u8; - out[k.strict_add(3)] = ((t2 >> 4) | (t3 << 6)) as u8; - out[k.strict_add(4)] = (t3 >> 2) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 2)); + out[k.strict_add(2)] = low_byte((t1 >> 6) | (t2 << 4)); + out[k.strict_add(3)] = low_byte((t2 >> 4) | (t3 << 6)); + out[k.strict_add(4)] = low_byte(t3 >> 2); } } @@ -7173,17 +7274,17 @@ fn compress_encode_11(input: &Poly, out: &mut [u8]) { let [t0, t1, t2, t3] = lo; let [t4, t5, t6, t7] = hi; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 3)) as u8; - out[k.strict_add(2)] = ((t1 >> 5) | (t2 << 6)) as u8; - out[k.strict_add(3)] = (t2 >> 2) as u8; - out[k.strict_add(4)] = ((t2 >> 10) | (t3 << 1)) as u8; - out[k.strict_add(5)] = ((t3 >> 7) | (t4 << 4)) as u8; - out[k.strict_add(6)] = ((t4 >> 4) | (t5 << 7)) as u8; - out[k.strict_add(7)] = (t5 >> 1) as u8; - out[k.strict_add(8)] = ((t5 >> 9) | (t6 << 2)) as u8; - out[k.strict_add(9)] = ((t6 >> 6) | (t7 << 5)) as u8; - out[k.strict_add(10)] = (t7 >> 3) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 3)); + out[k.strict_add(2)] = low_byte((t1 >> 5) | (t2 << 6)); + out[k.strict_add(3)] = low_byte(t2 >> 2); + out[k.strict_add(4)] = low_byte((t2 >> 10) | (t3 << 1)); + out[k.strict_add(5)] = low_byte((t3 >> 7) | (t4 << 4)); + out[k.strict_add(6)] = low_byte((t4 >> 4) | (t5 << 7)); + out[k.strict_add(7)] = low_byte(t5 >> 1); + out[k.strict_add(8)] = low_byte((t5 >> 9) | (t6 << 2)); + out[k.strict_add(9)] = low_byte((t6 >> 6) | (t7 << 5)); + out[k.strict_add(10)] = low_byte(t7 >> 3); } } @@ -7239,17 +7340,17 @@ fn byte_encode_11(input: &Poly, out: &mut [u8]) { let t6 = input[j.strict_add(6)] & 0x07ff; let t7 = input[j.strict_add(7)] & 0x07ff; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 3)) as u8; - out[k.strict_add(2)] = ((t1 >> 5) | (t2 << 6)) as u8; - out[k.strict_add(3)] = (t2 >> 2) as u8; - out[k.strict_add(4)] = ((t2 >> 10) | (t3 << 1)) as u8; - out[k.strict_add(5)] = ((t3 >> 7) | (t4 << 4)) as u8; - out[k.strict_add(6)] = ((t4 >> 4) | (t5 << 7)) as u8; - out[k.strict_add(7)] = (t5 >> 1) as u8; - out[k.strict_add(8)] = ((t5 >> 9) | (t6 << 2)) as u8; - out[k.strict_add(9)] = ((t6 >> 6) | (t7 << 5)) as u8; - out[k.strict_add(10)] = (t7 >> 3) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 3)); + out[k.strict_add(2)] = low_byte((t1 >> 5) | (t2 << 6)); + out[k.strict_add(3)] = low_byte(t2 >> 2); + out[k.strict_add(4)] = low_byte((t2 >> 10) | (t3 << 1)); + out[k.strict_add(5)] = low_byte((t3 >> 7) | (t4 << 4)); + out[k.strict_add(6)] = low_byte((t4 >> 4) | (t5 << 7)); + out[k.strict_add(7)] = low_byte(t5 >> 1); + out[k.strict_add(8)] = low_byte((t5 >> 9) | (t6 << 2)); + out[k.strict_add(9)] = low_byte((t6 >> 6) | (t7 << 5)); + out[k.strict_add(10)] = low_byte(t7 >> 3); } } @@ -7287,9 +7388,9 @@ fn byte_encode_12(input: &Poly, out: &mut [u8]) { let t0 = input[j]; let t1 = input[j.strict_add(1)]; - out[k] = t0 as u8; - out[k.strict_add(1)] = ((t0 >> 8) | (t1 << 4)) as u8; - out[k.strict_add(2)] = (t1 >> 4) as u8; + out[k] = low_byte(t0); + out[k.strict_add(1)] = low_byte((t0 >> 8) | (t1 << 4)); + out[k.strict_add(2)] = low_byte(t1 >> 4); } } @@ -7308,27 +7409,27 @@ fn byte_decode_12(input: &[u8], out: &mut Poly) { #[inline] fn add_mod(a: u16, b: u16) -> u16 { - let sum = u32::from(a) + u32::from(b); + let sum = u32::from(a).strict_add(u32::from(b)); let reduced = sum.wrapping_sub(Q_U32); - add_q_if_borrowed(reduced) as u16 + low_u16(add_q_if_borrowed(reduced)) } #[inline] fn sub_mod(a: u16, b: u16) -> u16 { let diff = u32::from(a).wrapping_sub(u32::from(b)); - add_q_if_borrowed(diff) as u16 + low_u16(add_q_if_borrowed(diff)) } #[inline] fn sub_if_ge_q(value: u16) -> u16 { let reduced = u32::from(value).wrapping_sub(Q_U32); - add_q_if_borrowed(reduced) as u16 + low_u16(add_q_if_borrowed(reduced)) } #[inline] #[cfg(test)] fn mul_mod(a: u16, b: u16) -> u16 { - reduce_u32(u32::from(a) * u32::from(b)) + reduce_u32(u32::from(a).strict_mul(u32::from(b))) } #[inline] @@ -7376,9 +7477,16 @@ fn from_montgomery_product_domain(value: u16) -> u16 { not(any(target_arch = "aarch64", target_arch = "x86_64")) ))] fn montgomery_reduce_i32(value: i32) -> i16 { - let k = mul_i32_secret(i32::from(value as i16), i32::from(Q_MONT_INV_U16 as i16)); - let c = (mul_i32_secret(i32::from(k as i16), Q_I32) >> 16) as i16; - ((value >> 16) as i16).wrapping_sub(c) + #[inline(always)] + fn low_i16(value: i32) -> i16 { + let [b0, b1, _, _] = value.to_le_bytes(); + i16::from_le_bytes([b0, b1]) + } + + let mont_inverse = i16::from_ne_bytes(Q_MONT_INV_U16.to_ne_bytes()); + let k = mul_i32_secret(i32::from(low_i16(value)), i32::from(mont_inverse)); + let c = low_i16(mul_i32_secret(i32::from(low_i16(k)), Q_I32) >> 16); + low_i16(value >> 16).wrapping_sub(c) } #[inline] @@ -7391,20 +7499,21 @@ fn montgomery_reduce_i32(value: i32) -> i16 { ))] fn signed_to_mod_q(value: i16) -> u16 { let value = i32::from(value); - (value + ((value >> 31) & Q_I32)) as u16 + let reduced = value.strict_add((value >> 31) & Q_I32); + low_u16(u32::from_ne_bytes(reduced.to_ne_bytes())) } #[inline] #[cfg(test)] fn reduce_u32(value: u32) -> u16 { let quotient = div_q_u32(value); - value.wrapping_sub(quotient * Q_U32) as u16 + low_u16(value.wrapping_sub(quotient.strict_mul(Q_U32))) } #[inline] #[cfg(test)] fn div_q_u32(value: u32) -> u32 { - ((u64::from(value) * Q_DIV_RECIP) >> Q_DIV_SHIFT) as u32 + low_u32(u64::from(value).strict_mul(Q_DIV_RECIP) >> Q_DIV_SHIFT) } #[inline] @@ -7415,12 +7524,15 @@ fn div_q_compress_u32(value: u32) -> u32 { } #[cfg(not(target_arch = "s390x"))] { - ((u64::from(value) * Q_COMPRESS_DIV_RECIP) >> Q_COMPRESS_DIV_SHIFT) as u32 + low_u32(u64::from(value).strict_mul(u64::from(Q_COMPRESS_DIV_RECIP)) >> Q_COMPRESS_DIV_SHIFT) } } #[inline] fn mul_u32_secret(a: u32, b: u32) -> u32 { + debug_assert!(u16::try_from(a).is_ok()); + debug_assert!(u16::try_from(b).is_ok()); + #[cfg(target_arch = "s390x")] { // IBM Z integer multiply latency is operand-dependent; keep secret-fed products on a fixed @@ -7429,7 +7541,7 @@ fn mul_u32_secret(a: u32, b: u32) -> u32 { } #[cfg(not(target_arch = "s390x"))] { - a * b + a.wrapping_mul(b) } } @@ -7442,13 +7554,16 @@ fn mul_u32_secret(a: u32, b: u32) -> u32 { not(any(target_arch = "aarch64", target_arch = "x86_64")) ))] fn mul_i32_secret(a: i32, b: i32) -> i32 { + debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&a)); + debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&b)); + #[cfg(target_arch = "s390x")] { mul_i32_16_ct(a, b) } #[cfg(not(target_arch = "s390x"))] { - a * b + a.wrapping_mul(b) } } @@ -7456,15 +7571,15 @@ fn mul_i32_secret(a: i32, b: i32) -> i32 { #[cfg_attr(not(target_arch = "s390x"), inline)] #[cfg(any(test, target_arch = "s390x"))] fn mul_u32_16_ct(a: u32, b: u32) -> u32 { - debug_assert!(a <= u32::from(u16::MAX)); - debug_assert!(b <= u32::from(u16::MAX)); + debug_assert!(u16::try_from(a).is_ok()); + debug_assert!(u16::try_from(b).is_ok()); let mut acc = 0u32; let mut bit = 0u32; while bit < 16 { let mask = 0u32.wrapping_sub((b >> bit) & 1); acc = acc.wrapping_add((a << bit) & mask); - bit += 1; + bit = bit.strict_add(1); } acc } @@ -7476,13 +7591,15 @@ fn mul_i32_16_ct(a: i32, b: i32) -> i32 { debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&a)); debug_assert!((i32::from(i16::MIN)..=i32::from(i16::MAX)).contains(&b)); - let a_sign = (a >> 31) as u32; - let b_sign = (b >> 31) as u32; - let abs_a = ((a as u32) ^ a_sign).wrapping_sub(a_sign); - let abs_b = ((b as u32) ^ b_sign).wrapping_sub(b_sign); + let a_bits = u32::from_ne_bytes(a.to_ne_bytes()); + let b_bits = u32::from_ne_bytes(b.to_ne_bytes()); + let a_sign = 0u32.wrapping_sub(a_bits >> 31); + let b_sign = 0u32.wrapping_sub(b_bits >> 31); + let abs_a = (a_bits ^ a_sign).wrapping_sub(a_sign); + let abs_b = (b_bits ^ b_sign).wrapping_sub(b_sign); let magnitude = mul_u32_16_ct(abs_a, abs_b); let sign = a_sign ^ b_sign; - ((magnitude ^ sign).wrapping_sub(sign)) as i32 + i32::from_ne_bytes((magnitude ^ sign).wrapping_sub(sign).to_ne_bytes()) } #[cfg_attr(target_arch = "s390x", inline(never))] @@ -7495,7 +7612,7 @@ fn div_q_compress_u32_ct(value: u32) -> u32 { let mut remainder = 0u32; let mut bit = 23u32; while bit > 0 { - bit -= 1; + bit = bit.strict_sub(1); remainder = (remainder << 1) | ((value >> bit) & 1); let reduced = remainder.wrapping_sub(Q_U32); let borrow = reduced >> 31; @@ -7529,13 +7646,14 @@ fn opaque_s390x_bit(value: u32) -> u32 { #[inline] #[cfg(test)] fn div_q_compress_u32_recip(value: u32) -> u32 { - ((u64::from(value) * Q_COMPRESS_DIV_RECIP) >> Q_COMPRESS_DIV_SHIFT) as u32 + low_u32(u64::from(value).strict_mul(u64::from(Q_COMPRESS_DIV_RECIP)) >> Q_COMPRESS_DIV_SHIFT) } #[inline] fn small_signed_to_mod_q(value: i16) -> u16 { let value = i32::from(value); - (value + ((value >> 31) & i32::from(Q))) as u16 + let reduced = value.strict_add((value >> 31) & i32::from(Q)); + low_u16(u32::from_ne_bytes(reduced.to_ne_bytes())) } fn h(input: &[u8]) -> [u8; HASH_BYTES] { @@ -7615,11 +7733,11 @@ mod tests { fn miri_mlkem512_portable_round_trip_and_rejection() { let mut key_random = [0u8; MlKem512::KEY_GENERATION_RANDOM_SIZE]; for (i, byte) in key_random.iter_mut().enumerate() { - *byte = (i.strict_mul(29).strict_add(7)) as u8; + *byte = test_low_byte(i.strict_mul(29).strict_add(7)); } let mut encapsulation_random = [0u8; MlKem512::ENCAPSULATION_RANDOM_SIZE]; for (i, byte) in encapsulation_random.iter_mut().enumerate() { - *byte = (i.strict_mul(31).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(31).strict_add(11)); } let (encapsulation_key, decapsulation_key) = MlKem512::generate_keypair(|out| { @@ -7646,7 +7764,7 @@ mod tests { fn ntt_round_trip_preserves_polynomial() { let mut poly = [0u16; N]; for (i, coeff) in poly.iter_mut().enumerate() { - *coeff = ((i.strict_mul(17).strict_add(91)) as u16) % Q; + *coeff = test_u16(i.strict_mul(17).strict_add(91)) % Q; } let original = poly; @@ -7781,22 +7899,22 @@ mod tests { #[cfg(all(target_arch = "aarch64", not(miri), not(feature = "portable-only")))] #[test] fn lazy_ntt_neon_finalizers_match_scalar_range() { - let min = -i32::from(Q) * 8; - let max = i32::from(Q) * 8; + let min = -i32::from(Q).strict_mul(8); + let max = i32::from(Q).strict_mul(8); let mut start = min; while start <= max { let lanes = [ start, - (start + 1).min(max), - (start + 2).min(max), - (start + 3).min(max), - (start + 4).min(max), - (start + 5).min(max), - (start + 6).min(max), - (start + 7).min(max), + start.strict_add(1).min(max), + start.strict_add(2).min(max), + start.strict_add(3).min(max), + start.strict_add(4).min(max), + start.strict_add(5).min(max), + start.strict_add(6).min(max), + start.strict_add(7).min(max), ]; - let lanes_i16 = lanes.map(|value| value as i16); + let lanes_i16 = lanes.map(|value| i16::try_from(value).expect("lazy NTT test lane fits in i16")); let mut canonical = [0u16; 8]; let mut product_domain = [0u16; 8]; @@ -7813,9 +7931,9 @@ mod tests { for lane in 0..8 { let mut expected = lanes[lane] % i32::from(Q); if expected < 0 { - expected += i32::from(Q); + expected = expected.strict_add(i32::from(Q)); } - let expected = expected as u16; + let expected = u16::try_from(expected).expect("canonical ML-KEM test coefficient fits in u16"); assert_eq!(canonical[lane], expected, "canonical start {start} lane {lane}"); assert_eq!( product_domain[lane], @@ -7824,7 +7942,7 @@ mod tests { ); } - start += 8; + start = start.strict_add(8); } } @@ -8012,7 +8130,7 @@ mod tests { fn sample_ntt_pair_matches_scalar_samplers() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(37).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(37).strict_add(11)); } let (left, right) = sample_ntt_pair(&rho, 0, 1, 2, 1); @@ -8025,7 +8143,7 @@ mod tests { fn sample_ntt_triple_matches_scalar_samplers() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(43).strict_add(17)) as u8; + *byte = test_low_byte(i.strict_mul(43).strict_add(17)); } let lanes = [(0, 0), (1, 2), (2, 1)]; @@ -8045,11 +8163,13 @@ mod tests { for seed in 0usize..64 { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (seed - .strict_mul(53) - .strict_add(i.strict_mul(47)) - .strict_add((seed >> 1).strict_mul(19)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(53) + .strict_add(i.strict_mul(47)) + .strict_add((seed >> 1).strict_mul(19)) + & 0xff, + ); } for j in 0u8..4 { @@ -8068,7 +8188,7 @@ mod tests { let mut actual = [0u16; N]; let blocks = sample_ntt_initial_3blocks_then_tail_into(&rho, j, i, &mut actual); assert!( - blocks >= SAMPLE_NTT_INITIAL_BLOCKS as u16, + blocks >= test_u16(SAMPLE_NTT_INITIAL_BLOCKS), "seed {seed}, lane ({j}, {i})" ); assert_eq!(actual, expected, "full sample seed {seed}, lane ({j}, {i})"); @@ -8090,11 +8210,13 @@ mod tests { for seed in 0usize..64 { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (seed - .strict_mul(59) - .strict_add(i.strict_mul(31)) - .strict_add((seed >> 2).strict_mul(23)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(59) + .strict_add(i.strict_mul(31)) + .strict_add((seed >> 2).strict_mul(23)) + & 0xff, + ); } for lanes in lane_sets { @@ -8164,11 +8286,13 @@ mod tests { for seed in 0usize..512 { let mut buf = [0u8; SHAKE128_RATE_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(109) - .strict_add(i.strict_mul(37)) - .strict_add((seed >> 2).strict_mul(41)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(109) + .strict_add(i.strict_mul(37)) + .strict_add((seed >> 2).strict_mul(41)) + & 0xff, + ); } for &start in &fill_offsets { @@ -8206,11 +8330,13 @@ mod tests { for seed in 0usize..512 { let mut buf = [0u8; SHAKE128_RATE_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(73) - .strict_add(i.strict_mul(29)) - .strict_add((seed >> 1).strict_mul(17)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(73) + .strict_add(i.strict_mul(29)) + .strict_add((seed >> 1).strict_mul(17)) + & 0xff, + ); } let mut expected = [0u16; MAX_CANDIDATES]; @@ -8242,12 +8368,14 @@ mod tests { let mut bufs = [[0u8; SHAKE128_RATE_BYTES]; 3]; for (lane, buf) in bufs.iter_mut().enumerate() { for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(97) - .strict_add(lane.strict_mul(43)) - .strict_add(i.strict_mul(31)) - .strict_add((seed >> 1).strict_mul(19)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(97) + .strict_add(lane.strict_mul(43)) + .strict_add(i.strict_mul(31)) + .strict_add((seed >> 1).strict_mul(19)) + & 0xff, + ); } } @@ -8297,11 +8425,13 @@ mod tests { for seed in 0usize..512 { let mut buf = [0u8; SAMPLE_NTT_INITIAL_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(83) - .strict_add(i.strict_mul(41)) - .strict_add((seed >> 1).strict_mul(37)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(83) + .strict_add(i.strict_mul(41)) + .strict_add((seed >> 1).strict_mul(37)) + & 0xff, + ); } let mut expected = [0u16; N]; @@ -8336,11 +8466,13 @@ mod tests { for seed in 0usize..256 { let mut buf = [0u8; SHAKE128_RATE_BYTES]; for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(91) - .strict_add(i.strict_mul(19)) - .strict_add((seed >> 2).strict_mul(23)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(91) + .strict_add(i.strict_mul(19)) + .strict_add((seed >> 2).strict_mul(23)) + & 0xff, + ); } for &start in &fill_offsets { @@ -8391,12 +8523,14 @@ mod tests { let mut bufs = [[0u8; SHAKE128_RATE_BYTES]; 3]; for (lane, buf) in bufs.iter_mut().enumerate() { for (i, byte) in buf.iter_mut().enumerate() { - *byte = (seed - .strict_mul(101) - .strict_add(lane.strict_mul(47)) - .strict_add(i.strict_mul(23)) - .strict_add((seed >> 2).strict_mul(29)) - & 0xff) as u8; + *byte = test_low_byte( + seed + .strict_mul(101) + .strict_add(lane.strict_mul(47)) + .strict_add(i.strict_mul(23)) + .strict_add((seed >> 2).strict_mul(29)) + & 0xff, + ); } } @@ -8452,7 +8586,7 @@ mod tests { fn sample_ntt_quad_matches_scalar_samplers() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(41).strict_add(13)) as u8; + *byte = test_low_byte(i.strict_mul(41).strict_add(13)); } let lanes = [(0, 0), (1, 3), (2, 1), (3, 2)]; @@ -8524,7 +8658,7 @@ mod tests { fn seeded_sample_ntt_matches_generic_xof_input() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(31).strict_add(7)) as u8; + *byte = test_low_byte(i.strict_mul(31).strict_add(7)); } for j in 0u8..4 { @@ -8543,7 +8677,7 @@ mod tests { fn seeded_prf_eta_matches_generic_xof_input() { let mut seed = [0u8; SEED_BYTES]; for (i, byte) in seed.iter_mut().enumerate() { - *byte = (i.strict_mul(43).strict_add(5)) as u8; + *byte = test_low_byte(i.strict_mul(43).strict_add(5)); } for nonce in 0u8..8 { @@ -8571,7 +8705,7 @@ mod tests { fn batched_sample_noise_pair_matches_scalar_sampling() { let mut seed = [0u8; SEED_BYTES]; for (i, byte) in seed.iter_mut().enumerate() { - *byte = (i.strict_mul(47).strict_add(13)) as u8; + *byte = test_low_byte(i.strict_mul(47).strict_add(13)); } let mut expected0 = [0u16; N]; @@ -8595,7 +8729,7 @@ mod tests { fn batched_sample_noise_quad_matches_scalar_sampling() { let mut seed = [0u8; SEED_BYTES]; for (i, byte) in seed.iter_mut().enumerate() { - *byte = (i.strict_mul(59).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(59).strict_add(19)); } let mut expected = [[0u16; N]; 4]; @@ -8604,14 +8738,14 @@ mod tests { sample_noise::(&seed, lane.strict_add(3), &mut expected[usize::from(lane)]); } let [actual0, actual1, actual2, actual3] = &mut actual; - sample_noise_quad::(&seed, 3, actual0, 4, actual1, 5, actual2, 6, actual3); + sample_noise_quad::(&seed, [3, 4, 5, 6], [actual0, actual1, actual2, actual3]); assert_eq!(actual, expected, "eta2 quad"); for lane in 0u8..4 { sample_noise::(&seed, lane.strict_add(8), &mut expected[usize::from(lane)]); } let [actual0, actual1, actual2, actual3] = &mut actual; - sample_noise_quad::(&seed, 8, actual0, 9, actual1, 10, actual2, 11, actual3); + sample_noise_quad::(&seed, [8, 9, 10, 11], [actual0, actual1, actual2, actual3]); assert_eq!(actual, expected, "eta3 quad"); } @@ -8619,15 +8753,15 @@ mod tests { fn fused_sample_ntt_accumulate_matches_sample_then_multiply() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(29).strict_add(17)) as u8; + *byte = test_low_byte(i.strict_mul(29).strict_add(17)); } for seed in 0usize..8 { let rhs = test_poly(seed.strict_add(100)); let base = test_poly(seed.strict_add(200)); let mut sampled = [0u16; N]; - let j = (seed % 4) as u8; - let i = ((seed.strict_mul(3)) % 4) as u8; + let j = test_u8(seed % 4); + let i = test_u8(seed.strict_mul(3) % 4); sample_ntt_into(&rho, j, i, &mut sampled); let mut expected = base; @@ -8644,17 +8778,17 @@ mod tests { fn fused_sample_ntt_pair_accumulate_matches_two_sampled_products() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(41).strict_add(23)) as u8; + *byte = test_low_byte(i.strict_mul(41).strict_add(23)); } for seed in 0usize..8 { let rhs0 = test_poly(seed.strict_add(300)); let rhs1 = test_poly(seed.strict_add(400)); let base = test_poly(seed.strict_add(500)); - let j0 = (seed % 4) as u8; - let i0 = ((seed.strict_mul(5).strict_add(1)) % 4) as u8; - let j1 = ((seed.strict_add(2)) % 4) as u8; - let i1 = ((seed.strict_mul(7).strict_add(3)) % 4) as u8; + let j0 = test_u8(seed % 4); + let i0 = test_u8(seed.strict_mul(5).strict_add(1) % 4); + let j1 = test_u8(seed.strict_add(2) % 4); + let i1 = test_u8(seed.strict_mul(7).strict_add(3) % 4); let mut sampled0 = [0u16; N]; let mut sampled1 = [0u16; N]; @@ -8675,7 +8809,7 @@ mod tests { fn fused_sample_ntt_quad_accumulate_matches_four_sampled_products() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(53).strict_add(29)) as u8; + *byte = test_low_byte(i.strict_mul(53).strict_add(29)); } for seed in 0usize..8 { @@ -8685,18 +8819,18 @@ mod tests { let rhs3 = test_poly(seed.strict_add(900)); let rhs = [&rhs0, &rhs1, &rhs2, &rhs3]; let coords = [ - ((seed % 4) as u8, ((seed.strict_mul(3).strict_add(1)) % 4) as u8), + (test_u8(seed % 4), test_u8(seed.strict_mul(3).strict_add(1) % 4)), ( - ((seed.strict_add(1)) % 4) as u8, - ((seed.strict_mul(5).strict_add(2)) % 4) as u8, + test_u8(seed.strict_add(1) % 4), + test_u8(seed.strict_mul(5).strict_add(2) % 4), ), ( - ((seed.strict_add(2)) % 4) as u8, - ((seed.strict_mul(7).strict_add(3)) % 4) as u8, + test_u8(seed.strict_add(2) % 4), + test_u8(seed.strict_mul(7).strict_add(3) % 4), ), ( - ((seed.strict_add(3)) % 4) as u8, - ((seed.strict_mul(11).strict_add(1)) % 4) as u8, + test_u8(seed.strict_add(3) % 4), + test_u8(seed.strict_mul(11).strict_add(1) % 4), ), ]; let base = test_poly(seed.strict_add(1000)); @@ -8728,7 +8862,7 @@ mod tests { fn materialized_k4_matrix_accumulate_matches_reference_layouts() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(61).strict_add(31)) as u8; + *byte = test_low_byte(i.strict_mul(61).strict_add(31)); } let rhs = [test_poly(0x10), test_poly(0x20), test_poly(0x30), test_poly(0x40)]; @@ -8758,7 +8892,7 @@ mod tests { fn materialized_k3_matrix_accumulate_matches_reference_layouts() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(47).strict_add(23)) as u8; + *byte = test_low_byte(i.strict_mul(47).strict_add(23)); } let rhs = [test_poly(0x10), test_poly(0x20), test_poly(0x30)]; @@ -8788,7 +8922,7 @@ mod tests { fn materialized_k2_matrix_accumulate_matches_reference_layouts() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(43).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(43).strict_add(19)); } let rhs = [test_poly(0x10), test_poly(0x20)]; @@ -8814,10 +8948,10 @@ mod tests { let mut rho = [0u8; SEED_BYTES]; let mut sigma = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(37).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(37).strict_add(11)); } for (i, byte) in sigma.iter_mut().enumerate() { - *byte = (i.strict_mul(53).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(53).strict_add(19)); } let mut s_hat = [[0u16; N]; 2]; @@ -8829,7 +8963,7 @@ mod tests { for (i, expected_i) in expected.iter_mut().enumerate() { for (j, rhs) in s_hat.iter().enumerate() { let mut sampled = [0u16; N]; - sample_ntt_into(&rho, j as u8, i as u8, &mut sampled); + sample_ntt_into(&rho, test_u8(j), test_u8(i), &mut sampled); multiply_ntts_add_assign_scalar(expected_i, &sampled, rhs); } } @@ -8850,7 +8984,7 @@ mod tests { fn assert_prepared_matrix_cache_matches_materialized_transpose() { let mut rho = [0u8; SEED_BYTES]; for (i, byte) in rho.iter_mut().enumerate() { - *byte = (i.strict_mul(67).strict_add(29)) as u8; + *byte = test_low_byte(i.strict_mul(67).strict_add(29)); } let mut rhs = [[0u16; N]; K]; @@ -8876,7 +9010,13 @@ mod tests { fn test_poly(seed: usize) -> Poly { let mut poly = [0u16; N]; for (i, coeff) in poly.iter_mut().enumerate() { - *coeff = ((seed.strict_mul(37).strict_add(i.strict_mul(19)).strict_add(11)) % usize::from(Q)) as u16; + let value = seed + .strict_mul(37) + .strict_add(i.strict_mul(19)) + .strict_add(11) + .checked_rem(usize::from(Q)) + .expect("ML-KEM modulus is nonzero"); + *coeff = test_u16(value); } poly } @@ -8941,13 +9081,13 @@ mod tests { let mut m = [0u8; SEED_BYTES]; let mut r = [0u8; SEED_BYTES]; for (i, byte) in key_random.iter_mut().enumerate() { - *byte = (i.strict_mul(13).strict_add(7)) as u8; + *byte = test_low_byte(i.strict_mul(13).strict_add(7)); } for (i, byte) in m.iter_mut().enumerate() { - *byte = (i.strict_mul(17).strict_add(23)) as u8; + *byte = test_low_byte(i.strict_mul(17).strict_add(23)); } for (i, byte) in r.iter_mut().enumerate() { - *byte = (i.strict_mul(19).strict_add(29)) as u8; + *byte = test_low_byte(i.strict_mul(19).strict_add(29)); } let (ek512, _) = keygen::<2, 2, 192, 768, 800, 1632>(&key_random); @@ -8979,7 +9119,7 @@ mod tests { ); for (i, byte) in key_random.iter_mut().enumerate() { - *byte = (i.strict_mul(31).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(31).strict_add(11)); } let (ek1024, _) = keygen::<4, 4, 128, 1536, 1568, 3168>(&key_random); let prepared1024 = prepare_encapsulation_key::<4, 1568>(&ek1024); @@ -9214,9 +9354,9 @@ mod tests { let mut b = [0u16; N]; for i in 0usize..N { - acc[i] = ((seed.strict_mul(19).strict_add(i.strict_mul(7))) % usize::from(Q)) as u16; - a[i] = ((seed.strict_mul(31).strict_add(i.strict_mul(11)).strict_add(5)) % usize::from(Q)) as u16; - b[i] = ((seed.strict_mul(43).strict_add(i.strict_mul(13)).strict_add(17)) % usize::from(Q)) as u16; + acc[i] = test_u16(seed.strict_mul(19).strict_add(i.strict_mul(7)) % usize::from(Q)); + a[i] = test_u16(seed.strict_mul(31).strict_add(i.strict_mul(11)).strict_add(5) % usize::from(Q)); + b[i] = test_u16(seed.strict_mul(43).strict_add(i.strict_mul(13)).strict_add(17) % usize::from(Q)); } let mut scalar = acc; @@ -9378,7 +9518,7 @@ mod tests { fn byte_encode_decode_round_trips_supported_widths() { let mut poly = [0u16; N]; for (i, coeff) in poly.iter_mut().enumerate() { - *coeff = ((i.strict_mul(19).strict_add(7)) as u16) % Q; + *coeff = test_u16(i.strict_mul(19).strict_add(7)) % Q; } let mut encoded_1 = [0u8; 32]; @@ -9455,7 +9595,7 @@ mod tests { fn fused_decode_decompress_matches_two_pass_codec() { let mut input = [0u8; 352]; for (i, byte) in input.iter_mut().enumerate() { - *byte = (i.strict_mul(73).strict_add(19)) as u8; + *byte = test_low_byte(i.strict_mul(73).strict_add(19)); } macro_rules! assert_width { @@ -9482,7 +9622,7 @@ mod tests { fn fused_message_decompress_add_matches_two_pass_codec() { let mut message = [0u8; SEED_BYTES]; for (i, byte) in message.iter_mut().enumerate() { - *byte = (i.strict_mul(37).strict_add(11)) as u8; + *byte = test_low_byte(i.strict_mul(37).strict_add(11)); } let base = test_poly(0x72); @@ -9517,8 +9657,8 @@ mod tests { for d in [1usize, 4, 5, 10, 11] { let max = 1u16 << d; for y in 0..max { - let x = ((Q_U32 * u32::from(y)) + (1u32 << (d - 1))) >> d; - let compressed = (div_q_compress_u32((x << d) + Q_HALF) & ((1u32 << d) - 1)) as u16; + let x = Q_U32.strict_mul(u32::from(y)).strict_add(1u32 << d.strict_sub(1)) >> d; + let compressed = low_u16(div_q_compress_u32((x << d).strict_add(Q_HALF)) & (1u32 << d).strict_sub(1)); assert_eq!(compressed, y, "d={d} y={y}"); } } diff --git a/src/auth/mlkem/portable/aarch64.rs b/src/auth/mlkem/portable/aarch64.rs index 1b007894..3e7f76e8 100644 --- a/src/auth/mlkem/portable/aarch64.rs +++ b/src/auth/mlkem/portable/aarch64.rs @@ -1,6 +1,3 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size ML-KEM native tables and assembly ABI. -#![allow(unsafe_code)] - use core::arch::global_asm; #[cfg(all(test, target_os = "linux"))] @@ -204,6 +201,10 @@ pub(super) unsafe fn sample_ntt_rej_uniform_3blocks_asm(out: *mut u16, input: *c #[inline] #[cfg(any(test, feature = "diag", target_os = "macos"))] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `acc`, `a`, and `b` must be valid, non-overlapping polynomials. pub(super) unsafe fn basemul_accumulate_asm(acc: &mut Poly, a: &Poly, b: &Poly) { #[cfg(target_os = "macos")] { @@ -251,6 +252,11 @@ pub(super) unsafe fn test_basemul_accumulate_asm(acc: &mut Poly, a: &Poly, b: &P #[cfg(any(target_os = "macos", target_os = "linux"))] #[inline] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `a` and `b` must each reference at least `2 * N` readable, +/// contiguous coefficients and must not overlap the uniquely borrowed `acc`. pub(super) unsafe fn basemul_accumulate_k2_asm_ptr(acc: &mut Poly, a: *const u16, b: *const u16) { #[cfg(target_os = "macos")] { @@ -283,6 +289,11 @@ pub(super) unsafe fn basemul_accumulate_k2_asm_ptr(acc: &mut Poly, a: *const u16 #[cfg(any(target_os = "macos", target_os = "linux"))] #[inline] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `a` and `b` must each reference at least `3 * N` readable, +/// contiguous coefficients and must not overlap the uniquely borrowed `acc`. pub(super) unsafe fn basemul_accumulate_k3_asm_ptr(acc: &mut Poly, a: *const u16, b: *const u16) { #[cfg(target_os = "macos")] { @@ -315,6 +326,11 @@ pub(super) unsafe fn basemul_accumulate_k3_asm_ptr(acc: &mut Poly, a: *const u16 #[cfg(any(target_os = "macos", target_os = "linux"))] #[inline] +/// # Safety +/// +/// The active platform must provide the matching ML-KEM AArch64 assembly +/// backend. `a` and `b` must each reference at least `4 * N` readable, +/// contiguous coefficients and must not overlap the uniquely borrowed `acc`. pub(super) unsafe fn basemul_accumulate_k4_asm_ptr(acc: &mut Poly, a: *const u16, b: *const u16) { #[cfg(target_os = "macos")] { diff --git a/src/auth/mlkem/portable/x86_64.rs b/src/auth/mlkem/portable/x86_64.rs index ecbeab37..b599b888 100644 --- a/src/auth/mlkem/portable/x86_64.rs +++ b/src/auth/mlkem/portable/x86_64.rs @@ -1,19 +1,98 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size ML-KEM polynomial kernels. - use core::arch::x86_64::{ - __m128i, __m256i, __m512i, _mm_loadu_si128, _mm256_add_epi16, _mm256_add_epi32, _mm256_and_si256, _mm256_cmpgt_epi16, - _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_loadu_si256, _mm256_mulhi_epi16, _mm256_mullo_epi16, - _mm256_mullo_epi32, _mm256_or_si256, _mm256_permutevar8x32_epi32, _mm256_set_epi16, _mm256_set_epi32, - _mm256_set1_epi16, _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, _mm256_srai_epi32, _mm256_srli_epi32, - _mm256_storeu_si256, _mm256_sub_epi16, _mm256_sub_epi32, _mm256_unpacklo_epi32, _mm512_add_epi32, _mm512_and_si512, - _mm512_cmpgt_epi32_mask, _mm512_cvtepi16_epi32, _mm512_loadu_si512, _mm512_mask_add_epi32, _mm512_mask_sub_epi32, - _mm512_mulhi_epi16, _mm512_mullo_epi16, _mm512_mullo_epi32, _mm512_or_si512, _mm512_set1_epi32, _mm512_setzero_si512, - _mm512_slli_epi32, _mm512_srai_epi32, _mm512_srli_epi32, _mm512_storeu_si512, _mm512_sub_epi16, + __m128i, __m256i, __m512i, _mm256_add_epi16, _mm256_add_epi32, _mm256_and_si256, _mm256_cmpgt_epi16, + _mm256_cmpgt_epi32, _mm256_cvtepi16_epi32, _mm256_mulhi_epi16, _mm256_mullo_epi16, _mm256_mullo_epi32, + _mm256_or_si256, _mm256_permutevar8x32_epi32, _mm256_set_epi16, _mm256_set_epi32, _mm256_set1_epi16, + _mm256_set1_epi32, _mm256_setzero_si256, _mm256_slli_epi32, _mm256_srai_epi32, _mm256_srli_epi32, _mm256_sub_epi16, + _mm256_sub_epi32, _mm256_unpacklo_epi32, _mm512_add_epi32, _mm512_and_si512, _mm512_cmpgt_epi32_mask, + _mm512_cvtepi16_epi32, _mm512_mask_add_epi32, _mm512_mask_sub_epi32, _mm512_mulhi_epi16, _mm512_mullo_epi16, + _mm512_mullo_epi32, _mm512_or_si512, _mm512_set1_epi32, _mm512_setzero_si512, _mm512_slli_epi32, _mm512_srai_epi32, + _mm512_srli_epi32, _mm512_sub_epi16, }; use super::{GAMMAS_MONT, N, Poly, Q_I16, Q_I32, Q_MONT_INV_U16, Q_U32, ZETAS_MONT}; +/// Load 16 potentially unaligned coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 16 initialized `u16` values. +#[inline] +unsafe fn load_u16x16(ptr: *const u16) -> __m256i { + // SAFETY: the caller provides 16 readable coefficients; `[u16; 16]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[u16; 16]>().read_unaligned() }; + // SAFETY: `[u16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 16], __m256i>(lanes) } +} + +/// Store 16 coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 16 `u16` values. +#[inline] +unsafe fn store_u16x16(ptr: *mut u16, value: __m256i) { + // SAFETY: `[u16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m256i, [u16; 16]>(value) }; + // SAFETY: the caller provides 16 writable coefficients; `[u16; 16]` retains the pointer's two-byte alignment. + unsafe { ptr.cast::<[u16; 16]>().write_unaligned(lanes) }; +} + +/// Load 32 potentially unaligned coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 32 initialized `u16` values. +#[inline] +unsafe fn load_u16x32(ptr: *const u16) -> __m512i { + // SAFETY: the caller provides 32 readable coefficients; `[u16; 32]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[u16; 32]>().read_unaligned() }; + // SAFETY: `[u16; 32]` and `__m512i` are 512-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[u16; 32], __m512i>(lanes) } +} + +/// Store 32 coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 32 `u16` values. +#[inline] +unsafe fn store_u16x32(ptr: *mut u16, value: __m512i) { + // SAFETY: `[u16; 32]` and `__m512i` are 512-bit values, and every bit pattern is valid for both. + let lanes = unsafe { core::mem::transmute::<__m512i, [u16; 32]>(value) }; + // SAFETY: the caller provides 32 writable coefficients; `[u16; 32]` retains the pointer's two-byte alignment. + unsafe { ptr.cast::<[u16; 32]>().write_unaligned(lanes) }; +} + +/// Load eight potentially unaligned signed coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read eight initialized `i16` values. +#[inline] +unsafe fn load_i16x8(ptr: *const i16) -> __m128i { + // SAFETY: the caller provides eight readable coefficients; `[i16; 8]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[i16; 8]>().read_unaligned() }; + // SAFETY: `[i16; 8]` and `__m128i` are 128-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[i16; 8], __m128i>(lanes) } +} + +/// Load 16 potentially unaligned signed coefficients without asserting SIMD alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 16 initialized `i16` values. +#[inline] +unsafe fn load_i16x16(ptr: *const i16) -> __m256i { + // SAFETY: the caller provides 16 readable coefficients; `[i16; 16]` retains the pointer's two-byte alignment. + let lanes = unsafe { ptr.cast::<[i16; 16]>().read_unaligned() }; + // SAFETY: `[i16; 16]` and `__m256i` are 256-bit values, and every bit pattern is valid for both. + unsafe { core::mem::transmute::<[i16; 16], __m256i>(lanes) } +} + #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut len = 128usize; while len >= 16 { @@ -32,16 +111,10 @@ pub(super) fn ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let u = _mm256_loadu_si256(poly.as_ptr().add(j).cast::<__m256i>()); - let t = mul_mont_mod_u16x16_avx2( - _mm256_loadu_si256(poly.as_ptr().add(j.strict_add(len)).cast::<__m256i>()), - zeta, - ); - _mm256_storeu_si256( - poly.as_mut_ptr().add(j.strict_add(len)).cast::<__m256i>(), - sub_mod_u16x16_avx2(u, t), - ); - _mm256_storeu_si256(poly.as_mut_ptr().add(j).cast::<__m256i>(), add_mod_u16x16_avx2(u, t)); + let u = load_u16x16(poly.as_ptr().add(j)); + let t = mul_mont_mod_u16x16_avx2(load_u16x16(poly.as_ptr().add(j.strict_add(len))), zeta); + store_u16x16(poly.as_mut_ptr().add(j.strict_add(len)), sub_mod_u16x16_avx2(u, t)); + store_u16x16(poly.as_mut_ptr().add(j), add_mod_u16x16_avx2(u, t)); } j = j.strict_add(16); } @@ -52,6 +125,9 @@ pub(super) fn ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn inverse_ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut len = 16usize; while len <= 128 { @@ -70,11 +146,11 @@ pub(super) fn inverse_ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let t = _mm256_loadu_si256(poly.as_ptr().add(j).cast::<__m256i>()); - let u = _mm256_loadu_si256(poly.as_ptr().add(j.strict_add(len)).cast::<__m256i>()); - _mm256_storeu_si256(poly.as_mut_ptr().add(j).cast::<__m256i>(), add_mod_u16x16_avx2(t, u)); - _mm256_storeu_si256( - poly.as_mut_ptr().add(j.strict_add(len)).cast::<__m256i>(), + let t = load_u16x16(poly.as_ptr().add(j)); + let u = load_u16x16(poly.as_ptr().add(j.strict_add(len))); + store_u16x16(poly.as_mut_ptr().add(j), add_mod_u16x16_avx2(t, u)); + store_u16x16( + poly.as_mut_ptr().add(j.strict_add(len)), mul_mont_mod_u16x16_avx2(sub_mod_u16x16_avx2(u, t), zeta), ); } @@ -87,6 +163,9 @@ pub(super) fn inverse_ntt_len_ge16_avx2(poly: &mut Poly, zeta_index: &mut usize) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -103,15 +182,15 @@ pub(super) fn ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let values = _mm256_loadu_si256(poly.as_ptr().add(start).cast::<__m256i>()); + let values = load_u16x16(poly.as_ptr().add(start)); let lower = deinterleave_len2_lower_avx2(values); let upper = deinterleave_len2_upper_avx2(values); let twiddles = duplicate_i16_quartet_lanes_avx2(zeta0, zeta1, zeta2, zeta3); let t = mul_mont_mod_u16x16_avx2(upper, twiddles); let lower_out = add_mod_u16x16_avx2(lower, t); let upper_out = sub_mod_u16x16_avx2(lower, t); - _mm256_storeu_si256( - poly.as_mut_ptr().add(start).cast::<__m256i>(), + store_u16x16( + poly.as_mut_ptr().add(start), _mm256_unpacklo_epi32(lower_out, upper_out), ); } @@ -121,6 +200,9 @@ pub(super) fn ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn inverse_ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { let mut start = 0usize; while start < N { @@ -140,14 +222,14 @@ pub(super) fn inverse_ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { // 4. The function is gated by `#[target_feature(enable = "avx2,sse4.1")]`, and the caller proves // AVX2 and SSE4.1 availability. unsafe { - let values = _mm256_loadu_si256(poly.as_ptr().add(start).cast::<__m256i>()); + let values = load_u16x16(poly.as_ptr().add(start)); let lower = deinterleave_len2_lower_avx2(values); let upper = deinterleave_len2_upper_avx2(values); let twiddles = duplicate_i16_quartet_lanes_avx2(zeta0, zeta1, zeta2, zeta3); let lower_out = add_mod_u16x16_avx2(lower, upper); let upper_out = mul_mont_mod_u16x16_avx2(sub_mod_u16x16_avx2(upper, lower), twiddles); - _mm256_storeu_si256( - poly.as_mut_ptr().add(start).cast::<__m256i>(), + store_u16x16( + poly.as_mut_ptr().add(start), _mm256_unpacklo_epi32(lower_out, upper_out), ); } @@ -157,6 +239,9 @@ pub(super) fn inverse_ntt_len2_avx2(poly: &mut Poly, zeta_index: &mut usize) { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn multiply_ntts_accumulate_k3_avx2(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -170,7 +255,7 @@ pub(super) fn multiply_ntts_accumulate_k3_avx2(acc: &mut Poly, a: [&Poly; 3], b: // AVX2 and SSE4.1 availability. unsafe { let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let acc_pairs = load_u16x16(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx2(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx2(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx2(a[2], b[2], gamma, coeff_offset); @@ -189,6 +274,9 @@ pub(super) fn multiply_ntts_accumulate_k3_avx2(acc: &mut Poly, a: [&Poly; 3], b: } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. pub(super) fn multiply_ntts_accumulate_k4_avx2(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4]) { for i in (0..GAMMAS_MONT.len()).step_by(8) { let coeff_offset = i.strict_mul(2); @@ -202,7 +290,7 @@ pub(super) fn multiply_ntts_accumulate_k4_avx2(acc: &mut Poly, a: [&Poly; 4], b: // AVX2 and SSE4.1 availability. unsafe { let gamma = load_i16x8_as_i32x8_avx2(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm256_loadu_si256(acc.as_ptr().add(coeff_offset).cast::<__m256i>()); + let acc_pairs = load_u16x16(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx2(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx2(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx2(a[2], b[2], gamma, coeff_offset); @@ -228,6 +316,9 @@ pub(super) fn multiply_ntts_accumulate_k4_avx2(acc: &mut Poly, a: [&Poly; 4], b: } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. pub(super) fn multiply_ntts_accumulate_k3_avx512(acc: &mut Poly, a: [&Poly; 3], b: [&Poly; 3]) { for i in (0..GAMMAS_MONT.len()).step_by(16) { let coeff_offset = i.strict_mul(2); @@ -242,7 +333,7 @@ pub(super) fn multiply_ntts_accumulate_k3_avx512(acc: &mut Poly, a: [&Poly; 3], // available. unsafe { let gamma = load_i16x16_as_i32x16_avx512(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm512_loadu_si512(acc.as_ptr().add(coeff_offset).cast::<__m512i>()); + let acc_pairs = load_u16x32(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx512(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx512(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx512(a[2], b[2], gamma, coeff_offset); @@ -261,6 +352,9 @@ pub(super) fn multiply_ntts_accumulate_k3_avx512(acc: &mut Poly, a: [&Poly; 3], } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. pub(super) fn multiply_ntts_accumulate_k4_avx512(acc: &mut Poly, a: [&Poly; 4], b: [&Poly; 4]) { for i in (0..GAMMAS_MONT.len()).step_by(16) { let coeff_offset = i.strict_mul(2); @@ -275,7 +369,7 @@ pub(super) fn multiply_ntts_accumulate_k4_avx512(acc: &mut Poly, a: [&Poly; 4], // available. unsafe { let gamma = load_i16x16_as_i32x16_avx512(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm512_loadu_si512(acc.as_ptr().add(coeff_offset).cast::<__m512i>()); + let acc_pairs = load_u16x32(acc.as_ptr().add(coeff_offset)); let (p00, p01) = base_multiply_chunk_avx512(a[0], b[0], gamma, coeff_offset); let (p10, p11) = base_multiply_chunk_avx512(a[1], b[1], gamma, coeff_offset); let (p20, p21) = base_multiply_chunk_avx512(a[2], b[2], gamma, coeff_offset); @@ -301,6 +395,9 @@ pub(super) fn multiply_ntts_accumulate_k4_avx512(acc: &mut Poly, a: [&Poly; 4], } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. pub(super) fn multiply_ntts_add_assign_avx512(acc: &mut Poly, a: &Poly, b: &Poly) { for i in (0..GAMMAS_MONT.len()).step_by(16) { let coeff_offset = i.strict_mul(2); @@ -315,7 +412,7 @@ pub(super) fn multiply_ntts_add_assign_avx512(acc: &mut Poly, a: &Poly, b: &Poly // available. unsafe { let gamma = load_i16x16_as_i32x16_avx512(GAMMAS_MONT.as_ptr().add(i)); - let acc_pairs = _mm512_loadu_si512(acc.as_ptr().add(coeff_offset).cast::<__m512i>()); + let acc_pairs = load_u16x32(acc.as_ptr().add(coeff_offset)); let (product0, product1) = base_multiply_chunk_avx512(a, b, gamma, coeff_offset); let out0 = add_mod_u32x16_avx512(acc_lanes0_avx512(acc_pairs), product0); @@ -326,21 +423,33 @@ pub(super) fn multiply_ntts_add_assign_avx512(acc: &mut Poly, a: &Poly, b: &Poly } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn deinterleave_len2_lower_avx2(values: __m256i) -> __m256i { _mm256_permutevar8x32_epi32(values, _mm256_set_epi32(0, 0, 6, 4, 0, 0, 2, 0)) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn deinterleave_len2_upper_avx2(values: __m256i) -> __m256i { _mm256_permutevar8x32_epi32(values, _mm256_set_epi32(0, 0, 7, 5, 0, 0, 3, 1)) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn duplicate_i16_quartet_lanes_avx2(a: i16, b: i16, c: i16, d: i16) -> __m256i { _mm256_set_epi16(0, 0, 0, 0, d, d, c, c, 0, 0, 0, 0, b, b, a, a) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn mul_mont_mod_u16x16_avx2(a: __m256i, b_mont: __m256i) -> __m256i { signed_to_mod_q_s16x16_avx2(montgomery_reduce_s16x16_avx2( _mm256_mullo_epi16(a, b_mont), @@ -349,19 +458,28 @@ fn mul_mont_mod_u16x16_avx2(a: __m256i, b_mont: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_s16x16_avx2(low: __m256i, high: __m256i) -> __m256i { - let k = _mm256_mullo_epi16(low, _mm256_set1_epi16(Q_MONT_INV_U16 as i16)); + let k = _mm256_mullo_epi16(low, _mm256_set1_epi16(Q_MONT_INV_U16.cast_signed())); let c = _mm256_mulhi_epi16(k, _mm256_set1_epi16(Q_I16)); _mm256_sub_epi16(high, c) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_s16x16_avx2(value: __m256i) -> __m256i { let negative = _mm256_cmpgt_epi16(_mm256_setzero_si256(), value); _mm256_add_epi16(value, _mm256_and_si256(negative, _mm256_set1_epi16(Q_I16))) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { let sum = _mm256_add_epi16(a, b); let ge_q = _mm256_cmpgt_epi16(sum, _mm256_set1_epi16(Q_I16 - 1)); @@ -369,6 +487,9 @@ fn add_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn sub_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { let diff = _mm256_sub_epi16(a, b); let borrowed = _mm256_cmpgt_epi16(b, a); @@ -376,6 +497,10 @@ fn sub_mod_u16x16_avx2(a: __m256i, b: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `coeff_offset..coeff_offset + 16` must be in +/// bounds for both polynomials. fn base_multiply_chunk_avx2(a: &Poly, b: &Poly, gamma: __m256i, coeff_offset: usize) -> (__m256i, __m256i) { let mask = _mm256_set1_epi32(0xffff); // SAFETY: fixed-size AVX2 polynomial chunk loads because: @@ -383,8 +508,8 @@ fn base_multiply_chunk_avx2(a: &Poly, b: &Poly, gamma: __m256i, coeff_offset: us // 2. Each load touches `coeff_offset..coeff_offset + 16`, within each 256-coefficient polynomial. // 3. `_mm256_loadu_si256` accepts arbitrary alignment. unsafe { - let a_pairs = _mm256_loadu_si256(a.as_ptr().add(coeff_offset).cast::<__m256i>()); - let b_pairs = _mm256_loadu_si256(b.as_ptr().add(coeff_offset).cast::<__m256i>()); + let a_pairs = load_u16x16(a.as_ptr().add(coeff_offset)); + let b_pairs = load_u16x16(b.as_ptr().add(coeff_offset)); let a0 = _mm256_and_si256(a_pairs, mask); let a1 = _mm256_srli_epi32::<16>(a_pairs); let b0 = _mm256_and_si256(b_pairs, mask); @@ -403,36 +528,52 @@ fn base_multiply_chunk_avx2(a: &Poly, b: &Poly, gamma: __m256i, coeff_offset: us } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn acc_lanes0_avx2(acc_pairs: __m256i) -> __m256i { _mm256_and_si256(acc_pairs, _mm256_set1_epi32(0xffff)) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn acc_lanes1_avx2(acc_pairs: __m256i) -> __m256i { _mm256_srli_epi32::<16>(acc_pairs) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to write 16 `u16` values. fn store_u32_pair_lanes_as_u16_avx2(ptr: *mut u16, lo: __m256i, hi: __m256i) { let packed = _mm256_or_si256(lo, _mm256_slli_epi32::<16>(hi)); // SAFETY: fixed-size AVX2 polynomial chunk store because: // 1. The caller passes a pointer to the start of a 16-coefficient in-bounds chunk. // 2. `lo` and `hi` are reduced modulo Q, so each lane fits in u16 before packing. // 3. `_mm256_storeu_si256` accepts arbitrary alignment. - unsafe { _mm256_storeu_si256(ptr.cast::<__m256i>(), packed) }; + unsafe { store_u16x16(ptr, packed) }; } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1, and `ptr` must be valid to read eight initialized +/// `i16` values. fn load_i16x8_as_i32x8_avx2(ptr: *const i16) -> __m256i { // SAFETY: unaligned 8-coefficient input load because: // 1. The caller proves `ptr..ptr + 8` is readable. // 2. `_mm_loadu_si128` accepts arbitrary alignment. // 3. `_mm256_cvtepi16_epi32` sign-extends the 8 i16 lanes into 8 i32 lanes. - let packed = unsafe { _mm_loadu_si128(ptr.cast::<__m128i>()) }; + let packed = unsafe { load_i16x8(ptr) }; _mm256_cvtepi16_epi32(packed) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { let k = _mm256_mullo_epi16(value, _mm256_set1_epi32(i32::from(Q_MONT_INV_U16))); let c = _mm256_mulhi_epi16(k, _mm256_set1_epi32(Q_I32)); @@ -442,19 +583,29 @@ fn montgomery_reduce_i32x8_avx2(value: __m256i) -> __m256i { } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn signed_to_mod_q_i32x8_avx2(value: __m256i) -> __m256i { let negative = _mm256_cmpgt_epi32(_mm256_setzero_si256(), value); _mm256_add_epi32(value, _mm256_and_si256(negative, _mm256_set1_epi32(Q_I32))) } #[target_feature(enable = "avx2,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2 and SSE4.1. fn add_mod_u32x8_avx2(a: __m256i, b: __m256i) -> __m256i { let sum = _mm256_add_epi32(a, b); - let ge_q = _mm256_cmpgt_epi32(sum, _mm256_set1_epi32((Q_U32 - 1) as i32)); + let ge_q = _mm256_cmpgt_epi32(sum, _mm256_set1_epi32((Q_U32 - 1).cast_signed())); _mm256_sub_epi32(sum, _mm256_and_si256(ge_q, _mm256_set1_epi32(Q_I32))) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1, and +/// `coeff_offset..coeff_offset + 32` must be in bounds for both polynomials. fn base_multiply_chunk_avx512(a: &Poly, b: &Poly, gamma: __m512i, coeff_offset: usize) -> (__m512i, __m512i) { let mask = _mm512_set1_epi32(0xffff); // SAFETY: fixed-size AVX-512 polynomial chunk loads because: @@ -462,8 +613,8 @@ fn base_multiply_chunk_avx512(a: &Poly, b: &Poly, gamma: __m512i, coeff_offset: // 2. Each load touches `coeff_offset..coeff_offset + 32`, within each 256-coefficient polynomial. // 3. `_mm512_loadu_si512` accepts arbitrary alignment. unsafe { - let a_pairs = _mm512_loadu_si512(a.as_ptr().add(coeff_offset).cast::<__m512i>()); - let b_pairs = _mm512_loadu_si512(b.as_ptr().add(coeff_offset).cast::<__m512i>()); + let a_pairs = load_u16x32(a.as_ptr().add(coeff_offset)); + let b_pairs = load_u16x32(b.as_ptr().add(coeff_offset)); let a0 = _mm512_and_si512(a_pairs, mask); let a1 = _mm512_srli_epi32::<16>(a_pairs); let b0 = _mm512_and_si512(b_pairs, mask); @@ -482,36 +633,53 @@ fn base_multiply_chunk_avx512(a: &Poly, b: &Poly, gamma: __m512i, coeff_offset: } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn acc_lanes0_avx512(acc_pairs: __m512i) -> __m512i { _mm512_and_si512(acc_pairs, _mm512_set1_epi32(0xffff)) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn acc_lanes1_avx512(acc_pairs: __m512i) -> __m512i { _mm512_srli_epi32::<16>(acc_pairs) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1, and `ptr` must be +/// valid to write 32 `u16` values. fn store_u32_pair_lanes_as_u16_avx512(ptr: *mut u16, lo: __m512i, hi: __m512i) { let packed = _mm512_or_si512(lo, _mm512_slli_epi32::<16>(hi)); // SAFETY: fixed-size AVX-512 polynomial chunk store because: // 1. The caller passes a pointer to the start of a 32-coefficient in-bounds chunk. // 2. `lo` and `hi` are reduced modulo Q, so each lane fits in u16 before packing. // 3. `_mm512_storeu_si512` accepts arbitrary alignment. - unsafe { _mm512_storeu_si512(ptr.cast::<__m512i>(), packed) }; + unsafe { store_u16x32(ptr, packed) }; } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1, and `ptr` must be +/// valid to read 16 initialized `i16` values. fn load_i16x16_as_i32x16_avx512(ptr: *const i16) -> __m512i { // SAFETY: unaligned 16-coefficient input load because: // 1. The caller proves `ptr..ptr + 16` is readable. // 2. `_mm256_loadu_si256` accepts arbitrary alignment. // 3. `_mm512_cvtepi16_epi32` sign-extends the 16 i16 lanes into 16 i32 lanes. - let packed = unsafe { _mm256_loadu_si256(ptr.cast::<__m256i>()) }; + let packed = unsafe { load_i16x16(ptr) }; _mm512_cvtepi16_epi32(packed) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn montgomery_reduce_i32x16_avx512(value: __m512i) -> __m512i { let k = _mm512_mullo_epi16(value, _mm512_set1_epi32(i32::from(Q_MONT_INV_U16))); let c = _mm512_mulhi_epi16(k, _mm512_set1_epi32(Q_I32)); @@ -521,14 +689,20 @@ fn montgomery_reduce_i32x16_avx512(value: __m512i) -> __m512i { } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn signed_to_mod_q_i32x16_avx512(value: __m512i) -> __m512i { let negative = _mm512_cmpgt_epi32_mask(_mm512_setzero_si512(), value); _mm512_mask_add_epi32(value, negative, value, _mm512_set1_epi32(Q_I32)) } #[target_feature(enable = "avx2,avx512f,avx512bw,avx512dq,sse4.1")] +/// # Safety +/// +/// The active CPU must support AVX2, AVX-512F, AVX-512BW, AVX-512DQ, and SSE4.1. fn add_mod_u32x16_avx512(a: __m512i, b: __m512i) -> __m512i { let sum = _mm512_add_epi32(a, b); - let ge_q = _mm512_cmpgt_epi32_mask(sum, _mm512_set1_epi32((Q_U32 - 1) as i32)); + let ge_q = _mm512_cmpgt_epi32_mask(sum, _mm512_set1_epi32((Q_U32 - 1).cast_signed())); _mm512_mask_sub_epi32(sum, ge_q, sum, _mm512_set1_epi32(Q_I32)) } diff --git a/src/auth/mod.rs b/src/auth/mod.rs index ae8e1925..f50505dc 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -47,28 +47,28 @@ //! ```toml //! [dependencies] //! # MAC bundle: HMAC-SHA-2/SHA-3, KMAC128/256, and standalone Poly1305 -//! rscrypto = { version = "0.6.4", default-features = false, features = ["macs"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["macs"] } //! //! # HKDF only -//! rscrypto = { version = "0.6.4", default-features = false, features = ["hkdf"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["hkdf"] } //! //! # Ed25519 only -//! rscrypto = { version = "0.6.4", default-features = false, features = ["ed25519"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["ed25519"] } //! //! # ECDSA P-256/P-384 signing and verification -//! rscrypto = { version = "0.6.4", default-features = false, features = ["ecdsa"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["ecdsa"] } //! //! # RSA -//! rscrypto = { version = "0.6.4", default-features = false, features = ["rsa"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["rsa"] } //! //! # Signature primitives -//! rscrypto = { version = "0.6.4", default-features = false, features = ["signatures"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["signatures"] } //! //! # X25519 only -//! rscrypto = { version = "0.6.4", default-features = false, features = ["key-exchange"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["key-exchange"] } //! //! # Everything in auth/key-derivation -//! rscrypto = { version = "0.6.4", default-features = false, features = ["auth"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["auth"] } //! ``` //! //! # API Conventions @@ -115,7 +115,24 @@ #[cfg(feature = "argon2")] pub mod argon2; -#[cfg(any(feature = "ed25519", feature = "x25519"))] +#[cfg(any( + feature = "ed25519", + all( + feature = "x25519", + any( + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ) +))] pub(crate) mod curve25519_edwards; #[cfg(any(feature = "ecdsa-p256", feature = "ecdsa-p384"))] pub mod ecdsa; @@ -133,7 +150,7 @@ pub mod kmac; pub mod mlkem; #[cfg(feature = "pbkdf2")] pub mod pbkdf2; -#[cfg(feature = "phc-strings")] +#[cfg(all(feature = "phc-strings", any(feature = "argon2", feature = "scrypt")))] pub(crate) mod phc; #[cfg(feature = "poly1305")] pub mod poly1305; @@ -297,10 +314,10 @@ pub use pbkdf2::{diag_pbkdf2_sha256_verify_portable, diag_pbkdf2_sha512_verify_p pub use poly1305::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; #[cfg(feature = "rsa")] pub use rsa::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, RsaKeyGenerationError, - RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, RsaPrivateScratch, - RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, RsaPublicKey, - RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, + RsaBlindingPair, RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, + RsaKeyGenerationError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, + RsaPrivateScratch, RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, + RsaPublicKey, RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, RsaSignatureVerifier, RsaTlsSignatureSchemes, RsaX509PublicKey, RsaX509PublicKeyAlgorithm, }; #[cfg(all(feature = "rsa", feature = "diag"))] diff --git a/src/auth/pbkdf2.rs b/src/auth/pbkdf2.rs index c55bab03..d84c627c 100644 --- a/src/auth/pbkdf2.rs +++ b/src/auth/pbkdf2.rs @@ -60,7 +60,6 @@ const SHA512_BLOCK_SIZE: usize = 128; const SHA512_INLINE_SALT_MAX: usize = SHA512_BLOCK_SIZE - 4 - 1 - 16; #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u32x8_be(dst: &mut [u8], words: &[u32; 8]) { dst[0..4].copy_from_slice(&words[0].to_be_bytes()); dst[4..8].copy_from_slice(&words[1].to_be_bytes()); @@ -73,7 +72,6 @@ fn write_u32x8_be(dst: &mut [u8], words: &[u32; 8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_u64x8_be(dst: &mut [u8], words: &[u64; 8]) { dst[0..8].copy_from_slice(&words[0].to_be_bytes()); dst[8..16].copy_from_slice(&words[1].to_be_bytes()); @@ -339,7 +337,6 @@ macro_rules! define_pbkdf2_sha2 { /// Pre-compute HMAC prefix states from `password`. #[must_use] - #[allow(clippy::indexing_slicing)] // password.len() <= block size in the else branch. pub fn new(password: &[u8]) -> Self { if let Some(state) = $new_fast_path(password) { return state; @@ -375,20 +372,17 @@ macro_rules! define_pbkdf2_sha2 { /// Derive a key into `okm`. #[inline] - #[allow(clippy::indexing_slicing)] pub fn derive(&self, salt: &[u8], iterations: u32, okm: &mut [u8]) -> Result<(), Pbkdf2Error> { Self::derive_with_prefixes(self.compress, &self.inner_init, &self.outer_init, salt, iterations, okm) } /// Derive a key into `okm` using validated password parameters. #[inline] - #[allow(clippy::indexing_slicing)] pub fn derive_with_params(&self, params: Pbkdf2Params<'_>, okm: &mut [u8]) -> Result<(), Pbkdf2Error> { self.derive(params.salt(), params.iterations(), okm) } #[inline] - #[allow(clippy::indexing_slicing)] fn derive_with_prefixes( compress: $compress_ty, inner_init: &[$word_ty; 8], @@ -403,23 +397,17 @@ macro_rules! define_pbkdf2_sha2 { if okm.is_empty() { return Ok(()); } - let num_blocks = okm.len().div_ceil($output_size_const); - if num_blocks as u64 > u32::MAX as u64 { - return Err(Pbkdf2Error::OutputTooLong); - } + let num_blocks = u32::try_from(okm.len().div_ceil($output_size_const)) + .map_err(|_| Pbkdf2Error::OutputTooLong)?; if iterations == 1 { - $iter1_fn(compress, inner_init, outer_init, salt, okm); + $iter1_fn(compress, inner_init, outer_init, salt, num_blocks, okm); return Ok(()); } - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut($output_size_const); + let (chunks, tail) = okm.as_chunks_mut::<$output_size_const>(); - for chunk in chunks.by_ref() { - // SAFETY: chunks_exact_mut yields slices whose length is exactly - // $output_size_const, so this cast is to the same initialized bytes. - let full_chunk = unsafe { &mut *(chunk.as_mut_ptr().cast::<[u8; $output_size_const]>()) }; + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { $f_fn( compress, inner_init, @@ -427,12 +415,10 @@ macro_rules! define_pbkdf2_sha2 { salt, iterations, block_index, - full_chunk, + chunk, ); - block_index = block_index.strict_add(1); } - let tail = chunks.into_remainder(); if !tail.is_empty() { let mut block_out = [0u8; $output_size_const]; $f_fn( @@ -441,7 +427,7 @@ macro_rules! define_pbkdf2_sha2 { outer_init, salt, iterations, - block_index, + num_blocks, &mut block_out, ); tail.copy_from_slice(&block_out[..tail.len()]); @@ -470,7 +456,6 @@ macro_rules! define_pbkdf2_sha2 { /// /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify(&self, salt: &[u8], iterations: u32, expected: &[u8]) -> Result<(), VerificationError> { self.verify_with_policy_bounded( @@ -491,7 +476,6 @@ macro_rules! define_pbkdf2_sha2 { /// /// Generated-code timing claims are configuration- and release-evidence-bound; /// see `ct.toml`. - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_with_policy( &self, @@ -506,7 +490,6 @@ macro_rules! define_pbkdf2_sha2 { /// Verify `expected` under an explicit lower-bound policy and /// caller-selected verification work limit. - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_with_policy_bounded( &self, @@ -536,24 +519,20 @@ macro_rules! define_pbkdf2_sha2 { /// verification should use [`verify`](Self::verify), /// [`verify_with_policy`](Self::verify_with_policy), or /// [`verify_password`](Self::verify_password). - #[allow(clippy::indexing_slicing)] #[must_use = "password verification must be checked; a dropped Result silently accepts the wrong password"] pub fn verify_primitive(&self, salt: &[u8], iterations: u32, expected: &[u8]) -> Result<(), VerificationError> { if iterations == 0 || expected.is_empty() { return Err(VerificationError::new()); } - let num_blocks = expected.len().div_ceil($output_size_const); - if num_blocks as u64 > u32::MAX as u64 { - return Err(VerificationError::new()); - } + let num_blocks = u32::try_from(expected.len().div_ceil($output_size_const)) + .map_err(|_| VerificationError::new())?; let compress = self.compress; let mut block_out = [0u8; $output_size_const]; let mut acc = 0u8; - for (i, chunk) in expected.chunks($output_size_const).enumerate() { - let block_index = (i as u32).strict_add(1); + for (block_index, chunk) in (1..=num_blocks).zip(expected.chunks($output_size_const)) { $f_fn( compress, &self.inner_init, @@ -716,8 +695,6 @@ macro_rules! define_pbkdf2_sha2 { /// Test-only: build with a specific digest compress function. #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] - #[allow(clippy::indexing_slicing)] pub(crate) fn new_with_compress_for_test(password: &[u8], compress: $compress_ty) -> Self { let mut key_block = [0u8; $block_size_const]; if password.len() > $block_size_const { @@ -804,7 +781,6 @@ define_pbkdf2_sha2! { /// Test-only: one-shot SHA-256 digest using a specific compress function. #[cfg(any(test, feature = "diag"))] -#[allow(clippy::indexing_slicing)] fn sha256_oneshot_with_compress(data: &[u8], compress: Sha256CompressBlocksFn) -> [u8; SHA256_OUTPUT_SIZE] { let mut state = SHA256_H0; let mut pos = 0usize; @@ -823,7 +799,8 @@ fn sha256_oneshot_with_compress(data: &[u8], compress: Sha256CompressBlocksFn) - block[56..64].copy_from_slice(&(data.len() as u64).strict_mul(8).to_be_bytes()); compress(&mut state, &block); let mut out = [0u8; SHA256_OUTPUT_SIZE]; - for (chunk, &word) in out.chunks_exact_mut(4).zip(state.iter()) { + let (chunks, _) = out.as_chunks_mut::<4>(); + for (chunk, &word) in chunks.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } ct::zeroize_words_no_fence(&mut state); @@ -834,6 +811,8 @@ fn sha256_oneshot_with_compress(data: &[u8], compress: Sha256CompressBlocksFn) - #[cfg(feature = "diag")] #[must_use] +/// Return whether portable PBKDF2-HMAC-SHA256 derives `expected` from `password` +/// using the salt `salt` and one iteration. pub fn diag_pbkdf2_sha256_verify_portable( password: &[u8; SHA256_OUTPUT_SIZE], expected: &[u8; SHA256_OUTPUT_SIZE], @@ -848,6 +827,8 @@ pub fn diag_pbkdf2_sha256_verify_portable( #[cfg(feature = "diag")] #[must_use] +/// Return whether portable PBKDF2-HMAC-SHA512 derives `expected` from `password` +/// using the salt `salt` and one iteration. pub fn diag_pbkdf2_sha512_verify_portable( password: &[u8; SHA512_OUTPUT_SIZE], expected: &[u8; SHA512_OUTPUT_SIZE], @@ -861,7 +842,6 @@ pub fn diag_pbkdf2_sha512_verify_portable( } #[inline] -#[allow(clippy::indexing_slicing)] fn pbkdf2_sha256_new_fast_path(password: &[u8]) -> Option { if !pbkdf2_sha256_spr_prefers_hmac_iter1() { return None; @@ -913,7 +893,6 @@ fn pbkdf2_sha256_spr_prefers_hmac_iter1() -> bool { } } -#[allow(clippy::indexing_slicing)] #[inline] fn pbkdf2_sha256_hmac_iter1_small(password: &[u8], salt: &[u8], okm: &mut [u8]) { debug_assert!(salt.len() <= SHA256_INLINE_SALT_MAX); @@ -923,8 +902,7 @@ fn pbkdf2_sha256_hmac_iter1_small(password: &[u8], salt: &[u8], okm: &mut [u8]) msg[..salt.len()].copy_from_slice(salt); let msg_len = salt.len().strict_add(4); - for (i, chunk) in okm.chunks_mut(SHA256_OUTPUT_SIZE).enumerate() { - let block_index = (i as u32).strict_add(1); + for (block_index, chunk) in (1u32..=2).zip(okm.chunks_mut(SHA256_OUTPUT_SIZE)) { msg[salt.len()..msg_len].copy_from_slice(&block_index.to_be_bytes()); let tag = HmacSha256::mac(password, &msg[..msg_len]); chunk.copy_from_slice(&tag.as_bytes()[..chunk.len()]); @@ -974,7 +952,6 @@ fn pbkdf2_sha256_derive_key_fast_path_with_preference( /// Each HMAC iteration in the hot loop runs exactly 2 SHA-256 compress calls /// using pre-padded block templates — no hash struct creation, no dispatch /// overhead, no padding recomputation. -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha256_f( compress: Sha256CompressBlocksFn, @@ -1104,28 +1081,23 @@ fn pbkdf2_sha256_f( core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha256_iter1( compress: Sha256CompressBlocksFn, inner_init: &[u32; 8], outer_init: &[u32; 8], salt: &[u8], + num_blocks: u32, okm: &mut [u8], ) { if salt.len() > SHA256_INLINE_SALT_MAX { - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA256_OUTPUT_SIZE); - for chunk in chunks.by_ref() { - // SAFETY: chunks_exact_mut yields slices whose length is exactly SHA256_OUTPUT_SIZE. - let full_chunk = unsafe { &mut *(chunk.as_mut_ptr().cast::<[u8; SHA256_OUTPUT_SIZE]>()) }; - pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, block_index, full_chunk); - block_index = block_index.strict_add(1); + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { + pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, block_index, chunk); } - let tail = chunks.into_remainder(); if !tail.is_empty() { let mut block_out = [0u8; SHA256_OUTPUT_SIZE]; - pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, block_index, &mut block_out); + pbkdf2_sha256_f(compress, inner_init, outer_init, salt, 1, num_blocks, &mut block_out); tail.copy_from_slice(&block_out[..tail.len()]); ct::zeroize(&mut block_out); } @@ -1147,10 +1119,9 @@ fn pbkdf2_sha256_iter1( outer_block[56..SHA256_BLOCK_SIZE].copy_from_slice(&768u64.to_be_bytes()); let mut state = [0u32; 8]; - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA256_OUTPUT_SIZE); - for chunk in chunks.by_ref() { + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); state = *inner_init; @@ -1161,12 +1132,10 @@ fn pbkdf2_sha256_iter1( compress(&mut state, &outer_block); write_u32x8_be(chunk, &state); - block_index = block_index.strict_add(1); } - let tail = chunks.into_remainder(); if !tail.is_empty() { - block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); + block[index_pos..pad_pos].copy_from_slice(&num_blocks.to_be_bytes()); state = *inner_init; compress(&mut state, &block); @@ -1239,8 +1208,6 @@ fn pbkdf2_sha512_derive_key_fast_path( /// Test-only: one-shot SHA-512 digest using a specific compress function. #[cfg(any(test, feature = "diag"))] -#[allow(dead_code)] -#[allow(clippy::indexing_slicing)] fn sha512_oneshot_with_compress(data: &[u8], compress: Sha512CompressBlocksFn) -> [u8; SHA512_OUTPUT_SIZE] { let mut state = SHA512_H0; let mut pos = 0usize; @@ -1259,7 +1226,8 @@ fn sha512_oneshot_with_compress(data: &[u8], compress: Sha512CompressBlocksFn) - block[112..128].copy_from_slice(&(data.len() as u128).strict_mul(8).to_be_bytes()); compress(&mut state, &block); let mut out = [0u8; SHA512_OUTPUT_SIZE]; - for (chunk, &word) in out.chunks_exact_mut(8).zip(state.iter()) { + let (chunks, _) = out.as_chunks_mut::<8>(); + for (chunk, &word) in chunks.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } ct::zeroize_words_no_fence(&mut state); @@ -1269,7 +1237,6 @@ fn sha512_oneshot_with_compress(data: &[u8], compress: Sha512CompressBlocksFn) - } /// Compute one PBKDF2-SHA512 block: `F(Password, Salt, c, i)`. -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha512_f( compress: Sha512CompressBlocksFn, @@ -1382,28 +1349,23 @@ fn pbkdf2_sha512_f( core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } -#[allow(clippy::indexing_slicing)] #[inline(always)] fn pbkdf2_sha512_iter1( compress: Sha512CompressBlocksFn, inner_init: &[u64; 8], outer_init: &[u64; 8], salt: &[u8], + num_blocks: u32, okm: &mut [u8], ) { if salt.len() > SHA512_INLINE_SALT_MAX { - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA512_OUTPUT_SIZE); - for chunk in chunks.by_ref() { - // SAFETY: chunks_exact_mut yields slices whose length is exactly SHA512_OUTPUT_SIZE. - let full_chunk = unsafe { &mut *(chunk.as_mut_ptr().cast::<[u8; SHA512_OUTPUT_SIZE]>()) }; - pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, block_index, full_chunk); - block_index = block_index.strict_add(1); + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { + pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, block_index, chunk); } - let tail = chunks.into_remainder(); if !tail.is_empty() { let mut block_out = [0u8; SHA512_OUTPUT_SIZE]; - pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, block_index, &mut block_out); + pbkdf2_sha512_f(compress, inner_init, outer_init, salt, 1, num_blocks, &mut block_out); tail.copy_from_slice(&block_out[..tail.len()]); ct::zeroize(&mut block_out); } @@ -1425,10 +1387,9 @@ fn pbkdf2_sha512_iter1( outer_block[112..SHA512_BLOCK_SIZE].copy_from_slice(&1536u128.to_be_bytes()); let mut state = [0u64; 8]; - let mut block_index = 1u32; - let mut chunks = okm.chunks_exact_mut(SHA512_OUTPUT_SIZE); - for chunk in chunks.by_ref() { + let (chunks, tail) = okm.as_chunks_mut::(); + for (block_index, chunk) in (1..=num_blocks).zip(chunks) { block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); state = *inner_init; @@ -1439,12 +1400,10 @@ fn pbkdf2_sha512_iter1( compress(&mut state, &outer_block); write_u64x8_be(chunk, &state); - block_index = block_index.strict_add(1); } - let tail = chunks.into_remainder(); if !tail.is_empty() { - block[index_pos..pad_pos].copy_from_slice(&block_index.to_be_bytes()); + block[index_pos..pad_pos].copy_from_slice(&num_blocks.to_be_bytes()); state = *inner_init; compress(&mut state, &block); @@ -1477,7 +1436,7 @@ mod tests { #[test] fn rfc7914_sha256_vector_1() { let mut dk = [0u8; 64]; - Pbkdf2Sha256::derive_key_primitive(b"passwd", b"salt", 1, &mut dk).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"passwd", b"salt", 1, &mut dk).expect("RFC 7914 vector parameters are valid"); assert_eq!( dk, [ @@ -1498,7 +1457,9 @@ mod tests { for len in [1usize, 16, 31, 32, 33, 63, 64] { let mut expected = vec![0u8; len]; let mut actual = vec![0u8; len]; - state.derive(&salt, 1, &mut expected).unwrap(); + state + .derive(&salt, 1, &mut expected) + .expect("nonempty output and one iteration are valid"); pbkdf2_sha256_hmac_iter1_small(&password, &salt, &mut actual); assert_eq!(actual, expected, "len={len}"); } @@ -1512,20 +1473,33 @@ mod tests { let mut actual = [0u8; 64]; oracle_sha256(&password, &salt, 1, &mut expected); - assert!(pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, true).unwrap()); + assert!( + pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, true) + .expect("fast-path parameters are valid") + ); assert_eq!(actual, expected); let mut rejected = [0u8; 65]; - assert!(!pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut rejected, true).unwrap()); - assert!(!pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 2, &mut actual, true).unwrap()); - assert!(!pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, false).unwrap()); + assert!( + !pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut rejected, true) + .expect("oversized fast-path output is a supported fallback") + ); + assert!( + !pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 2, &mut actual, true) + .expect("multi-iteration fast-path input is a supported fallback") + ); + assert!( + !pbkdf2_sha256_derive_key_fast_path_with_preference(&password, &salt, 1, &mut actual, false) + .expect("disabled fast-path input is a supported fallback") + ); } #[cfg(not(miri))] #[test] fn rfc7914_sha256_vector_2() { let mut dk = [0u8; 64]; - Pbkdf2Sha256::derive_key_primitive(b"Password", b"NaCl", 80000, &mut dk).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"Password", b"NaCl", 80000, &mut dk) + .expect("RFC 7914 vector parameters are valid"); assert_eq!( dk, [ @@ -1589,7 +1563,8 @@ mod tests { oracle_sha256(password, salt, iterations, &mut expected); let mut actual = vec![0u8; dk_len]; - Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut actual).unwrap(); + Pbkdf2Sha256::derive_key_primitive(password, salt, iterations, &mut actual) + .expect("oracle case parameters are valid"); assert_eq!( actual, @@ -1642,7 +1617,8 @@ mod tests { oracle_sha512(password, salt, iterations, &mut expected); let mut actual = vec![0u8; dk_len]; - Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut actual).unwrap(); + Pbkdf2Sha512::derive_key_primitive(password, salt, iterations, &mut actual) + .expect("oracle case parameters are valid"); assert_eq!( actual, @@ -1660,37 +1636,45 @@ mod tests { #[test] fn sha256_verify_correct_password() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); - assert!(Pbkdf2Sha256::verify_password_primitive(b"password", b"salt", 100, &dk).is_ok()); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); + Pbkdf2Sha256::verify_password_primitive(b"password", b"salt", 100, &dk) + .expect("matching PBKDF2-SHA256 output must verify"); } #[test] fn sha256_verify_wrong_password() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha256::verify_password_primitive(b"wrong", b"salt", 100, &dk).is_err()); } #[test] fn sha256_verify_wrong_salt() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha256::verify_password_primitive(b"password", b"wrong", 100, &dk).is_err()); } #[test] fn sha256_verify_wrong_iterations() { - let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha256::verify_password_primitive(b"password", b"salt", 101, &dk).is_err()); } #[test] fn sha512_verify_correct_password() { - let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100).unwrap(); - assert!(Pbkdf2Sha512::verify_password_primitive(b"password", b"salt", 100, &dk).is_ok()); + let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); + Pbkdf2Sha512::verify_password_primitive(b"password", b"salt", 100, &dk) + .expect("matching PBKDF2-SHA512 output must verify"); } #[test] fn sha512_verify_wrong_password() { - let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100).unwrap(); + let dk = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"password", b"salt", 100) + .expect("test derivation parameters are valid"); assert!(Pbkdf2Sha512::verify_password_primitive(b"wrong", b"salt", 100, &dk).is_err()); } @@ -1728,15 +1712,13 @@ mod tests { ), Err(Pbkdf2Error::SaltTooShort) ); - assert!( - Pbkdf2Sha256::derive_key( - b"pw", - &strong_salt, - Pbkdf2Sha256::MIN_RECOMMENDED_ITERATIONS, - &mut empty - ) - .is_ok() - ); + Pbkdf2Sha256::derive_key( + b"pw", + &strong_salt, + Pbkdf2Sha256::MIN_RECOMMENDED_ITERATIONS, + &mut empty, + ) + .expect("the default PBKDF2-SHA256 policy must accept its minimums"); assert!( Pbkdf2Sha256::verify_password( b"pw", @@ -1751,12 +1733,14 @@ mod tests { #[test] fn pbkdf2_params_can_use_explicit_policy_for_migrations() { let policy = Pbkdf2VerifyPolicy::new(1, 0); - let params = Pbkdf2Sha256::params_with_policy(b"", 1, &policy).unwrap(); + let params = + Pbkdf2Sha256::params_with_policy(b"", 1, &policy).expect("the explicit migration policy permits these inputs"); assert_eq!(params.salt(), b""); assert_eq!(params.iterations(), 1); let mut empty = []; - assert!(Pbkdf2Sha256::derive_key_with_params(b"pw", params, &mut empty).is_ok()); + Pbkdf2Sha256::derive_key_with_params(b"pw", params, &mut empty) + .expect("validated migration parameters permit empty output"); } #[test] @@ -1769,11 +1753,12 @@ mod tests { ] { let params = Pbkdf2Sha256::params_with_policy_bounded(b"", iterations, &policy, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS) - .unwrap(); + .expect("iterations at or below the verification limit are valid"); assert!(policy.allows_bounded(¶ms, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS)); } let excessive_sha256 = Pbkdf2Sha256::MAX_VERIFY_ITERATIONS.strict_add(1); - let lower_bound_only = Pbkdf2Sha256::params_with_policy(b"", excessive_sha256, &policy).unwrap(); + let lower_bound_only = Pbkdf2Sha256::params_with_policy(b"", excessive_sha256, &policy) + .expect("the lower-bound-only policy has no upper iteration limit"); assert!(policy.allows(&lower_bound_only)); assert!(!policy.allows_bounded(&lower_bound_only, Pbkdf2Sha256::MAX_VERIFY_ITERATIONS)); assert_eq!( @@ -1802,11 +1787,12 @@ mod tests { ] { let params = Pbkdf2Sha512::params_with_policy_bounded(b"", iterations, &policy, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS) - .unwrap(); + .expect("iterations at or below the verification limit are valid"); assert!(policy.allows_bounded(¶ms, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS)); } let excessive_sha512 = Pbkdf2Sha512::MAX_VERIFY_ITERATIONS.strict_add(1); - let lower_bound_only = Pbkdf2Sha512::params_with_policy(b"", excessive_sha512, &policy).unwrap(); + let lower_bound_only = Pbkdf2Sha512::params_with_policy(b"", excessive_sha512, &policy) + .expect("the lower-bound-only policy has no upper iteration limit"); assert!(policy.allows(&lower_bound_only)); assert!(!policy.allows_bounded(&lower_bound_only, Pbkdf2Sha512::MAX_VERIFY_ITERATIONS)); assert_eq!( @@ -1841,12 +1827,12 @@ mod tests { #[test] fn sha256_empty_output_ok() { - assert!(Pbkdf2Sha256::derive_key_primitive(b"pw", b"salt", 1, &mut []).is_ok()); + Pbkdf2Sha256::derive_key_primitive(b"pw", b"salt", 1, &mut []).expect("PBKDF2-SHA256 permits empty output"); } #[test] fn sha512_empty_output_ok() { - assert!(Pbkdf2Sha512::derive_key_primitive(b"pw", b"salt", 1, &mut []).is_ok()); + Pbkdf2Sha512::derive_key_primitive(b"pw", b"salt", 1, &mut []).expect("PBKDF2-SHA512 permits empty output"); } #[test] @@ -1870,8 +1856,10 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - Pbkdf2Sha256::derive_key_primitive(&password, &salt, 2, &mut expected).unwrap(); - assert!(Pbkdf2Sha256::verify_password_primitive(&password, &salt, 2, &expected).is_ok()); + Pbkdf2Sha256::derive_key_primitive(&password, &salt, 2, &mut expected) + .expect("coverage case parameters are valid"); + Pbkdf2Sha256::verify_password_primitive(&password, &salt, 2, &expected) + .expect("matching PBKDF2-SHA256 output must verify"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; @@ -1895,8 +1883,10 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - Pbkdf2Sha512::derive_key_primitive(&password, &salt, 2, &mut expected).unwrap(); - assert!(Pbkdf2Sha512::verify_password_primitive(&password, &salt, 2, &expected).is_ok()); + Pbkdf2Sha512::derive_key_primitive(&password, &salt, 2, &mut expected) + .expect("coverage case parameters are valid"); + Pbkdf2Sha512::verify_password_primitive(&password, &salt, 2, &expected) + .expect("matching PBKDF2-SHA512 output must verify"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; @@ -1914,11 +1904,17 @@ mod tests { #[test] fn sha256_state_reuse_matches_oneshot() { let state = Pbkdf2Sha256::new(b"password"); - let dk1 = state.derive_array::<32>(b"salt1", 100).unwrap(); - let dk2 = state.derive_array::<32>(b"salt2", 100).unwrap(); - - let oneshot1 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt1", 100).unwrap(); - let oneshot2 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt2", 100).unwrap(); + let dk1 = state + .derive_array::<32>(b"salt1", 100) + .expect("first state-reuse derivation parameters are valid"); + let dk2 = state + .derive_array::<32>(b"salt2", 100) + .expect("second state-reuse derivation parameters are valid"); + + let oneshot1 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt1", 100) + .expect("first one-shot derivation parameters are valid"); + let oneshot2 = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"password", b"salt2", 100) + .expect("second one-shot derivation parameters are valid"); assert_eq!(dk1, oneshot1); assert_eq!(dk2, oneshot2); @@ -1931,7 +1927,8 @@ mod tests { fn sha256_single_iteration() { let mut expected = [0u8; 32]; oracle_sha256(b"pw", b"salt", 1, &mut expected); - let actual = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"pw", b"salt", 1).unwrap(); + let actual = Pbkdf2Sha256::derive_key_array_primitive::<32>(b"pw", b"salt", 1) + .expect("single-iteration PBKDF2-SHA256 parameters are valid"); assert_eq!(actual, expected); } @@ -1939,7 +1936,8 @@ mod tests { fn sha512_single_iteration() { let mut expected = [0u8; 64]; oracle_sha512(b"pw", b"salt", 1, &mut expected); - let actual = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"pw", b"salt", 1).unwrap(); + let actual = Pbkdf2Sha512::derive_key_array_primitive::<64>(b"pw", b"salt", 1) + .expect("single-iteration PBKDF2-SHA512 parameters are valid"); assert_eq!(actual, expected); } @@ -2054,7 +2052,9 @@ mod tests { let state = Pbkdf2Sha256::new_with_compress_for_test(password, compress); let mut actual = vec![0u8; dk_len]; - state.derive(salt, iterations, &mut actual).unwrap(); + state + .derive(salt, iterations, &mut actual) + .expect("forced SHA-256 kernel case parameters are valid"); assert_eq!( actual, @@ -2089,7 +2089,9 @@ mod tests { let state = Pbkdf2Sha512::new_with_compress_for_test(password, compress); let mut actual = vec![0u8; dk_len]; - state.derive(salt, iterations, &mut actual).unwrap(); + state + .derive(salt, iterations, &mut actual) + .expect("forced SHA-512 kernel case parameters are valid"); assert_eq!( actual, @@ -2136,10 +2138,12 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - state.derive(&salt, 3, &mut expected).unwrap(); + state + .derive(&salt, 3, &mut expected) + .expect("SHA-256 work-count case parameters are valid"); let (ok, ok_blocks) = counted_sha256_verify(&state, &salt, 3, &expected); - assert!(ok.is_ok(), "sha256 verify must accept correct output_len={out_len}"); + ok.expect("SHA-256 verification must accept matching output"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; @@ -2182,10 +2186,12 @@ mod tests { for out_len in output_lengths { let mut expected = vec![0u8; out_len]; - state.derive(&salt, 3, &mut expected).unwrap(); + state + .derive(&salt, 3, &mut expected) + .expect("SHA-512 work-count case parameters are valid"); let (ok, ok_blocks) = counted_sha512_verify(&state, &salt, 3, &expected); - assert!(ok.is_ok(), "sha512 verify must accept correct output_len={out_len}"); + ok.expect("SHA-512 verification must accept matching output"); let mut wrong_first = expected.clone(); wrong_first[0] ^= 1; diff --git a/src/auth/phc.rs b/src/auth/phc.rs index 73d8b83f..c6115b93 100644 --- a/src/auth/phc.rs +++ b/src/auth/phc.rs @@ -11,16 +11,6 @@ //! //! [phc]: https://github.com/P-H-C/phc-string-format/blob/master/phc-sf-spec.md -#![allow(clippy::indexing_slicing)] -// Base64 and decimal helpers are only reachable when a -// PHC-aware hasher (argon2 or scrypt) is enabled. Without either, the -// helpers are dead code — silence the warning rather than cfg-gate every -// symbol individually. -#![cfg_attr( - all(feature = "phc-strings", not(any(feature = "argon2", feature = "scrypt"))), - allow(dead_code) -)] - use alloc::string::String; use core::fmt; @@ -45,39 +35,27 @@ const B64_DECODE_TABLE: [u8; 256] = { /// Appends to `out` — callers managing multi-segment PHC strings reuse the /// same `String` buffer without intermediate allocation. pub(crate) fn base64_encode_into(bytes: &[u8], out: &mut String) { - let full_triples = bytes.len() / 3; - let tail = bytes.len() % 3; - - for i in 0..full_triples { - let off = i.strict_mul(3); - // SAFETY: off + 3 <= bytes.len() by construction of full_triples. - let b0 = bytes[off] as u32; - let b1 = bytes[off.strict_add(1)] as u32; - let b2 = bytes[off.strict_add(2)] as u32; - let word = (b0 << 16) | (b1 << 8) | b2; - - out.push(B64_ENCODE_TABLE[((word >> 18) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word >> 12) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word >> 6) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[(word & 0x3F) as usize] as char); - } - - let off = full_triples.strict_mul(3); - match tail { - 1 => { - let b0 = bytes[off] as u32; - out.push(B64_ENCODE_TABLE[((b0 >> 2) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((b0 << 4) & 0x3F) as usize] as char); - } - 2 => { - let b0 = bytes[off] as u32; - let b1 = bytes[off.strict_add(1)] as u32; - let word = (b0 << 8) | b1; - out.push(B64_ENCODE_TABLE[((word >> 10) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word >> 4) & 0x3F) as usize] as char); - out.push(B64_ENCODE_TABLE[((word << 2) & 0x3F) as usize] as char); - } - _ => {} + let (triples, tail) = bytes.as_chunks::<3>(); + for &[b0, b1, b2] in triples { + out.push(char::from(B64_ENCODE_TABLE[usize::from(b0 >> 2)])); + out.push(char::from( + B64_ENCODE_TABLE[usize::from(((b0 & 0x03) << 4) | (b1 >> 4))], + )); + out.push(char::from( + B64_ENCODE_TABLE[usize::from(((b1 & 0x0F) << 2) | (b2 >> 6))], + )); + out.push(char::from(B64_ENCODE_TABLE[usize::from(b2 & 0x3F)])); + } + + if let &[b0, b1] = tail { + out.push(char::from(B64_ENCODE_TABLE[usize::from(b0 >> 2)])); + out.push(char::from( + B64_ENCODE_TABLE[usize::from(((b0 & 0x03) << 4) | (b1 >> 4))], + )); + out.push(char::from(B64_ENCODE_TABLE[usize::from((b1 & 0x0F) << 2)])); + } else if let &[b0] = tail { + out.push(char::from(B64_ENCODE_TABLE[usize::from(b0 >> 2)])); + out.push(char::from(B64_ENCODE_TABLE[usize::from((b0 & 0x03) << 4)])); } } @@ -106,41 +84,30 @@ pub(crate) const fn base64_decoded_len(encoded_len: usize) -> usize { /// round-trip). pub(crate) fn base64_decode_into(s: &str, out: &mut [u8]) -> Result { let bytes = s.as_bytes(); - let full = bytes.len() / 4; - let tail = bytes.len() % 4; - - if tail == 1 { + let (groups, tail) = bytes.as_chunks::<4>(); + if tail.len() == 1 { return Err(PhcError::InvalidBase64); } - let expected_out = base64_decoded_len(bytes.len()); - if out.len() < expected_out { - return Err(PhcError::OutputBufferTooSmall); - } - - let mut written = 0usize; - for i in 0..full { - let off = i.strict_mul(4); - let d0 = B64_DECODE_TABLE[bytes[off] as usize]; - let d1 = B64_DECODE_TABLE[bytes[off.strict_add(1)] as usize]; - let d2 = B64_DECODE_TABLE[bytes[off.strict_add(2)] as usize]; - let d3 = B64_DECODE_TABLE[bytes[off.strict_add(3)] as usize]; + let destination = out.get_mut(..expected_out).ok_or(PhcError::OutputBufferTooSmall)?; + let (output_groups, output_tail) = destination.as_chunks_mut::<3>(); + + for (&[b0, b1, b2, b3], output) in groups.iter().zip(output_groups) { + let d0 = B64_DECODE_TABLE[usize::from(b0)]; + let d1 = B64_DECODE_TABLE[usize::from(b1)]; + let d2 = B64_DECODE_TABLE[usize::from(b2)]; + let d3 = B64_DECODE_TABLE[usize::from(b3)]; if (d0 | d1 | d2 | d3) == 0xFF { return Err(PhcError::InvalidBase64); } - let word = ((d0 as u32) << 18) | ((d1 as u32) << 12) | ((d2 as u32) << 6) | (d3 as u32); - out[written] = (word >> 16) as u8; - out[written.strict_add(1)] = (word >> 8) as u8; - out[written.strict_add(2)] = word as u8; - written = written.strict_add(3); + *output = [(d0 << 2) | (d1 >> 4), (d1 << 4) | (d2 >> 2), (d2 << 6) | d3]; } - let off = full.strict_mul(4); - match tail { - 0 => {} - 2 => { - let d0 = B64_DECODE_TABLE[bytes[off] as usize]; - let d1 = B64_DECODE_TABLE[bytes[off.strict_add(1)] as usize]; + match (tail, output_tail) { + ([], []) => {} + (&[b0, b1], [output]) => { + let d0 = B64_DECODE_TABLE[usize::from(b0)]; + let d1 = B64_DECODE_TABLE[usize::from(b1)]; if (d0 | d1) == 0xFF { return Err(PhcError::InvalidBase64); } @@ -148,13 +115,12 @@ pub(crate) fn base64_decode_into(s: &str, out: &mut [u8]) -> Result> 4); - written = written.strict_add(1); + *output = (d0 << 2) | (d1 >> 4); } - 3 => { - let d0 = B64_DECODE_TABLE[bytes[off] as usize]; - let d1 = B64_DECODE_TABLE[bytes[off.strict_add(1)] as usize]; - let d2 = B64_DECODE_TABLE[bytes[off.strict_add(2)] as usize]; + (&[b0, b1, b2], [output0, output1]) => { + let d0 = B64_DECODE_TABLE[usize::from(b0)]; + let d1 = B64_DECODE_TABLE[usize::from(b1)]; + let d2 = B64_DECODE_TABLE[usize::from(b2)]; if (d0 | d1 | d2) == 0xFF { return Err(PhcError::InvalidBase64); } @@ -162,15 +128,13 @@ pub(crate) fn base64_decode_into(s: &str, out: &mut [u8]) -> Result> 2); - out[written] = (word >> 8) as u8; - out[written.strict_add(1)] = word as u8; - written = written.strict_add(2); + *output0 = (d0 << 2) | (d1 >> 4); + *output1 = (d1 << 4) | (d2 >> 2); } _ => return Err(PhcError::InvalidBase64), } - Ok(written) + Ok(expected_out) } /// Append `n` as base-10 decimal (no leading zero) to `out`. @@ -189,12 +153,13 @@ pub(crate) fn push_u32_decimal(out: &mut String, n: u32) { let mut len = 0usize; let mut v = n; while v > 0 { - digits[len] = b'0' + (v % 10) as u8; + let [digit, _, _, _] = (v % 10).to_le_bytes(); + digits[len] = b'0'.wrapping_add(digit); v /= 10; len = len.strict_add(1); } for i in (0..len).rev() { - out.push(digits[i] as char); + out.push(char::from(digits[i])); } } @@ -206,15 +171,13 @@ pub(crate) fn push_u32_decimal(out: &mut String, n: u32) { /// keys, empty values, missing `=`, and empty pair segments are reported as /// `PhcError::MalformedParams`. pub(crate) struct PhcParamIter<'a> { - rest: &'a str, - done: bool, + rest: Option<&'a str>, } impl<'a> PhcParamIter<'a> { pub(crate) fn new(params: &'a str) -> Self { Self { - rest: params, - done: params.is_empty(), + rest: (!params.is_empty()).then_some(params), } } } @@ -223,35 +186,20 @@ impl<'a> Iterator for PhcParamIter<'a> { type Item = Result<(&'a str, &'a str), PhcError>; fn next(&mut self) -> Option { - if self.done { - return None; - } - let (pair, advance) = match self.rest.find(',') { - Some(idx) => { - // SAFETY: idx is a valid char boundary (',' is ASCII). - let pair = &self.rest[..idx]; - self.rest = &self.rest[idx.strict_add(1)..]; - (pair, false) - } - None => { - let pair = self.rest; - self.rest = ""; - (pair, true) - } + let rest = self.rest?; + let (pair, remaining) = match rest.split_once(',') { + Some((pair, remaining)) => (pair, Some(remaining)), + None => (rest, None), }; - if advance { - self.done = true; - } + self.rest = remaining; if pair.is_empty() { return Some(Err(PhcError::MalformedParams)); } - let eq = match pair.find('=') { - Some(i) => i, + let (key, value) = match pair.split_once('=') { + Some(fields) => fields, None => return Some(Err(PhcError::MalformedParams)), }; - let key = &pair[..eq]; - let value = &pair[eq.strict_add(1)..]; if key.is_empty() || value.is_empty() { return Some(Err(PhcError::MalformedParams)); } @@ -267,20 +215,21 @@ pub(crate) fn parse_param_u32(value: &str) -> Result { } // Reject leading zeros (e.g. "01") and leading sign (e.g. "-1", "+1"). let bytes = value.as_bytes(); - if bytes.len() > 1 && bytes[0] == b'0' { + if matches!(bytes, [b'0', _, ..]) { return Err(PhcError::MalformedParams); } - let mut acc: u64 = 0; + let mut acc = 0u32; for &b in bytes { if !b.is_ascii_digit() { return Err(PhcError::MalformedParams); } - acc = acc.strict_mul(10).strict_add((b - b'0') as u64); - if acc > u32::MAX as u64 { - return Err(PhcError::ParamOutOfRange); - } + let digit = u32::from(b.wrapping_sub(b'0')); + acc = acc + .checked_mul(10) + .and_then(|prefix| prefix.checked_add(digit)) + .ok_or(PhcError::ParamOutOfRange)?; } - Ok(u32::try_from(acc).unwrap_or_else(|_| unreachable!("acc <= u32::MAX, enforced inside the loop"))) + Ok(acc) } // ─── Segmented PHC parser ─────────────────────────────────────────────────── @@ -440,13 +389,13 @@ mod tests { #[test] fn base64_roundtrip_all_lengths_0_to_64() { - for len in 0..=64 { - let input: Vec = (0..len).map(|i| ((i * 31 + 7) & 0xff) as u8).collect(); + for len in 0u8..=64 { + let input: Vec = (0..len).map(|i| i.wrapping_mul(31).wrapping_add(7)).collect(); let mut encoded = String::new(); base64_encode_into(&input, &mut encoded); let mut decoded = vec![0u8; base64_decoded_len(encoded.len())]; - let n = base64_decode_into(&encoded, &mut decoded).unwrap(); + let n = base64_decode_into(&encoded, &mut decoded).expect("encoder output must be canonical base64"); decoded.truncate(n); assert_eq!(decoded, input, "roundtrip failed at len={len}"); } @@ -478,6 +427,13 @@ mod tests { let mut out = [0u8; 32]; assert_eq!(base64_decode_into("A", &mut out), Err(PhcError::InvalidBase64)); assert_eq!(base64_decode_into("AAAAA", &mut out), Err(PhcError::InvalidBase64)); + + let mut empty = []; + assert_eq!( + base64_decode_into("A", &mut empty), + Err(PhcError::InvalidBase64), + "an invalid tail takes precedence over destination sizing" + ); } #[test] @@ -506,16 +462,26 @@ mod tests { #[test] fn param_iter_single_pair() { let mut it = PhcParamIter::new("m=65536"); - assert_eq!(it.next().unwrap().unwrap(), ("m", "65536")); + assert_eq!( + it.next() + .expect("one parameter must be present") + .expect("the parameter must be well-formed"), + ("m", "65536") + ); assert!(it.next().is_none()); } #[test] fn param_iter_multiple_pairs() { let mut it = PhcParamIter::new("m=65536,t=3,p=4"); - assert_eq!(it.next().unwrap().unwrap(), ("m", "65536")); - assert_eq!(it.next().unwrap().unwrap(), ("t", "3")); - assert_eq!(it.next().unwrap().unwrap(), ("p", "4")); + for expected in [("m", "65536"), ("t", "3"), ("p", "4")] { + assert_eq!( + it.next() + .expect("the expected parameter must be present") + .expect("the parameter must be well-formed"), + expected + ); + } assert!(it.next().is_none()); } @@ -528,34 +494,69 @@ mod tests { #[test] fn param_iter_rejects_missing_equals() { let mut it = PhcParamIter::new("mX65536"); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next().expect("the malformed parameter must be emitted"), + Err(PhcError::MalformedParams) + ); } #[test] fn param_iter_rejects_empty_pair_segment() { let mut it = PhcParamIter::new("m=1,,p=2"); - assert_eq!(it.next().unwrap().unwrap(), ("m", "1")); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next() + .expect("the first parameter must be present") + .expect("the first parameter must be well-formed"), + ("m", "1") + ); + assert_eq!( + it.next().expect("the empty parameter must be emitted"), + Err(PhcError::MalformedParams) + ); + } + + #[test] + fn param_iter_rejects_trailing_empty_pair_segment() { + let mut it = PhcParamIter::new("m=1,"); + assert_eq!( + it.next() + .expect("the first parameter must be present") + .expect("the first parameter must be well-formed"), + ("m", "1") + ); + assert_eq!( + it.next().expect("the trailing empty parameter must be emitted"), + Err(PhcError::MalformedParams) + ); + assert!(it.next().is_none()); } #[test] fn param_iter_rejects_empty_key() { let mut it = PhcParamIter::new("=65536"); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next().expect("the malformed parameter must be emitted"), + Err(PhcError::MalformedParams) + ); } #[test] fn param_iter_rejects_empty_value() { let mut it = PhcParamIter::new("m="); - assert_eq!(it.next().unwrap(), Err(PhcError::MalformedParams)); + assert_eq!( + it.next().expect("the malformed parameter must be emitted"), + Err(PhcError::MalformedParams) + ); } #[test] fn parse_param_u32_accepts_valid() { - assert_eq!(parse_param_u32("0").unwrap(), 0); - assert_eq!(parse_param_u32("1").unwrap(), 1); - assert_eq!(parse_param_u32("65536").unwrap(), 65_536); - assert_eq!(parse_param_u32("4294967295").unwrap(), u32::MAX); + for (encoded, expected) in [("0", 0), ("1", 1), ("65536", 65_536), ("4294967295", u32::MAX)] { + assert_eq!( + parse_param_u32(encoded).expect("canonical u32 text must parse"), + expected + ); + } } #[test] @@ -574,7 +575,7 @@ mod tests { #[test] fn parse_argon2id_canonical() { let encoded = "$argon2id$v=19$m=65536,t=3,p=4$c29tZXNhbHQ$c29tZWhhc2g"; - let parts = parse(encoded).unwrap(); + let parts = parse(encoded).expect("canonical Argon2id PHC text must parse"); assert_eq!(parts.algorithm, "argon2id"); assert_eq!(parts.version, Some("19")); assert_eq!(parts.parameters, "m=65536,t=3,p=4"); @@ -585,7 +586,7 @@ mod tests { #[test] fn parse_scrypt_no_version() { let encoded = "$scrypt$ln=14,r=8,p=1$c29tZXNhbHQ$c29tZWhhc2g"; - let parts = parse(encoded).unwrap(); + let parts = parse(encoded).expect("canonical scrypt PHC text must parse"); assert_eq!(parts.algorithm, "scrypt"); assert_eq!(parts.version, None); assert_eq!(parts.parameters, "ln=14,r=8,p=1"); @@ -636,7 +637,7 @@ mod tests { #[test] fn parse_without_version_segment_returns_none() { - let parts = parse("$argon2id$m=1,t=1,p=1$c29tZQ$c29tZQ").unwrap(); + let parts = parse("$argon2id$m=1,t=1,p=1$c29tZQ$c29tZQ").expect("PHC text without a version segment must parse"); assert_eq!(parts.version, None); } diff --git a/src/auth/poly1305.rs b/src/auth/poly1305.rs index b8bee637..06f8ddc2 100644 --- a/src/auth/poly1305.rs +++ b/src/auth/poly1305.rs @@ -1,7 +1,5 @@ //! Standalone Poly1305 one-time authenticator (RFC 8439). -#![allow(clippy::indexing_slicing)] // Poly1305 uses fixed 16-byte block and limb offsets. - use core::fmt; use crate::{ @@ -22,6 +20,22 @@ fn load_u32_le(input: &[u8]) -> u32 { u32::from_le_bytes(bytes) } +#[inline(always)] +fn low_u32(value: u64) -> u32 { + let bytes = value.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + +#[inline(always)] +fn limb_product(left: u32, right: u32) -> u64 { + u64::from(left).wrapping_mul(u64::from(right)) +} + +#[inline(always)] +fn limb_multiply_accumulate(accumulator: u64, left: u32, right: u32) -> u64 { + accumulator.wrapping_add(limb_product(left, right)) +} + /// Poly1305 one-time key. /// /// This type is intentionally not `Clone` or `Copy`. Poly1305 keys must be @@ -231,10 +245,10 @@ impl State { let r3 = self.r[3]; let r4 = self.r[4]; - let s1 = r1 * 5; - let s2 = r2 * 5; - let s3 = r3 * 5; - let s4 = r4 * 5; + let s1 = r1.wrapping_mul(5); + let s2 = r2.wrapping_mul(5); + let s3 = r3.wrapping_mul(5); + let s4 = r4.wrapping_mul(5); let mut h0 = self.h[0]; let mut h1 = self.h[1]; @@ -248,51 +262,55 @@ impl State { h3 = h3.wrapping_add((load_u32_le(&block[9..13]) >> 6) & LIMB_MASK); h4 = h4.wrapping_add((load_u32_le(&block[12..16]) >> 8) | hibit); - let d0 = (u64::from(h0) * u64::from(r0)) - + (u64::from(h1) * u64::from(s4)) - + (u64::from(h2) * u64::from(s3)) - + (u64::from(h3) * u64::from(s2)) - + (u64::from(h4) * u64::from(s1)); - let mut d1 = (u64::from(h0) * u64::from(r1)) - + (u64::from(h1) * u64::from(r0)) - + (u64::from(h2) * u64::from(s4)) - + (u64::from(h3) * u64::from(s3)) - + (u64::from(h4) * u64::from(s2)); - let mut d2 = (u64::from(h0) * u64::from(r2)) - + (u64::from(h1) * u64::from(r1)) - + (u64::from(h2) * u64::from(r0)) - + (u64::from(h3) * u64::from(s4)) - + (u64::from(h4) * u64::from(s3)); - let mut d3 = (u64::from(h0) * u64::from(r3)) - + (u64::from(h1) * u64::from(r2)) - + (u64::from(h2) * u64::from(r1)) - + (u64::from(h3) * u64::from(r0)) - + (u64::from(h4) * u64::from(s4)); - let mut d4 = (u64::from(h0) * u64::from(r4)) - + (u64::from(h1) * u64::from(r3)) - + (u64::from(h2) * u64::from(r2)) - + (u64::from(h3) * u64::from(r1)) - + (u64::from(h4) * u64::from(r0)); - - let mut c = (d0 >> 26) as u32; - h0 = (d0 as u32) & LIMB_MASK; - d1 += u64::from(c); - - c = (d1 >> 26) as u32; - h1 = (d1 as u32) & LIMB_MASK; - d2 += u64::from(c); - - c = (d2 >> 26) as u32; - h2 = (d2 as u32) & LIMB_MASK; - d3 += u64::from(c); - - c = (d3 >> 26) as u32; - h3 = (d3 as u32) & LIMB_MASK; - d4 += u64::from(c); - - c = (d4 >> 26) as u32; - h4 = (d4 as u32) & LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + let mut d0 = limb_product(h0, r0); + d0 = limb_multiply_accumulate(d0, h1, s4); + d0 = limb_multiply_accumulate(d0, h2, s3); + d0 = limb_multiply_accumulate(d0, h3, s2); + d0 = limb_multiply_accumulate(d0, h4, s1); + + let mut d1 = limb_product(h0, r1); + d1 = limb_multiply_accumulate(d1, h1, r0); + d1 = limb_multiply_accumulate(d1, h2, s4); + d1 = limb_multiply_accumulate(d1, h3, s3); + d1 = limb_multiply_accumulate(d1, h4, s2); + + let mut d2 = limb_product(h0, r2); + d2 = limb_multiply_accumulate(d2, h1, r1); + d2 = limb_multiply_accumulate(d2, h2, r0); + d2 = limb_multiply_accumulate(d2, h3, s4); + d2 = limb_multiply_accumulate(d2, h4, s3); + + let mut d3 = limb_product(h0, r3); + d3 = limb_multiply_accumulate(d3, h1, r2); + d3 = limb_multiply_accumulate(d3, h2, r1); + d3 = limb_multiply_accumulate(d3, h3, r0); + d3 = limb_multiply_accumulate(d3, h4, s4); + + let mut d4 = limb_product(h0, r4); + d4 = limb_multiply_accumulate(d4, h1, r3); + d4 = limb_multiply_accumulate(d4, h2, r2); + d4 = limb_multiply_accumulate(d4, h3, r1); + d4 = limb_multiply_accumulate(d4, h4, r0); + + let mut c = low_u32(d0 >> 26); + h0 = low_u32(d0) & LIMB_MASK; + d1 = d1.wrapping_add(u64::from(c)); + + c = low_u32(d1 >> 26); + h1 = low_u32(d1) & LIMB_MASK; + d2 = d2.wrapping_add(u64::from(c)); + + c = low_u32(d2 >> 26); + h2 = low_u32(d2) & LIMB_MASK; + d3 = d3.wrapping_add(u64::from(c)); + + c = low_u32(d3 >> 26); + h3 = low_u32(d3) & LIMB_MASK; + d4 = d4.wrapping_add(u64::from(c)); + + c = low_u32(d4 >> 26); + h4 = low_u32(d4) & LIMB_MASK; + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -323,7 +341,7 @@ impl State { c = h4 >> 26; h4 &= LIMB_MASK; - h0 = h0.wrapping_add(c * 5); + h0 = h0.wrapping_add(c.wrapping_mul(5)); c = h0 >> 26; h0 &= LIMB_MASK; @@ -366,14 +384,14 @@ impl State { h2 = (h2 >> 12) | (h3 << 14); h3 = (h3 >> 18) | (h4 << 8); - let mut f = u64::from(h0) + u64::from(self.pad[0]); - h0 = f as u32; - f = u64::from(h1) + u64::from(self.pad[1]) + (f >> 32); - h1 = f as u32; - f = u64::from(h2) + u64::from(self.pad[2]) + (f >> 32); - h2 = f as u32; - f = u64::from(h3) + u64::from(self.pad[3]) + (f >> 32); - h3 = f as u32; + let mut f = u64::from(h0).wrapping_add(u64::from(self.pad[0])); + h0 = low_u32(f); + f = u64::from(h1).wrapping_add(u64::from(self.pad[1])).wrapping_add(f >> 32); + h1 = low_u32(f); + f = u64::from(h2).wrapping_add(u64::from(self.pad[2])).wrapping_add(f >> 32); + h2 = low_u32(f); + f = u64::from(h3).wrapping_add(u64::from(self.pad[3])).wrapping_add(f >> 32); + h3 = low_u32(f); let mut tag = [0u8; TAG_SIZE]; tag[0..4].copy_from_slice(&h0.to_le_bytes()); @@ -433,7 +451,7 @@ impl Poly1305 { #[inline] pub fn update(&mut self, mut data: &[u8]) { if self.buffer_len != 0 { - let take = core::cmp::min(TAG_SIZE - self.buffer_len, data.len()); + let take = core::cmp::min(TAG_SIZE.strict_sub(self.buffer_len), data.len()); self.buffer[self.buffer_len..self.buffer_len.strict_add(take)].copy_from_slice(&data[..take]); self.buffer_len = self.buffer_len.strict_add(take); data = &data[take..]; @@ -445,15 +463,11 @@ impl Poly1305 { } } - let mut blocks = data.chunks_exact(TAG_SIZE); - for chunk in &mut blocks { - let mut block = [0u8; TAG_SIZE]; - block.copy_from_slice(chunk); - self.state.compute_block(&block, false); - ct::zeroize_no_fence(&mut block); + let (blocks, rem) = data.as_chunks::(); + for block in blocks { + self.state.compute_block(block, false); } - let rem = blocks.remainder(); if !rem.is_empty() { self.buffer[..rem.len()].copy_from_slice(rem); self.buffer_len = rem.len(); @@ -512,7 +526,7 @@ impl Drop for Poly1305 { fn drop(&mut self) { ct::zeroize(&mut self.buffer); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.buffer_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.buffer_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } diff --git a/src/auth/rsa.rs b/src/auth/rsa.rs index da035f87..248f5059 100644 --- a/src/auth/rsa.rs +++ b/src/auth/rsa.rs @@ -1275,6 +1275,21 @@ pub struct RsaPrivateKeyParts<'a> { pub coefficient: &'a [u8], } +impl fmt::Debug for RsaPrivateKeyParts<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RsaPrivateKeyParts") + .field("modulus_bits", &unsigned_be_bit_len(self.modulus)) + .field("public_exponent", &self.public_exponent) + .field("private_exponent", &"****") + .field("prime_p", &"****") + .field("prime_q", &"****") + .field("exponent_p", &"****") + .field("exponent_q", &"****") + .field("coefficient", &"****") + .finish() + } +} + impl RsaPrivateKey { /// Key-generation contract used by [`Self::generate`] and /// [`Self::generate_with_policy`]. @@ -2053,11 +2068,7 @@ impl RsaPrivateKey { clear_decryption_output_on_error(result, out) } - /// Sign a message using RSASSA-PKCS1-v1_5 with a caller-supplied blinding factor. - /// - /// `blinding_factor` and `blinding_factor_inverse` must be fixed-width - /// modulus-sized representatives satisfying - /// `blinding_factor * blinding_factor_inverse == 1 mod n`. + /// Sign a message using RSASSA-PKCS1-v1_5 with caller-supplied blinding. /// /// # Errors /// @@ -2068,23 +2079,19 @@ impl RsaPrivateKey { &self, profile: RsaPkcs1v15Profile, message: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pkcs1v15_with_blinding_factor( - profile, - message, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .sign_pkcs1v15_with_blinding_factor(profile, message, blinding, out); clear_output_on_error(result, out) } /// Sign using RSASSA-PKCS1-v1_5 with caller-supplied blinding and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The blinding-factor requirements are the same as + /// after setup. The blinding-pair requirements are the same as /// [`Self::sign_pkcs1v15_with_blinding_factor`]. /// /// # Errors @@ -2097,18 +2104,13 @@ impl RsaPrivateKey { &self, profile: RsaPkcs1v15Profile, message: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pkcs1v15_with_blinding_factor_and_scratch( - profile, - message, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .sign_pkcs1v15_with_blinding_factor_and_scratch(profile, message, blinding, out, scratch); clear_output_on_error(result, out) } @@ -2125,24 +2127,19 @@ impl RsaPrivateKey { profile: RsaPssProfile, message: &[u8], salt: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pss_with_salt_and_blinding_factor( - profile, - message, - salt, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .sign_pss_with_salt_and_blinding_factor(profile, message, salt, blinding, out); clear_output_on_error(result, out) } /// Sign using RSASSA-PSS with explicit salt, caller blinding, and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The salt and blinding-factor requirements are the same as + /// after setup. The salt and blinding-pair requirements are the same as /// [`Self::sign_pss_with_salt_and_blinding_factor`]. /// /// # Errors @@ -2151,26 +2148,19 @@ impl RsaPrivateKey { /// allocated for a different modulus width, the blinding pair is invalid, the /// key is too small for the salt/profile, or the post-signing public fault /// check fails. - #[allow(clippy::too_many_arguments)] #[must_use = "RSA signing failure must be checked; a dropped Result silently discards a failed signature"] pub fn sign_pss_with_salt_and_blinding_factor_and_scratch( &self, profile: RsaPssProfile, message: &[u8], salt: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result<(), RsaPrivateOpError> { - let result = self.components.sign_pss_with_salt_and_blinding_factor_and_scratch( - profile, - message, - salt, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .sign_pss_with_salt_and_blinding_factor_and_scratch(profile, message, salt, blinding, out, scratch); clear_output_on_error(result, out) } @@ -2190,17 +2180,12 @@ impl RsaPrivateKey { profile: RsaOaepProfile, label: &[u8], ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result { - let result = self.components.decrypt_oaep_with_blinding_factor( - profile, - label, - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .decrypt_oaep_with_blinding_factor(profile, label, ciphertext, blinding, out); clear_decryption_output_on_error(result, out) } @@ -2218,22 +2203,19 @@ impl RsaPrivateKey { pub fn decrypt_pkcs1v15_with_blinding_factor( &self, ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], ) -> Result { - let result = self.components.decrypt_pkcs1v15_with_blinding_factor( - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - ); + let result = self + .components + .decrypt_pkcs1v15_with_blinding_factor(ciphertext, blinding, out); clear_decryption_output_on_error(result, out) } /// Decrypt RSAES-OAEP with caller-supplied blinding and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The blinding-factor requirements are the same as + /// after setup. The blinding-pair requirements are the same as /// [`Self::decrypt_oaep_with_blinding_factor`]. /// /// # Errors @@ -2241,33 +2223,26 @@ impl RsaPrivateKey { /// Returns [`RsaPrivateOpError`] if lengths are invalid, `scratch` was /// allocated for a different modulus width, the blinding pair is invalid, the /// ciphertext representative is out of range, or OAEP decoding fails. - #[allow(clippy::too_many_arguments)] #[must_use = "RSA decryption failure must be checked; a dropped Result silently discards plaintext"] pub fn decrypt_oaep_with_blinding_factor_and_scratch( &self, profile: RsaOaepProfile, label: &[u8], ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result { - let result = self.components.decrypt_oaep_with_blinding_factor_and_scratch( - profile, - label, - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .decrypt_oaep_with_blinding_factor_and_scratch(profile, label, ciphertext, blinding, out, scratch); clear_decryption_output_on_error(result, out) } /// Decrypt RSAES-PKCS1-v1_5 with caller-supplied blinding and scratch. /// /// Reusing scratch avoids top-level private-operation temporary allocation - /// after setup. The blinding-factor requirements are the same as + /// after setup. The blinding-pair requirements are the same as /// [`Self::decrypt_pkcs1v15_with_blinding_factor`]. /// /// # Errors @@ -2279,17 +2254,13 @@ impl RsaPrivateKey { pub fn decrypt_pkcs1v15_with_blinding_factor_and_scratch( &self, ciphertext: &[u8], - blinding_factor: &[u8], - blinding_factor_inverse: &[u8], + blinding: RsaBlindingPair<'_>, out: &mut [u8], scratch: &mut RsaPrivateScratch, ) -> Result { - let result = self.components.decrypt_pkcs1v15_with_blinding_factor_and_scratch( - ciphertext, - RsaBlindingPair::caller_supplied(blinding_factor, blinding_factor_inverse), - out, - scratch, - ); + let result = self + .components + .decrypt_pkcs1v15_with_blinding_factor_and_scratch(ciphertext, blinding, out, scratch); clear_decryption_output_on_error(result, out) } } @@ -2612,7 +2583,6 @@ impl Drop for RsaPrivateScratch { // separate from private-operation exponentiation so validation arithmetic does // not become signing, decryption, blinding, CRT recombination, or key // generation arithmetic by accident. -#[allow(dead_code)] struct RsaPrivateKeyComponents { public: RsaPublicKey, private_exponent: SecretBigEndianInteger, @@ -2625,7 +2595,6 @@ struct RsaPrivateKeyComponents { coefficient: SecretBigEndianInteger, } -#[allow(dead_code)] impl RsaPrivateKeyComponents { #[inline] fn public_key(&self) -> &RsaPublicKey { @@ -2991,6 +2960,7 @@ impl RsaPrivateKeyComponents { result } + #[cfg(feature = "getrandom")] fn sign_pkcs1v15_with_stored_blinding_and_scratch( &self, profile: RsaPkcs1v15Profile, @@ -3084,6 +3054,7 @@ impl RsaPrivateKeyComponents { result } + #[cfg(feature = "getrandom")] fn sign_pss_with_stored_salt_and_blinding_and_scratch( &self, profile: RsaPssProfile, @@ -3162,6 +3133,7 @@ impl RsaPrivateKeyComponents { self.private_operation_with_blinding_factor_and_scratch(blinding, out, scratch) } + #[cfg(feature = "getrandom")] fn sign_encoded_message_with_stored_blinding_and_scratch( &self, out: &mut [u8], @@ -3213,6 +3185,7 @@ impl RsaPrivateKeyComponents { clear_decryption_output_on_error(result, out) } + #[cfg(feature = "getrandom")] fn decrypt_oaep_with_stored_blinding_and_scratch( &self, profile: RsaOaepProfile, @@ -3262,6 +3235,7 @@ impl RsaPrivateKeyComponents { clear_decryption_output_on_error(result, out) } + #[cfg(feature = "getrandom")] fn decrypt_pkcs1v15_with_stored_blinding_and_scratch( &self, ciphertext: &[u8], @@ -3457,6 +3431,7 @@ impl RsaPrivateKeyComponents { } } + #[cfg(feature = "getrandom")] fn private_operation_from_scratch_encoded_with_stored_blinding( &self, scratch: &mut RsaPrivateScratch, @@ -3690,12 +3665,10 @@ impl fmt::Debug for RsaPrivateKeyComponents { } } -#[allow(dead_code)] struct SecretBigEndianInteger { bytes: Box<[u8]>, } -#[allow(dead_code)] impl SecretBigEndianInteger { fn new(bytes: &[u8]) -> Result { if is_zero_unsigned_be(bytes) { @@ -3706,6 +3679,7 @@ impl SecretBigEndianInteger { }) } + #[cfg(feature = "getrandom")] fn from_vec(mut bytes: Vec) -> Result { if is_zero_unsigned_be(&bytes) { ct::zeroize(&mut bytes); @@ -3804,29 +3778,50 @@ impl RsaBlindingFactor { } } -struct RsaBlindingPair<'a> { +/// Borrowed inputs for a caller-blinded RSA private operation. +/// +/// The factor and inverse must be fixed-width, modulus-sized representatives +/// satisfying `factor * inverse == 1 mod n`. Construction only borrows the +/// inputs; each private operation validates them against its key before secret +/// exponentiation. This type does not own, copy, or zeroize the borrowed bytes. +#[derive(Clone, Copy)] +pub struct RsaBlindingPair<'a> { factor: &'a [u8], inverse: &'a [u8], validate: bool, } +impl fmt::Debug for RsaBlindingPair<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RsaBlindingPair") + .field("factor", &"****") + .field("inverse", &"****") + .finish() + } +} + impl<'a> RsaBlindingPair<'a> { - #[cfg(feature = "getrandom")] + /// Borrow a factor and its claimed modular inverse. + /// + /// Validation requires the target RSA modulus and is therefore deferred to + /// the private operation that consumes this value. #[inline] - const fn trusted(factor: &'a [u8], inverse: &'a [u8]) -> Self { + #[must_use] + pub const fn new(factor: &'a [u8], inverse: &'a [u8]) -> Self { Self { factor, inverse, - validate: false, + validate: true, } } + #[cfg(feature = "getrandom")] #[inline] - const fn caller_supplied(factor: &'a [u8], inverse: &'a [u8]) -> Self { + const fn trusted(factor: &'a [u8], inverse: &'a [u8]) -> Self { Self { factor, inverse, - validate: true, + validate: false, } } } @@ -5768,11 +5763,8 @@ impl RsaPublicModulus { } } - fn montgomery_r2(&self) -> &[u64] { - match self.r2.as_deref() { - Some(r2) => r2, - None => unreachable!("private RSA modulus missing Montgomery R^2"), - } + fn montgomery_r2(&self) -> Option<&[u64]> { + self.r2.as_deref() } fn public_operation( @@ -5831,7 +5823,7 @@ impl RsaPublicModulus { && rsa_aarch64_asm::supports_bignum_mont_words(limbs) && t.len() >= rsa_aarch64_asm::bignum_mont_scratch_words(limbs) { - rsa_aarch64_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, limbs, t); + rsa_aarch64_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, t); limbs_to_be(base, out); return Ok(()); } @@ -5846,7 +5838,7 @@ impl RsaPublicModulus { && rsa_aarch64_linux_asm::supports_bignum_mont_words(limbs) && t.len() >= rsa_aarch64_linux_asm::bignum_mont_scratch_words(limbs) { - rsa_aarch64_linux_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, limbs, t); + rsa_aarch64_linux_asm::public_e65537_mont_words(base, x, r2, acc, &self.limbs, self.n0, t); limbs_to_be(base, out); return Ok(()); } @@ -6463,23 +6455,20 @@ fn der_len_len(len: usize) -> usize { 1usize.strict_add(significant) } -#[allow(clippy::cast_possible_truncation)] fn der_push_len(len: usize, out: &mut Vec) { if len < 128 { - out.push(len as u8); + out.push(u8::try_from(len).expect("a short DER length must fit in one byte")); return; } - let mut started = false; - let len_len = der_len_len(len).strict_sub(1); - out.push(0x80 | (len_len as u8)); - for index in (0..core::mem::size_of::()).rev() { - let byte = (len >> index.strict_mul(8)) as u8; - if byte != 0 || started { - out.push(byte); - started = true; - } - } + let bytes = len.to_be_bytes(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); + let len_bytes = &bytes[first_nonzero..]; + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); + out.extend_from_slice(len_bytes); } fn parse_rsa_algorithm_identifier(der: &[u8]) -> Result<(), RsaKeyError> { @@ -6496,7 +6485,6 @@ fn parse_rsa_algorithm_identifier(der: &[u8]) -> Result<(), RsaKeyError> { reader.finish() } -#[allow(dead_code)] fn parse_pkcs8_private_key_der_with_policy( der: &[u8], policy: &RsaPublicKeyPolicy, @@ -6522,7 +6510,6 @@ fn parse_pkcs8_private_key_der_parts_with_policy<'a>( parse_pkcs1_private_key_der_parts_with_policy(private_key, policy) } -#[allow(dead_code)] fn parse_pkcs1_private_key_der_with_policy( der: &[u8], policy: &RsaPublicKeyPolicy, @@ -7292,7 +7279,7 @@ fn keygen_mask_unused_top_bits(bytes: &mut [u8], bits: usize) { return; } if let Some(first) = bytes.first_mut() { - *first &= (1u8 << used_top_bits) - 1; + *first &= u8::MAX >> 8usize.strict_sub(used_top_bits); } } @@ -7596,7 +7583,10 @@ fn keygen_inverse_small_mod_odd(exponent: u64, modulus: &[u8]) -> Result } for k in 1..exponent { - if (1u128 + u128::from(k).strict_mul(u128::from(modulus_mod_exponent))).is_multiple_of(u128::from(exponent)) { + if 1u128 + .strict_add(u128::from(k).strict_mul(u128::from(modulus_mod_exponent))) + .is_multiple_of(u128::from(exponent)) + { return keygen_mul_u64_add_one_div_u64(modulus, k, exponent); } } @@ -7614,11 +7604,11 @@ fn keygen_mul_u64_add_one_div_u64( let mut carry = 1u128; for &byte in value.iter().rev() { let acc = u128::from(byte).strict_mul(u128::from(multiplier)).strict_add(carry); - product_rev.push(acc as u8); + product_rev.push(acc.to_le_bytes()[0]); carry = acc >> 8; } while carry != 0 { - product_rev.push(carry as u8); + product_rev.push(carry.to_le_bytes()[0]); carry >>= 8; } product_rev.reverse(); @@ -7628,8 +7618,8 @@ fn keygen_mul_u64_add_one_div_u64( let divisor = u128::from(divisor); for &byte in &product_rev { let acc = (remainder << 8).strict_add(u128::from(byte)); - quotient.push((acc / divisor) as u8); - remainder = acc % divisor; + quotient.push(u8::try_from(acc.div_euclid(divisor)).map_err(|_| RsaKeyGenerationError::ArithmeticFailure)?); + remainder = acc.rem_euclid(divisor); } ct::zeroize(&mut product_rev); if remainder != 0 { @@ -7699,9 +7689,9 @@ fn unsigned_be_mod_u64(bytes: &[u8], modulus: u64) -> u64 { let mut remainder = 0u128; let modulus = u128::from(modulus); for &byte in bytes { - remainder = ((remainder << 8).strict_add(u128::from(byte))) % modulus; + remainder = (remainder << 8).strict_add(u128::from(byte)).rem_euclid(modulus); } - remainder as u64 + split_u128(remainder).0 } const fn tls13_signature_scheme_profile_and_key_algorithm( @@ -8102,7 +8092,6 @@ fn validate_modulus(modulus: &[u8], policy: &RsaPublicKeyPolicy) -> Result(message: &[u8], digest_info_prefix: &[u8], out: &mut [u8]) -> Result<(), RsaPrivateOpError> where D: Digest, @@ -8154,7 +8143,6 @@ fn fill_pkcs1v15_nonzero_padding_with( Ok(()) } -#[allow(clippy::indexing_slicing)] fn encode_pkcs1v15_encryption_with_seed(message: &[u8], seed: &[u8], out: &mut [u8]) -> Result<(), RsaEncryptionError> { let ps_len = pkcs1v15_encryption_padding_len(out.len(), message.len())?; if seed.len() != ps_len { @@ -8222,7 +8210,6 @@ fn clear_output_on_error(result: Result, out: &mut [u8]) -> Result(message: &[u8], salt: &[u8], em_bits: usize, out: &mut [u8]) -> Result<(), RsaPrivateOpError> where D: Digest, @@ -8239,7 +8226,6 @@ where result } -#[allow(clippy::indexing_slicing)] fn encode_pss_with_mask( message: &[u8], salt: &[u8], @@ -8301,7 +8287,6 @@ where Ok(()) } -#[allow(clippy::indexing_slicing)] fn encode_oaep_with_masks( label: &[u8], message: &[u8], @@ -8353,7 +8338,6 @@ where Ok(()) } -#[allow(clippy::indexing_slicing)] fn decode_oaep(label: &[u8], encoded: &mut [u8], out: &mut [u8]) -> Result where D: Digest, @@ -8372,7 +8356,6 @@ where result } -#[allow(clippy::indexing_slicing)] fn decode_oaep_with_masks( label: &[u8], encoded: &mut [u8], @@ -8491,7 +8474,7 @@ where return Err(VerificationError::new()); } if unused_bits > 0 { - let mask = 0xffu8 << (8usize.strict_sub(unused_bits) as u32); + let mask = 0xffu8 << 8usize.strict_sub(unused_bits); if masked_db.first().copied().unwrap_or(0) & mask != 0 { return Err(VerificationError::new()); } @@ -8589,7 +8572,6 @@ where } } -#[allow(clippy::indexing_slicing)] fn limbs_from_be(bytes: &[u8]) -> Vec { let limbs = bytes.len().strict_add(7) / 8; let mut out = vec![0u64; limbs]; @@ -8597,11 +8579,10 @@ fn limbs_from_be(bytes: &[u8]) -> Vec { out } -#[allow(clippy::indexing_slicing)] fn limbs_from_be_into(bytes: &[u8], out: &mut [u64]) { let full_limbs = bytes.len() / 8; let leading = bytes.len() % 8; - let needed_limbs = full_limbs + usize::from(leading != 0); + let needed_limbs = bytes.len().div_ceil(8); if out.len() != needed_limbs { out.fill(0); } @@ -8622,7 +8603,6 @@ fn limbs_from_be_into(bytes: &[u8], out: &mut [u64]) { } } -#[allow(clippy::indexing_slicing)] fn limbs_to_be(limbs: &[u64], out: &mut [u8]) { let full_limbs = out.len() / 8; let leading = out.len() % 8; @@ -8741,7 +8721,6 @@ fn left_pad_be(src: &[u8], out: &mut [u8]) -> Result<(), RsaPrivateOpError> { Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_sub_mod_unsigned_be_to_fixed( left: &[u8], right: &[u8], @@ -8757,7 +8736,7 @@ fn private_sub_mod_unsigned_be_to_fixed( let src = out.len().strict_sub(index).strict_sub(1); let subtrahend = u16::from(right[src]).strict_add(borrow); let difference = u16::from(left[src]).wrapping_sub(subtrahend); - out[src] = difference as u8; + out[src] = difference.to_le_bytes()[0]; borrow = (difference >> 15) & 1; } @@ -8767,14 +8746,13 @@ fn private_sub_mod_unsigned_be_to_fixed( let dst = out.len().strict_sub(index).strict_sub(1); let addend = u16::from(modulus[dst]) & mask; let sum = u16::from(out[dst]).strict_add(addend).strict_add(carry); - out[dst] = sum as u8; + out[dst] = sum.to_le_bytes()[0]; carry = sum >> 8; } Ok(()) } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "getrandom")] fn private_sub_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: &mut [u8]) -> Result<(), RsaPrivateOpError> { if left.len() != right.len() || left.len() != out.len() || unsigned_be_cmp(left, right) == core::cmp::Ordering::Less { @@ -8784,14 +8762,16 @@ fn private_sub_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: &mut [u8]) - let mut borrow = 0i16; for index in 0..out.len() { let src = out.len().strict_sub(index).strict_sub(1); - let mut difference = i16::from(left[src]) - i16::from(right[src]) - borrow; + let mut difference = i16::from(left[src]) + .strict_sub(i16::from(right[src])) + .strict_sub(borrow); if difference < 0 { - difference += 256; + difference = difference.strict_add(256); borrow = 1; } else { borrow = 0; } - out[src] = difference as u8; + out[src] = difference.to_le_bytes()[0]; } if borrow == 0 { @@ -8801,7 +8781,6 @@ fn private_sub_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: &mut [u8]) - } } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "getrandom")] fn private_sub_unsigned_be_to_len( left: &[u8], @@ -8827,15 +8806,17 @@ fn private_sub_unsigned_be_to_len( .and_then(|src| right.get(src)) .copied() .unwrap_or(0); - let mut difference = i16::from(left_byte) - i16::from(right_byte) - borrow; + let mut difference = i16::from(left_byte) + .strict_sub(i16::from(right_byte)) + .strict_sub(borrow); if difference < 0 { - difference += 256; + difference = difference.strict_add(256); borrow = 1; } else { borrow = 0; } let dst = out.len().strict_sub(index).strict_sub(1); - out[dst] = difference as u8; + out[dst] = difference.to_le_bytes()[0]; } if borrow != 0 { @@ -8844,7 +8825,6 @@ fn private_sub_unsigned_be_to_len( Ok(private_import_canonical_unsigned_be(out)) } -#[allow(clippy::indexing_slicing)] fn private_exponentiate_representative( modulus: &RsaPublicModulus, exponent: &[u8], @@ -8856,6 +8836,9 @@ fn private_exponentiate_representative( if input.len() != bytes || out.len() != bytes || exponent.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let mut t = SecretLimbs::zeroed(limbs.strict_mul(2).strict_add(2)); let mut representative = SecretLimbs::zeroed(limbs); @@ -8876,17 +8859,11 @@ fn private_exponentiate_representative( private_mont_mul( base.as_mut_slice(), representative.as_slice(), - modulus.montgomery_r2(), - modulus, - t.as_mut_slice(), - ); - private_mont_mul( - acc.as_mut_slice(), - one.as_slice(), - modulus.montgomery_r2(), + r2, modulus, t.as_mut_slice(), ); + private_mont_mul(acc.as_mut_slice(), one.as_slice(), r2, modulus, t.as_mut_slice()); let mut table = private_fixed_window_table(base.as_slice(), acc.as_slice(), modulus, t.as_mut_slice()); let mut index = 0; @@ -8896,23 +8873,27 @@ fn private_exponentiate_representative( &table, exponent_byte >> 4, modulus, - t.as_mut_slice(), - acc.as_mut_slice(), - squared.as_mut_slice(), - multiplied.as_mut_slice(), - selected.as_mut_slice(), + RsaPrivateWindowWorkspace { + t: t.as_mut_slice(), + acc: acc.as_mut_slice(), + squared: squared.as_mut_slice(), + multiplied: multiplied.as_mut_slice(), + selected: selected.as_mut_slice(), + }, ); private_exponentiate_window( &table, exponent_byte & 0x0f, modulus, - t.as_mut_slice(), - acc.as_mut_slice(), - squared.as_mut_slice(), - multiplied.as_mut_slice(), - selected.as_mut_slice(), + RsaPrivateWindowWorkspace { + t: t.as_mut_slice(), + acc: acc.as_mut_slice(), + squared: squared.as_mut_slice(), + multiplied: multiplied.as_mut_slice(), + selected: selected.as_mut_slice(), + }, ); - index += 1; + index = index.strict_add(1); } ct::zeroize_words(table.as_mut_slice()); @@ -8921,7 +8902,6 @@ fn private_exponentiate_representative( Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_exponentiate_representative_with_scratch( modulus: &RsaPublicModulus, exponent: &[u8], @@ -8934,6 +8914,9 @@ fn private_exponentiate_representative_with_scratch( if input.len() != bytes || out.len() != bytes || exponent.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let RsaPrivateExponentWorkspace { t, representative, @@ -8954,8 +8937,8 @@ fn private_exponentiate_representative_with_scratch( one.fill(0); one[0] = 1; - private_mont_mul(base, representative, modulus.montgomery_r2(), modulus, t); - private_mont_mul(acc, one, modulus.montgomery_r2(), modulus, t); + private_mont_mul(base, representative, r2, modulus, t); + private_mont_mul(acc, one, r2, modulus, t); private_fixed_window_table_into(table, base, acc, modulus, t); let mut index = 0; @@ -8965,23 +8948,27 @@ fn private_exponentiate_representative_with_scratch( table, exponent_byte >> 4, modulus, - t, - acc, - squared, - multiplied, - selected, + RsaPrivateWindowWorkspace { + t: &mut *t, + acc: &mut *acc, + squared: &mut *squared, + multiplied: &mut *multiplied, + selected: &mut *selected, + }, ); private_exponentiate_window( table, exponent_byte & 0x0f, modulus, - t, - acc, - squared, - multiplied, - selected, + RsaPrivateWindowWorkspace { + t: &mut *t, + acc: &mut *acc, + squared: &mut *squared, + multiplied: &mut *multiplied, + selected: &mut *selected, + }, ); - index += 1; + index = index.strict_add(1); } private_mont_reduce(reduced, acc, modulus, t); @@ -8989,7 +8976,6 @@ fn private_exponentiate_representative_with_scratch( Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_fixed_window_table( base: &[u64], one_montgomery: &[u64], @@ -9002,7 +8988,6 @@ fn private_fixed_window_table( table } -#[allow(clippy::indexing_slicing)] fn private_fixed_window_table_into( table: &mut [u64], base: &[u64], @@ -9025,17 +9010,27 @@ fn private_fixed_window_table_into( } } -#[allow(clippy::too_many_arguments)] +struct RsaPrivateWindowWorkspace<'a> { + t: &'a mut [u64], + acc: &'a mut [u64], + squared: &'a mut [u64], + multiplied: &'a mut [u64], + selected: &'a mut [u64], +} + fn private_exponentiate_window( table: &[u64], window: u8, modulus: &RsaPublicModulus, - t: &mut [u64], - acc: &mut [u64], - squared: &mut [u64], - multiplied: &mut [u64], - selected: &mut [u64], + workspace: RsaPrivateWindowWorkspace<'_>, ) { + let RsaPrivateWindowWorkspace { + t, + acc, + squared, + multiplied, + selected, + } = workspace; for _ in 0..4 { private_mont_mul(squared, acc, acc, modulus, t); acc.copy_from_slice(squared); @@ -9046,13 +9041,14 @@ fn private_exponentiate_window( acc.copy_from_slice(multiplied); } -#[allow(clippy::indexing_slicing)] #[inline(always)] fn private_select_window_power(out: &mut [u64], table: &[u64], window: u8) { let limbs = out.len(); debug_assert_eq!(table.len(), limbs.strict_mul(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES)); out.fill(0); - for index in 0..PRIVATE_FIXED_WINDOW_TABLE_ENTRIES as u8 { + let entry_count = + u8::try_from(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES).expect("private fixed-window table entry count must fit u8"); + for index in 0..entry_count { let start = usize::from(index).strict_mul(limbs); let entry = &table[start..start.strict_add(limbs)]; let mask = core::hint::black_box(private_choice_eq_mask_u8(window, index)); @@ -9069,6 +9065,7 @@ fn private_select_window_power(out: &mut [u64], table: &[u64], window: u8) { #[cfg(feature = "diag")] #[inline(always)] +/// Run the four-limb RSA private fixed-window selector for diagnostic harnesses. pub fn diag_rsa_private_select_window_power_4(table: &[u64; 64], window: u8) -> [u64; 4] { let mut out = [0u64; 4]; private_select_window_power(&mut out, table, window); @@ -9077,6 +9074,7 @@ pub fn diag_rsa_private_select_window_power_4(table: &[u64; 64], window: u8) -> #[cfg(feature = "diag")] #[inline(always)] +/// Evaluate the fixed-width RSA private-component checks used by diagnostic harnesses. pub fn diag_rsa_private_component_validation_32(component: &[u8; 32], upper_bound: &[u8; 32], other: &[u8; 32]) -> u8 { let canonical = u8::from(is_canonical_positive_unsigned_be(component)); let less_than_bound = u8::from(ct_unsigned_be_lt_public_shape(component, upper_bound)); @@ -9102,6 +9100,9 @@ fn mod_mul_representatives( if left.len() != bytes || right.len() != bytes || out.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let mut t = SecretLimbs::zeroed(limbs.strict_mul(2).strict_add(2)); let mut left_limbs = SecretLimbs::zeroed(limbs); @@ -9120,14 +9121,14 @@ fn mod_mul_representatives( private_mont_mul( left_mont.as_mut_slice(), left_limbs.as_slice(), - modulus.montgomery_r2(), + r2, modulus, t.as_mut_slice(), ); private_mont_mul( right_mont.as_mut_slice(), right_limbs.as_slice(), - modulus.montgomery_r2(), + r2, modulus, t.as_mut_slice(), ); @@ -9160,6 +9161,9 @@ fn mod_mul_representatives_with_scratch( if left.len() != bytes || right.len() != bytes || out.len() != bytes { return Err(RsaPrivateOpError::InvalidLength); } + let r2 = modulus + .montgomery_r2() + .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; scratch.ensure_limb_count(limbs)?; let t_len = limbs.strict_mul(2).strict_add(2); @@ -9205,8 +9209,8 @@ fn mod_mul_representatives_with_scratch( return Err(RsaPrivateOpError::RepresentativeOutOfRange); } - private_mont_mul(left_mont, left_limbs, modulus.montgomery_r2(), modulus, t); - private_mont_mul(right_mont, right_limbs, modulus.montgomery_r2(), modulus, t); + private_mont_mul(left_mont, left_limbs, r2, modulus, t); + private_mont_mul(right_mont, right_limbs, r2, modulus, t); private_mont_mul(product_mont, left_mont, right_mont, modulus, t); private_mont_reduce(product, product_mont, modulus, t); limbs_to_be(product, out); @@ -9229,6 +9233,14 @@ fn private_mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulu } } +#[inline(always)] +fn split_u128(value: u128) -> (u64, u64) { + ( + u64::try_from(value & u128::from(u64::MAX)).expect("masked low half of u128 must fit u64"), + u64::try_from(value >> u64::BITS).expect("shifted high half of u128 must fit u64"), + ) +} + #[cfg(feature = "getrandom")] fn private_import_product_unsigned_be(left: &[u8], right: &[u8]) -> Option { let left = SecretLimbs::from_be(left); @@ -9236,22 +9248,24 @@ fn private_import_product_unsigned_be(left: &[u8], right: &[u8]) -> Option> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } let index = left_index.strict_add(right.as_slice().len()); for limb in product.as_mut_slice().get_mut(index..)?.iter_mut() { - let acc = u128::from(*limb).strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + let acc = u128::from(*limb).strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } if carry != 0 { return None; @@ -9274,7 +9288,7 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & let mut product = SecretLimbs::zeroed(out_limb_count); for (left_index, &left_limb) in left.as_slice().iter().enumerate() { - let mut carry = 0u128; + let mut carry = 0u64; for (right_index, &right_limb) in right.as_slice().iter().enumerate() { let index = left_index.strict_add(right_index); let Some(limb) = product.as_mut_slice().get_mut(index) else { @@ -9282,9 +9296,10 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & }; let acc = u128::from(*limb) .strict_add(u128::from(left_limb).strict_mul(u128::from(right_limb))) - .strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } let index = left_index.strict_add(right.as_slice().len()); @@ -9292,9 +9307,10 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & return Err(RsaKeyError::InvalidModulus); }; for limb in carry_limbs { - let acc = u128::from(*limb).strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + let acc = u128::from(*limb).strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } if carry != 0 { return Err(RsaKeyError::InvalidModulus); @@ -9321,7 +9337,6 @@ fn private_import_product_unsigned_be_to_fixed(left: &[u8], right: &[u8], out: & Ok(()) } -#[allow(clippy::indexing_slicing)] fn private_product_add_unsigned_be_to_fixed( left: &[u8], right: &[u8], @@ -9362,7 +9377,7 @@ fn private_product_add_unsigned_be_to_fixed( product.fill(0); for (left_index, &left_limb) in left_limbs.iter().enumerate() { - let mut carry = 0u128; + let mut carry = 0u64; for (right_index, &right_limb) in right_limbs.iter().enumerate() { let index = left_index.strict_add(right_index); let limb = product @@ -9370,9 +9385,10 @@ fn private_product_add_unsigned_be_to_fixed( .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; let acc = u128::from(*limb) .strict_add(u128::from(left_limb).strict_mul(u128::from(right_limb))) - .strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } let index = left_index.strict_add(right_limbs.len()); @@ -9380,9 +9396,10 @@ fn private_product_add_unsigned_be_to_fixed( .get_mut(index..) .ok_or(RsaPrivateOpError::RepresentativeOutOfRange)?; for limb in carry_limbs { - let acc = u128::from(*limb).strict_add(carry); - *limb = acc as u64; - carry = acc >> 64; + let acc = u128::from(*limb).strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *limb = acc_low; + carry = acc_high; } if carry != 0 { return Err(RsaPrivateOpError::RepresentativeOutOfRange); @@ -9401,7 +9418,7 @@ fn private_product_add_unsigned_be_to_fixed( .copied() .unwrap_or(0); let sum = u16::from(out[dst]).strict_add(u16::from(add_byte)).strict_add(carry); - out[dst] = sum as u8; + out[dst] = sum.to_le_bytes()[0]; carry = sum >> 8; } @@ -9439,7 +9456,6 @@ fn private_import_decrement_unsigned_be_to_fixed(bytes: &[u8], out: &mut [u8]) - } } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "getrandom")] fn private_sub_small_unsigned_be_to_fixed( bytes: &[u8], @@ -9527,7 +9543,6 @@ fn private_import_unsigned_be_mod_to_fixed( Ok(()) } -#[allow(clippy::indexing_slicing)] fn add_bit_mod_in_place(value: &mut [u64], modulus: &[u64], bit: u8) { debug_assert_eq!(value.len(), modulus.len()); @@ -9559,7 +9574,6 @@ fn private_import_canonical_unsigned_be(mut bytes: Vec) -> SecretBigEndianBu SecretBigEndianBuffer::new(canonical) } -#[allow(clippy::indexing_slicing)] fn cmp_limbs(a: &[u64], b: &[u64]) -> core::cmp::Ordering { debug_assert_eq!(a.len(), b.len()); for index in (0..a.len()).rev() { @@ -9582,7 +9596,6 @@ fn ct_limbs_lt(left: &[u64], right: &[u64]) -> bool { borrow == 1 } -#[allow(clippy::indexing_slicing)] fn limb_bit_len(limbs: &[u64]) -> usize { for index in (0..limbs.len()).rev() { let limb = limbs[index]; @@ -9631,7 +9644,6 @@ fn ct_nonzero_u64(value: u64) -> u64 { (value | value.wrapping_neg()) >> 63 } -#[allow(clippy::indexing_slicing)] fn sub_modulus_in_place(value: &mut [u64], modulus: &[u64]) -> u64 { debug_assert_eq!(value.len(), modulus.len()); let mut borrow = 0u64; @@ -9644,7 +9656,6 @@ fn sub_modulus_in_place(value: &mut [u64], modulus: &[u64]) -> u64 { borrow } -#[allow(clippy::indexing_slicing)] fn add_modulus_masked(value: &mut [u64], modulus: &[u64], choice: u64) { debug_assert_eq!(value.len(), modulus.len()); let mask = 0u64.wrapping_sub(choice & 1); @@ -9658,7 +9669,6 @@ fn add_modulus_masked(value: &mut [u64], modulus: &[u64], choice: u64) { } } -#[allow(clippy::indexing_slicing)] fn subtract_modulus_if_needed(value: &mut [u64], modulus: &[u64], extra: u64) { debug_assert_eq!(value.len(), modulus.len()); let borrow = sub_modulus_in_place(value, modulus); @@ -9667,7 +9677,6 @@ fn subtract_modulus_if_needed(value: &mut [u64], modulus: &[u64], extra: u64) { } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn add_mod_in_place(value: &mut [u64], addend: &[u64], modulus: &[u64]) { debug_assert_eq!(value.len(), addend.len()); debug_assert_eq!(value.len(), modulus.len()); @@ -9683,7 +9692,6 @@ fn add_mod_in_place(value: &mut [u64], addend: &[u64], modulus: &[u64]) { subtract_modulus_if_needed(value, modulus, carry); } -#[allow(clippy::indexing_slicing)] fn double_mod_in_place(value: &mut [u64], modulus: &[u64]) { debug_assert_eq!(value.len(), modulus.len()); let mut carry = 0u64; @@ -9715,7 +9723,6 @@ fn mul_mod_bitserial(out: &mut [u64], a: &[u64], b: &[u64], modulus: &[u64], add } } -#[allow(clippy::indexing_slicing)] fn pow2_mod_into(out: &mut [u64], bits: usize, modulus: &[u64]) { out.fill(0); let modulus_bits = limb_bit_len(modulus); @@ -9736,27 +9743,23 @@ fn montgomery_n0(n0: u64) -> u64 { inv.wrapping_neg() } -#[allow(clippy::indexing_slicing)] fn mont_square_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let _ = tmp; mont_square_product(value, modulus, t); } -#[allow(clippy::indexing_slicing)] fn mont_mul_in_place_left(left: &mut [u64], right: &[u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { copy_limbs(tmp, left); mont_mul(left, tmp, right, modulus, t); } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_square_comba_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { copy_limbs(tmp, value); mont_mul_comba(value, tmp, tmp, modulus, t); } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_mul_comba_in_place_left( left: &mut [u64], right: &[u64], @@ -9768,7 +9771,6 @@ fn mont_mul_comba_in_place_left( mont_mul_comba(left, tmp, right, modulus, t); } -#[allow(clippy::indexing_slicing)] fn mont_square_cios_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { #[cfg(all( target_arch = "aarch64", @@ -9816,7 +9818,6 @@ fn mont_square_cios_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPu mont_mul_cios(value, tmp, tmp, modulus, t); } -#[allow(clippy::indexing_slicing)] fn mont_mul_cios_in_place_left( left: &mut [u64], right: &[u64], @@ -9881,14 +9882,12 @@ fn mont_mul_cios_in_place_left( } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_square_cios_portable_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { copy_limbs(tmp, value); mont_mul_cios_portable(value, tmp, tmp, modulus, t); } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn mont_mul_cios_portable_in_place_left( left: &mut [u64], right: &[u64], @@ -9913,7 +9912,6 @@ fn use_public_cios_montgomery(modulus: &RsaPublicModulus) -> bool { modulus.limbs.len() <= 128 } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "diag")] fn mont_square_auto_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { if use_public_cios_montgomery(modulus) { @@ -9923,7 +9921,6 @@ fn mont_square_auto_in_place(value: &mut [u64], tmp: &mut [u64], modulus: &RsaPu } } -#[allow(clippy::indexing_slicing)] #[cfg(feature = "diag")] fn mont_mul_auto_in_place_left( left: &mut [u64], @@ -9957,7 +9954,6 @@ fn mont_reduce_auto(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, } } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul_cios(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10003,7 +9999,6 @@ fn mont_mul_cios(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModul mont_mul_cios_portable(out, a, b, modulus, t); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10013,14 +10008,15 @@ fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPu t[..n.strict_add(2)].fill(0); - for i in 0..n { + for &b_limb in &b[..n] { let mut carry = 0u64; - for j in 0..n { - let acc = u128::from(t[j]) - .strict_add(u128::from(a[j]).strict_mul(u128::from(b[i]))) + for (t_limb, &a_limb) in t[..n].iter_mut().zip(&a[..n]) { + let acc = u128::from(*t_limb) + .strict_add(u128::from(a_limb).strict_mul(u128::from(b_limb))) .strict_add(u128::from(carry)); - t[j] = acc as u64; - carry = (acc >> 64) as u64; + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } let (sum, overflow) = t[n].overflowing_add(carry); t[n] = sum; @@ -10028,19 +10024,20 @@ fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPu let q = t[0].wrapping_mul(modulus.n0); carry = 0; - for j in 0..n { - let acc = u128::from(t[j]) - .strict_add(u128::from(q).strict_mul(u128::from(modulus.limbs[j]))) + for (t_limb, &modulus_limb) in t[..n].iter_mut().zip(modulus.limbs.iter()) { + let acc = u128::from(*t_limb) + .strict_add(u128::from(q).strict_mul(u128::from(modulus_limb))) .strict_add(u128::from(carry)); - t[j] = acc as u64; - carry = (acc >> 64) as u64; + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } let (sum, overflow) = t[n].overflowing_add(carry); t[n] = sum; t[n.strict_add(1)] = t[n.strict_add(1)].strict_add(u64::from(overflow)); - for j in 0..=n { - t[j] = t[j.strict_add(1)]; + for index in 0..=n { + t[index] = t[index.strict_add(1)]; } t[n.strict_add(1)] = 0; } @@ -10052,7 +10049,6 @@ fn mont_mul_cios_portable(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPu subtract_modulus_if_needed(out, &modulus.limbs, t[n]); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_reduce_cios(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10106,7 +10102,6 @@ fn mont_reduce_cios(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, mont_reduce_cios_portable(out, value, modulus, t); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_reduce_cios_portable(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10119,19 +10114,20 @@ fn mont_reduce_cios_portable(out: &mut [u64], value: &[u64], modulus: &RsaPublic for _ in 0..n { let q = t[0].wrapping_mul(modulus.n0); let mut carry = 0u64; - for j in 0..n { - let acc = u128::from(t[j]) - .strict_add(u128::from(q).strict_mul(u128::from(modulus.limbs[j]))) + for (t_limb, &modulus_limb) in t[..n].iter_mut().zip(modulus.limbs.iter()) { + let acc = u128::from(*t_limb) + .strict_add(u128::from(q).strict_mul(u128::from(modulus_limb))) .strict_add(u128::from(carry)); - t[j] = acc as u64; - carry = (acc >> 64) as u64; + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } let (sum, overflow) = t[n].overflowing_add(carry); t[n] = sum; t[n.strict_add(1)] = t[n.strict_add(1)].strict_add(u64::from(overflow)); - for j in 0..=n { - t[j] = t[j.strict_add(1)]; + for index in 0..=n { + t[index] = t[index.strict_add(1)]; } t[n.strict_add(1)] = 0; } @@ -10144,7 +10140,6 @@ fn mont_reduce_cios_portable(out: &mut [u64], value: &[u64], modulus: &RsaPublic } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul_comba(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10156,17 +10151,18 @@ fn mont_mul_comba(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModu for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); let acc = u128::from(q) .strict_mul(u128::from(modulus.limbs[j])) .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10178,7 +10174,6 @@ fn mont_mul_comba(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModu } #[cfg(feature = "diag")] -#[allow(clippy::indexing_slicing)] fn comba_mul_into(out: &mut [u64], a: &[u64], b: &[u64]) { debug_assert_eq!(a.len(), b.len()); let n = a.len(); @@ -10211,8 +10206,7 @@ fn comba_mul_into(out: &mut [u64], a: &[u64], b: &[u64]) { fn add_product_to_acc(acc_lo: &mut u64, acc_mid: &mut u64, acc_hi: &mut u64, a: u64, b: u64) { let product = u128::from(a).strict_mul(u128::from(b)); - let product_lo = product as u64; - let product_hi = (product >> 64) as u64; + let (product_lo, product_hi) = split_u128(product); let (lo, lo_overflow) = acc_lo.overflowing_add(product_lo); *acc_lo = lo; @@ -10222,7 +10216,6 @@ fn add_product_to_acc(acc_lo: &mut u64, acc_mid: &mut u64, acc_hi: &mut u64, a: *acc_hi = acc_hi.strict_add(u64::from(product_hi_overflow).strict_add(u64::from(lo_carry_overflow))); } -#[allow(clippy::indexing_slicing)] fn square_into_wide_product(out: &mut [u64], value: &[u64]) { let n = value.len(); debug_assert!(out.len() >= n.strict_mul(2).strict_add(2)); @@ -10261,7 +10254,6 @@ fn square_into_wide_product(out: &mut [u64], value: &[u64]) { out[product_limbs] = carry_hi; } -#[allow(clippy::indexing_slicing)] fn mont_square_product(out: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10271,14 +10263,18 @@ fn mont_square_product(out: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64 for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); - let acc = u128::from(q) * u128::from(modulus.limbs[j]) + u128::from(t[index]) + carry; - t[index] = acc as u64; - carry = acc >> 64; + let acc = u128::from(q) + .strict_mul(u128::from(modulus.limbs[j])) + .strict_add(u128::from(t[index])) + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10289,7 +10285,6 @@ fn mont_square_product(out: &mut [u64], modulus: &RsaPublicModulus, t: &mut [u64 subtract_modulus_if_needed(out, &modulus.limbs, extra); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_mul(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10299,33 +10294,35 @@ fn mont_mul(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t t.fill(0); - for i in 0..n { - let mut carry = 0u128; - for j in 0..n { - let index = i.strict_add(j); - let acc = u128::from(a[j]) - .strict_mul(u128::from(b[i])) - .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + for (i, &b_limb) in b[..n].iter().enumerate() { + let mut carry = 0u64; + let row_end = i.strict_add(n); + for (t_limb, &a_limb) in t[i..row_end].iter_mut().zip(&a[..n]) { + let acc = u128::from(a_limb) + .strict_mul(u128::from(b_limb)) + .strict_add(u128::from(*t_limb)) + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + *t_limb = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, row_end, carry); } for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); let acc = u128::from(q) .strict_mul(u128::from(modulus.limbs[j])) .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10336,7 +10333,6 @@ fn mont_mul(out: &mut [u64], a: &[u64], b: &[u64], modulus: &RsaPublicModulus, t subtract_modulus_if_needed(out, &modulus.limbs, extra); } -#[allow(clippy::indexing_slicing, clippy::needless_range_loop)] fn mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &mut [u64]) { let n = modulus.limbs.len(); debug_assert_eq!(out.len(), n); @@ -10348,17 +10344,18 @@ fn mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &m for i in 0..n { let q = t[i].wrapping_mul(modulus.n0); - let mut carry = 0u128; + let mut carry = 0u64; for j in 0..n { let index = i.strict_add(j); let acc = u128::from(q) .strict_mul(u128::from(modulus.limbs[j])) .strict_add(u128::from(t[index])) - .strict_add(carry); - t[index] = acc as u64; - carry = acc >> 64; + .strict_add(u128::from(carry)); + let (acc_low, acc_high) = split_u128(acc); + t[index] = acc_low; + carry = acc_high; } - add_carry(t, i.strict_add(n), carry as u64); + add_carry(t, i.strict_add(n), carry); } for (dst, src) in out.iter_mut().zip(t[n..n.strict_add(n)].iter().copied()) { @@ -10369,7 +10366,6 @@ fn mont_reduce(out: &mut [u64], value: &[u64], modulus: &RsaPublicModulus, t: &m subtract_modulus_if_needed(out, &modulus.limbs, extra); } -#[allow(clippy::indexing_slicing)] fn add_carry(t: &mut [u64], index: usize, mut carry: u64) { for limb in &mut t[index..] { let (sum, overflow) = limb.overflowing_add(carry); @@ -10481,45 +10477,52 @@ mod tests { include_str!("../../testdata/rsa/nist_cavp/rsa_keygen_186_3_probable_prime_subset.json"); fn hex_to_vec(hex: &str) -> Vec { - assert_eq!(hex.len() % 2, 0); - let mut out = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let (chunks, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty(), "hex input must contain complete byte pairs"); + let mut out = Vec::with_capacity(chunks.len()); + for &[hi, lo] in chunks { + let hi = hex_value(hi).expect("hex fixtures must contain only ASCII hexadecimal digits"); + let lo = hex_value(lo).expect("hex fixtures must contain only ASCII hexadecimal digits"); + out.push((hi << 4) | lo); } out } - const fn hex_value(byte: u8) -> u8 { + const fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } #[cfg(feature = "getrandom")] fn json_field<'a>(value: &'a Value, name: &'static str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("the CAVP fixture field must be a string") } #[cfg(feature = "getrandom")] fn test_keygen_drbg(label: &'static [u8]) -> RsaKeygenDrbg { - let mut seed = [0u8; RSA_KEYGEN_DRBG_ENTROPY_BYTES + RSA_KEYGEN_DRBG_NONCE_BYTES]; + let mut seed = [0u8; RSA_KEYGEN_DRBG_ENTROPY_BYTES.strict_add(RSA_KEYGEN_DRBG_NONCE_BYTES)]; for (index, byte) in seed.iter_mut().enumerate() { - *byte = (index as u8).wrapping_mul(17).wrapping_add(0xa5); + *byte = u8::try_from(index) + .expect("the RSA keygen DRBG fixture seed length must fit in one byte") + .wrapping_mul(17) + .wrapping_add(0xa5); } RsaKeygenDrbg::new(&seed, label) } fn rsa_miri_private_limb_widths() -> core::ops::RangeInclusive { - let min_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.min_modulus_bits.strict_add(1) / 2; - let max_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.max_modulus_bits.strict_add(1) / 2; + let min_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.min_modulus_bits.div_ceil(2); + let max_factor_bits = RsaPublicKeyPolicy::LEGACY_VERIFICATION.max_modulus_bits.div_ceil(2); let bits_per_limb = u64::BITS as usize; - let min_limbs = min_factor_bits.strict_add(bits_per_limb.strict_sub(1)) / bits_per_limb; - let max_limbs = max_factor_bits.strict_add(bits_per_limb.strict_sub(1)) / bits_per_limb; + let min_limbs = min_factor_bits.div_ceil(bits_per_limb); + let max_limbs = max_factor_bits.div_ceil(bits_per_limb); min_limbs..=max_limbs } @@ -10570,7 +10573,9 @@ mod tests { rsa_miri_fill_exponent_scratch(&mut scratch, MARKER); { - let workspace = scratch.workspace(limbs).unwrap(); + let workspace = scratch + .workspace(limbs) + .expect("valid RSA private-exponent scratch must bind to its limb width"); assert_eq!(workspace.t.len(), limbs.strict_mul(2).strict_add(2)); assert_eq!(workspace.representative.len(), limbs); assert_eq!(workspace.one.len(), limbs); @@ -10592,7 +10597,9 @@ mod tests { } } - for window in 0..PRIVATE_FIXED_WINDOW_TABLE_ENTRIES as u8 { + let window_count = u8::try_from(PRIVATE_FIXED_WINDOW_TABLE_ENTRIES) + .expect("the RSA fixed-window table entry count must fit in one byte"); + for window in 0..window_count { workspace.selected.fill(MARKER); private_select_window_power(workspace.selected, workspace.table, window); let start = usize::from(window).strict_mul(limbs); @@ -10657,7 +10664,9 @@ mod tests { fn rsa_miri_proof_miller_rabin_covers_small_prime_and_composite_paths() { let prime = 1009u16.to_be_bytes(); assert!(!has_small_prime_factor(&prime)); - assert!(private_import_is_probable_prime(&prime).unwrap()); + assert!( + private_import_is_probable_prime(&prime).expect("the small-prime RSA fixture must be accepted for testing") + ); let composite = [0x0f, 0x98, 0xa5]; assert_eq!( @@ -10665,7 +10674,10 @@ mod tests { 1009 * 1013 ); assert!(!has_small_prime_factor(&composite)); - assert!(!private_import_is_probable_prime(&composite).unwrap()); + assert!( + !private_import_is_probable_prime(&composite) + .expect("the composite RSA fixture must be evaluated without an arithmetic failure") + ); } #[test] @@ -10677,19 +10689,21 @@ mod tests { ([0x00], [0x00], [0x01], [0x00]), ] { let mut out = [0u8; 1]; - private_sub_mod_unsigned_be_to_fixed(&left, &right, &modulus, &mut out).unwrap(); + private_sub_mod_unsigned_be_to_fixed(&left, &right, &modulus, &mut out) + .expect("the bounded modular-subtraction fixture must be valid"); assert_eq!(out, expected); } let mut out = [0u8; 2]; - private_sub_mod_unsigned_be_to_fixed(&[0x00, 0x01], &[0x01, 0x00], &[0x01, 0x01], &mut out).unwrap(); + private_sub_mod_unsigned_be_to_fixed(&[0x00, 0x01], &[0x01, 0x00], &[0x01, 0x01], &mut out) + .expect("the borrow-propagation modular-subtraction fixture must be valid"); assert_eq!(out, [0x00, 0x02]); } fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -10747,7 +10761,11 @@ mod tests { params.extend_from_slice(&context_constructed(1, &x509_mgf1_algorithm(profile))); params.extend_from_slice(&context_constructed( 2, - &integer_unsigned(&u64::try_from(salt_len).unwrap().to_be_bytes()), + &integer_unsigned( + &u64::try_from(salt_len) + .expect("an RSA-PSS test salt length must fit in an unsigned DER integer") + .to_be_bytes(), + ), )); algorithm_identifier(ID_RSASSA_PSS_OID, Some(&tlv(TAG_SEQUENCE, ¶ms))) } @@ -10845,7 +10863,8 @@ ca5b455045218c7e196209c1c651702ece090a15e3cbcc265971300023a86fe9d34ad527e9ef03b7 }; RsaPrivateKey { - components: private_key_components_from_prevalidated_parts(&components, unsigned_be_bit_len(&modulus)).unwrap(), + components: private_key_components_from_prevalidated_parts(&components, unsigned_be_bit_len(&modulus)) + .expect("the prevalidated RSA private-key fixture must satisfy component invariants"), } } @@ -10985,18 +11004,20 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } fn tlv_with_noncanonical_short_len(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(3 + value.len()); + let mut out = Vec::with_capacity(3usize.strict_add(value.len())); out.push(tag); out.push(0x81); - out.push(value.len() as u8); + out.push(u8::try_from(value.len()).expect("a noncanonical short DER fixture length must fit in one byte")); out.extend_from_slice(value); out } @@ -11009,7 +11030,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut out = Vec::with_capacity(der.len().strict_add(1)); out.push(tag); - out.push(0x80 | (len_len.strict_add(1) as u8)); + out.push(0x80 | u8::try_from(len_len.strict_add(1)).expect("a DER length-of-length must fit in one byte")); out.push(0); out.extend_from_slice(&der[2..]); out @@ -11017,14 +11038,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("a short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("a DER length of at least 128 must contain a nonzero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); out.extend_from_slice(len_bytes); out } @@ -11044,14 +11068,19 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 } fn pss_encoded_sha256(message: &[u8], em_bits: usize, salt_len: usize) -> Vec { - let em_len = em_bits.strict_add(7) / 8; + let em_len = em_bits.div_ceil(8); let h_len = Sha256::OUTPUT_SIZE; let db_len = em_len.strict_sub(h_len).strict_sub(1); let ps_len = db_len.strict_sub(salt_len).strict_sub(1); let mut salt = Vec::with_capacity(salt_len); for index in 0..salt_len { - salt.push((index as u8).wrapping_mul(17).wrapping_add(0xa5)); + salt.push( + u8::try_from(index) + .expect("the RSA-PSS test salt length must fit in one byte") + .wrapping_mul(17) + .wrapping_add(0xa5), + ); } let m_hash = Sha256::digest(message); @@ -11149,7 +11178,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn pkcs1_private_key_parser_preserves_components_and_public_key() { let der = test_pkcs1_private_key(); - let key = parse_pkcs1_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = parse_pkcs1_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the canonical PKCS#1 private-key fixture must parse"); assert_eq!(prevalidated_test_private_key().to_pkcs1_der().as_bytes(), der); @@ -11263,7 +11293,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!(unsigned_be_bit_len(&prime_q), 1024); assert_eq!(prime_p, composite_p.to_bytes_be()); assert!(!has_small_prime_factor(&prime_p)); - assert!(!private_import_is_probable_prime(&prime_p).unwrap()); + assert!( + !private_import_is_probable_prime(&prime_p) + .expect("the balanced composite fixture must be evaluated without an arithmetic failure") + ); assert_eq!( RsaPrivateKey::from_components_with_policy( @@ -11340,6 +11373,20 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn private_key_components_debug_redacts_secret_material() { let key = prevalidated_test_private_key(); let debug = format!("{:?}", key.components); + let (blinding_factor, blinding_inverse) = factor_two_and_inverse(key.public_key().modulus()); + let blinding_debug = format!("{:?}", RsaBlindingPair::new(&blinding_factor, &blinding_inverse)); + let parts_debug = format!( + "{:?}", + rsa_private_key_parts( + &rsa_private_modulus(), + &rsa_private_exponent(), + &rsa_private_prime_p(), + &rsa_private_prime_q(), + &rsa_private_exponent_p(), + &rsa_private_exponent_q(), + &rsa_private_coefficient(), + ) + ); assert!(debug.contains("modulus_bits")); assert!(debug.contains("public_exponent")); @@ -11347,6 +11394,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert!(debug.contains("prime_p: \"****\"")); assert!(debug.contains("prime_q: \"****\"")); assert!(debug.contains("coefficient: \"****\"")); + assert!(parts_debug.contains("modulus_bits")); + assert!(parts_debug.contains("public_exponent: 65537")); + assert!(parts_debug.contains("private_exponent: \"****\"")); + assert!(parts_debug.contains("prime_p: \"****\"")); + assert!(parts_debug.contains("prime_q: \"****\"")); + assert!(parts_debug.contains("coefficient: \"****\"")); + assert_eq!( + blinding_debug, + "RsaBlindingPair { factor: \"****\", inverse: \"****\" }" + ); + assert!(!blinding_debug.contains(&format!("{blinding_factor:?}"))); + assert!(!blinding_debug.contains(&format!("{blinding_inverse:?}"))); for secret in [ rsa_private_exponent(), rsa_private_prime_p(), @@ -11356,6 +11415,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_coefficient(), ] { assert!(!debug.contains(&format!("{secret:?}"))); + assert!(!parts_debug.contains(&format!("{secret:?}"))); } assert_eq!( @@ -11382,15 +11442,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, ) - .unwrap(); + .expect("PKCS#1 v1.5 signing with a valid blinding pair must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("the generated PKCS#1 v1.5 signature must verify"); assert_eq!( key .public_key() @@ -11404,15 +11463,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pss_signature, ) - .unwrap(); + .expect("PSS signing with explicit salt and a valid blinding pair must succeed"); key .public_key() .verify_pss_with_salt_len(RsaPssProfile::Sha256, salt.len(), message, &pss_signature) - .unwrap(); + .expect("the generated PSS signature must verify with its explicit salt length"); assert_eq!( key .public_key() @@ -11426,7 +11484,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption with a correctly sized deterministic seed must succeed"); let short_seed = [0x53; Sha256::OUTPUT_SIZE - 1]; ciphertext.fill(0xa5); assert_eq!( @@ -11443,26 +11501,24 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("OAEP fixture encryption must succeed after the rejected short seed"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ) - .unwrap(); + .expect("OAEP decryption with the matching label and blinding pair must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); assert_eq!( key.decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, b"wrong label", &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11473,27 +11529,25 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_padding, &mut pkcs1v15_ciphertext) - .unwrap(); + .expect("PKCS#1 v1.5 encryption with a valid deterministic padding string must succeed"); let mut pkcs1v15_decrypted = vec![0u8; key.signature_len()]; let pkcs1v15_decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor( &pkcs1v15_ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_decrypted, ) - .unwrap(); + .expect("PKCS#1 v1.5 decryption with a valid blinding pair must succeed"); assert_eq!(&pkcs1v15_decrypted[..pkcs1v15_decrypted_len], pkcs1v15_plaintext); let mut scratch = key.private_scratch(); let pkcs1v15_decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &pkcs1v15_ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_decrypted, &mut scratch, ) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 decryption with a valid blinding pair must succeed"); assert_eq!(&pkcs1v15_decrypted[..pkcs1v15_decrypted_len], pkcs1v15_plaintext); } @@ -11534,9 +11588,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn private_key_invalid_blinding_clears_signing_and_decryption_outputs() { let key = prevalidated_test_private_key(); let message = b"rscrypto RSA invalid blinding output clearing"; - let (blinding_factor, blinding_factor_inverse) = factor_two_and_inverse(key.public_key().modulus()); - let mut bad_blinding_inverse = blinding_factor_inverse.clone(); - *bad_blinding_inverse.last_mut().unwrap() ^= 1; + let (blinding_factor, mut bad_blinding_inverse) = factor_two_and_inverse(key.public_key().modulus()); + *bad_blinding_inverse + .last_mut() + .expect("the fixed-width blinding inverse must contain at least one byte") ^= 1; let mut scratch = key.private_scratch(); let mut pkcs1v15_signature = vec![0xa5; key.signature_len()]; @@ -11544,8 +11599,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_signature, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11557,8 +11611,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_signature, &mut scratch, ), @@ -11573,8 +11626,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pss_signature, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11587,8 +11639,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pss_signature, &mut scratch, ), @@ -11603,7 +11654,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("the OAEP invalid-blinding fixture must encrypt successfully"); let mut decrypted = vec![0xa5; key.signature_len()]; assert_eq!( @@ -11611,8 +11662,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut decrypted, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11625,8 +11675,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut decrypted, &mut scratch, ), @@ -11640,14 +11689,13 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut pkcs1v15_ciphertext) - .unwrap(); + .expect("the PKCS#1 v1.5 invalid-blinding fixture must encrypt successfully"); let mut pkcs1v15_decrypted = vec![0xa5; key.signature_len()]; assert_eq!( key.decrypt_pkcs1v15_with_blinding_factor( &pkcs1v15_ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_decrypted, ), Err(RsaPrivateOpError::InvalidBlindingFactor) @@ -11658,8 +11706,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!( key.decrypt_pkcs1v15_with_blinding_factor_and_scratch( &pkcs1v15_ciphertext, - &blinding_factor, - &bad_blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &bad_blinding_inverse), &mut pkcs1v15_decrypted, &mut scratch, ), @@ -11677,7 +11724,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut valid = masked_oaep_sha256_from_decoded_db(&seed, &decoded_db); let mut out = vec![0u8; message.len()]; - let len = decode_oaep::(label, &mut valid, &mut out).unwrap(); + let len = decode_oaep::(label, &mut valid, &mut out) + .expect("the canonical OAEP encoded-message fixture must decode"); assert_eq!(len, message.len()); assert_eq!(out, message); @@ -11725,10 +11773,12 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let message = b"abc"; let seed = [0x7bu8; 10]; let mut encoded = [0u8; 16]; - encode_pkcs1v15_encryption_with_seed(message, &seed, &mut encoded).unwrap(); + encode_pkcs1v15_encryption_with_seed(message, &seed, &mut encoded) + .expect("the canonical PKCS#1 v1.5 encryption-padding fixture must encode"); let mut out = [0u8; 8]; - let len = decode_pkcs1v15_encryption(&encoded, &mut out).unwrap(); + let len = decode_pkcs1v15_encryption(&encoded, &mut out) + .expect("the canonical PKCS#1 v1.5 encryption-padding fixture must decode"); assert_eq!(&out[..len], message); let mut zero_seed = seed; @@ -11781,8 +11831,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ) .unwrap(); @@ -11794,8 +11843,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ) @@ -11819,8 +11867,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, b"wrong label", &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11832,8 +11879,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, b"wrong label", &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -11865,8 +11911,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &tampered_tail, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11878,8 +11923,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &tampered_tail, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -11910,8 +11954,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &zero_representative, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, ), Err(RsaPrivateOpError::DecryptionFailed) @@ -11923,8 +11966,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 $profile, label, &zero_representative, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -11966,36 +12008,41 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("valid PKCS#1 v1.5 test plaintext and seed must encrypt"); let mut out = vec![0u8; key.signature_len()]; let decrypted_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_factor_inverse, &mut out) - .unwrap(); + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut out, + ) + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with explicit blinding"); assert_eq!(&out[..decrypted_len], plaintext); let mut scratch = key.private_scratch(); let decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ) - .unwrap(); + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with caller scratch"); assert_eq!(&out[..decrypted_len], plaintext); #[cfg(feature = "getrandom")] { - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut out).unwrap(); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut out) + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with random blinding"); assert_eq!(&out[..decrypted_len], plaintext); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut out, &mut scratch) - .unwrap(); + .expect("valid PKCS#1 v1.5 ciphertext must decrypt with random blinding and caller scratch"); assert_eq!(&out[..decrypted_len], plaintext); } - let mut zero_seed = seed.clone(); + let mut zero_seed = seed; zero_seed[3] = 0; ciphertext.fill(0xa5); assert_eq!( @@ -12021,11 +12068,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 bad_block[2..10].fill(0xc5); bad_block[10] = 0; bad_block[11..11usize.strict_add(plaintext.len())].copy_from_slice(plaintext); - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("malformed encoded block must remain a valid RSA representative"); let mut assert_decrypt_error = |ciphertext: &[u8]| { out.fill(0xa5); assert_eq!( - key.decrypt_pkcs1v15_with_blinding_factor(ciphertext, &blinding_factor, &blinding_factor_inverse, &mut out,), + key.decrypt_pkcs1v15_with_blinding_factor( + ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut out, + ), Err(RsaPrivateOpError::DecryptionFailed) ); assert!(is_zero_unsigned_be(&out)); @@ -12033,8 +12087,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!( key.decrypt_pkcs1v15_with_blinding_factor_and_scratch( ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut out, &mut scratch, ), @@ -12059,24 +12112,34 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 }; bad_block[1] = 1; - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("wrong block type must remain a valid RSA representative"); assert_decrypt_error(&ciphertext); bad_block[1] = 2; bad_block[5] = 0; - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("short padding block must remain a valid RSA representative"); assert_decrypt_error(&ciphertext); bad_block[5] = 0xc5; bad_block[10..].fill(0xc5); - key.public_key().public_operation(&bad_block, &mut ciphertext).unwrap(); + key + .public_key() + .public_operation(&bad_block, &mut ciphertext) + .expect("missing separator block must remain a valid RSA representative"); assert_decrypt_error(&ciphertext); } #[test] fn key_der_exports_roundtrip_through_strict_importers() { let pkcs1 = test_pkcs1_private_key(); - let key = RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("PKCS#1 fixture must import"); let rsa_algorithm = algorithm_identifier(RSA_ENCRYPTION_OID, Some(&null())); assert_eq!( @@ -12090,22 +12153,25 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let exported_pkcs1 = key.to_pkcs1_der(); let imported_pkcs1 = - RsaPrivateKey::from_pkcs1_der_with_policy(&exported_pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + RsaPrivateKey::from_pkcs1_der_with_policy(&exported_pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported PKCS#1 private key must re-import"); assert_eq!(imported_pkcs1.public_key(), key.public_key()); let exported_pkcs8 = key.to_pkcs8_der(); let imported_pkcs8 = - RsaPrivateKey::from_pkcs8_der_with_policy(&exported_pkcs8, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + RsaPrivateKey::from_pkcs8_der_with_policy(&exported_pkcs8, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported PKCS#8 private key must re-import"); assert_eq!(imported_pkcs8.public_key(), key.public_key()); let public_pkcs1 = key.public_key().to_pkcs1_der(); let imported_public_pkcs1 = - RsaPublicKey::from_pkcs1_der_with_policy(&public_pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + RsaPublicKey::from_pkcs1_der_with_policy(&public_pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported PKCS#1 public key must re-import"); assert_eq!(imported_public_pkcs1, *key.public_key()); let spki = key.public_key().to_spki_der(); - let imported_spki = - RsaPublicKey::from_spki_der_with_policy(&spki, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let imported_spki = RsaPublicKey::from_spki_der_with_policy(&spki, &RsaPublicKeyPolicy::legacy_verification()) + .expect("exported SPKI public key must re-import"); assert_eq!(imported_spki, *key.public_key()); } @@ -12120,7 +12186,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let coefficient = rsa_private_coefficient(); let policy = RsaPublicKeyPolicy::legacy_verification(); - let public = RsaPublicKey::from_modulus_exponent_with_policy(&modulus, 65_537, &policy).unwrap(); + let public = RsaPublicKey::from_modulus_exponent_with_policy(&modulus, 65_537, &policy) + .expect("valid raw public components must import"); assert_eq!(public.modulus(), modulus); assert_eq!(public.public_exponent().as_u64(), 65_537); @@ -12136,7 +12203,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ), &policy, ) - .unwrap(); + .expect("valid raw private components must import"); assert_eq!(key.public_key(), &public); let message = b"rscrypto raw RSA component import signing roundtrip"; @@ -12146,17 +12213,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) - .unwrap(); + .expect("imported raw private components must sign"); public .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &signature) - .unwrap(); + .expect("signature from imported raw components must verify"); let mut bad_coefficient = coefficient; - *bad_coefficient.last_mut().unwrap() ^= 1; + *bad_coefficient + .last_mut() + .expect("RSA coefficient fixture must be nonempty") ^= 1; assert_eq!( RsaPrivateKey::from_components_with_policy( rsa_private_key_parts( @@ -12285,7 +12353,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 Some(RsaKeyError::InvalidModulus) ); - let mut bad_coefficient = coefficient.clone(); + let mut bad_coefficient = coefficient; bad_coefficient.insert(0, 0); assert_eq!( RsaPrivateKey::from_components_with_policy( @@ -12315,11 +12383,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 signing with random blinding must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 signature must verify"); key .sign_pkcs1v15_with_scratch( RsaPkcs1v15Profile::Sha384, @@ -12327,35 +12395,35 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 &mut pkcs1v15_signature, &mut scratch, ) - .unwrap(); + .expect("PKCS#1 v1.5 signing with caller scratch must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &pkcs1v15_signature) - .unwrap(); + .expect("caller-scratch PKCS#1 v1.5 signature must verify"); let mut pss_signature = vec![0u8; key.signature_len()]; key .sign_pss(RsaPssProfile::Sha256, message, &mut pss_signature) - .unwrap(); + .expect("PSS signing with random salt and blinding must succeed"); key .public_key() .verify_pss(RsaPssProfile::Sha256, message, &pss_signature) - .unwrap(); + .expect("PSS signature must verify"); key .sign_pss_with_scratch(RsaPssProfile::Sha384, message, &mut pss_signature, &mut scratch) - .unwrap(); + .expect("PSS signing with caller scratch must succeed"); key .public_key() .verify_pss(RsaPssProfile::Sha384, message, &pss_signature) - .unwrap(); + .expect("caller-scratch PSS signature must verify"); let pss_zero_salt = RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 0); key .sign_signature_with_scratch(pss_zero_salt, message, &mut pss_signature, &mut scratch) - .unwrap(); + .expect("zero-salt PSS signing with caller scratch must succeed"); key .public_key() .verify_signature(pss_zero_salt, message, &pss_signature) - .unwrap(); + .expect("zero-salt PSS signature must verify"); let label = b"rscrypto-getrandom-oaep"; let plaintext = b"normal oaep api"; @@ -12363,27 +12431,29 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .encrypt_oaep(RsaOaepProfile::Sha256, label, plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption with random seed must succeed"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted) - .unwrap(); + .expect("OAEP ciphertext must decrypt with random blinding"); assert_eq!(&decrypted[..decrypted_len], plaintext); let decrypted_len = key .decrypt_oaep_with_scratch(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("OAEP ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"normal pkcs1v15 encryption api"; key .public_key() .encrypt_pkcs1v15(pkcs1v15_plaintext, &mut ciphertext) - .unwrap(); - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut decrypted).unwrap(); + .expect("PKCS#1 v1.5 encryption with random padding must succeed"); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut decrypted) + .expect("PKCS#1 v1.5 ciphertext must decrypt with random blinding"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("PKCS#1 v1.5 ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); } @@ -12397,36 +12467,38 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_signature(pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("typed PKCS#1 v1.5 profile must sign"); key .public_key() .verify_signature(pkcs1_profile, message, &pkcs1v15_signature) - .unwrap(); + .expect("typed PKCS#1 v1.5 signature must verify"); let pss_zero_salt = RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 0); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_signature(pss_zero_salt, message, &mut pss_signature).unwrap(); + key + .sign_signature(pss_zero_salt, message, &mut pss_signature) + .expect("typed zero-salt PSS profile must sign"); key .public_key() .verify_signature(pss_zero_salt, message, &pss_signature) - .unwrap(); + .expect("typed zero-salt PSS signature must verify"); key .sign_pss_with_salt_len(RsaPssProfile::Sha512, 24, message, &mut pss_signature) - .unwrap(); + .expect("explicit-salt-length PSS signing must succeed"); key .public_key() .verify_pss_with_salt_len(RsaPssProfile::Sha512, 24, message, &pss_signature) - .unwrap(); + .expect("explicit-salt-length PSS signature must verify"); let mut scratch = key.private_scratch(); key .sign_pss_with_salt_len_and_scratch(RsaPssProfile::Sha384, 16, message, &mut pss_signature, &mut scratch) - .unwrap(); + .expect("explicit-salt-length PSS signing with caller scratch must succeed"); key .public_key() .verify_pss_with_salt_len(RsaPssProfile::Sha384, 16, message, &pss_signature) - .unwrap(); + .expect("caller-scratch explicit-salt-length PSS signature must verify"); pss_signature.fill(0xa5); assert_eq!( @@ -12460,114 +12532,122 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let x509_pkcs1v15 = algorithm_identifier(SHA384_WITH_RSA_ENCRYPTION_OID, Some(&null())); key .sign_x509_signature_algorithm_der(&x509_pkcs1v15, message, &mut signature) - .unwrap(); - let x509_pkcs1v15_profile = RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pkcs1v15).unwrap(); + .expect("X.509 PKCS#1 v1.5 algorithm must sign"); + let x509_pkcs1v15_profile = RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pkcs1v15) + .expect("X.509 PKCS#1 v1.5 algorithm must parse"); key .public_key() .verify_signature(x509_pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("X.509 PKCS#1 v1.5 signature must verify"); key .sign_x509_signature_algorithm_der_with_scratch(&x509_pkcs1v15, message, &mut signature, &mut scratch) - .unwrap(); + .expect("X.509 PKCS#1 v1.5 algorithm must sign with caller scratch"); key .public_key() .verify_signature(x509_pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("caller-scratch X.509 PKCS#1 v1.5 signature must verify"); let x509_pss = x509_pss_algorithm(RsaPssProfile::Sha256, 20); key .sign_x509_signature_algorithm_der(&x509_pss, message, &mut signature) - .unwrap(); - let x509_pss_profile = RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss).unwrap(); + .expect("X.509 PSS algorithm must sign"); + let x509_pss_profile = + RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss).expect("X.509 PSS algorithm must parse"); key .public_key() .verify_signature(x509_pss_profile, message, &signature) - .unwrap(); + .expect("X.509 PSS signature must verify"); key .sign_x509_signature_algorithm_der_with_scratch(&x509_pss, message, &mut signature, &mut scratch) - .unwrap(); + .expect("X.509 PSS algorithm must sign with caller scratch"); key .public_key() .verify_signature(x509_pss_profile, message, &signature) - .unwrap(); + .expect("caller-scratch X.509 PSS signature must verify"); key .sign_tls13_signature_scheme(0x0804, message, &mut signature) - .unwrap(); + .expect("TLS 1.3 rsa_pss_rsae_sha256 scheme must sign"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), + RsaSignatureProfile::from_tls13_signature_scheme(0x0804) + .expect("TLS 1.3 rsa_pss_rsae_sha256 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("TLS 1.3 rsa_pss_rsae_sha256 signature must verify"); key .sign_tls13_signature_scheme_with_scratch(0x0804, message, &mut signature, &mut scratch) - .unwrap(); + .expect("TLS 1.3 scheme must sign with caller scratch"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), + RsaSignatureProfile::from_tls13_signature_scheme(0x0804) + .expect("TLS 1.3 rsa_pss_rsae_sha256 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("caller-scratch TLS 1.3 signature must verify"); key .sign_tls_certificate_signature_scheme(0x0501, message, &mut signature) - .unwrap(); + .expect("TLS certificate rsa_pkcs1_sha384 scheme must sign"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501).unwrap(), + RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501) + .expect("TLS certificate rsa_pkcs1_sha384 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("TLS certificate rsa_pkcs1_sha384 signature must verify"); key .sign_tls_certificate_signature_scheme_with_scratch(0x0501, message, &mut signature, &mut scratch) - .unwrap(); + .expect("TLS certificate scheme must sign with caller scratch"); key .public_key() .verify_signature( - RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501).unwrap(), + RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0501) + .expect("TLS certificate rsa_pkcs1_sha384 scheme must map to a profile"), message, &signature, ) - .unwrap(); + .expect("caller-scratch TLS certificate signature must verify"); let jwt_algorithm = RsaJwtAlgorithm::Ps512; key .jwt_signer(jwt_algorithm) .try_sign_into(message, &mut signature) - .unwrap(); + .expect("JWT PS512 signer must sign"); key .public_key() .jwt_verifier(jwt_algorithm) .verify("PS512", message, &signature) - .unwrap(); + .expect("JWT PS512 signature must verify"); key .sign_signature_with_scratch(jwt_algorithm.signature_profile(), message, &mut signature, &mut scratch) - .unwrap(); + .expect("JWT PS512 profile must sign with caller scratch"); key .public_key() .jwt_verifier(jwt_algorithm) .verify("PS512", message, &signature) - .unwrap(); + .expect("caller-scratch JWT PS512 signature must verify"); - key.sign_cose_algorithm_id(-257, message, &mut signature).unwrap(); + key + .sign_cose_algorithm_id(-257, message, &mut signature) + .expect("COSE RS256 algorithm must sign"); key .public_key() .verify_cose_algorithm_id(-257, message, &signature) - .unwrap(); + .expect("COSE RS256 signature must verify"); key .sign_cose_algorithm_id_with_scratch(-257, message, &mut signature, &mut scratch) - .unwrap(); + .expect("COSE RS256 algorithm must sign with caller scratch"); key .public_key() .verify_cose_algorithm_id(-257, message, &signature) - .unwrap(); + .expect("caller-scratch COSE RS256 signature must verify"); signature.fill(0xa5); assert_eq!( @@ -12637,7 +12717,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn private_key_random_blinding_factor_has_valid_crt_inverse() { let key = prevalidated_test_private_key(); - let blinding = key.components.random_blinding_factor().unwrap(); + let blinding = key + .components + .random_blinding_factor() + .expect("test key must yield an invertible random blinding factor"); let mut check = vec![0u8; key.signature_len()]; mod_mul_representatives( @@ -12646,7 +12729,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 blinding.inverse(), &mut check, ) - .unwrap(); + .expect("blinding factor and inverse must multiply modulo n"); assert!(check[..check.len().strict_sub(1)].iter().all(|&byte| byte == 0)); assert_eq!(check.last().copied(), Some(1)); @@ -12657,7 +12740,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 fn private_key_blinding_inverse_rejects_non_invertible_factor() { let key = prevalidated_test_private_key(); let mut factor = vec![0u8; key.signature_len()]; - left_pad_be(&rsa_private_prime_p(), &mut factor).unwrap(); + left_pad_be(&rsa_private_prime_p(), &mut factor).expect("prime fixture must fit the RSA representative width"); let mut inverse = vec![0u8; key.signature_len()]; assert_eq!( @@ -12678,16 +12761,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, &mut scratch, ) - .unwrap(); + .expect("PKCS#1 v1.5 signing with explicit blinding and caller scratch must succeed"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("caller-scratch PKCS#1 v1.5 signature must verify"); let salt = [0x42; Sha384::OUTPUT_SIZE]; let mut pss_signature = vec![0u8; key.signature_len()]; @@ -12696,16 +12778,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha384, message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pss_signature, &mut scratch, ) - .unwrap(); + .expect("PSS signing with explicit salt, blinding, and caller scratch must succeed"); key .public_key() .verify_pss(RsaPssProfile::Sha384, message, &pss_signature) - .unwrap(); + .expect("caller-scratch PSS signature must verify"); let label = b"rscrypto-private-scratch-oaep"; let plaintext = b"private scratch OAEP roundtrip"; @@ -12714,19 +12795,18 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("valid OAEP fixture inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor_and_scratch( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("OAEP ciphertext must decrypt with explicit blinding and caller scratch"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"private scratch RSAES-PKCS1-v1_5 roundtrip"; @@ -12734,16 +12814,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("valid PKCS#1 v1.5 fixture inputs must encrypt"); let decrypted_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("PKCS#1 v1.5 ciphertext must decrypt with explicit blinding and caller scratch"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); let mut smaller_scratch = wrong_width_private_scratch(&key); @@ -12751,8 +12830,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, &mut smaller_scratch, ), @@ -12772,8 +12850,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key.sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ), @@ -12824,22 +12901,27 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn keygen_generated_prime_satisfies_fips_a1_3_prime_constraints() { let mut drbg = test_keygen_drbg(b"generated-prime-constraints"); - let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)).unwrap(); + let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)) + .expect("deterministic prime search must complete"); assert!(keygen_probable_prime_meets_fips_lower_bound(&prime, 128)); assert_eq!(prime.last().copied().unwrap_or_default() & 1, 1); assert!(!has_small_prime_factor(&prime)); assert!(!keygen_conflicts_with_public_exponent(&prime)); - assert!(keygen_is_probable_prime(&mut drbg, &prime).unwrap()); + assert!(keygen_is_probable_prime(&mut drbg, &prime).expect("generated prime must pass a valid primality test")); } #[cfg(feature = "getrandom")] #[test] fn keygen_lcm_private_exponent_contract_is_enforced() { - let p_minus_one = private_import_decrement_unsigned_be(&rsa_private_prime_p()).unwrap(); - let q_minus_one = private_import_decrement_unsigned_be(&rsa_private_prime_q()).unwrap(); - let lambda = keygen_lcm_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()).unwrap(); - let phi = private_import_product_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()).unwrap(); + let p_minus_one = + private_import_decrement_unsigned_be(&rsa_private_prime_p()).expect("fixture prime p must be greater than zero"); + let q_minus_one = + private_import_decrement_unsigned_be(&rsa_private_prime_q()).expect("fixture prime q must be greater than zero"); + let lambda = keygen_lcm_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()) + .expect("fixture prime totients must have an LCM"); + let phi = private_import_product_unsigned_be(p_minus_one.as_slice(), q_minus_one.as_slice()) + .expect("fixture prime totient product must fit the supported width"); assert_eq!( private_import_unsigned_be_mod(phi.as_slice(), lambda.as_slice()).as_slice(), @@ -12863,7 +12945,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 accepted = Some(candidate); } } - let candidate = accepted.unwrap(); + let candidate = accepted.expect("deterministic search must find a FIPS-shaped candidate"); assert!( keygen_candidate_has_fixed_shape(&candidate, 128), "accepted FIPS A.1.3 candidate must be odd and full-width" @@ -12903,16 +12985,19 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn keygen_miller_rabin_accepts_prime_and_rejects_composite_for_fixed_bases() { let prime = 1009u16.to_be_bytes(); - let prime_modulus = private_component_modulus(&prime).unwrap(); + let prime_modulus = private_component_modulus(&prime).expect("nonzero prime fixture must define a modulus"); assert!( - keygen_miller_rabin_accepts_base(&prime_modulus, &[0, 63], 4, &1008u16.to_be_bytes(), &[0, 11]).unwrap(), + keygen_miller_rabin_accepts_base(&prime_modulus, &[0, 63], 4, &1008u16.to_be_bytes(), &[0, 11]) + .expect("valid prime fixture must admit a Miller-Rabin round"), "1009 must pass a direct Miller-Rabin round for base 11" ); let composite = 341u16.to_be_bytes(); - let composite_modulus = private_component_modulus(&composite).unwrap(); + let composite_modulus = + private_component_modulus(&composite).expect("nonzero composite fixture must define a modulus"); assert!( - !keygen_miller_rabin_accepts_base(&composite_modulus, &[0, 85], 2, &340u16.to_be_bytes(), &[0, 2]).unwrap(), + !keygen_miller_rabin_accepts_base(&composite_modulus, &[0, 85], 2, &340u16.to_be_bytes(), &[0, 2]) + .expect("valid composite fixture must admit a Miller-Rabin round"), "341 must fail a direct Miller-Rabin round for base 2" ); } @@ -12927,15 +13012,15 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_prime_p(), rsa_private_prime_q(), ) - .unwrap() - .unwrap(); + .expect("fixture primes must derive without arithmetic failure") + .expect("fixture primes must produce the requested modulus width"); let key = RsaPrivateKey { components }; assert_eq!(key.public_key().modulus(), modulus); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let exported = key.to_pkcs1_der(); - let imported = - RsaPrivateKey::from_pkcs1_der_with_policy(&exported, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let imported = RsaPrivateKey::from_pkcs1_der_with_policy(&exported, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated PKCS#1 private key must re-import"); assert_eq!(imported.public_key(), key.public_key()); let message = b"rscrypto generated RSA component signing roundtrip"; @@ -12945,15 +13030,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) - .unwrap(); + .expect("key derived from fixture primes must sign"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &signature) - .unwrap(); + .expect("signature from fixture-prime key must verify"); let label = b"rscrypto-keygen-oaep"; let plaintext = b"generated component oaep roundtrip"; @@ -12962,18 +13046,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("fixture-prime OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ) - .unwrap(); + .expect("fixture-prime OAEP ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"generated component RSAES-PKCS1-v1_5 roundtrip"; @@ -12981,10 +13064,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("fixture-prime PKCS#1 v1.5 inputs must encrypt"); let decrypted_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_factor_inverse, &mut decrypted) - .unwrap(); + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut decrypted, + ) + .expect("fixture-prime PKCS#1 v1.5 ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); } @@ -13003,7 +13090,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut covered = Vec::new(); for test in tests { - let modulus_bits = test["mod"].as_u64().expect("CAVP modulus size must be numeric") as usize; + let modulus_bits = usize::try_from(test["mod"].as_u64().expect("CAVP modulus size must be numeric")) + .expect("CAVP modulus size must fit usize"); let components = keygen_build_private_key_from_primes( modulus_bits, &policy, @@ -13016,8 +13104,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_eq!(key.public_key().modulus_bits(), modulus_bits); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let mut primality_drbg = test_keygen_drbg(b"cavp-primality"); - assert!(keygen_is_probable_prime(&mut primality_drbg, key.components.prime_p.as_bytes()).unwrap()); - assert!(keygen_is_probable_prime(&mut primality_drbg, key.components.prime_q.as_bytes()).unwrap()); + assert!( + keygen_is_probable_prime(&mut primality_drbg, key.components.prime_p.as_bytes()) + .expect("CAVP prime p must admit a primality test") + ); + assert!( + keygen_is_probable_prime(&mut primality_drbg, key.components.prime_q.as_bytes()) + .expect("CAVP prime q must admit a primality test") + ); let (blinding_factor, blinding_factor_inverse) = factor_two_and_inverse(key.public_key().modulus()); let message = b"rscrypto NIST CAVP keygen candidate private operation"; @@ -13027,15 +13121,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pkcs1v15_signature, ) - .unwrap(); + .expect("CAVP-derived key must sign PKCS#1 v1.5"); key .public_key() .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &pkcs1v15_signature) - .unwrap(); + .expect("CAVP-derived PKCS#1 v1.5 signature must verify"); let salt = [0xa5; Sha256::OUTPUT_SIZE]; let mut pss_signature = vec![0u8; key.signature_len()]; @@ -13044,15 +13137,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 RsaPssProfile::Sha256, message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut pss_signature, ) - .unwrap(); + .expect("CAVP-derived key must sign PSS with explicit salt and blinding"); key .public_key() .verify_pss(RsaPssProfile::Sha256, message, &pss_signature) - .unwrap(); + .expect("CAVP-derived PSS signature must verify"); let label = b"rscrypto-cavp-keygen-oaep"; let plaintext = b"NIST CAVP keygen candidate OAEP"; @@ -13061,18 +13153,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("CAVP-derived OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_blinding_factor( RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, ) - .unwrap(); + .expect("CAVP-derived OAEP ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"NIST CAVP keygen candidate RSAES-PKCS1-v1_5"; @@ -13080,10 +13171,14 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("CAVP-derived PKCS#1 v1.5 inputs must encrypt"); let decrypted_len = key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_factor_inverse, &mut decrypted) - .unwrap(); + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), + &mut decrypted, + ) + .expect("CAVP-derived PKCS#1 v1.5 ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); covered.push(modulus_bits); @@ -13095,16 +13190,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(feature = "getrandom")] #[test] fn keygen_generate_with_policy_produces_usable_private_key_end_to_end() { - let key = RsaPrivateKey::generate_with_policy(2048, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = RsaPrivateKey::generate_with_policy(2048, &RsaPublicKeyPolicy::legacy_verification()) + .expect("legacy test policy must generate a 2048-bit key"); assert_eq!(key.public_key().modulus_bits(), 2048); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let pkcs1 = key.to_pkcs1_der(); let pkcs8 = key.to_pkcs8_der(); - let pkcs1_imported = - RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); - let pkcs8_imported = - RsaPrivateKey::from_pkcs8_der_with_policy(&pkcs8, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let pkcs1_imported = RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated PKCS#1 key must re-import"); + let pkcs8_imported = RsaPrivateKey::from_pkcs8_der_with_policy(&pkcs8, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated PKCS#8 key must re-import"); assert_eq!(pkcs1_imported.public_key(), key.public_key()); assert_eq!(pkcs8_imported.public_key(), key.public_key()); @@ -13114,29 +13210,31 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15($pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("generated key must sign PKCS#1 v1.5"); key .public_key() .verify_pkcs1v15($pkcs1_profile, message, &pkcs1v15_signature) - .unwrap(); + .expect("generated-key PKCS#1 v1.5 signature must verify"); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_pss($pss_profile, message, &mut pss_signature).unwrap(); + key + .sign_pss($pss_profile, message, &mut pss_signature) + .expect("generated key must sign PSS"); key .public_key() .verify_pss($pss_profile, message, &pss_signature) - .unwrap(); + .expect("generated-key PSS signature must verify"); let label = b"rscrypto-generated-key-oaep"; let mut ciphertext = vec![0u8; key.signature_len()]; key .public_key() .diag_encrypt_oaep_with_seed($oaep_profile, label, $plaintext, &$seed, &mut ciphertext) - .unwrap(); + .expect("generated-key OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep($oaep_profile, label, &ciphertext, &mut decrypted) - .unwrap(); + .expect("generated-key OAEP ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], $plaintext); let pkcs1v15_plaintext = b"generated key RSAES-PKCS1-v1_5"; @@ -13144,8 +13242,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut decrypted).unwrap(); + .expect("generated-key PKCS#1 v1.5 inputs must encrypt"); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut decrypted) + .expect("generated-key PKCS#1 v1.5 ciphertext must decrypt"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); }}; } @@ -13176,18 +13276,22 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(feature = "getrandom")] #[test] fn keygen_generate_default_modern_key_produces_usable_private_key_end_to_end() { - let key = RsaPrivateKey::generate(3072).unwrap(); + let key = RsaPrivateKey::generate(3072).expect("default policy must generate a 3072-bit key"); assert_eq!(key.public_key().modulus_bits(), 3072); assert_eq!(key.public_key().public_exponent().as_u64(), RSA_KEYGEN_PUBLIC_EXPONENT); let pkcs1 = key.to_pkcs1_der(); let pkcs8 = key.to_pkcs8_der(); assert_eq!( - RsaPrivateKey::from_pkcs1_der(&pkcs1).unwrap().public_key(), + RsaPrivateKey::from_pkcs1_der(&pkcs1) + .expect("generated PKCS#1 key must re-import") + .public_key(), key.public_key() ); assert_eq!( - RsaPrivateKey::from_pkcs8_der(&pkcs8).unwrap().public_key(), + RsaPrivateKey::from_pkcs8_der(&pkcs8) + .expect("generated PKCS#8 key must re-import") + .public_key(), key.public_key() ); @@ -13198,20 +13302,20 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let pkcs1v15_profile = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); key .sign_signature_with_scratch(pkcs1v15_profile, message, &mut signature, &mut scratch) - .unwrap(); + .expect("generated modern key must sign PKCS#1 v1.5 with caller scratch"); key .public_key() .verify_signature(pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 signature must verify"); let pss_profile = RsaSignatureProfile::pss(RsaPssProfile::Sha256); key .sign_signature_with_scratch(pss_profile, message, &mut signature, &mut scratch) - .unwrap(); + .expect("generated modern key must sign PSS with caller scratch"); key .public_key() .verify_signature(pss_profile, message, &signature) - .unwrap(); + .expect("generated modern-key PSS signature must verify"); let label = b"rscrypto-generated-modern-key-oaep"; let plaintext = b"modern generated key OAEP roundtrip"; @@ -13220,11 +13324,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_oaep_with_seed(RsaOaepProfile::Sha256, label, plaintext, &seed, &mut ciphertext) - .unwrap(); + .expect("generated modern-key OAEP inputs must encrypt"); let mut decrypted = vec![0u8; key.signature_len()]; let decrypted_len = key .decrypt_oaep_with_scratch(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("generated modern-key OAEP ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], plaintext); let pkcs1v15_plaintext = b"modern generated key RSAES-PKCS1-v1_5"; @@ -13232,10 +13336,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_key() .diag_encrypt_pkcs1v15_with_seed(pkcs1v15_plaintext, &pkcs1v15_seed, &mut ciphertext) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 inputs must encrypt"); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 ciphertext must decrypt with caller scratch"); assert_eq!(&decrypted[..decrypted_len], pkcs1v15_plaintext); } @@ -13278,7 +13382,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert!( keygen_build_private_key_from_primes(modulus_bits, &policy, rsa_private_prime_p(), rsa_private_prime_p()) - .unwrap() + .expect("duplicate prime fixtures must complete validation") .is_none() ); assert!( @@ -13288,7 +13392,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_prime_p(), rsa_private_prime_q() ) - .unwrap() + .expect("undersized modulus candidates must complete validation") .is_none() ); assert!( @@ -13298,7 +13402,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 rsa_private_prime_p(), rsa_private_prime_q() ) - .unwrap() + .expect("oversized modulus candidates must complete validation") .is_none() ); @@ -13306,7 +13410,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut carry = 2u16; for byte in close_prime_q.iter_mut().rev() { let sum = u16::from(*byte).strict_add(carry); - *byte = sum as u8; + *byte = sum.to_le_bytes()[0]; carry = sum >> 8; if carry == 0 { break; @@ -13319,7 +13423,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); assert!( keygen_build_private_key_from_primes(modulus_bits, &policy, rsa_private_prime_p(), close_prime_q) - .unwrap() + .expect("close-prime candidates must complete validation") .is_none() ); @@ -13331,7 +13435,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 break; } let sum = u64::from(*byte).strict_add(delta & 0xff); - *byte = sum as u8; + *byte = sum.to_le_bytes()[0]; delta = (delta >> 8).strict_add(sum >> 8); } assert_eq!(delta, 0); @@ -13341,7 +13445,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); assert!( keygen_build_private_key_from_primes(modulus_bits, &policy, public_exponent_conflict, rsa_private_prime_q()) - .unwrap() + .expect("public-exponent-conflicting candidates must complete validation") .is_none() ); } @@ -13350,20 +13454,21 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn keygen_random_prime_search_returns_probable_prime() { let mut drbg = test_keygen_drbg(b"random-prime-search"); - let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)).unwrap(); + let prime = keygen_generate_prime(&mut drbg, 128, 256, None, 256usize.strict_mul(5)) + .expect("deterministic prime search must complete"); assert_eq!(unsigned_be_bit_len(&prime), 128); assert_eq!(prime.last().copied().unwrap_or_default() & 1, 1); assert!(!has_small_prime_factor(&prime)); assert!(!keygen_conflicts_with_public_exponent(&prime)); assert!(keygen_probable_prime_meets_fips_lower_bound(&prime, 128)); - assert!(keygen_is_probable_prime(&mut drbg, &prime).unwrap()); + assert!(keygen_is_probable_prime(&mut drbg, &prime).expect("generated prime must admit a primality test")); } #[test] fn pkcs1_private_key_parser_applies_modulus_policy_before_component_checks() { let mut bad_q = rsa_private_prime_q(); - *bad_q.last_mut().unwrap() ^= 0x02; + *bad_q.last_mut().expect("prime q fixture must be nonempty") ^= 0x02; let der = test_pkcs1_private_key_with_components( &[0x01, 0x00, 0x01], &rsa_private_exponent(), @@ -13386,7 +13491,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let pkcs1 = test_pkcs1_private_key(); let rsa_algorithm = algorithm_identifier(RSA_ENCRYPTION_OID, Some(&null())); let der = test_pkcs8_private_key(&pkcs1, &rsa_algorithm); - let key = parse_pkcs8_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = parse_pkcs8_private_key_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("valid PKCS#8 fixture must parse"); assert_eq!(key.public_key().modulus(), rsa_private_modulus()); @@ -13495,7 +13601,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); let mut bad_q = rsa_private_prime_q(); - let last = bad_q.last_mut().unwrap(); + let last = bad_q.last_mut().expect("prime q fixture must be nonempty"); *last ^= 0x02; let mut inconsistent_body = Vec::new(); for field in [ @@ -13521,7 +13627,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn pkcs1_private_key_parser_rejects_crt_congruence_mismatches() { let mut bad_exponent_p = rsa_private_exponent_p(); - *bad_exponent_p.last_mut().unwrap() ^= 0x02; + *bad_exponent_p + .last_mut() + .expect("CRT exponent p fixture must be nonempty") ^= 0x02; let bad_dp = test_pkcs1_private_key_with_crt( &[0x01, 0x00, 0x01], &bad_exponent_p, @@ -13534,7 +13642,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); let mut bad_exponent_q = rsa_private_exponent_q(); - *bad_exponent_q.last_mut().unwrap() ^= 0x02; + *bad_exponent_q + .last_mut() + .expect("CRT exponent q fixture must be nonempty") ^= 0x02; let bad_dq = test_pkcs1_private_key_with_crt( &[0x01, 0x00, 0x01], &rsa_private_exponent_p(), @@ -13547,7 +13657,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 ); let mut bad_coefficient = rsa_private_coefficient(); - *bad_coefficient.last_mut().unwrap() ^= 0x02; + *bad_coefficient + .last_mut() + .expect("CRT coefficient fixture must be nonempty") ^= 0x02; let bad_qinv = test_pkcs1_private_key_with_crt( &[0x01, 0x00, 0x01], &rsa_private_exponent_p(), @@ -13914,7 +14026,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -13923,18 +14039,22 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let mut bytes = vec![0u8; words.strict_mul(8)]; for (i, byte) in bytes.iter_mut().enumerate() { *byte = 0xa5u8 - .wrapping_add((i as u8).wrapping_mul(0x3d)) - .wrapping_add((words as u8).wrapping_mul(0x17)); + .wrapping_add(i.to_le_bytes()[0].wrapping_mul(0x3d)) + .wrapping_add(words.to_le_bytes()[0].wrapping_mul(0x17)); } bytes[0] = 0xff; - *bytes.last_mut().unwrap() |= 1; + *bytes.last_mut().expect("RSA Montgomery test width must be nonzero") |= 1; RsaPublicModulus::new(&bytes, words.strict_mul(64)) } #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -13948,7 +14068,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 state ^= state << 17; *limb = state; } - *limbs.last_mut().unwrap() &= 0x7fff_ffff_ffff_ffff; + *limbs.last_mut().expect("RSA Montgomery test width must be nonzero") &= 0x7fff_ffff_ffff_ffff; limbs[0] |= 1; limbs } @@ -13956,7 +14076,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -13972,7 +14096,11 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[cfg(all( any( all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")), - all(target_arch = "x86_64", target_os = "linux") + all( + target_arch = "x86_64", + target_os = "linux", + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) + ) ), not(feature = "portable-only"), not(miri) @@ -14057,7 +14185,6 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 &mut asm_acc, &modulus.limbs, modulus.n0, - words, &mut asm_t, ); public_e65537_cios_portable(&mut portable_out, &a, &r2, &mut portable_acc, &modulus, &mut portable_t); @@ -14088,11 +14215,17 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 assert_aarch64_rsa_montgomery_backend_matches_portable!(rsa_aarch64_asm); } + // x86-64 BMI2/ADX discovery is `std`-only: without it `caps()` reports just + // the compile-time feature set, so the assembly backend is unreachable and + // there is no second implementation to differentially test. Compile this + // evidence test exactly when the backend can be observed as available, and + // keep the assertion hard inside it. #[cfg(all( target_arch = "x86_64", target_os = "linux", not(feature = "portable-only"), - not(miri) + not(miri), + any(feature = "std", all(target_feature = "bmi2", target_feature = "adx")) ))] #[test] fn x86_64_linux_rsa_montgomery_asm_matches_portable_across_supported_widths() { @@ -14243,9 +14376,10 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn diag_spki_public_key_validation_matches_import_metadata() { let spki = include_bytes!("../../benches/rsa_fixtures/rsa3072_spki.der"); - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("RSA-3072 SPKI fixture must import"); let (modulus_len, modulus_bits, exponent) = - diag_rsa_validate_spki_public_key_der(spki, &RsaPublicKeyPolicy::default()).unwrap(); + diag_rsa_validate_spki_public_key_der(spki, &RsaPublicKeyPolicy::default()) + .expect("RSA-3072 SPKI fixture must pass diagnostic validation"); assert_eq!(modulus_len, key.modulus().len()); assert_eq!(modulus_bits, key.modulus_bits()); @@ -14256,7 +14390,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 #[test] fn diag_montgomery_r2_precompute_matches_imported_key() { let spki = include_bytes!("../../benches/rsa_fixtures/rsa3072_spki.der"); - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("RSA-3072 SPKI fixture must import"); let scratch = key.public_scratch(); let (_, scratch_r2) = scratch.montgomery.split_at(scratch.limb_count); @@ -14275,7 +14409,8 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); for exponent in [&[0x03][..], &[0x11][..], &[0x01, 0x00, 0x01][..]] { - let key = RsaPublicKey::from_pkcs1_der_with_policy(&test_pkcs1_public_key(&modulus, exponent), &policy).unwrap(); + let key = RsaPublicKey::from_pkcs1_der_with_policy(&test_pkcs1_public_key(&modulus, exponent), &policy) + .expect("test exponent must produce a valid public key"); let mut specialized = vec![0u8; key.modulus().len()]; let mut generic = vec![0u8; key.modulus().len()]; let mut specialized_scratch = key.public_scratch(); @@ -14283,8 +14418,9 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 key .public_operation_with_scratch(&input, &mut specialized, &mut specialized_scratch) - .unwrap(); - diag_rsa_public_operation_generic_exponent(&key, &input, &mut generic, &mut generic_scratch).unwrap(); + .expect("specialized public operation must accept the test representative"); + diag_rsa_public_operation_generic_exponent(&key, &input, &mut generic, &mut generic_scratch) + .expect("generic public operation must accept the test representative"); assert_eq!( generic, specialized, @@ -14302,7 +14438,7 @@ f70203010001a3533051301d0603551d0e04160414fd0e576ce3f05b08884ad67ef3e8b4d39039c6 &test_pkcs1_public_key(&modulus, &exponent), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("diagnostic backend test key must import"); let representative = key.modulus().to_vec(); let mut out = vec![0xa5; key.modulus().len()]; @@ -14416,13 +14552,13 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ &test_pkcs1_public_key(&modulus, &[0x01, 0x00, 0x01]), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("independent-vector public key must import"); let mut out = vec![0u8; key.modulus().len()]; let mut scratch = key.public_scratch(); key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("independent-vector representative must exponentiate"); assert_eq!(out, expected); } @@ -14506,10 +14642,12 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ &x509_certificate_fixture_public_key(), &RsaPublicKeyPolicy::legacy_verification(), ) - .unwrap(); + .expect("X.509 issuer fixture must import"); let certificate = x509_pkcs1v15_certificate_fixture(); - assert!(issuer.verify_x509_certificate_signature_der(&certificate).is_ok()); + issuer + .verify_x509_certificate_signature_der(&certificate) + .expect("fixture certificate signature must verify"); let mut tampered = certificate; if let Some(last) = tampered.last_mut() { @@ -14577,9 +14715,15 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ left in any::(), right in any::(), ) { - let left = left % modulus; - let right = right % modulus; - let expected = ((u64::from(left) + u64::from(modulus) - u64::from(right)) % u64::from(modulus)) as u32; + let left = left.rem_euclid(modulus); + let right = right.rem_euclid(modulus); + let expected = u32::try_from( + u64::from(left) + .strict_add(u64::from(modulus)) + .strict_sub(u64::from(right)) + .rem_euclid(u64::from(modulus)), + ) + .expect("reduced u32 modular difference must fit u32"); let mut out = [0u8; 4]; private_sub_mod_unsigned_be_to_fixed( @@ -14588,7 +14732,7 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ &modulus.to_be_bytes(), &mut out, ) - .unwrap(); + .expect("valid u32 modular subtraction inputs must succeed"); prop_assert_eq!(out, expected.to_be_bytes()); } @@ -14598,7 +14742,8 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ modulus[0] |= 0x80; modulus[255] |= 0x01; let exponent = RsaPublicExponent(65_537); - let bits = validate_modulus(&modulus, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let bits = validate_modulus(&modulus, &RsaPublicKeyPolicy::legacy_verification()) + .expect("generated full-width odd modulus must satisfy the legacy test policy"); let key = RsaPublicKey { modulus: RsaPublicModulus::new(&modulus, bits), exponent, diff --git a/src/auth/rsa_aarch64_asm.rs b/src/auth/rsa_aarch64_asm.rs index 2a158ced..c8d02b7e 100644 --- a/src/auth/rsa_aarch64_asm.rs +++ b/src/auth/rsa_aarch64_asm.rs @@ -3,8 +3,6 @@ //! This module owns the ABI boundary. `rsa.rs` owns all RSA validation, //! representative range checks, dispatch, and fallback semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_rsa_aarch64_apple_darwin.s")); @@ -74,7 +72,14 @@ pub(super) fn mont_mul_cios_words( // read/write ranges. The assembly does not retain pointers after returning. // 4. `out` does not alias `a`, `b`, or `modulus` in current non-in-place callers. unsafe { - rscrypto_rsa_bn_mul_mont_words_apple(out.as_mut_ptr(), a.as_ptr(), b.as_ptr(), modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_mul_mont_words_apple( + out.as_mut_ptr(), + a.as_ptr(), + b.as_ptr(), + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -104,7 +109,7 @@ pub(super) fn mont_square_cios_words_in_place( value.as_ptr(), value.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -139,7 +144,7 @@ pub(super) fn mont_mul_cios_words_in_place_left( left.as_ptr(), right.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -162,8 +167,8 @@ pub(super) fn mont_reduce_cios_words( // SAFETY: RSA Montgomery REDC assembly call because: // 1. This module is compiled only for macOS AArch64 and embeds the matching Darwin symbol. - // 2. The caller checks `out`, `value`, and `modulus` are 48 or 64 `u64` limbs and `t` has `2 * - // words + 2` scratch limbs. + // 2. The caller checks `out`, `value`, and `modulus` are 48, 64, or 128 `u64` limbs and `t` has + // `2 * words + 2` scratch limbs. // 3. The assembly writes only caller-provided scratch and performs final conditional subtraction // with mask selection, not data-dependent branches. // 4. All pointers are derived from live Rust slices and the assembly does not retain them. @@ -180,7 +185,6 @@ pub(super) fn mont_reduce_cios_words( } #[inline] -#[allow(clippy::too_many_arguments)] // Assembly ABI boundary; a wrapper struct would only repack these operands. pub(super) fn public_e65537_mont_words( out: &mut [u64], input: &[u64], @@ -188,9 +192,9 @@ pub(super) fn public_e65537_mont_words( acc: &mut [u64], modulus: &[u64], n0: u64, - words: usize, t: &mut [u64], ) { + let words = out.len(); debug_assert!(supports_bignum_mont_words(words)); debug_assert_eq!(out.len(), words); debug_assert_eq!(input.len(), words); @@ -212,7 +216,7 @@ pub(super) fn public_e65537_mont_words( input.as_ptr(), r2.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); acc.copy_from_slice(out); @@ -222,7 +226,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), acc.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); } @@ -231,7 +235,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), out.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); diff --git a/src/auth/rsa_aarch64_linux_asm.rs b/src/auth/rsa_aarch64_linux_asm.rs index f32c6180..73872ed8 100644 --- a/src/auth/rsa_aarch64_linux_asm.rs +++ b/src/auth/rsa_aarch64_linux_asm.rs @@ -3,8 +3,6 @@ //! This module owns the ABI boundary. `rsa.rs` owns all RSA validation, //! representative range checks, dispatch, and fallback semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; global_asm!(include_str!("asm/rscrypto_rsa_aarch64_linux_elf.s")); @@ -74,7 +72,14 @@ pub(super) fn mont_mul_cios_words( // read/write ranges. The assembly does not retain pointers after returning. // 4. `out` does not alias `a`, `b`, or `modulus` in current non-in-place callers. unsafe { - rscrypto_rsa_bn_mul_mont_words_aarch64_elf(out.as_mut_ptr(), a.as_ptr(), b.as_ptr(), modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_mul_mont_words_aarch64_elf( + out.as_mut_ptr(), + a.as_ptr(), + b.as_ptr(), + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -104,7 +109,7 @@ pub(super) fn mont_square_cios_words_in_place( value.as_ptr(), value.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -139,7 +144,7 @@ pub(super) fn mont_mul_cios_words_in_place_left( left.as_ptr(), right.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; @@ -162,8 +167,8 @@ pub(super) fn mont_reduce_cios_words( // SAFETY: RSA Montgomery REDC assembly call because: // 1. This module is compiled only for Linux AArch64 and embeds the matching ELF symbol. - // 2. The caller checks `out`, `value`, and `modulus` are 48 or 64 `u64` limbs and `t` has `2 * - // words + 2` scratch limbs. + // 2. The caller checks `out`, `value`, and `modulus` are 48, 64, or 128 `u64` limbs and `t` has + // `2 * words + 2` scratch limbs. // 3. The assembly writes only caller-provided scratch and performs final conditional subtraction // with mask selection, not data-dependent branches. // 4. All pointers are derived from live Rust slices and the assembly does not retain them. @@ -180,7 +185,6 @@ pub(super) fn mont_reduce_cios_words( } #[inline] -#[allow(clippy::too_many_arguments)] // Assembly ABI boundary; a wrapper struct would only repack these operands. pub(super) fn public_e65537_mont_words( out: &mut [u64], input: &[u64], @@ -188,9 +192,9 @@ pub(super) fn public_e65537_mont_words( acc: &mut [u64], modulus: &[u64], n0: u64, - words: usize, t: &mut [u64], ) { + let words = out.len(); debug_assert!(supports_bignum_mont_words(words)); debug_assert_eq!(out.len(), words); debug_assert_eq!(input.len(), words); @@ -212,7 +216,7 @@ pub(super) fn public_e65537_mont_words( input.as_ptr(), r2.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); acc.copy_from_slice(out); @@ -222,7 +226,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), acc.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); } @@ -231,7 +235,7 @@ pub(super) fn public_e65537_mont_words( acc.as_ptr(), out.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ); diff --git a/src/auth/rsa_x86_64_asm.rs b/src/auth/rsa_x86_64_asm.rs index fdb81119..224e2e9a 100644 --- a/src/auth/rsa_x86_64_asm.rs +++ b/src/auth/rsa_x86_64_asm.rs @@ -3,8 +3,6 @@ //! This module owns the ABI boundary. `rsa.rs` owns all RSA validation, //! representative range checks, dispatch, and fallback semantics. -#![allow(unsafe_code)] - use core::arch::global_asm; use crate::platform::{caps, caps::x86}; @@ -73,7 +71,14 @@ pub(super) fn mont_mul_cios_words( // 4. All pointers are derived from live Rust slices and are valid for the assembly's fixed // read/write ranges. The assembly does not retain pointers after returning. let _ = unsafe { - rscrypto_rsa_bn_mulx4x_mont_x86_64_elf(out.as_mut_ptr(), a.as_ptr(), b.as_ptr(), modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_mulx4x_mont_x86_64_elf( + out.as_mut_ptr(), + a.as_ptr(), + b.as_ptr(), + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -92,7 +97,7 @@ pub(super) fn mont_square_cios_words_in_place( // SAFETY: RSA in-place Montgomery square ADX/BMI2 assembly call because: // 1. This module is compiled only for Linux x86-64 and embeds the matching ELF symbol. - // 2. `supports_bignum_mont_words` checks BMI2 and ADX before this call, matching the kernel's + // 2. `supports_bignum_mont_square_words` checks BMI2 and ADX before this call, matching the kernel's // `mulx/adcx/adox` instruction requirements. // 3. The caller checks `value` and `modulus` are supported `u64` limb widths satisfying the square // kernel's `num >= 8 && num % 8 == 0` precondition. @@ -101,7 +106,14 @@ pub(super) fn mont_square_cios_words_in_place( // `value` as both `out` and `a` preserves the in-place helper contract. // 5. All pointers are derived from live Rust slices and the assembly does not retain them. let _ = unsafe { - rscrypto_rsa_bn_sqr8x_mont_x86_64_elf(value.as_mut_ptr(), value.as_ptr(), 1, modulus.as_ptr(), &n0, words) + rscrypto_rsa_bn_sqr8x_mont_x86_64_elf( + value.as_mut_ptr(), + value.as_ptr(), + 1, + modulus.as_ptr(), + &raw const n0, + words, + ) }; } @@ -134,7 +146,7 @@ pub(super) fn mont_mul_cios_words_in_place_left( left.as_ptr(), right.as_ptr(), modulus.as_ptr(), - &n0, + &raw const n0, words, ) }; diff --git a/src/auth/scrypt.rs b/src/auth/scrypt.rs index d60879ac..deb05c2b 100644 --- a/src/auth/scrypt.rs +++ b/src/auth/scrypt.rs @@ -48,12 +48,6 @@ //! [`crate::Pbkdf2Sha256`] (alloc-free) or the `argon2` / `phc-strings` //! features under the same caveats. -#![allow(clippy::indexing_slicing)] -// `unwrap_used` applies to slice→array conversions whose lengths are fixed -// by construction (BLOCK_SIZE / BLOCK_WORDS slicing); every site is bounded -// by compile-time constants and cannot fail at runtime. -#![allow(clippy::unwrap_used)] - use alloc::vec::Vec; use core::fmt; @@ -102,8 +96,8 @@ const MAX_R_TIMES_P: u64 = (1u64 << 30) - 1; /// use rscrypto::{ScryptParams, auth::scrypt::ScryptError}; /// /// assert_eq!( -/// ScryptParams::new(0, 8, 1).unwrap_err(), -/// ScryptError::InvalidLogN +/// ScryptParams::new(0, 8, 1), +/// Err(ScryptError::InvalidLogN) /// ); /// ``` #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -342,9 +336,8 @@ mod x86_sse2 { debug_assert_eq!(blocks.len() % super::BLOCK_SIZE, 0); for chunk in blocks.chunks_exact_mut(super::BLOCK_SIZE) { let mut words = [0u32; super::BLOCK_WORDS]; - for (src, word) in chunk.chunks_exact(4).zip(words.iter_mut()) { - let arr: [u8; 4] = src.try_into().unwrap(); - *word = u32::from_le_bytes(arr); + for (src, word) in chunk.as_chunks::<4>().0.iter().zip(&mut words) { + *word = u32::from_le_bytes(*src); } for (i, dst) in chunk.chunks_exact_mut(4).enumerate() { dst.copy_from_slice(&words[pivot[i]].to_le_bytes()); @@ -438,8 +431,9 @@ mod x86_sse2 { fn integerify_pivot_low64(chunk: &[u8]) -> u64 { debug_assert_eq!(chunk.len() % super::BLOCK_SIZE, 0); let last = &chunk[chunk.len().strict_sub(super::BLOCK_SIZE)..]; - let lo = u32::from_le_bytes(last[0..4].try_into().unwrap()) as u64; - let hi = u32::from_le_bytes(last[52..56].try_into().unwrap()) as u64; + let words = last.as_chunks::<4>().0; + let lo = u64::from(u32::from_le_bytes(words[0])); + let hi = u64::from(u32::from_le_bytes(words[13])); lo | (hi << 32) } @@ -463,9 +457,12 @@ mod x86_sse2 { block_mix(v_chunk, chunk); } - let n_mask = (n as u64).wrapping_sub(1); + let n_mask = u64::try_from(n) + .expect("scrypt allocation length must fit u64") + .strict_sub(1); for _ in 0..n { - let j = (integerify_pivot_low64(chunk) & n_mask) as usize; + let j = + usize::try_from(integerify_pivot_low64(chunk) & n_mask).expect("masked scrypt ROMix index must fit usize"); let v_start = j.strict_mul(len); xor_into(chunk, &v[v_start..v_start.strict_add(len)], scratch); block_mix(scratch, chunk); @@ -495,7 +492,6 @@ impl SalsaBlock { /// then adds the original input back word-wise. Additions are `u32` /// modular-wraparound per the spec. #[inline(always)] -#[allow(clippy::too_many_lines)] fn salsa20_8(block: &mut SalsaBlock) { let input = block.0; let mut y = block.0; @@ -649,10 +645,13 @@ fn ro_mix(chunk: &mut [SalsaBlock], v: &mut [SalsaBlock], scratch: &mut [SalsaBl // ping-pong; each pair XORs V[j] into the current X-buffer before // BlockMix targets the other buffer. X ends in `chunk`, which the // caller re-serialises for the final PBKDF2 leg. - let n_mask = (n as u64).wrapping_sub(1); + let n_mask = u64::try_from(n) + .expect("scrypt allocation length must fit u64") + .strict_sub(1); for _ in 0..pairs { // Even iteration: X lives in `chunk`; write into `scratch`. - let j = (integerify_low64(&chunk[two_r.strict_sub(1)]) & n_mask) as usize; + let j = usize::try_from(integerify_low64(&chunk[two_r.strict_sub(1)]) & n_mask) + .expect("masked scrypt ROMix index must fit usize"); let v_off = j.strict_mul(two_r); for k in 0..two_r { xor_block_into(&mut chunk[k], &v[v_off.strict_add(k)]); @@ -660,7 +659,8 @@ fn ro_mix(chunk: &mut [SalsaBlock], v: &mut [SalsaBlock], scratch: &mut [SalsaBl block_mix_into(chunk, scratch, r); // Odd iteration: X lives in `scratch`; write into `chunk`. - let j = (integerify_low64(&scratch[two_r.strict_sub(1)]) & n_mask) as usize; + let j = usize::try_from(integerify_low64(&scratch[two_r.strict_sub(1)]) & n_mask) + .expect("masked scrypt ROMix index must fit usize"); let v_off = j.strict_mul(two_r); for k in 0..two_r { xor_block_into(&mut scratch[k], &v[v_off.strict_add(k)]); @@ -671,24 +671,10 @@ fn ro_mix(chunk: &mut [SalsaBlock], v: &mut [SalsaBlock], scratch: &mut [SalsaBl // ─── Zeroisation helpers ──────────────────────────────────────────────────── -#[inline] -fn zeroize_u32_slice_no_fence(words: &mut [u32]) { - let mut chunks = words.chunks_exact_mut(16); - for chunk in &mut chunks { - // SAFETY: chunk has exactly 16 initialized u32s and [u32; 16] has the - // same alignment requirement as u32. - unsafe { core::ptr::write_volatile(chunk.as_mut_ptr().cast::<[u32; 16]>(), [0u32; 16]) }; - } - for w in chunks.into_remainder() { - // SAFETY: w is a valid, aligned, dereferenceable pointer to initialized u32. - unsafe { core::ptr::write_volatile(w, 0) }; - } -} - #[inline] fn zeroize_blocks_no_fence(blocks: &mut [SalsaBlock]) { for block in blocks { - zeroize_u32_slice_no_fence(&mut block.0); + ct::zeroize_words_no_fence(&mut block.0); } } @@ -848,10 +834,9 @@ fn scrypt_hash_portable( .map_err(|_| ScryptError::InvalidOutputLen)?; // Decode byte form into little-endian u32 blocks. - for (block, chunk) in state.b_u32.iter_mut().zip(state.b_bytes.chunks_exact(BLOCK_SIZE)) { - for (word, bytes) in block.0.iter_mut().zip(chunk.chunks_exact(4)) { - let arr: [u8; 4] = bytes.try_into().unwrap(); - *word = u32::from_le_bytes(arr); + for (block, chunk) in state.b_u32.iter_mut().zip(state.b_bytes.as_chunks::().0) { + for (word, bytes) in block.0.iter_mut().zip(chunk.as_chunks::<4>().0) { + *word = u32::from_le_bytes(*bytes); } } @@ -865,8 +850,8 @@ fn scrypt_hash_portable( // Re-serialise the mixed B back into the byte buffer for the final // PBKDF2 leg (the spec treats B as a byte string at this point). - for (block, chunk) in state.b_u32.iter().zip(state.b_bytes.chunks_exact_mut(BLOCK_SIZE)) { - for (word, bytes) in block.0.iter().zip(chunk.chunks_exact_mut(4)) { + for (block, chunk) in state.b_u32.iter().zip(state.b_bytes.as_chunks_mut::().0) { + for (word, bytes) in block.0.iter().zip(chunk.as_chunks_mut::<4>().0) { bytes.copy_from_slice(&word.to_le_bytes()); } } @@ -934,12 +919,13 @@ fn scrypt_hash(params: &ScryptParams, password: &[u8], salt: &[u8], out: &mut [u /// ```rust /// use rscrypto::{Scrypt, ScryptParams}; /// -/// // Small CI-friendly params — production deployments should use -/// let params = ScryptParams::new(10, 8, 1).unwrap(); +/// // Small CI-friendly parameters. +/// let params = ScryptParams::new(10, 8, 1)?; /// /// let mut hash = [0u8; 32]; -/// Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut hash).unwrap(); +/// Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut hash)?; /// assert!(Scrypt::verify(¶ms, b"password", b"random-salt-1234", &hash).is_ok()); +/// # Ok::<(), rscrypto::ScryptError>(()) /// ``` #[derive(Debug, Clone, Copy, Default)] pub struct Scrypt; @@ -1134,7 +1120,7 @@ mod password_phc { } impl ApprovedPhc { - pub fn salt(&self) -> &[u8] { + pub(super) fn salt(&self) -> &[u8] { &self.salt[..self.salt_len as usize] } } @@ -1175,7 +1161,8 @@ mod password_phc { if log_n > u8::MAX as u32 { return Err(PhcError::ParamOutOfRange); } - let params = ScryptParams::new(log_n as u8, r, p).map_err(|_| PhcError::ParamOutOfRange)?; + let log_n = u8::try_from(log_n).map_err(|_| PhcError::ParamOutOfRange)?; + let params = ScryptParams::new(log_n, r, p).map_err(|_| PhcError::ParamOutOfRange)?; scrypt_shape(¶ms).map_err(|_| PhcError::ParamOutOfRange)?; if !limits.allows(params) { return Err(PhcError::ParamOutOfRange); @@ -1198,7 +1185,7 @@ mod password_phc { Ok(ApprovedPhc { params, salt, - salt_len: decoded_salt_len as u8, + salt_len: u8::try_from(decoded_salt_len).map_err(|_| PhcError::InvalidLength)?, expected, }) } @@ -1235,20 +1222,21 @@ mod tests { ]; fn small_params() -> ScryptParams { - ScryptParams::new(4, 1, 1).unwrap() + ScryptParams::new(4, 1, 1).expect("small scrypt test profile must be valid") } #[test] fn rfc7914_vector_1_empty_inputs() { let mut output = [0u8; 64]; - Scrypt::derive(&small_params(), b"", b"", &mut output).unwrap(); + Scrypt::derive(&small_params(), b"", b"", &mut output) + .expect("RFC 7914 empty-input vector derivation must succeed"); assert_eq!(output, RFC_V1_EXPECTED); } fn oracle_scrypt(password: &[u8], salt: &[u8], log_n: u8, r: u32, p: u32, output_len: usize) -> alloc::vec::Vec { - let params = scrypt::Params::new(log_n, r, p).unwrap(); + let params = scrypt::Params::new(log_n, r, p).expect("oracle scrypt profile must be valid"); let mut output = vec![0u8; output_len]; - scrypt::scrypt(password, salt, ¶ms, &mut output).unwrap(); + scrypt::scrypt(password, salt, ¶ms, &mut output).expect("oracle scrypt derivation must succeed"); output } @@ -1256,9 +1244,10 @@ mod tests { fn matches_the_oracle_across_shapes_and_output_lengths() { let cases: &[(u8, u32, u32, usize)] = &[(4, 1, 1, 16), (5, 2, 1, 32), (6, 2, 2, 64)]; for &(log_n, r, p, output_len) in cases { - let params = ScryptParams::new(log_n, r, p).unwrap(); + let params = ScryptParams::new(log_n, r, p).expect("differential scrypt profile must be valid"); let mut actual = vec![0u8; output_len]; - Scrypt::derive(¶ms, b"password", b"salty-salty-salt", &mut actual).unwrap(); + Scrypt::derive(¶ms, b"password", b"salty-salty-salt", &mut actual) + .expect("rscrypto scrypt derivation must succeed"); assert_eq!( actual, oracle_scrypt(b"password", b"salty-salty-salt", log_n, r, p, output_len), @@ -1272,11 +1261,13 @@ mod tests { fn sse2_backend_matches_portable() { let cases: &[(u8, u32, u32, usize)] = &[(4, 1, 1, 32), (5, 2, 1, 48), (6, 2, 2, 32), (7, 8, 1, 64)]; for &(log_n, r, p, output_len) in cases { - let params = ScryptParams::new(log_n, r, p).unwrap(); + let params = ScryptParams::new(log_n, r, p).expect("backend differential scrypt profile must be valid"); let mut portable = vec![0u8; output_len]; let mut sse2 = vec![0u8; output_len]; - scrypt_hash_portable(¶ms, b"password", b"salty-salty-salt", &mut portable).unwrap(); - scrypt_hash_x86_sse2(¶ms, b"password", b"salty-salty-salt", &mut sse2).unwrap(); + scrypt_hash_portable(¶ms, b"password", b"salty-salty-salt", &mut portable) + .expect("portable scrypt derivation must succeed"); + scrypt_hash_x86_sse2(¶ms, b"password", b"salty-salty-salt", &mut sse2) + .expect("SSE2 scrypt derivation must succeed"); assert_eq!(sse2, portable); } } @@ -1285,16 +1276,29 @@ mod tests { fn raw_verify_accepts_only_the_exact_inputs() { let params = small_params(); let mut expected = [0u8; 32]; - Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut expected).unwrap(); + Scrypt::derive(¶ms, b"password", b"random-salt-1234", &mut expected) + .expect("scrypt verification fixture derivation must succeed"); - assert!(Scrypt::verify(¶ms, b"password", b"random-salt-1234", &expected).is_ok()); - assert!(Scrypt::verify(¶ms, b"wrong", b"random-salt-1234", &expected).is_err()); - assert!(Scrypt::verify(¶ms, b"password", b"other-salt-00000", &expected).is_err()); + assert_eq!( + Scrypt::verify(¶ms, b"password", b"random-salt-1234", &expected), + Ok(()) + ); + assert_eq!( + Scrypt::verify(¶ms, b"wrong", b"random-salt-1234", &expected), + Err(VerificationError::new()) + ); + assert_eq!( + Scrypt::verify(¶ms, b"password", b"other-salt-00000", &expected), + Err(VerificationError::new()) + ); for position in 0..expected.len() { let mut tampered = expected; tampered[position] ^= 1; - assert!(Scrypt::verify(¶ms, b"password", b"random-salt-1234", &tampered).is_err()); + assert_eq!( + Scrypt::verify(¶ms, b"password", b"random-salt-1234", &tampered), + Err(VerificationError::new()) + ); } } @@ -1305,7 +1309,7 @@ mod tests { assert_eq!(ScryptParams::new(4, 0, 1), Err(ScryptError::InvalidR)); assert_eq!(ScryptParams::new(4, 1, 0), Err(ScryptError::InvalidP)); assert_eq!(ScryptParams::new(4, 1 << 15, 1 << 15), Err(ScryptError::InvalidP)); - assert!(ScryptParams::new(4, 1, 1).is_ok()); + assert_eq!(ScryptParams::new(4, 1, 1), Ok(ScryptParams { log_n: 4, r: 1, p: 1 })); } #[test] @@ -1320,7 +1324,7 @@ mod tests { #[cfg(target_pointer_width = "64")] #[test] fn derive_rejects_impossible_memory_shape_before_allocation() { - let params = ScryptParams::new(63, 1 << 20, 1).unwrap(); + let params = ScryptParams::new(63, 1 << 20, 1).expect("overflow regression profile is structurally valid"); let mut output = [0u8; 32]; assert_eq!( Scrypt::derive(¶ms, b"password", b"salt", &mut output), @@ -1381,14 +1385,14 @@ mod tests { fn encode(params: ScryptParams, password: &[u8], salt: &[u8]) -> alloc::string::String { let mut verifier = [0u8; PASSWORD_OUTPUT_LEN]; - Scrypt::derive(¶ms, password, salt, &mut verifier).unwrap(); + Scrypt::derive(¶ms, password, salt, &mut verifier).expect("scrypt PHC fixture derivation must succeed"); password_phc::encode(params, salt, &verifier) } #[test] fn canonical_password_record_round_trips() { let params = small_params(); - let password = ScryptPassword::new(params).unwrap(); + let password = ScryptPassword::new(params).expect("small scrypt password profile must be valid"); let encoded = encode(params, b"password", &[0xaa; 16]); assert!(encoded.starts_with("$scrypt$ln=4,r=1,p=1$")); @@ -1396,13 +1400,16 @@ mod tests { password.verify_password(b"password", &encoded), Ok(PasswordStatus::Current) ); - assert!(password.verify_password(b"wrong", &encoded).is_err()); + assert_eq!( + password.verify_password(b"wrong", &encoded), + Err(VerificationError::new()) + ); } #[test] fn accepted_older_profile_requests_rehash() { - let generation = ScryptParams::new(5, 1, 1).unwrap(); - let password = ScryptPassword::new(generation).unwrap(); + let generation = ScryptParams::new(5, 1, 1).expect("rehash target profile must be valid"); + let password = ScryptPassword::new(generation).expect("rehash target resource shape must be valid"); let encoded = encode(small_params(), b"password", &[0xbb; 16]); assert_eq!( @@ -1414,7 +1421,7 @@ mod tests { #[test] fn accepted_noncurrent_salt_length_requests_rehash() { let params = small_params(); - let password = ScryptPassword::new(params).unwrap(); + let password = ScryptPassword::new(params).expect("small scrypt password profile must be valid"); let encoded = encode(params, b"password", &[0xbb; 8]); assert_eq!( @@ -1443,8 +1450,8 @@ mod tests { fn limits_bound_both_memory_and_work() { let limits = ScryptVerificationLimits::for_profile(small_params()); assert!(limits.allows(small_params())); - assert!(!limits.allows(ScryptParams::new(5, 1, 1).unwrap())); - assert!(!limits.allows(ScryptParams::new(4, 1, 2).unwrap())); + assert!(!limits.allows(ScryptParams::new(5, 1, 1).expect("memory-limit profile must be valid"))); + assert!(!limits.allows(ScryptParams::new(4, 1, 2).expect("work-limit profile must be valid"))); } #[test] @@ -1453,24 +1460,41 @@ mod tests { let salt = "AAAAAAAAAAAAAAAAAAAAAA"; let hash = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; let cases = [ - format!("$argon2id$v=19$m=32,t=2,p=1$${salt}$${hash}"), - format!("$scrypt$v=1$ln=4,r=1,p=1$${salt}$${hash}"), - format!("$scrypt$r=1,ln=4,p=1$${salt}$${hash}"), - format!("$scrypt$ln=4,ln=4,p=1$${salt}$${hash}"), - format!("$scrypt$ln=4,r=1,x=1$${salt}$${hash}"), - format!("$scrypt$ln=04,r=1,p=1$${salt}$${hash}"), + ( + format!("$argon2id$v=19$m=32,t=2,p=1${salt}${hash}"), + PhcError::AlgorithmMismatch, + ), + ( + format!("$scrypt$v=1$ln=4,r=1,p=1${salt}${hash}"), + PhcError::UnsupportedVersion, + ), + (format!("$scrypt$r=1,ln=4,p=1${salt}${hash}"), PhcError::DuplicateParam), + (format!("$scrypt$ln=4,ln=4,p=1${salt}${hash}"), PhcError::DuplicateParam), + (format!("$scrypt$ln=4,r=1,x=1${salt}${hash}"), PhcError::UnknownParam), + ( + format!("$scrypt$ln=04,r=1,p=1${salt}${hash}"), + PhcError::MalformedParams, + ), ]; - for encoded in cases { - assert!(password_phc::approve(&encoded, limits).is_err(), "{encoded}"); + for (encoded, expected) in cases { + assert_eq!( + password_phc::approve(&encoded, limits).err(), + Some(expected), + "{encoded}" + ); } } #[cfg(feature = "getrandom")] #[test] fn generated_records_use_fresh_salts() { - let password = ScryptPassword::new(small_params()).unwrap(); - let first = password.hash_password(b"password").unwrap(); - let second = password.hash_password(b"password").unwrap(); + let password = ScryptPassword::new(small_params()).expect("small scrypt password profile must be valid"); + let first = password + .hash_password(b"password") + .expect("first scrypt password record must be generated"); + let second = password + .hash_password(b"password") + .expect("second scrypt password record must be generated"); assert_ne!(first, second); assert_eq!( diff --git a/src/auth/x25519.rs b/src/auth/x25519.rs index e0b9c4a9..1f8a35ff 100644 --- a/src/auth/x25519.rs +++ b/src/auth/x25519.rs @@ -281,7 +281,6 @@ impl X25519SecretKey { X25519SharedSecret::diffie_hellman(self, public) } - #[allow(dead_code)] #[cfg(any( not(any( all( @@ -352,7 +351,6 @@ impl Drop for X25519SecretKey { #[derive(Clone, Copy)] pub struct X25519PublicKey { bytes: [u8; Self::LENGTH], - #[allow(dead_code)] #[cfg(any( not(any( all( @@ -362,8 +360,7 @@ pub struct X25519PublicKey { ), all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) )), - miri, - test + miri ))] u: FieldElement, } @@ -389,8 +386,7 @@ impl X25519PublicKey { ), all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) )), - miri, - test + miri ))] u: decode_u_coordinate(&bytes), } @@ -405,7 +401,6 @@ impl X25519PublicKey { #[inline] #[must_use] - #[allow(dead_code)] #[cfg(any( not(any( all( @@ -430,8 +425,7 @@ impl X25519PublicKey { ), all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) )), - miri, - test + miri ))] u, } @@ -596,7 +590,6 @@ impl Drop for X25519SharedSecret { } } -#[allow(clippy::indexing_slicing)] #[must_use] #[cfg(any( not(any( @@ -715,7 +708,7 @@ fn decode_u_coordinate(bytes: &[u8; POINT_LENGTH]) -> FieldElement { acc_bits = acc_bits.wrapping_add(8); } - *limb = (acc & u128::from(MASK51)) as u64; + *limb = u64::try_from(acc & u128::from(MASK51)).expect("masked field limb fits u64"); acc >>= RADIX_BITS; acc_bits = acc_bits.wrapping_sub(RADIX_BITS); } @@ -760,18 +753,18 @@ mod tests { fn scalar(seed: u8) -> [u8; POINT_LENGTH] { let mut out = [0u8; POINT_LENGTH]; - for (index, byte) in out.iter_mut().enumerate() { - *byte = seed.wrapping_mul(37).wrapping_add((index as u8).wrapping_mul(19)); + for (index, byte) in (0u8..).zip(&mut out) { + *byte = seed.wrapping_mul(37).wrapping_add(index.wrapping_mul(19)); } out } fn peer(seed: u8) -> [u8; POINT_LENGTH] { let mut out = [0u8; POINT_LENGTH]; - for (index, byte) in out.iter_mut().enumerate() { + for (index, byte) in (0u8..).zip(&mut out) { *byte = seed .wrapping_mul(53) - .wrapping_add((index as u8).wrapping_mul(11)) + .wrapping_add(index.wrapping_mul(11)) .wrapping_add(7); } out[POINT_LENGTH - 1] |= 0x80; diff --git a/src/auth/x25519/aarch64_asm.rs b/src/auth/x25519/aarch64_asm.rs index 3d6ed5d6..fbef2588 100644 --- a/src/auth/x25519/aarch64_asm.rs +++ b/src/auth/x25519/aarch64_asm.rs @@ -4,8 +4,6 @@ //! byte backends. This module owns the ABI boundary and keeps the public //! rscrypto API semantics in `x25519.rs`. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::POINT_LENGTH; diff --git a/src/auth/x25519/x86_64_asm.rs b/src/auth/x25519/x86_64_asm.rs index 0df729e3..709a13ec 100644 --- a/src/auth/x25519/x86_64_asm.rs +++ b/src/auth/x25519/x86_64_asm.rs @@ -4,8 +4,6 @@ //! byte backends. This module owns the ABI boundary and keeps the public //! rscrypto API semantics in `x25519.rs`. -#![allow(unsafe_code)] - use core::arch::global_asm; use super::POINT_LENGTH; diff --git a/src/backend/ascon.rs b/src/backend/ascon.rs index cb1b3b26..9a7327e8 100644 --- a/src/backend/ascon.rs +++ b/src/backend/ascon.rs @@ -67,8 +67,7 @@ pub(crate) fn permute_12_portable(s: &mut [u64; 5]) { /// /// Fully unrolled for the same register-allocation benefits as /// [`permute_12_portable`]. -#[cfg(any(feature = "ascon-aead", test))] -#[cfg_attr(test, allow(dead_code))] +#[cfg(feature = "ascon-aead")] #[inline(always)] pub(crate) fn permute_8_portable(s: &mut [u64; 5]) { let mut x0 = s[0]; diff --git a/src/backend/cache.rs b/src/backend/cache.rs index 59027bcd..6034b496 100644 --- a/src/backend/cache.rs +++ b/src/backend/cache.rs @@ -106,7 +106,7 @@ impl AtomicOnceCache { core::hint::spin_loop(); } } - _ => unreachable!("atomic cache state is private and has three values"), + _ => core::hint::spin_loop(), } } } @@ -116,7 +116,7 @@ impl AtomicOnceCache { /// /// Building block for dispatcher caching with proper synchronization. /// See module documentation for platform-specific behavior. -pub struct OnceCache { +pub(crate) struct OnceCache { #[cfg(feature = "std")] inner: std::sync::OnceLock, @@ -130,7 +130,7 @@ pub struct OnceCache { impl OnceCache { /// Create a new empty cache. #[must_use] - pub const fn new() -> Self { + pub(crate) const fn new() -> Self { Self { #[cfg(feature = "std")] inner: std::sync::OnceLock::new(), @@ -151,7 +151,7 @@ impl OnceCache { /// /// Returns the cached value by copy (since T is Copy). #[inline] - pub fn get_or_init(&self, f: impl FnOnce() -> T) -> T { + pub(crate) fn get_or_init(&self, f: impl FnOnce() -> T) -> T { #[cfg(feature = "std")] { *self.inner.get_or_init(f) @@ -213,13 +213,13 @@ mod tests { } #[cfg(feature = "std")] - #[allow(clippy::std_instead_of_core, clippy::std_instead_of_alloc)] mod threading_tests { - use std::{ + use alloc::{boxed::Box, vec::Vec}; + use core::{ + panic::AssertUnwindSafe, sync::atomic::{AtomicUsize, Ordering}, - thread, - vec::Vec, }; + use std::thread; use super::*; @@ -243,7 +243,7 @@ mod tests { .collect(); for handle in handles { - handle.join().unwrap(); + handle.join().expect("cache worker thread must not panic"); } assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1); @@ -253,20 +253,19 @@ mod tests { fn test_atomic_once_cache_recovers_after_initializer_panic() { let cache = AtomicOnceCache::::new(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.get_or_init(|| panic!("controlled initializer panic")); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + cache.get_or_init(|| std::panic::resume_unwind(Box::new("controlled initializer panic"))); })); - assert!(result.is_err()); + result.expect_err("panicking initializer must unwind"); assert_eq!(cache.get_or_init(|| 23), 23); } #[test] fn test_atomic_once_cache_waiter_retries_after_initializer_panic() { - use std::{ - sync::{Arc, Barrier, mpsc}, - time::Duration, - }; + use alloc::sync::Arc; + use core::time::Duration; + use std::sync::{Barrier, mpsc}; let cache = Arc::new(AtomicOnceCache::::new()); let initializer_entered = Arc::new(Barrier::new(2)); @@ -276,11 +275,11 @@ mod tests { let panicking_entered = Arc::clone(&initializer_entered); let panicking_release = Arc::clone(&release_initializer); let panicking = thread::spawn(move || { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + std::panic::catch_unwind(AssertUnwindSafe(|| { panicking_cache.get_or_init(|| { panicking_entered.wait(); panicking_release.wait(); - panic!("controlled initializer panic"); + std::panic::resume_unwind(Box::new("controlled initializer panic")) }); })) }); @@ -289,13 +288,23 @@ mod tests { let waiting_cache = Arc::clone(&cache); let (sender, receiver) = mpsc::channel(); let waiting = thread::spawn(move || { - sender.send(waiting_cache.get_or_init(|| 29)).unwrap(); + sender + .send(waiting_cache.get_or_init(|| 29)) + .expect("cache waiter result receiver must remain connected"); }); release_initializer.wait(); - assert!(panicking.join().unwrap().is_err()); - assert_eq!(receiver.recv_timeout(Duration::from_secs(2)).unwrap(), 29); - waiting.join().unwrap(); + panicking + .join() + .expect("panicking cache thread must unwind only inside catch_unwind") + .expect_err("cache initializer must panic"); + assert_eq!( + receiver + .recv_timeout(Duration::from_secs(2)) + .expect("cache waiter must publish its result before the timeout"), + 29 + ); + waiting.join().expect("cache waiter thread must not panic"); } #[test] @@ -318,7 +327,7 @@ mod tests { .collect(); for handle in handles { - handle.join().unwrap(); + handle.join().expect("cache worker thread must not panic"); } // Selector called exactly once diff --git a/src/backend/curve25519.rs b/src/backend/curve25519.rs index bf15b085..c30a6e38 100644 --- a/src/backend/curve25519.rs +++ b/src/backend/curve25519.rs @@ -32,8 +32,7 @@ const MODULUS_N: u64 = MASK51; /// chains. const SUB_BIAS_0: u64 = ((1u64 << RADIX_BITS) - 19).wrapping_mul(16); const SUB_BIAS_N: u64 = MASK51.wrapping_mul(16); -#[allow(dead_code)] -#[cfg(any(feature = "ed25519", feature = "x25519"))] +#[cfg(feature = "ed25519")] const SQRT_M1: FieldElement = FieldElement::from_limbs([ 1_718_705_420_411_056, 234_908_883_556_509, @@ -63,7 +62,6 @@ impl FieldElement { /// Construct a small field element without reduction. #[inline] #[must_use] - #[allow(dead_code)] pub(crate) const fn from_small(value: u64) -> Self { Self([value, 0, 0, 0, 0]) } @@ -71,7 +69,6 @@ impl FieldElement { /// Borrow the raw radix-51 limbs. #[inline] #[must_use] - #[allow(dead_code)] #[cfg(any(feature = "ed25519", feature = "x25519"))] pub(crate) const fn limbs(&self) -> &[u64; FIELD_LIMBS] { &self.0 @@ -180,7 +177,6 @@ impl FieldElement { /// Negate the field element modulo `2^255 - 19`. #[inline] #[must_use] - #[allow(dead_code)] #[cfg(any(feature = "ed25519", feature = "x25519"))] pub(crate) fn neg(&self) -> Self { Self::ZERO.sub(self) @@ -217,8 +213,7 @@ impl FieldElement { /// Decode a canonical 32-byte field element. #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn from_bytes(bytes: &[u8; 32]) -> Option { let mut acc = 0u128; let mut acc_bits = 0u32; @@ -235,7 +230,7 @@ impl FieldElement { } } - *limb = (acc & u128::from(MASK51)) as u64; + *limb = u64::try_from(acc & u128::from(MASK51)).expect("masked field limb fits u64"); acc >>= RADIX_BITS; acc_bits = acc_bits.wrapping_sub(RADIX_BITS); } @@ -267,7 +262,7 @@ impl FieldElement { while acc_bits >= 8 { if let Some(byte) = out_iter.next() { - *byte = acc as u8; + *byte = u8::try_from(acc & 0xff).expect("masked field byte fits u8"); } acc >>= 8; acc_bits = acc_bits.wrapping_sub(8); @@ -275,7 +270,7 @@ impl FieldElement { } if let Some(byte) = out_iter.next() { - *byte = acc as u8; + *byte = u8::try_from(acc & 0xff).expect("masked field byte fits u8"); } out @@ -283,7 +278,6 @@ impl FieldElement { /// Return `true` when the canonical field element is zero. #[must_use] - #[allow(dead_code)] #[cfg(any(feature = "ed25519", feature = "x25519"))] pub(crate) fn is_zero(&self) -> bool { self.normalize().0.iter().all(|&limb| limb == 0) @@ -291,8 +285,22 @@ impl FieldElement { /// Branchless conditional swap at the Rust source level. #[inline] - #[cfg(feature = "x25519")] - #[allow(dead_code)] + #[cfg(all( + feature = "x25519", + any( + feature = "diag", + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ))] pub(crate) fn conditional_swap(lhs: &mut Self, rhs: &mut Self, swap: u8) { let mask = 0u64.wrapping_sub(u64::from(swap & 1)); for (lhs_limb, rhs_limb) in lhs.0.iter_mut().zip(rhs.0.iter_mut()) { @@ -304,16 +312,14 @@ impl FieldElement { /// Return the low-bit sign of the canonical encoding. #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn is_negative(&self) -> bool { ((*self).to_bytes()[0] & 1) == 1 } /// Square root in `GF(2^255 - 19)` when one exists. #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn sqrt(&self) -> Option { let normalized = self.normalize(); self @@ -360,16 +366,14 @@ impl FieldElement { } #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] fn pow_p58(&self) -> Self { let (t19, _) = self.pow22501(); self.mul(&t19.pow2k(2)) } #[must_use] - #[allow(dead_code)] - #[cfg(any(feature = "ed25519", feature = "x25519"))] + #[cfg(feature = "ed25519")] pub(crate) fn sqrt_ratio_i(&self, denominator: &Self) -> Option { let numerator = self.normalize(); let denominator = denominator.normalize(); @@ -454,7 +458,7 @@ fn reduce_wide(wide: [u128; FIELD_LIMBS]) -> [u64; FIELD_LIMBS] { h0 = h0.wrapping_add((h4 >> RADIX_BITS).wrapping_mul(19)); h4 &= mask; - [h0 as u64, h1 as u64, h2 as u64, h3 as u64, h4 as u64] + [h0, h1, h2, h3, h4].map(|limb| u64::try_from(limb).expect("reduced field limb fits u64")) } /// Single-round carry propagation on u64 limbs (sub output, normalize). @@ -518,7 +522,7 @@ mod tests { fn from_u128(mut value: u128) -> FieldElement { let mut limbs = [0u64; 5]; for limb in &mut limbs { - *limb = (value & u128::from(MASK51)) as u64; + *limb = u64::try_from(value & u128::from(MASK51)).expect("masked test limb fits u64"); value >>= 51; } FieldElement::from_limbs(limbs) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 4f03f7db..d64af482 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -5,7 +5,7 @@ //! so algorithms can reuse implementation building blocks without creating //! public-surface or feature-graph coupling. #[cfg(any(feature = "ascon-hash", feature = "ascon-aead"))] -pub mod ascon; +pub(crate) mod ascon; #[cfg(any( feature = "crc16", feature = "crc24", @@ -19,13 +19,30 @@ pub mod ascon; not(all(target_arch = "aarch64", target_os = "macos")) ), feature = "blake3", - feature = "ascon-hash", + all(feature = "ascon-hash", feature = "diag"), feature = "xxh3", feature = "aes-gcm", feature = "aes-gcm-siv", feature = "chacha20poly1305", feature = "xchacha20poly1305" ))] -pub mod cache; -#[cfg(any(feature = "ed25519", feature = "x25519"))] -pub mod curve25519; +pub(crate) mod cache; +#[cfg(any( + feature = "ed25519", + all( + feature = "x25519", + any( + test, + miri, + not(any( + all( + target_arch = "aarch64", + any(target_os = "macos", target_os = "linux"), + not(feature = "portable-only") + ), + all(target_arch = "x86_64", target_os = "linux", not(feature = "portable-only")) + )) + ) + ) +))] +pub(crate) mod curve25519; diff --git a/src/checksum/common/clmul.rs b/src/checksum/common/clmul.rs index 9c6cea00..96e582f2 100644 --- a/src/checksum/common/clmul.rs +++ b/src/checksum/common/clmul.rs @@ -42,7 +42,7 @@ const fn clmul64(a: u64, b: u64) -> (u64, u64) { lo ^= b; } else { lo ^= b << i; - hi ^= b >> (64 - i); + hi ^= b >> 64u32.strict_sub(i); } } i = i.strict_add(1); @@ -79,7 +79,7 @@ pub(crate) const fn reduce128(hi: u64, lo: u64, poly: u64) -> u64 { } else { // XOR poly shifted by i result_lo ^= poly << i; - result_hi ^= (poly >> (64 - i)) | (1 << i); + result_hi ^= (poly >> 64i32.strict_sub(i)) | (1 << i); } } i = i.strict_sub(1); @@ -176,14 +176,14 @@ pub(crate) const fn fold16_coeff_for_bytes(reflected_poly: u64, shift_bytes: u32 } let normal = normal_poly(reflected_poly); - let d = shift_bytes * 8; + let d = shift_bytes.strict_mul(8); // `d >= 8`, so `d - 1` is valid. - (fold_k(normal, d - 1), fold_k(normal, d + 63)) + (fold_k(normal, d.strict_sub(1)), fold_k(normal, d.strict_add(63))) } impl Crc64ClmulConstants { #[must_use] - pub const fn new(reflected_poly: u64) -> Self { + const fn new(reflected_poly: u64) -> Self { let poly = reciprocal_poly(reflected_poly); let normal = normal_poly(reflected_poly); let mu = compute_tikv_mu(poly); @@ -227,7 +227,7 @@ const fn compute_tikv_mu(poly: u64) -> u64 { let mut i: u32 = 1; while i <= k { let p_i = (poly >> i) & 1; - let q_j = (inv >> (k - i)) & 1; + let q_j = (inv >> k.strict_sub(i)) & 1; s ^= p_i & q_j; i = i.strict_add(1); } @@ -262,23 +262,29 @@ pub(crate) const CRC64_NVME_CLMUL: Crc64ClmulConstants = Crc64ClmulConstants::ne /// - `combine_8way`: merge coefficients for 8-way (x86_64) #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64 vs aarch64 stream widths). pub(crate) struct Crc64StreamConstants { /// 2-way fold coefficient (256B = 2×128B). pub fold_256b: (u64, u64), /// 3-way fold coefficient (384B = 3×128B). + #[cfg(target_arch = "aarch64")] pub fold_384b: (u64, u64), /// 4-way fold coefficient (512B = 4×128B). + #[cfg(target_arch = "x86_64")] pub fold_512b: (u64, u64), /// 7-way fold coefficient (896B = 7×128B). + #[cfg(target_arch = "x86_64")] pub fold_896b: (u64, u64), /// 8-way fold coefficient (1024B = 8×128B). + #[cfg(target_arch = "x86_64")] pub fold_1024b: (u64, u64), /// 4-way combine coefficients: shifts by 384B, 256B, 128B. + #[cfg(target_arch = "x86_64")] pub combine_4way: [(u64, u64); 3], /// 7-way combine coefficients: shifts by 768B, 640B, 512B, 384B, 256B, 128B. + #[cfg(target_arch = "x86_64")] pub combine_7way: [(u64, u64); 6], /// 8-way combine coefficients: shifts by 896B, 768B, 640B, 512B, 384B, 256B, 128B. + #[cfg(target_arch = "x86_64")] pub combine_8way: [(u64, u64); 7], } @@ -286,18 +292,24 @@ pub(crate) struct Crc64StreamConstants { impl Crc64StreamConstants { /// Compute all multi-stream folding constants for a given polynomial. #[must_use] - pub const fn new(reflected_poly: u64) -> Self { + const fn new(reflected_poly: u64) -> Self { Self { fold_256b: fold16_coeff_for_bytes(reflected_poly, 256), + #[cfg(target_arch = "aarch64")] fold_384b: fold16_coeff_for_bytes(reflected_poly, 384), + #[cfg(target_arch = "x86_64")] fold_512b: fold16_coeff_for_bytes(reflected_poly, 512), + #[cfg(target_arch = "x86_64")] fold_896b: fold16_coeff_for_bytes(reflected_poly, 896), + #[cfg(target_arch = "x86_64")] fold_1024b: fold16_coeff_for_bytes(reflected_poly, 1024), + #[cfg(target_arch = "x86_64")] combine_4way: [ fold16_coeff_for_bytes(reflected_poly, 384), fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_7way: [ fold16_coeff_for_bytes(reflected_poly, 768), fold16_coeff_for_bytes(reflected_poly, 640), @@ -306,6 +318,7 @@ impl Crc64StreamConstants { fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_8way: [ fold16_coeff_for_bytes(reflected_poly, 896), fold16_coeff_for_bytes(reflected_poly, 768), diff --git a/src/checksum/common/combine.rs b/src/checksum/common/combine.rs index d15ee498..269102c3 100644 --- a/src/checksum/common/combine.rs +++ b/src/checksum/common/combine.rs @@ -17,7 +17,6 @@ // SAFETY: All array indexing in this module uses bounded loop indices (0..N where N is the // array size). Clippy cannot prove this in const fn contexts, but the bounds are statically // guaranteed by the loop conditions. -#![allow(clippy::indexing_slicing)] // Unified GF(2) matrix macro @@ -55,12 +54,12 @@ macro_rules! define_gf2_combine { ) => { #[doc = $doc_matrix] #[derive(Clone, Copy)] - pub struct $Name([$T; $DIM]); + pub(in crate::checksum) struct $Name([$T; $DIM]); impl $Name { /// Create the identity matrix. #[must_use] - pub const fn identity() -> Self { + pub(in crate::checksum) const fn identity() -> Self { let mut m = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -73,7 +72,7 @@ macro_rules! define_gf2_combine { /// Multiply matrix by a vector. #[inline] #[must_use] - pub const fn mul_vec(self, vec: $T) -> $T { + pub(in crate::checksum) const fn mul_vec(self, vec: $T) -> $T { let mut result = 0 as $T; let mut i: u32 = 0; while i < $DIM as u32 { @@ -87,7 +86,7 @@ macro_rules! define_gf2_combine { /// Multiply two matrices. #[must_use] - pub const fn mul_mat(self, other: Self) -> Self { + pub(in crate::checksum) const fn mul_mat(self, other: Self) -> Self { let mut result = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -100,14 +99,14 @@ macro_rules! define_gf2_combine { /// Square the matrix. #[inline] #[must_use] - pub const fn square(self) -> Self { + pub(in crate::checksum) const fn square(self) -> Self { self.mul_mat(self) } } #[doc = $doc_shift1] #[must_use] - pub const fn $shift1_fn(poly: $T) -> $Name { + const fn $shift1_fn(poly: $T) -> $Name { let mut m = [0 as $T; $DIM]; m[0] = poly; let mut j: u32 = 1; @@ -120,7 +119,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_shift8] #[must_use] - pub const fn $shift8_fn(poly: $T) -> $Name { + pub(in crate::checksum) const fn $shift8_fn(poly: $T) -> $Name { let shift1 = $shift1_fn(poly); let shift2 = shift1.square(); let shift4 = shift2.square(); @@ -129,7 +128,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_combine] #[must_use] - pub const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name) -> $T { + pub(in crate::checksum) const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name) -> $T { if len_b == 0 { return crc_a; } @@ -170,12 +169,12 @@ macro_rules! define_gf2_combine { ) => { #[doc = $doc_matrix] #[derive(Clone, Copy)] - pub struct $Name([$T; $DIM]); + pub(in crate::checksum) struct $Name([$T; $DIM]); impl $Name { /// Create the identity matrix. #[must_use] - pub const fn identity() -> Self { + const fn identity() -> Self { let mut m = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -188,7 +187,7 @@ macro_rules! define_gf2_combine { /// Multiply matrix by a vector. #[inline] #[must_use] - pub const fn mul_vec(self, vec: $T) -> $T { + const fn mul_vec(self, vec: $T) -> $T { let mut result = 0 as $T; let mut i: u32 = 0; while i < $DIM as u32 { @@ -202,7 +201,7 @@ macro_rules! define_gf2_combine { /// Multiply two matrices. #[must_use] - pub const fn mul_mat(self, other: Self) -> Self { + const fn mul_mat(self, other: Self) -> Self { let mut result = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -215,14 +214,14 @@ macro_rules! define_gf2_combine { /// Square the matrix. #[inline] #[must_use] - pub const fn square(self) -> Self { + const fn square(self) -> Self { self.mul_mat(self) } } #[doc = $doc_shift1] #[must_use] - pub const fn $shift1_fn(poly: $T) -> $Name { + const fn $shift1_fn(poly: $T) -> $Name { let mut m = [0 as $T; $DIM]; m[0] = poly; let mut j: u32 = 1; @@ -235,7 +234,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_shift8] #[must_use] - pub const fn $shift8_fn(poly: $T) -> $Name { + pub(in crate::checksum) const fn $shift8_fn(poly: $T) -> $Name { let shift1 = $shift1_fn(poly); let shift2 = shift1.square(); let shift4 = shift2.square(); @@ -246,7 +245,13 @@ macro_rules! define_gf2_combine { /// This works for any init/xorout as long as the caller supplies /// `init_xorout = init ^ xorout` for the CRC variant being combined. #[must_use] - pub const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name, init_xorout: $T) -> $T { + pub(in crate::checksum) const fn $combine_fn( + crc_a: $T, + crc_b: $T, + len_b: usize, + shift8_matrix: $Name, + init_xorout: $T, + ) -> $T { if len_b == 0 { return crc_a; } @@ -288,15 +293,15 @@ macro_rules! define_gf2_combine { ) => { #[doc = $doc_matrix] #[derive(Clone, Copy)] - pub struct $Name([$T; $DIM]); + pub(in crate::checksum) struct $Name([$T; $DIM]); impl $Name { /// Bit mask that keeps only the low `DIM` bits of the backing type. - pub const MASK: $T = $MASK; + const MASK: $T = $MASK; /// Create the identity matrix. #[must_use] - pub const fn identity() -> Self { + const fn identity() -> Self { let mut m = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -309,7 +314,7 @@ macro_rules! define_gf2_combine { /// Multiply matrix by a vector (low bits only). #[inline] #[must_use] - pub const fn mul_vec(self, vec: $T) -> $T { + const fn mul_vec(self, vec: $T) -> $T { let vec = vec & Self::MASK; let mut result = 0 as $T; let mut i: u32 = 0; @@ -324,7 +329,7 @@ macro_rules! define_gf2_combine { /// Multiply two matrices. #[must_use] - pub const fn mul_mat(self, other: Self) -> Self { + const fn mul_mat(self, other: Self) -> Self { let mut result = [0 as $T; $DIM]; let mut i: u32 = 0; while i < $DIM as u32 { @@ -337,14 +342,14 @@ macro_rules! define_gf2_combine { /// Square the matrix. #[inline] #[must_use] - pub const fn square(self) -> Self { + const fn square(self) -> Self { self.mul_mat(self) } } #[doc = $doc_shift1] #[must_use] - pub const fn $shift1_fn(poly: $T) -> $Name { + const fn $shift1_fn(poly: $T) -> $Name { let poly = poly & $Name::MASK; let mut m = [0 as $T; $DIM]; @@ -361,7 +366,7 @@ macro_rules! define_gf2_combine { #[doc = $doc_shift8] #[must_use] - pub const fn $shift8_fn(poly: $T) -> $Name { + pub(in crate::checksum) const fn $shift8_fn(poly: $T) -> $Name { let shift1 = $shift1_fn(poly); let shift2 = shift1.square(); let shift4 = shift2.square(); @@ -372,7 +377,13 @@ macro_rules! define_gf2_combine { /// This works for any init/xorout as long as the caller supplies /// `init_xorout = init ^ xorout` for the CRC variant being combined. #[must_use] - pub const fn $combine_fn(crc_a: $T, crc_b: $T, len_b: usize, shift8_matrix: $Name, init_xorout: $T) -> $T { + pub(in crate::checksum) const fn $combine_fn( + crc_a: $T, + crc_b: $T, + len_b: usize, + shift8_matrix: $Name, + init_xorout: $T, + ) -> $T { if len_b == 0 { return crc_a & $Name::MASK; } @@ -465,7 +476,7 @@ define_gf2_combine! { #[inline] #[must_use] #[cfg(all(feature = "crc32", target_arch = "aarch64"))] -pub const fn pow_shift8_matrix_32(len_bytes: usize, shift8_matrix: Gf2Matrix32) -> Gf2Matrix32 { +pub(in crate::checksum) const fn pow_shift8_matrix_32(len_bytes: usize, shift8_matrix: Gf2Matrix32) -> Gf2Matrix32 { if len_bytes == 0 { return Gf2Matrix32::identity(); } diff --git a/src/checksum/common/kernels.rs b/src/checksum/common/kernels.rs index 6c89e1fc..a6ee6a50 100644 --- a/src/checksum/common/kernels.rs +++ b/src/checksum/common/kernels.rs @@ -1,12 +1,12 @@ //! Shared CRC kernel names. /// Reference (bitwise) kernel name - canonical implementation for verification. -pub const REFERENCE: &str = "reference/bitwise"; +pub(in crate::checksum) const REFERENCE: &str = "reference/bitwise"; /// Portable fallback kernel name (used by all CRC widths). #[cfg(any(feature = "crc32", feature = "crc64"))] -pub const PORTABLE_SLICE16: &str = "portable/slice16"; +pub(in crate::checksum) const PORTABLE_SLICE16: &str = "portable/slice16"; /// Portable slice-by-8 kernel name. #[cfg(any(feature = "crc16", feature = "crc24"))] -pub const PORTABLE_SLICE8: &str = "portable/slice8"; +pub(in crate::checksum) const PORTABLE_SLICE8: &str = "portable/slice8"; diff --git a/src/checksum/common/mod.rs b/src/checksum/common/mod.rs index 15032ae4..54178a67 100644 --- a/src/checksum/common/mod.rs +++ b/src/checksum/common/mod.rs @@ -17,16 +17,46 @@ target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", - target_arch = "s390x", - target_arch = "riscv64" + target_arch = "s390x" ) ))] -pub mod clmul; -pub mod combine; -pub mod kernels; -pub mod portable; -pub mod prefetch; -pub mod reference; -pub mod tables; +pub(in crate::checksum) mod clmul; +pub(in crate::checksum) mod combine; +pub(in crate::checksum) mod kernels; +pub(in crate::checksum) mod portable; +#[cfg(any( + all( + target_arch = "x86_64", + any(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64") + ), + all( + target_arch = "aarch64", + any(feature = "crc16", feature = "crc24", feature = "crc64") + ), + test +))] +pub(in crate::checksum) mod prefetch; +pub(in crate::checksum) mod reference; +pub(in crate::checksum) mod tables; #[cfg(test)] -pub mod tests; +pub(in crate::checksum) mod tests; + +#[inline] +#[cfg(all( + feature = "crc16", + any(target_arch = "powerpc64", target_arch = "riscv64", target_arch = "s390x") +))] +pub(in crate::checksum) const fn low_u16(value: u32) -> u16 { + let [b0, b1, ..] = value.to_le_bytes(); + u16::from_le_bytes([b0, b1]) +} + +#[inline] +#[cfg(all( + any(feature = "crc16", feature = "crc24", feature = "crc32"), + any(target_arch = "powerpc64", target_arch = "riscv64", target_arch = "s390x") +))] +pub(in crate::checksum) const fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, ..] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} diff --git a/src/checksum/common/portable.rs b/src/checksum/common/portable.rs index cafbcbc9..ba7dfc43 100644 --- a/src/checksum/common/portable.rs +++ b/src/checksum/common/portable.rs @@ -20,8 +20,6 @@ // - chunks_exact guarantees chunk sizes // - Table indices use `& 0xFF` (0..255) or explicit byte extraction // Clippy cannot prove this in const fn contexts, but bounds are statically guaranteed. -#![allow(clippy::indexing_slicing)] -#![cfg_attr(all(target_arch = "wasm32", target_feature = "simd128"), allow(unsafe_code))] #[cfg(any(feature = "crc16", feature = "crc32", feature = "crc64"))] macro_rules! tail_step { @@ -241,7 +239,7 @@ pub(crate) fn slice8_24(crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 /// * `tables` - 8 lookup tables (256 entries each) #[cfg(all(test, feature = "crc32"))] #[inline] -pub fn slice8_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 { +fn slice8_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 { let (chunks, remainder) = data.as_chunks::<8>(); for chunk in chunks { @@ -275,9 +273,9 @@ pub fn slice8_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 8]) -> u32 { #[inline] fn slice16_32_scalar(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { let (chunks4, remainder) = data.as_chunks::<4>(); - let mut quads = chunks4.chunks_exact(4); + let (quads, trailing_chunks) = chunks4.as_chunks::<4>(); - for quad in quads.by_ref() { + for quad in quads { let a = u32::from_le_bytes(quad[0]) ^ crc; let b = u32::from_le_bytes(quad[1]); let c = u32::from_le_bytes(quad[2]); @@ -302,7 +300,7 @@ fn slice16_32_scalar(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u3 } // Handle a 4-byte tail (one to three u32 chunks) - for chunk in quads.remainder() { + for chunk in trailing_chunks { let val = u32::from_le_bytes(*chunk) ^ crc; crc = tables[3][(val & 0xFF) as usize] ^ tables[2][((val >> 8) & 0xFF) as usize] @@ -325,7 +323,7 @@ fn slice16_32_scalar(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u3 /// * `tables` - 16 lookup tables (256 entries each) #[cfg(all(feature = "crc32", not(all(target_arch = "wasm32", target_feature = "simd128"))))] #[inline] -pub fn slice16_32(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { +pub(in crate::checksum) fn slice16_32(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { slice16_32_scalar(crc, data, tables) } @@ -335,7 +333,7 @@ pub fn slice16_32(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { /// `v128` loads when `target_feature = "simd128"` is enabled. #[cfg(all(feature = "crc32", target_arch = "wasm32", target_feature = "simd128"))] #[inline] -pub fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { +pub(in crate::checksum) fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { use core::arch::wasm32::v128; let mut ptr = data.as_ptr(); @@ -343,7 +341,7 @@ pub fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { while len >= 16 { // SAFETY: `ptr` is within `data` and `len >= 16`. - let v = unsafe { core::ptr::read_unaligned(ptr as *const v128) }; + let v = unsafe { core::ptr::read_unaligned(ptr.cast::()) }; // Extract lanes using WASM SIMD intrinsics (avoids transmute layout assumptions). let a = core::arch::wasm32::u32x4_extract_lane::<0>(v) ^ crc; let b = core::arch::wasm32::u32x4_extract_lane::<1>(v); @@ -390,7 +388,7 @@ pub fn slice16_32(mut crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { /// * `tables` - 8 lookup tables (256 entries each) #[cfg(all(feature = "crc64", any(target_arch = "x86_64", target_arch = "aarch64", test)))] #[inline] -pub fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { +pub(in crate::checksum) fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { let (chunks, remainder) = data.as_chunks::<8>(); for chunk in chunks { @@ -423,9 +421,9 @@ pub fn slice8_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { #[inline] fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { let (chunks8, remainder) = data.as_chunks::<8>(); - let mut pairs = chunks8.chunks_exact(2); + let (pairs, trailing_chunks) = chunks8.as_chunks::<2>(); - for pair in pairs.by_ref() { + for pair in pairs { let a = u64::from_le_bytes(pair[0]) ^ crc; let b = u64::from_le_bytes(pair[1]); @@ -448,7 +446,7 @@ fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u6 } // Handle an odd 8-byte tail - if let [chunk] = pairs.remainder() { + if let [chunk] = trailing_chunks { let val = u64::from_le_bytes(*chunk) ^ crc; crc = tables[7][(val & 0xFF) as usize] ^ tables[6][((val >> 8) & 0xFF) as usize] @@ -475,7 +473,7 @@ fn slice16_64_scalar(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u6 /// * `tables` - 16 lookup tables (256 entries each) #[cfg(all(feature = "crc64", not(all(target_arch = "wasm32", target_feature = "simd128"))))] #[inline] -pub fn slice16_64(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { +pub(in crate::checksum) fn slice16_64(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { slice16_64_scalar(crc, data, tables) } @@ -485,7 +483,7 @@ pub fn slice16_64(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { /// loads when `target_feature = "simd128"` is enabled. #[cfg(all(feature = "crc64", target_arch = "wasm32", target_feature = "simd128"))] #[inline] -pub fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { +pub(in crate::checksum) fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { use core::arch::wasm32::v128; let mut ptr = data.as_ptr(); @@ -493,7 +491,7 @@ pub fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { while len >= 16 { // SAFETY: `ptr` is within `data` and `len >= 16`. - let v = unsafe { core::ptr::read_unaligned(ptr as *const v128) }; + let v = unsafe { core::ptr::read_unaligned(ptr.cast::()) }; // Extract lanes using WASM SIMD intrinsics (avoids transmute layout assumptions). let a = core::arch::wasm32::u64x2_extract_lane::<0>(v) ^ crc; let b = core::arch::wasm32::u64x2_extract_lane::<1>(v); @@ -530,45 +528,12 @@ pub fn slice16_64(mut crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { #[cfg(test)] mod tests { use super::*; - - // CRC-16 Tests - - /// Generate CRC-16 tables for testing (CCITT polynomial 0x8408 reflected). #[cfg(feature = "crc16")] - const fn crc16_table_entry(poly: u16, index: u8) -> u16 { - let mut crc = index as u16; - let mut i: u32 = 0; - while i < 8 { - if crc & 1 != 0 { - crc = (crc >> 1) ^ poly; - } else { - crc >>= 1; - } - i = i.strict_add(1); - } - crc - } + use crate::checksum::common::tables::{CRC16_CCITT_POLY, generate_crc16_tables_8}; + #[cfg(feature = "crc64")] + use crate::checksum::common::tables::{CRC64_XZ_POLY, generate_crc64_tables_8, generate_crc64_tables_16}; - #[cfg(feature = "crc16")] - const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { - let mut tables = [[0u16; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc16_table_entry(poly, i as u8); - i = i.strict_add(1); - } - let mut k = 1usize; - while k < 8 { - i = 0; - while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); - i = i.strict_add(1); - } - k = k.strict_add(1); - } - tables - } + // CRC-16 Tests // CRC-32 Tests @@ -600,9 +565,6 @@ mod tests { assert_eq!(slice8_32(!0, data, &tables8), slice16_32(!0, data, &tables16)); } - #[cfg(feature = "crc16")] - const CRC16_CCITT_POLY: u16 = 0x8408; // Reflected - #[test] #[cfg(feature = "crc16")] fn test_slice8_16_empty() { @@ -634,67 +596,6 @@ mod tests { // CRC-64 Tests - /// Generate CRC-64 tables for testing. - #[cfg(feature = "crc64")] - const fn crc64_table_entry(poly: u64, index: u8) -> u64 { - let mut crc = index as u64; - let mut i: u32 = 0; - while i < 8 { - if crc & 1 != 0 { - crc = (crc >> 1) ^ poly; - } else { - crc >>= 1; - } - i = i.strict_add(1); - } - crc - } - - #[cfg(feature = "crc64")] - const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { - let mut tables = [[0u64; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); - } - let mut k = 1usize; - while k < 8 { - i = 0; - while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); - i = i.strict_add(1); - } - k = k.strict_add(1); - } - tables - } - - #[cfg(feature = "crc64")] - const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { - let mut tables = [[0u64; 256]; 16]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); - } - let mut k = 1usize; - while k < 16 { - i = 0; - while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); - i = i.strict_add(1); - } - k = k.strict_add(1); - } - tables - } - - #[cfg(feature = "crc64")] - const CRC64_XZ_POLY: u64 = 0xC96C_5795_D787_0F42; // Reflected - #[test] #[cfg(feature = "crc64")] fn test_slice8_64_empty() { diff --git a/src/checksum/common/prefetch.rs b/src/checksum/common/prefetch.rs index ea4dba90..ca781e0b 100644 --- a/src/checksum/common/prefetch.rs +++ b/src/checksum/common/prefetch.rs @@ -3,15 +3,6 @@ // SAFETY: This module provides low-level prefetch intrinsics that require unsafe. // Prefetch instructions are hints to the CPU and cannot cause memory unsafety; // invalid addresses are silently ignored. -#![allow(unsafe_code)] -#![cfg_attr( - all( - target_arch = "aarch64", - feature = "crc32", - not(any(feature = "crc16", feature = "crc24", feature = "crc64", test)) - ), - allow(dead_code, unused_imports) -)] //! This module provides architecture-specific prefetch distances and inline //! helpers for large-buffer CRC computation. //! @@ -27,18 +18,20 @@ //! ```text //! use crate::checksum::common::prefetch::{prefetch_read_l1, LARGE_BLOCK_DISTANCE}; //! -//! // In a double-unrolled loop processing 512B per iteration: -//! while ptr.add(DOUBLE_BLOCK) <= end { -//! // Prefetch 2 iterations ahead (1KB for 512B blocks) -//! prefetch_read_l1(ptr.add(LARGE_BLOCK_DISTANCE)); +//! if end.offset_from_unsigned(ptr) >= DOUBLE_BLOCK { +//! let last_double_block = end.sub(DOUBLE_BLOCK); +//! while ptr <= last_double_block { +//! // Prefetch accepts arbitrary addresses, so use wrapping addition. +//! prefetch_read_l1(ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); //! -//! // Process first 256B block -//! // ... fold operations ... +//! // Process first 256B block +//! // ... fold operations ... //! -//! // Process second 256B block -//! // ... fold operations ... +//! // Process second 256B block +//! // ... fold operations ... //! -//! ptr = ptr.add(DOUBLE_BLOCK); +//! ptr = ptr.add(DOUBLE_BLOCK); +//! } //! } //! ``` @@ -48,13 +41,13 @@ /// /// The x86-64 folding loops use this 1,024-byte lookahead. #[cfg(target_arch = "x86_64")] -pub const LARGE_BLOCK_DISTANCE: usize = 1024; +pub(in crate::checksum) const LARGE_BLOCK_DISTANCE: usize = 1024; /// Prefetch distance for large buffer kernels on ARM64. /// /// The AArch64 folding loops use this 768-byte lookahead. #[cfg(target_arch = "aarch64")] -pub const LARGE_BLOCK_DISTANCE: usize = 768; +pub(in crate::checksum) const LARGE_BLOCK_DISTANCE: usize = 768; // x86-64 Prefetch Intrinsics @@ -72,7 +65,7 @@ mod x86_64_impl { /// The pointer does not need to be valid or aligned. Prefetch is a hint; /// invalid addresses are silently ignored by the CPU. #[inline(always)] - pub(crate) unsafe fn prefetch_read_l1(ptr: *const u8) { + pub(in crate::checksum) unsafe fn prefetch_read_l1(ptr: *const u8) { // SAFETY: Prefetch is a CPU hint; invalid addresses are silently ignored. // The _mm_prefetch intrinsic cannot cause memory unsafety. unsafe { @@ -101,7 +94,7 @@ mod aarch64_impl { /// The pointer does not need to be valid or aligned. Prefetch is a hint; /// invalid addresses are silently ignored by the CPU. #[inline(always)] - pub(crate) unsafe fn prefetch_read_l1(ptr: *const u8) { + pub(in crate::checksum) unsafe fn prefetch_read_l1(ptr: *const u8) { // SAFETY: Inline assembly for PRFM prefetch hint. Prefetch instructions // are CPU hints that cannot cause memory unsafety; invalid addresses // are silently ignored by the hardware. @@ -120,13 +113,13 @@ mod aarch64_impl { // Public API #[cfg(target_arch = "aarch64")] -pub(crate) use aarch64_impl::prefetch_read_l1; +pub(in crate::checksum) use aarch64_impl::prefetch_read_l1; #[cfg(target_arch = "x86_64")] -pub(crate) use x86_64_impl::prefetch_read_l1; +pub(in crate::checksum) use x86_64_impl::prefetch_read_l1; // Fallback for other architectures (no-op) #[cfg(all(not(any(target_arch = "x86_64", target_arch = "aarch64")), test))] -pub const LARGE_BLOCK_DISTANCE: usize = 512; +pub(in crate::checksum) const LARGE_BLOCK_DISTANCE: usize = 512; #[cfg(all(not(any(target_arch = "x86_64", target_arch = "aarch64")), test))] #[inline(always)] @@ -135,7 +128,7 @@ pub const LARGE_BLOCK_DISTANCE: usize = 512; /// # Safety /// /// This function performs no memory access and is always safe to call. -pub(crate) unsafe fn prefetch_read_l1(_ptr: *const u8) {} +pub(in crate::checksum) unsafe fn prefetch_read_l1(_ptr: *const u8) {} // Tests diff --git a/src/checksum/common/reference.rs b/src/checksum/common/reference.rs index 92826a08..261ac18f 100644 --- a/src/checksum/common/reference.rs +++ b/src/checksum/common/reference.rs @@ -35,7 +35,6 @@ // SAFETY: All array indexing uses bounded loop indices (0..data.len()). // Clippy cannot prove this in const fn contexts, but bounds are statically guaranteed. -#![allow(clippy::indexing_slicing)] // CRC-16 Reference Implementation @@ -55,7 +54,7 @@ /// The raw CRC register state (caller applies final XOR if needed). #[cfg(feature = "crc16")] #[must_use] -pub const fn crc16_bitwise(poly: u16, init: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) const fn crc16_bitwise(poly: u16, init: u16, data: &[u8]) -> u16 { let mut crc = init; let mut i: usize = 0; while i < data.len() { @@ -88,7 +87,7 @@ pub const fn crc16_bitwise(poly: u16, init: u16, data: &[u8]) -> u16 { /// The CRC value in the low 24 bits. #[cfg(feature = "crc24")] #[must_use] -pub const fn crc24_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) const fn crc24_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { // Work in expanded form: CRC in top 24 bits of u32 let poly_expanded = poly.strict_shl(8); let mut crc = (init & 0x00FF_FFFF).strict_shl(8); @@ -130,7 +129,7 @@ pub const fn crc24_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { /// The raw CRC register state (caller applies final XOR if needed). #[cfg(feature = "crc32")] #[must_use] -pub const fn crc32_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) const fn crc32_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { let mut crc = init; let mut i: usize = 0; while i < data.len() { @@ -163,7 +162,7 @@ pub const fn crc32_bitwise(poly: u32, init: u32, data: &[u8]) -> u32 { /// The raw CRC register state (caller applies final XOR if needed). #[cfg(feature = "crc64")] #[must_use] -pub const fn crc64_bitwise(poly: u64, init: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) const fn crc64_bitwise(poly: u64, init: u64, data: &[u8]) -> u64 { let mut crc = init; let mut i: usize = 0; while i < data.len() { @@ -397,11 +396,16 @@ mod tests { #[cfg(all(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64"))] fn all_widths_handle_large_input() { // Verify all widths can handle larger inputs without panic - let data: [u8; 1024] = core::array::from_fn(|i| (i as u8).wrapping_mul(17)); - - let _ = crc16_bitwise(CRC16_CCITT_POLY, !0u16, &data); - let _ = crc24_bitwise(CRC24_OPENPGP_POLY, 0x00B7_04CE, &data); - let _ = crc32_bitwise(CRC32_IEEE_POLY, !0u32, &data); - let _ = crc64_bitwise(CRC64_XZ_POLY, !0u64, &data); + let mut byte = 0u8; + let data: [u8; 1024] = core::array::from_fn(|_| { + let value = byte.wrapping_mul(17); + byte = byte.wrapping_add(1); + value + }); + + let _crc16 = crc16_bitwise(CRC16_CCITT_POLY, !0u16, &data); + let _crc24 = crc24_bitwise(CRC24_OPENPGP_POLY, 0x00B7_04CE, &data); + let _crc32 = crc32_bitwise(CRC32_IEEE_POLY, !0u32, &data); + let _crc64 = crc64_bitwise(CRC64_XZ_POLY, !0u64, &data); } } diff --git a/src/checksum/common/tables.rs b/src/checksum/common/tables.rs index 581edfd0..f70b6545 100644 --- a/src/checksum/common/tables.rs +++ b/src/checksum/common/tables.rs @@ -19,7 +19,6 @@ // SAFETY: All array indexing in this module uses bounded loop indices (0..256, 0..N). // Clippy cannot prove this in const fn contexts, but bounds are statically guaranteed. -#![allow(clippy::indexing_slicing)] // CRC-16 Table Generation @@ -28,7 +27,7 @@ /// Uses bit-by-bit computation with the reflected polynomial. #[cfg(feature = "crc16")] #[must_use] -pub const fn crc16_table_entry(poly: u16, index: u8) -> u16 { +pub(super) const fn crc16_table_entry(poly: u16, index: u8) -> u16 { let mut crc = index as u16; let mut i: u32 = 0; while i < 8 { @@ -49,21 +48,24 @@ pub const fn crc16_table_entry(poly: u16, index: u8) -> u16 { /// * `poly` - The reflected polynomial #[cfg(feature = "crc16")] #[must_use] -pub const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { +pub(in crate::checksum) const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { let mut tables = [[0u16; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc16_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc16_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -84,7 +86,7 @@ pub const fn generate_crc16_tables_8(poly: u16) -> [[u16; 256]; 8] { /// The returned entry has its low 8 bits cleared. #[cfg(feature = "crc24")] #[must_use] -pub const fn crc24_table_entry(poly: u32, index: u8) -> u32 { +const fn crc24_table_entry(poly: u32, index: u8) -> u32 { // Expand a 24-bit CRC into the top 24 bits of a u32. // Polynomial is aligned to the same position (<< 8). let poly = poly.strict_shl(8); @@ -108,21 +110,24 @@ pub const fn crc24_table_entry(poly: u32, index: u8) -> u32 { /// * `poly` - The normal polynomial (low 24 bits), e.g. 0x864CFB for OpenPGP #[cfg(feature = "crc24")] #[must_use] -pub const fn generate_crc24_tables_8(poly: u32) -> [[u32; 256]; 8] { +pub(in crate::checksum) const fn generate_crc24_tables_8(poly: u32) -> [[u32; 256]; 8] { let mut tables = [[0u32; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc24_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc24_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev >> 24) as usize] ^ (prev.strict_shl(8)); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev >> 24) as usize] ^ (prev.strict_shl(8)); i = i.strict_add(1); } k = k.strict_add(1); @@ -138,7 +143,7 @@ pub const fn generate_crc24_tables_8(poly: u32) -> [[u32; 256]; 8] { /// Uses bit-by-bit computation with the reflected polynomial. #[cfg(feature = "crc32")] #[must_use] -pub const fn crc32_table_entry(poly: u32, index: u8) -> u32 { +const fn crc32_table_entry(poly: u32, index: u8) -> u32 { let mut crc = index as u32; let mut i: u32 = 0; while i < 8 { @@ -159,21 +164,24 @@ pub const fn crc32_table_entry(poly: u32, index: u8) -> u32 { /// * `poly` - The reflected polynomial #[cfg(all(test, feature = "crc32"))] #[must_use] -pub const fn generate_crc32_tables_8(poly: u32) -> [[u32; 256]; 8] { +pub(super) const fn generate_crc32_tables_8(poly: u32) -> [[u32; 256]; 8] { let mut tables = [[0u32; 256]; 8]; - let mut i: u16 = 0; - while i < 256 { - tables[0][i as usize] = crc32_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc32_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k: usize = 1; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -189,21 +197,24 @@ pub const fn generate_crc32_tables_8(poly: u32) -> [[u32; 256]; 8] { /// * `poly` - The reflected polynomial #[cfg(feature = "crc32")] #[must_use] -pub const fn generate_crc32_tables_16(poly: u32) -> [[u32; 256]; 16] { +pub(in crate::checksum) const fn generate_crc32_tables_16(poly: u32) -> [[u32; 256]; 16] { let mut tables = [[0u32; 256]; 16]; - let mut i: u16 = 0; - while i < 256 { - tables[0][i as usize] = crc32_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc32_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k: usize = 1; while k < 16 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -219,7 +230,7 @@ pub const fn generate_crc32_tables_16(poly: u32) -> [[u32; 256]; 16] { /// Uses bit-by-bit computation with the reflected polynomial. #[cfg(feature = "crc64")] #[must_use] -pub const fn crc64_table_entry(poly: u64, index: u8) -> u64 { +pub(super) const fn crc64_table_entry(poly: u64, index: u8) -> u64 { let mut crc = index as u64; let mut i: u32 = 0; while i < 8 { @@ -240,21 +251,24 @@ pub const fn crc64_table_entry(poly: u64, index: u8) -> u64 { /// * `poly` - The reflected polynomial #[cfg(all(feature = "crc64", any(target_arch = "x86_64", target_arch = "aarch64", test)))] #[must_use] -pub const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { +pub(in crate::checksum) const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { let mut tables = [[0u64; 256]; 8]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc64_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 8 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -273,21 +287,24 @@ pub const fn generate_crc64_tables_8(poly: u64) -> [[u64; 256]; 8] { /// * `poly` - The reflected polynomial #[cfg(feature = "crc64")] #[must_use] -pub const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { +pub(in crate::checksum) const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { let mut tables = [[0u64; 256]; 16]; - let mut i = 0u16; - while i < 256 { - tables[0][i as usize] = crc64_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + tables[0][index as usize] = crc64_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } let mut k = 1usize; while k < 16 { - i = 0; + let mut i = 0usize; while i < 256 { - let prev = tables[k - 1][i as usize]; - tables[k][i as usize] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); + let prev = tables[k.strict_sub(1)][i]; + tables[k][i] = tables[0][(prev & 0xFF) as usize] ^ (prev >> 8); i = i.strict_add(1); } k = k.strict_add(1); @@ -303,19 +320,19 @@ pub const fn generate_crc64_tables_16(poly: u64) -> [[u64; 256]; 16] { /// CRC-16-CCITT polynomial (0x1021) in reflected form. /// Used by X.25, V.41, HDLC, XMODEM, Bluetooth, PACTOR, SD, etc. #[cfg(feature = "crc16")] -pub const CRC16_CCITT_POLY: u16 = 0x8408; +pub(in crate::checksum) const CRC16_CCITT_POLY: u16 = 0x8408; /// CRC-16-IBM polynomial (0x8005) in reflected form. /// Used by Modbus, USB, ANSI X3.28, etc. #[cfg(feature = "crc16")] -pub const CRC16_IBM_POLY: u16 = 0xA001; +pub(in crate::checksum) const CRC16_IBM_POLY: u16 = 0xA001; // CRC-24 Polynomials /// CRC-24-OPENPGP polynomial (0x864CFB) in normal form. /// Used by OpenPGP (RFC 4880). #[cfg(feature = "crc24")] -pub const CRC24_OPENPGP_POLY: u32 = 0x0086_4CFB; +pub(in crate::checksum) const CRC24_OPENPGP_POLY: u32 = 0x0086_4CFB; // CRC-64 Polynomials @@ -325,23 +342,23 @@ pub const CRC24_OPENPGP_POLY: u32 = 0x0086_4CFB; /// /// Used by Ethernet, gzip, zip, PNG, etc. #[cfg(feature = "crc32")] -pub const CRC32_IEEE_POLY: u32 = 0xEDB8_8320; +pub(in crate::checksum) const CRC32_IEEE_POLY: u32 = 0xEDB8_8320; /// CRC-32C (Castagnoli) polynomial (0x1EDC6F41) in reflected form. /// /// Used by iSCSI, SCTP, ext4, Btrfs, SSE4.2 `crc32`, etc. #[cfg(feature = "crc32")] -pub const CRC32C_POLY: u32 = 0x82F6_3B78; +pub(in crate::checksum) const CRC32C_POLY: u32 = 0x82F6_3B78; /// CRC-64-XZ polynomial (0x42F0E1EBA9EA3693) in reflected form. /// Used by XZ Utils, 7-Zip, LZMA. #[cfg(feature = "crc64")] -pub const CRC64_XZ_POLY: u64 = 0xC96C_5795_D787_0F42; +pub(in crate::checksum) const CRC64_XZ_POLY: u64 = 0xC96C_5795_D787_0F42; /// CRC-64-NVME polynomial (0xAD93D23594C93659) in reflected form. /// Used by NVMe specification. #[cfg(feature = "crc64")] -pub const CRC64_NVME_POLY: u64 = 0x9A6C_9329_AC4B_C9B5; +pub(in crate::checksum) const CRC64_NVME_POLY: u64 = 0x9A6C_9329_AC4B_C9B5; // Tests diff --git a/src/checksum/common/tests.rs b/src/checksum/common/tests.rs index a4f32683..06c86df8 100644 --- a/src/checksum/common/tests.rs +++ b/src/checksum/common/tests.rs @@ -13,15 +13,13 @@ //! //! Used via [`define_crc_property_tests!`] macro in algorithm modules. -#![cfg_attr(miri, allow(dead_code))] - use crate::traits::{Checksum, ChecksumCombine}; /// Generic test harness for CRC algorithms. /// /// Provides a suite of property-based tests that verify fundamental CRC invariants. /// These tests can be run against any type implementing `Checksum + ChecksumCombine`. -pub struct CrcTestHarness { +pub(in crate::checksum) struct CrcTestHarness { _phantom: core::marker::PhantomData, } @@ -36,8 +34,8 @@ where /// /// This is the fundamental combine property that enables parallel CRC computation. #[inline] - pub fn test_combine_property(data: &[u8], split: usize) { - let split = if data.is_empty() { 0 } else { split % data.len() }; + pub(in crate::checksum) fn test_combine_property(data: &[u8], split: usize) { + let split = split.checked_rem(data.len()).unwrap_or(0); let (a, b) = data.split_at(split); let crc_a = C::checksum(a); @@ -49,9 +47,9 @@ where } /// Test combine property at all possible split points for a given buffer. + #[cfg(feature = "crc64")] #[inline] - #[cfg_attr(test, allow(dead_code))] - pub fn test_combine_all_splits(data: &[u8]) { + pub(in crate::checksum) fn test_combine_all_splits(data: &[u8]) { let full = C::checksum(data); for split in 0..=data.len() { @@ -66,7 +64,7 @@ where /// Test combine with empty second part (identity case). #[inline] - pub fn test_combine_empty_suffix(data: &[u8]) { + pub(in crate::checksum) fn test_combine_empty_suffix(data: &[u8]) { let crc_data = C::checksum(data); let crc_empty = C::checksum(&[]); let combined = C::combine(crc_data, crc_empty, 0); @@ -76,7 +74,7 @@ where /// Test combine with empty first part. #[inline] - pub fn test_combine_empty_prefix(data: &[u8]) { + pub(in crate::checksum) fn test_combine_empty_prefix(data: &[u8]) { let crc_empty = C::checksum(&[]); let crc_data = C::checksum(data); let combined = C::combine(crc_empty, crc_data, data.len()); @@ -87,9 +85,9 @@ where // Streaming Consistency Tests /// Test that streaming updates produce the same result as one-shot. + #[cfg(feature = "crc64")] #[inline] - #[cfg_attr(test, allow(dead_code))] - pub fn test_streaming_equals_oneshot(data: &[u8]) { + pub(in crate::checksum) fn test_streaming_equals_oneshot(data: &[u8]) { let oneshot = C::checksum(data); let mut hasher = C::new(); @@ -101,7 +99,7 @@ where /// Test streaming with byte-at-a-time updates. #[inline] - pub fn test_streaming_byte_at_a_time(data: &[u8]) { + pub(in crate::checksum) fn test_streaming_byte_at_a_time(data: &[u8]) { let oneshot = C::checksum(data); let mut hasher = C::new(); @@ -115,7 +113,7 @@ where /// Test streaming across a specific chunk size boundary. #[inline] - pub fn test_streaming_chunked(data: &[u8], chunk_size: usize) { + pub(in crate::checksum) fn test_streaming_chunked(data: &[u8], chunk_size: usize) { if chunk_size == 0 { return; } @@ -133,7 +131,7 @@ where /// Test that finalize is idempotent (can be called multiple times). #[inline] - pub fn test_finalize_idempotent(data: &[u8]) { + pub(in crate::checksum) fn test_finalize_idempotent(data: &[u8]) { let mut hasher = C::new(); hasher.update(data); @@ -149,7 +147,7 @@ where /// Test that reset returns the hasher to its initial state. #[inline] - pub fn test_reset(data: &[u8]) { + pub(in crate::checksum) fn test_reset(data: &[u8]) { let fresh = C::checksum(data); let mut hasher = C::new(); @@ -170,7 +168,7 @@ where /// 2. Computing second half separately /// 3. Combining the results #[inline] - pub fn test_streaming_and_combine(data: &[u8]) { + pub(in crate::checksum) fn test_streaming_and_combine(data: &[u8]) { if data.is_empty() { return; } @@ -197,7 +195,7 @@ where /// Test empty input. #[inline] - pub fn test_empty_input() { + pub(in crate::checksum) fn test_empty_input() { let oneshot = C::checksum(&[]); let hasher = C::new(); @@ -208,7 +206,7 @@ where /// Test single byte inputs for all byte values. #[inline] - pub fn test_single_bytes() { + pub(in crate::checksum) fn test_single_bytes() { for byte in 0u8..=255 { let oneshot = C::checksum(&[byte]); @@ -302,7 +300,7 @@ macro_rules! define_crc_property_tests { /// Used by all CRC cross-check test modules to exercise edge cases around /// lane widths, cache line boundaries, and page boundaries. #[cfg(not(miri))] -pub const TEST_LENGTHS: &[usize] = &[ +pub(in crate::checksum) const TEST_LENGTHS: &[usize] = &[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // Tiny 16, 17, 31, 32, 33, 63, 64, 65, // SSE/NEON boundaries 127, 128, 129, 255, 256, 257, // Cache line boundaries @@ -311,25 +309,23 @@ pub const TEST_LENGTHS: &[usize] = &[ 8192, 16384, 32768, 65536, // Large buffers ]; #[cfg(miri)] -pub const TEST_LENGTHS: &[usize] = &[ +pub(in crate::checksum) const TEST_LENGTHS: &[usize] = &[ 0, 1, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 511, 512, 513, 1023, 1024, 1025, ]; /// Prime-sized chunk patterns for streaming cross-check tests. #[cfg(not(miri))] -pub const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 17, 31, 37, 61, 127, 251]; +pub(in crate::checksum) const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 17, 31, 37, 61, 127, 251]; #[cfg(miri)] -pub const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 31, 61]; +pub(in crate::checksum) const STREAMING_CHUNK_SIZES: &[usize] = &[1, 3, 7, 13, 31, 61]; /// Generate deterministic test data of a given length. /// /// Uses a simple mixing function to produce non-trivial byte patterns /// that avoid accidentally passing due to regularity. -#[cfg_attr(test, allow(dead_code))] -pub fn generate_test_data(len: usize) -> alloc::vec::Vec { - (0..len) - .map(|i| (i as u64).wrapping_mul(17).wrapping_add(i as u64) as u8) - .collect() +#[cfg(any(feature = "crc16", feature = "crc24"))] +pub(in crate::checksum) fn generate_test_data(len: usize) -> alloc::vec::Vec { + (0u8..=u8::MAX).cycle().take(len).map(|i| i.wrapping_mul(18)).collect() } // Tests for the test harness itself diff --git a/src/checksum/crc16/aarch64.rs b/src/checksum/crc16/aarch64.rs index 64ce2ed1..4c185675 100644 --- a/src/checksum/crc16/aarch64.rs +++ b/src/checksum/crc16/aarch64.rs @@ -6,10 +6,8 @@ //! //! # Safety //! -//! Uses `unsafe` for ARM SIMD intrinsics. Callers must ensure PMULL is -//! available before executing these kernels (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! Uses `unsafe` for ARM SIMD intrinsics. Callers must establish NEON, AES +//! (PMULL), and SHA3 (EOR3) support as required by each accelerated path. use core::{ arch::aarch64::*, @@ -42,33 +40,57 @@ impl BitXorAssign for Simd { } impl Simd { + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn new(high: u64, low: u64) -> Self { Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high))) } + /// Loads 16 bytes from `ptr` without requiring alignment. + /// + /// # Safety + /// + /// The current CPU must support NEON, and `ptr` must address at least 16 initialized readable + /// bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn load(ptr: *const u8) -> Self { - // SAFETY: Caller guarantees: - // 1. NEON target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: The caller guarantees NEON support and 16 initialized readable bytes at `ptr`. unsafe { Self(vld1q_u8(ptr)) } } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn and(self, mask: Self) -> Self { Self(vandq_u8(self.0, mask.0)) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_right_8(self) -> Self { Self(vextq_u8(self.0, vdupq_n_u8(0), 8)) } + /// Moves the low 32-bit lane to the high 32-bit lane and clears the rest. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_left_12(self) -> Self { @@ -77,6 +99,11 @@ impl Simd { Self(vreinterpretq_u8_u32(result)) } + /// Multiplies the low polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul00(self, other: Self) -> Self { @@ -85,6 +112,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's high polynomial lane by the other vector's low lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul01(self, other: Self) -> Self { @@ -93,6 +125,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's low polynomial lane by the other vector's high lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul10(self, other: Self) -> Self { @@ -101,6 +138,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies the high polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul11(self, other: Self) -> Self { @@ -109,6 +151,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { @@ -122,6 +169,11 @@ impl Simd { } } + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -146,9 +198,14 @@ impl Simd { } } + /// Applies Barrett reduction and returns the low CRC-16 state. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] - unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { + unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u16 { // SAFETY: Caller guarantees: // 1. AES + NEON target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -159,14 +216,19 @@ impl Simd { let xorred = self ^ clmul2; let hi = xorred.shift_right_8(); - vgetq_lane_u32(vreinterpretq_u32_u8(hi.0), 0) + vgetq_lane_u16(vreinterpretq_u16_u8(hi.0), 0) } } } +/// Reduces eight folded SIMD lanes to a CRC-16 state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] -unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { +unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u16 { // SAFETY: Caller guarantees: // 1. AES + NEON target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -185,6 +247,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// Folds one reflected 128-byte block into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_block_128_width32_reflected(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { @@ -203,8 +270,17 @@ unsafe fn fold_block_128_width32_reflected(x: &mut [Simd; 8], chunk: &[Simd; 8], } } +/// Folds one reflected 16-byte lane with a three-input XOR. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn fold_lane_width32_reflected_eor3(x: Simd, coeff: Simd, data_to_xor: Simd) -> Simd { // SAFETY: Caller guarantees: @@ -217,8 +293,17 @@ unsafe fn fold_lane_width32_reflected_eor3(x: Simd, coeff: Simd, data_to_xor: Si } } +/// Folds one reflected 128-byte block with EOR3 into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn fold_block_128_width32_reflected_eor3(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { // SAFETY: Caller guarantees: @@ -236,13 +321,18 @@ unsafe fn fold_block_128_width32_reflected_eor3(x: &mut [Simd; 8], chunk: &[Simd } } +/// Folds reflected 128-byte blocks through two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_2way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -250,10 +340,11 @@ unsafe fn update_simd_width32_reflected_2way( // 2. All pointer arithmetic stays within bounds via loop guards. // 3. All SIMD operations are pure register computations after loads. unsafe { - debug_assert!(blocks.len() >= 2); if blocks.len() < 2 { - // SAFETY: this function is only called when there are at least 2 blocks. - core::hint::unreachable_unchecked() + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected(state, first, rest, keys); } let coeff_256b = Simd::new(fold_256b.0, fold_256b.1); @@ -267,7 +358,7 @@ unsafe fn update_simd_width32_reflected_2way( let mut s1 = *blocks_ptr.add(1); // Inject CRC into stream 0 (block 0). - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); // Pointer-walk variant to keep the hot loop free of index bounds checks. const BLOCK_SIZE: usize = 128; @@ -276,7 +367,7 @@ unsafe fn update_simd_width32_reflected_2way( let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(2); - let mut rem = blocks.len() - 2; + let mut rem = blocks.len().strict_sub(2); while rem >= DOUBLE_GROUP { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -293,7 +384,7 @@ unsafe fn update_simd_width32_reflected_2way( fold_block_128_width32_reflected(&mut s1, &*ptr.add(3), coeff_256b); ptr = ptr.add(DOUBLE_GROUP); - rem -= DOUBLE_GROUP; + rem = rem.strict_sub(DOUBLE_GROUP); } // Handle remaining pairs. @@ -301,7 +392,7 @@ unsafe fn update_simd_width32_reflected_2way( fold_block_128_width32_reflected(&mut s0, &*ptr, coeff_256b); fold_block_128_width32_reflected(&mut s1, &*ptr.add(1), coeff_256b); ptr = ptr.add(2); - rem -= 2; + rem = rem.strict_sub(2); } // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). @@ -324,14 +415,19 @@ unsafe fn update_simd_width32_reflected_2way( } } +/// Folds reflected 128-byte blocks through three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_3way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_384b: (u64, u64), fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -356,7 +452,7 @@ unsafe fn update_simd_width32_reflected_3way( let mut s2 = blocks[2]; // Inject CRC into stream 0 (block 0). - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); // Double-unrolled main loop: process 6 blocks (768B) per iteration. const BLOCK_SIZE: usize = 128; @@ -366,7 +462,12 @@ unsafe fn update_simd_width32_reflected_3way( let blocks_ptr = blocks.as_ptr(); let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(3); - let double_end = blocks_ptr.add(3 + ((blocks.len() - 3) / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_blocks = blocks + .len() + .strict_sub(3) + .strict_div(DOUBLE_GROUP) + .strict_mul(DOUBLE_GROUP); + let double_end = blocks_ptr.add(3usize.strict_add(double_blocks)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -388,7 +489,7 @@ unsafe fn update_simd_width32_reflected_3way( } // Handle remaining triplets. - let triple_end = blocks_ptr.add((blocks.len() / 3) * 3); + let triple_end = blocks_ptr.add(blocks.len().strict_div(3).strict_mul(3)); while ptr < triple_end { fold_block_128_width32_reflected(&mut s0, &*ptr, coeff_384b); fold_block_128_width32_reflected(&mut s1, &*ptr.add(1), coeff_384b); @@ -427,9 +528,14 @@ unsafe fn update_simd_width32_reflected_3way( } } +/// Folds a reflected sequence of 128-byte blocks and reduces it to CRC-16. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] -unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[Simd; 8]], keys: &[u64; 23]) -> u32 { +unsafe fn update_simd_width32_reflected(state: u16, first: &[Simd; 8], rest: &[[Simd; 8]], keys: &[u64; 23]) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -439,7 +545,7 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ unsafe { let mut x = *first; - x[0] ^= Simd::new(0, state as u64); + x[0] ^= Simd::new(0, u64::from(state)); let coeff_128b = Simd::new(keys[4], keys[3]); @@ -451,7 +557,7 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ let rest_ptr = rest.as_ptr(); let rest_end = rest_ptr.add(rest.len()); let mut ptr = rest_ptr; - let double_end = rest_ptr.add((rest.len() / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_end = rest_ptr.add(rest.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -499,15 +605,24 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ } } +/// Folds a reflected sequence of 128-byte blocks with EOR3 and reduces it to CRC-16. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_width32_reflected_eor3( - state: u32, + state: u16, first: &[Simd; 8], rest: &[[Simd; 8]], keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -517,7 +632,7 @@ unsafe fn update_simd_width32_reflected_eor3( unsafe { let mut x = *first; - x[0] ^= Simd::new(0, state as u64); + x[0] ^= Simd::new(0, u64::from(state)); let coeff_128b = Simd::new(keys[4], keys[3]); @@ -528,7 +643,7 @@ unsafe fn update_simd_width32_reflected_eor3( let rest_ptr = rest.as_ptr(); let rest_end = rest_ptr.add(rest.len()); let mut ptr = rest_ptr; - let double_end = rest_ptr.add((rest.len() / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_end = rest_ptr.add(rest.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -550,14 +665,23 @@ unsafe fn update_simd_width32_reflected_eor3( } } -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +/// Folds reflected 128-byte blocks through two EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_width32_reflected_eor3_2way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -565,10 +689,11 @@ unsafe fn update_simd_width32_reflected_eor3_2way( // 2. All pointer arithmetic stays within bounds via loop guards. // 3. All SIMD operations are pure register computations after loads. unsafe { - debug_assert!(blocks.len() >= 2); if blocks.len() < 2 { - // SAFETY: this function is only called when there are at least 2 blocks. - core::hint::unreachable_unchecked() + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected_eor3(state, first, rest, keys); } let coeff_256b = Simd::new(fold_256b.0, fold_256b.1); @@ -578,7 +703,7 @@ unsafe fn update_simd_width32_reflected_eor3_2way( let mut s0 = *blocks_ptr; let mut s1 = *blocks_ptr.add(1); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); const BLOCK_SIZE: usize = 128; const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B @@ -586,7 +711,7 @@ unsafe fn update_simd_width32_reflected_eor3_2way( let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(2); - let mut rem = blocks.len() - 2; + let mut rem = blocks.len().strict_sub(2); while rem >= DOUBLE_GROUP { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -600,14 +725,14 @@ unsafe fn update_simd_width32_reflected_eor3_2way( fold_block_128_width32_reflected_eor3(&mut s1, &*ptr.add(3), coeff_256b); ptr = ptr.add(DOUBLE_GROUP); - rem -= DOUBLE_GROUP; + rem = rem.strict_sub(DOUBLE_GROUP); } while rem >= 2 { fold_block_128_width32_reflected_eor3(&mut s0, &*ptr, coeff_256b); fold_block_128_width32_reflected_eor3(&mut s1, &*ptr.add(1), coeff_256b); ptr = ptr.add(2); - rem -= 2; + rem = rem.strict_sub(2); } let mut combined = s1; @@ -628,15 +753,24 @@ unsafe fn update_simd_width32_reflected_eor3_2way( } } -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +/// Folds reflected 128-byte blocks through three EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_width32_reflected_eor3_3way( - state: u32, + state: u16, blocks: &[[Simd; 8]], fold_384b: (u64, u64), fold_256b: (u64, u64), keys: &[u64; 23], -) -> u32 { +) -> u16 { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; // SAFETY: Caller guarantees: @@ -659,7 +793,7 @@ unsafe fn update_simd_width32_reflected_eor3_3way( let mut s1 = blocks[1]; let mut s2 = blocks[2]; - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, u64::from(state)); const BLOCK_SIZE: usize = 128; const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B @@ -668,7 +802,12 @@ unsafe fn update_simd_width32_reflected_eor3_3way( let blocks_ptr = blocks.as_ptr(); let blocks_end = blocks_ptr.add(blocks.len()); let mut ptr = blocks_ptr.add(3); - let double_end = blocks_ptr.add(3 + ((blocks.len() - 3) / DOUBLE_GROUP) * DOUBLE_GROUP); + let double_blocks = blocks + .len() + .strict_sub(3) + .strict_div(DOUBLE_GROUP) + .strict_mul(DOUBLE_GROUP); + let double_end = blocks_ptr.add(3usize.strict_add(double_blocks)); while ptr < double_end { let prefetch_ptr = ptr.add(PREFETCH_BLOCKS); @@ -686,7 +825,7 @@ unsafe fn update_simd_width32_reflected_eor3_3way( ptr = ptr.add(DOUBLE_GROUP); } - let triple_end = blocks_ptr.add((blocks.len() / 3) * 3); + let triple_end = blocks_ptr.add(blocks.len().strict_div(3).strict_mul(3)); while ptr < triple_end { fold_block_128_width32_reflected_eor3(&mut s0, &*ptr, coeff_384b); fold_block_128_width32_reflected_eor3(&mut s1, &*ptr.add(1), coeff_384b); @@ -722,6 +861,11 @@ unsafe fn update_simd_width32_reflected_eor3_3way( } } +/// Computes CRC-16 with single-lane PMULL folding for small buffers. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull_small( @@ -745,7 +889,7 @@ unsafe fn crc16_width32_pmull_small( let coeff_16b = Simd::new(keys[2], keys[1]); let mut x0 = Simd::load(buf); - x0 ^= Simd::new(0, state as u64); + x0 ^= Simd::new(0, u64::from(state)); buf = buf.add(16); len = len.strict_sub(16); @@ -757,13 +901,18 @@ unsafe fn crc16_width32_pmull_small( } let x0 = x0.fold_width32_reflected(keys[6], keys[5]); - state = x0.barrett_width32_reflected(keys[8], keys[7]) as u16; + state = x0.barrett_width32_reflected(keys[8], keys[7]); let tail = core::slice::from_raw_parts(buf, len); portable(state, tail) } } +/// Computes CRC-16 with PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { @@ -777,12 +926,16 @@ unsafe fn crc16_width32_pmull(mut state: u16, data: &[u8], keys: &[u64; 23], por }; state = portable(state, left); - let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = update_simd_width32_reflected(state, first, rest, keys); portable(state, right) } } +/// Computes CRC-16 with two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull_2way( @@ -802,19 +955,23 @@ unsafe fn crc16_width32_pmull_2way( } state = portable(state, left); - let state32 = if middle.len() >= 2 { - update_simd_width32_reflected_2way(state as u32, middle, fold_256b, keys) + state = if middle.len() >= 2 { + update_simd_width32_reflected_2way(state, middle, fold_256b, keys) } else { let Some((first, rest)) = middle.split_first() else { return crc16_width32_pmull_small(state, data, keys, portable); }; - update_simd_width32_reflected(state as u32, first, rest, keys) + update_simd_width32_reflected(state, first, rest, keys) }; - state = state32 as u16; portable(state, right) } } +/// Computes CRC-16 with three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc16_width32_pmull_3way( @@ -835,14 +992,22 @@ unsafe fn crc16_width32_pmull_3way( } state = portable(state, left); - let state32 = update_simd_width32_reflected_3way(state as u32, middle, fold_384b, fold_256b, keys); - state = state32 as u16; + state = update_simd_width32_reflected_3way(state, middle, fold_384b, fold_256b, keys); portable(state, right) } } +/// Computes CRC-16 with EOR3-assisted PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc16_width32_pmull_eor3( mut state: u16, @@ -860,14 +1025,22 @@ unsafe fn crc16_width32_pmull_eor3( }; state = portable(state, left); - let state32 = update_simd_width32_reflected_eor3(state as u32, first, rest, keys); - state = state32 as u16; + state = update_simd_width32_reflected_eor3(state, first, rest, keys); portable(state, right) } } +/// Computes CRC-16 with two EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc16_width32_pmull_eor3_2way( mut state: u16, @@ -886,21 +1059,29 @@ unsafe fn crc16_width32_pmull_eor3_2way( } state = portable(state, left); - let state32 = if middle.len() >= 2 { - update_simd_width32_reflected_eor3_2way(state as u32, middle, fold_256b, keys) + state = if middle.len() >= 2 { + update_simd_width32_reflected_eor3_2way(state, middle, fold_256b, keys) } else { let Some((first, rest)) = middle.split_first() else { return crc16_width32_pmull_small(state, data, keys, portable); }; - update_simd_width32_reflected_eor3(state as u32, first, rest, keys) + update_simd_width32_reflected_eor3(state, first, rest, keys) }; - state = state32 as u16; portable(state, right) } } +/// Computes CRC-16 with three EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL), NEON, and SHA3 (EOR3). #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc16_width32_pmull_eor3_3way( mut state: u16, @@ -920,22 +1101,17 @@ unsafe fn crc16_width32_pmull_eor3_3way( } state = portable(state, left); - let state32 = update_simd_width32_reflected_eor3_3way(state as u32, middle, fold_384b, fold_256b, keys); - state = state32 as u16; + state = update_simd_width32_reflected_eor3_3way(state, middle, fold_384b, fold_256b, keys); portable(state, right) } } -// Public Safe Kernels (matching CRC-64 pure fn(u16, &[u8]) -> u16 signature) +// Safe kernel wrappers matching the CRC-64 `fn(u16, &[u8]) -> u16` signature. /// CRC-16/CCITT PMULL kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull( crc, @@ -949,13 +1125,9 @@ pub fn crc16_ccitt_pmull_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PMULL small-buffer kernel. /// /// Optimized for inputs smaller than a folding block (128 bytes). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_small( crc, @@ -967,13 +1139,9 @@ pub fn crc16_ccitt_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_2way( crc, @@ -986,13 +1154,9 @@ pub fn crc16_ccitt_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ccitt_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ccitt_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_3way( crc, @@ -1006,14 +1170,14 @@ pub fn crc16_ccitt_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL+EOR3 kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ccitt_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ccitt_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3( crc, @@ -1025,14 +1189,14 @@ pub fn crc16_ccitt_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL+EOR3 kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ccitt_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ccitt_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_2way( crc, @@ -1045,14 +1209,14 @@ pub fn crc16_ccitt_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT PMULL+EOR3 kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ccitt_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ccitt_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_3way( crc, @@ -1066,37 +1230,25 @@ pub fn crc16_ccitt_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PMULL small-buffer kernel. /// /// Optimized for inputs smaller than a folding block (128 bytes). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_small_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_small(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PMULL kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_2way( crc, @@ -1109,13 +1261,9 @@ pub fn crc16_ibm_pmull_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc16_ibm_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. +pub(super) fn crc16_ibm_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL support before invoking this private wrapper. unsafe { crc16_width32_pmull_3way( crc, @@ -1129,26 +1277,26 @@ pub fn crc16_ibm_pmull_3way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL+EOR3 kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ibm_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ibm_pmull_eor3_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PMULL+EOR3 kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ibm_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ibm_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_2way( crc, @@ -1161,14 +1309,14 @@ pub fn crc16_ibm_pmull_eor3_2way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM PMULL+EOR3 kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL+SHA3 before selecting this kernel. #[inline] -#[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] -pub fn crc16_ibm_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { - // SAFETY: Dispatcher verifies PMULL+SHA3 before selecting this kernel. +#[cfg(all( + any(test, feature = "std"), + not(miri), + any(target_os = "linux", target_os = "android") +))] +pub(super) fn crc16_ibm_pmull_eor3_3way_safe(crc: u16, data: &[u8]) -> u16 { + // SAFETY: All callers establish PMULL and SHA3 support before invoking this private wrapper. unsafe { crc16_width32_pmull_eor3_3way( crc, @@ -1194,8 +1342,11 @@ mod tests { const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; fn data() -> Vec { - (0..4111) - .map(|i| (i as u8).wrapping_mul(29).wrapping_add((i >> 8) as u8)) + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(29).wrapping_add(high) + }) .collect() } @@ -1204,7 +1355,7 @@ mod tests { for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), portable(state, slice), diff --git a/src/checksum/crc16/config.rs b/src/checksum/crc16/config.rs index 2649ddb1..da879f6a 100644 --- a/src/checksum/crc16/config.rs +++ b/src/checksum/crc16/config.rs @@ -95,8 +95,16 @@ fn parse_force_value(value: &str) -> Crc16Force { #[inline] #[must_use] -#[allow(unused_variables)] fn clamp_force_to_caps(requested: Crc16Force, caps: Caps) -> Crc16Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc16Force::Auto | Crc16Force::Reference | Crc16Force::Portable => requested, Crc16Force::Clmul => { @@ -188,7 +196,7 @@ fn config_ibm(caps: Caps) -> Crc16Config { /// detected platform capabilities. #[inline] #[must_use] -pub fn get_ccitt() -> Crc16Config { +pub(super) fn get_ccitt() -> Crc16Config { #[cfg(feature = "std")] { use std::sync::OnceLock; @@ -208,7 +216,7 @@ pub fn get_ccitt() -> Crc16Config { /// detected platform capabilities. #[inline] #[must_use] -pub fn get_ibm() -> Crc16Config { +pub(super) fn get_ibm() -> Crc16Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc16/kernels.rs b/src/checksum/crc16/kernels.rs index 2ea447b6..09589ac2 100644 --- a/src/checksum/crc16/kernels.rs +++ b/src/checksum/crc16/kernels.rs @@ -14,24 +14,24 @@ //! - Tier 4 (Wide): VPCLMUL (x86_64 AVX-512), Zvbc (riscv64) /// Reference (bitwise) kernel name. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; /// Portable slice-by-8 kernel name. -pub const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; +pub(in crate::checksum) const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; // Kernel Name Tables and Functions (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc16Fn; // CRC-16/CCITT Kernel Functions /// CCITT PCLMUL kernel. - pub const CCITT_PCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_PCLMUL: [Crc16Fn; 5] = [ arch::crc16_ccitt_pclmul_safe, arch::crc16_ccitt_pclmul_2way_safe, arch::crc16_ccitt_pclmul_4way_safe, @@ -40,10 +40,10 @@ pub mod x86_64 { ]; /// CCITT PCLMUL small-buffer kernel. - pub const CCITT_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pclmul_small_safe; + pub(in crate::checksum) const CCITT_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pclmul_small_safe; /// CCITT VPCLMUL kernel. - pub const CCITT_VPCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_VPCLMUL: [Crc16Fn; 5] = [ arch::crc16_ccitt_vpclmul_safe, arch::crc16_ccitt_vpclmul_2way_safe, arch::crc16_ccitt_vpclmul_4way_safe, @@ -54,7 +54,7 @@ pub mod x86_64 { // CRC-16/IBM Kernel Functions /// IBM PCLMUL kernel. - pub const IBM_PCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_PCLMUL: [Crc16Fn; 5] = [ arch::crc16_ibm_pclmul_safe, arch::crc16_ibm_pclmul_2way_safe, arch::crc16_ibm_pclmul_4way_safe, @@ -63,10 +63,10 @@ pub mod x86_64 { ]; /// IBM PCLMUL small-buffer kernel. - pub const IBM_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pclmul_small_safe; + pub(in crate::checksum) const IBM_PCLMUL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pclmul_small_safe; /// IBM VPCLMUL kernel. - pub const IBM_VPCLMUL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_VPCLMUL: [Crc16Fn; 5] = [ arch::crc16_ibm_vpclmul_safe, arch::crc16_ibm_vpclmul_2way_safe, arch::crc16_ibm_vpclmul_4way_safe, @@ -76,14 +76,14 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc16Fn; // CRC-16/CCITT Kernel Functions /// CCITT PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - pub const CCITT_PMULL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_PMULL: [Crc16Fn; 5] = [ arch::crc16_ccitt_pmull_safe, arch::crc16_ccitt_pmull_2way_safe, arch::crc16_ccitt_pmull_3way_safe, @@ -92,8 +92,8 @@ pub mod aarch64 { ]; /// CCITT PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub const CCITT_PMULL_EOR3: [Crc16Fn; 5] = [ + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(in crate::checksum) const CCITT_PMULL_EOR3: [Crc16Fn; 5] = [ arch::crc16_ccitt_pmull_eor3_safe, arch::crc16_ccitt_pmull_eor3_2way_safe, arch::crc16_ccitt_pmull_eor3_3way_safe, @@ -102,12 +102,12 @@ pub mod aarch64 { ]; /// CCITT PMULL small-buffer kernel. - pub const CCITT_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pmull_small_safe; + pub(in crate::checksum) const CCITT_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ccitt_pmull_small_safe; // CRC-16/IBM Kernel Functions /// IBM PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - pub const IBM_PMULL: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_PMULL: [Crc16Fn; 5] = [ arch::crc16_ibm_pmull_safe, arch::crc16_ibm_pmull_2way_safe, arch::crc16_ibm_pmull_3way_safe, @@ -116,8 +116,8 @@ pub mod aarch64 { ]; /// IBM PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub const IBM_PMULL_EOR3: [Crc16Fn; 5] = [ + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(in crate::checksum) const IBM_PMULL_EOR3: [Crc16Fn; 5] = [ arch::crc16_ibm_pmull_eor3_safe, arch::crc16_ibm_pmull_eor3_2way_safe, arch::crc16_ibm_pmull_eor3_3way_safe, @@ -126,16 +126,16 @@ pub mod aarch64 { ]; /// IBM PMULL small-buffer kernel. - pub const IBM_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pmull_small_safe; + pub(in crate::checksum) const IBM_PMULL_SMALL_KERNEL: Crc16Fn = arch::crc16_ibm_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc16Fn; /// CCITT VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const CCITT_VPMSUM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_VPMSUM: [Crc16Fn; 5] = [ arch::crc16_ccitt_vpmsum_safe, arch::crc16_ccitt_vpmsum_2way_safe, arch::crc16_ccitt_vpmsum_4way_safe, @@ -144,7 +144,7 @@ pub mod power { ]; /// IBM VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const IBM_VPMSUM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_VPMSUM: [Crc16Fn; 5] = [ arch::crc16_ibm_vpmsum_safe, arch::crc16_ibm_vpmsum_2way_safe, arch::crc16_ibm_vpmsum_4way_safe, @@ -154,12 +154,12 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc16Fn; /// CCITT VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CCITT_VGFM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_VGFM: [Crc16Fn; 5] = [ arch::crc16_ccitt_vgfm_safe, arch::crc16_ccitt_vgfm_2way_safe, arch::crc16_ccitt_vgfm_4way_safe, @@ -168,7 +168,7 @@ pub mod s390x { ]; /// IBM VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const IBM_VGFM: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_VGFM: [Crc16Fn; 5] = [ arch::crc16_ibm_vgfm_safe, arch::crc16_ibm_vgfm_2way_safe, arch::crc16_ibm_vgfm_4way_safe, @@ -178,31 +178,12 @@ pub mod s390x { } #[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { +pub(in crate::checksum) mod riscv64 { use super::super::riscv64 as arch; use crate::checksum::dispatchers::Crc16Fn; - /// Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - /// CCITT Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CCITT_ZBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_ZBC: [Crc16Fn; 5] = [ arch::crc16_ccitt_zbc_safe, arch::crc16_ccitt_zbc_2way_safe, arch::crc16_ccitt_zbc_4way_safe, @@ -211,7 +192,7 @@ pub mod riscv64 { ]; /// CCITT Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CCITT_ZVBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const CCITT_ZVBC: [Crc16Fn; 5] = [ arch::crc16_ccitt_zvbc_safe, arch::crc16_ccitt_zvbc_2way_safe, arch::crc16_ccitt_zvbc_4way_safe, @@ -220,7 +201,7 @@ pub mod riscv64 { ]; /// IBM Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const IBM_ZBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_ZBC: [Crc16Fn; 5] = [ arch::crc16_ibm_zbc_safe, arch::crc16_ibm_zbc_2way_safe, arch::crc16_ibm_zbc_4way_safe, @@ -229,7 +210,7 @@ pub mod riscv64 { ]; /// IBM Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const IBM_ZVBC: [Crc16Fn; 5] = [ + pub(in crate::checksum) const IBM_ZVBC: [Crc16Fn; 5] = [ arch::crc16_ibm_zvbc_safe, arch::crc16_ibm_zvbc_2way_safe, arch::crc16_ibm_zvbc_4way_safe, diff --git a/src/checksum/crc16/keys.rs b/src/checksum/crc16/keys.rs index 29b3ea09..cf4e8809 100644 --- a/src/checksum/crc16/keys.rs +++ b/src/checksum/crc16/keys.rs @@ -10,7 +10,7 @@ use crate::checksum::common::tables::{CRC16_CCITT_POLY, CRC16_IBM_POLY}; /// Key schedule for CRC-16/CCITT (X.25 / IBM-SDLC), reflected polynomial. #[rustfmt::skip] -pub(crate) const CRC16_CCITT_KEYS_REFLECTED: [u64; 23] = [ +pub(super) const CRC16_CCITT_KEYS_REFLECTED: [u64; 23] = [ 0x0000000000000000, 0x00000000000189ae, 0x0000000000008e10, @@ -38,7 +38,7 @@ pub(crate) const CRC16_CCITT_KEYS_REFLECTED: [u64; 23] = [ /// Key schedule for CRC-16/IBM (ARC), reflected polynomial. #[rustfmt::skip] -pub(crate) const CRC16_IBM_KEYS_REFLECTED: [u64; 23] = [ +pub(super) const CRC16_IBM_KEYS_REFLECTED: [u64; 23] = [ 0x0000000000000000, 0x0000000000018cc2, 0x000000000001d0c2, @@ -71,41 +71,73 @@ pub(crate) const CRC16_IBM_KEYS_REFLECTED: [u64; 23] = [ /// - advance a stream by `N * 128B` per iteration (striping) /// - merge streams back together (combine coefficients) #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64/aarch64/power/s390x/riscv64 stream widths). -pub(crate) struct Width32StreamConstants { +pub(super) struct Width32StreamConstants { /// 2-way fold coefficient (256B = 2×128B). - pub fold_256b: (u64, u64), + pub(super) fold_256b: (u64, u64), /// 3-way fold coefficient (384B = 3×128B). - pub fold_384b: (u64, u64), + #[cfg(target_arch = "aarch64")] + pub(super) fold_384b: (u64, u64), /// 4-way fold coefficient (512B = 4×128B). - pub fold_512b: (u64, u64), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) fold_512b: (u64, u64), /// 7-way fold coefficient (896B = 7×128B). - pub fold_896b: (u64, u64), + #[cfg(target_arch = "x86_64")] + pub(super) fold_896b: (u64, u64), /// 8-way fold coefficient (1024B = 8×128B). - pub fold_1024b: (u64, u64), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) fold_1024b: (u64, u64), /// 4-way combine coefficients: shifts by 384B, 256B, 128B. - pub combine_4way: [(u64, u64); 3], + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) combine_4way: [(u64, u64); 3], /// 7-way combine coefficients: shifts by 768B, 640B, 512B, 384B, 256B, 128B. - pub combine_7way: [(u64, u64); 6], + #[cfg(target_arch = "x86_64")] + pub(super) combine_7way: [(u64, u64); 6], /// 8-way combine coefficients: shifts by 896B, 768B, 640B, 512B, 384B, 256B, 128B. - pub combine_8way: [(u64, u64); 7], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) combine_8way: [(u64, u64); 7], } impl Width32StreamConstants { /// Compute all multi-stream folding constants for a given reflected polynomial. #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + const fn new(reflected_poly: u32) -> Self { Self { fold_256b: fold16_coeff_for_bytes(reflected_poly, 256), + #[cfg(target_arch = "aarch64")] fold_384b: fold16_coeff_for_bytes(reflected_poly, 384), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] fold_512b: fold16_coeff_for_bytes(reflected_poly, 512), + #[cfg(target_arch = "x86_64")] fold_896b: fold16_coeff_for_bytes(reflected_poly, 896), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] fold_1024b: fold16_coeff_for_bytes(reflected_poly, 1024), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] combine_4way: [ fold16_coeff_for_bytes(reflected_poly, 384), fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_7way: [ fold16_coeff_for_bytes(reflected_poly, 768), fold16_coeff_for_bytes(reflected_poly, 640), @@ -114,6 +146,7 @@ impl Width32StreamConstants { fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] combine_8way: [ fold16_coeff_for_bytes(reflected_poly, 896), fold16_coeff_for_bytes(reflected_poly, 768), @@ -127,9 +160,9 @@ impl Width32StreamConstants { } } -pub(crate) const CRC16_CCITT_STREAM_REFLECTED: Width32StreamConstants = +pub(super) const CRC16_CCITT_STREAM_REFLECTED: Width32StreamConstants = Width32StreamConstants::new(CRC16_CCITT_POLY as u32); -pub(crate) const CRC16_IBM_STREAM_REFLECTED: Width32StreamConstants = +pub(super) const CRC16_IBM_STREAM_REFLECTED: Width32StreamConstants = Width32StreamConstants::new(CRC16_IBM_POLY as u32); // Constant Generation (compile-time) @@ -162,16 +195,16 @@ const fn reduce128(hi: u64, lo: u64, poly: u32) -> u32 { let poly_full: u128 = (1u128.strict_shl(32)) | (poly as u128); let mut val: u128 = (hi as u128).strict_shl(64) | (lo as u128); - let mut bit: i32 = 127; + let mut bit: u32 = 127; while bit >= 32 { - let b = bit as u32; - if ((val.strict_shr(b)) & 1) != 0 { - val ^= poly_full.strict_shl(b.strict_sub(32)); + if ((val.strict_shr(bit)) & 1) != 0 { + val ^= poly_full.strict_shl(bit.strict_sub(32)); } bit = bit.strict_sub(1); } - val as u32 + let [b0, b1, b2, b3, ..] = val.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } /// Compute x^n mod (x^width + poly) in GF(2) where `poly` is the normal CRC polynomial diff --git a/src/checksum/crc16/mod.rs b/src/checksum/crc16/mod.rs index cdf4af50..210d31a2 100644 --- a/src/checksum/crc16/mod.rs +++ b/src/checksum/crc16/mod.rs @@ -4,26 +4,18 @@ //! - [`Crc16Ccitt`] - CRC-16/X25 (also known as IBM-SDLC) //! - [`Crc16Ibm`] - CRC-16/ARC (also known as CRC-16/IBM) //! -//! # Quick Start -//! -//! ```rust -//! use rscrypto::checksum::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm}; -//! -//! let data = b"123456789"; -//! assert_eq!(Crc16Ccitt::checksum(data), 0x906E); -//! assert_eq!(Crc16Ibm::checksum(data), 0xBB3D); -//! -//! let (a, b) = data.split_at(4); -//! let combined = Crc16Ccitt::combine(Crc16Ccitt::checksum(a), Crc16Ccitt::checksum(b), b.len()); -//! assert_eq!(combined, Crc16Ccitt::checksum(data)); -//! ``` - pub(crate) mod config; pub(crate) mod kernels; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" +))] pub(crate) mod keys; pub(crate) mod portable; -#[allow(unused_imports)] pub use config::{Crc16Config, Crc16Force}; #[cfg(any(test, feature = "std"))] @@ -32,10 +24,6 @@ use crate::checksum::common::{ combine::{Gf2Matrix16, combine_crc16, generate_shift8_matrix_16}, tables::{CRC16_CCITT_POLY, CRC16_IBM_POLY, generate_crc16_tables_8}, }; -// Re-export traits for test modules (`use super::*`). -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; - #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "powerpc64")] @@ -52,9 +40,9 @@ mod x86_64; mod kernel_tables { use super::*; - pub static CCITT_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_CCITT_POLY); + pub(super) static CCITT_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_CCITT_POLY); - pub static IBM_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_IBM_POLY); + pub(super) static IBM_TABLES_8: [[u16; 256]; 8] = generate_crc16_tables_8(CRC16_IBM_POLY); } // Reference Kernel Wrappers @@ -404,6 +392,7 @@ impl crate::traits::ChecksumCombine for Crc16Ccitt { #[cfg(feature = "alloc")] impl Crc16Ccitt { + /// Creates a buffering wrapper that coalesces short updates before CRC-16/X25 dispatch. #[must_use] pub fn buffered() -> BufferedCrc16Ccitt { BufferedCrc16Ccitt::new() @@ -546,6 +535,7 @@ impl crate::traits::ChecksumCombine for Crc16Ibm { #[cfg(feature = "alloc")] impl Crc16Ibm { + /// Creates a buffering wrapper that coalesces short updates before CRC-16/ARC dispatch. #[must_use] pub fn buffered() -> BufferedCrc16Ibm { BufferedCrc16Ibm::new() @@ -611,6 +601,7 @@ mod tests { extern crate std; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; #[test] fn test_vectors_crc16_ccitt_x25() { @@ -694,6 +685,7 @@ mod cross_check { use super::*; use crate::checksum::common::tests::{STREAMING_CHUNK_SIZES, TEST_LENGTHS, generate_test_data}; + use crate::traits::{Checksum, ChecksumCombine}; // CRC-16/CCITT Cross-Check Tests diff --git a/src/checksum/crc16/portable.rs b/src/checksum/crc16/portable.rs index 650b3db5..1da6d283 100644 --- a/src/checksum/crc16/portable.rs +++ b/src/checksum/crc16/portable.rs @@ -5,13 +5,13 @@ use crate::checksum::common::portable; /// CRC-16/CCITT (X25 / IBM-SDLC) slice-by-8 computation. #[inline] -pub fn crc16_ccitt_slice8(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ccitt_slice8(crc: u16, data: &[u8]) -> u16 { portable::slice8_16(crc, data, &kernel_tables::CCITT_TABLES_8) } /// CRC-16/IBM (ARC) slice-by-8 computation. #[inline] -pub fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { portable::slice8_16(crc, data, &kernel_tables::IBM_TABLES_8) } @@ -21,7 +21,7 @@ pub fn crc16_ibm_slice8(crc: u16, data: &[u8]) -> u16 { /// /// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] -pub fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { crc16_bytewise(crc, data, &kernel_tables::CCITT_TABLES_8[0]) } @@ -29,13 +29,12 @@ pub fn crc16_ccitt_bytewise(crc: u16, data: &[u8]) -> u16 { /// /// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] -pub fn crc16_ibm_bytewise(crc: u16, data: &[u8]) -> u16 { +pub(in crate::checksum) fn crc16_ibm_bytewise(crc: u16, data: &[u8]) -> u16 { crc16_bytewise(crc, data, &kernel_tables::IBM_TABLES_8[0]) } /// Update CRC-16 state using a byte-at-a-time lookup table. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by mask, table is [u16; 256] fn crc16_bytewise(mut crc: u16, data: &[u8], table: &[u16; 256]) -> u16 { for &b in data { let index = ((crc ^ (b as u16)) & 0xFF) as usize; diff --git a/src/checksum/crc16/power.rs b/src/checksum/crc16/power.rs index 5e145fa5..aa8f0720 100644 --- a/src/checksum/crc16/power.rs +++ b/src/checksum/crc16/power.rs @@ -8,10 +8,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -19,6 +16,8 @@ use core::{ simd::i64x2, }; +use crate::checksum::common::{low_u16, low_u32}; + use super::keys::{ CRC16_CCITT_KEYS_REFLECTED, CRC16_CCITT_STREAM_REFLECTED, CRC16_IBM_KEYS_REFLECTED, CRC16_IBM_STREAM_REFLECTED, }; @@ -80,12 +79,13 @@ impl Simd { /// On `powerpc64le` this is a no-op. On big-endian `powerpc64`, we byte-swap /// each 64-bit lane so the folding algorithm sees the same lane values as on /// little-endian platforms. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { - // SAFETY: Caller guarantees: - // 1. ALTIVEC + VSX + POWER8-VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. #[cfg(target_endian = "little")] { self @@ -107,6 +107,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -132,6 +135,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -153,6 +159,10 @@ impl Simd { /// Fold 16 bytes (reflected width32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -170,6 +180,9 @@ impl Simd { unsafe { Self(Self::vpmsumd(self.0, coeff.swap_lanes().0)) } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -186,6 +199,10 @@ impl Simd { } /// Fold 16 bytes down to the "width32" reduction state (reflected mode). + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -216,6 +233,10 @@ impl Simd { } /// Barrett reduction for reflected width32; returns the updated CRC state. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -232,11 +253,14 @@ impl Simd { unsafe { let t1 = Self::mul64(self.low_64(), mu); let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 + low_u32((self ^ l).high_64()) } } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -264,6 +288,9 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -305,6 +332,9 @@ unsafe fn update_simd_width32_reflected(state: u32, first: &[Simd; 8], rest: &[[ } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -329,6 +359,9 @@ unsafe fn fold_block_128_reflected(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -354,6 +387,9 @@ unsafe fn normalize_block_le(mut block: [Simd; 8]) -> [Simd; 8] { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -416,6 +452,9 @@ unsafe fn update_simd_width32_reflected_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -444,7 +483,7 @@ unsafe fn update_simd_width32_reflected_4way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -505,6 +544,9 @@ unsafe fn update_simd_width32_reflected_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -533,7 +575,7 @@ unsafe fn update_simd_width32_reflected_8way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -642,6 +684,9 @@ unsafe fn update_simd_width32_reflected_8way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -662,11 +707,14 @@ unsafe fn crc16_width32_vpmsum(mut state: u16, data: &[u8], keys: &[u64; 23], po state = portable(state, left); let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -693,11 +741,14 @@ unsafe fn crc16_width32_vpmsum_2way( state = portable(state, left); let state32 = update_simd_width32_reflected_2way(state as u32, middle, fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -725,11 +776,14 @@ unsafe fn crc16_width32_vpmsum_4way( state = portable(state, left); let state32 = update_simd_width32_reflected_4way(state as u32, middle, fold_512b, combine, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -757,7 +811,7 @@ unsafe fn crc16_width32_vpmsum_8way( state = portable(state, left); let state32 = update_simd_width32_reflected_8way(state as u32, middle, fold_1024b, combine, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -766,11 +820,9 @@ unsafe fn crc16_width32_vpmsum_8way( /// CRC-16/CCITT VPMSUMD kernel. /// -/// # Safety -/// -/// Dispatcher verifies VPMSUMD before selecting this kernel. +/// Runtime dispatch selects this kernel only when VPMSUMD is available. #[inline] -pub fn crc16_ccitt_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum( @@ -783,7 +835,7 @@ pub fn crc16_ccitt_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_2way( @@ -797,7 +849,7 @@ pub fn crc16_ccitt_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_4way( @@ -812,7 +864,7 @@ pub fn crc16_ccitt_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_8way( @@ -828,17 +880,15 @@ pub fn crc16_ccitt_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPMSUMD kernel. /// -/// # Safety -/// -/// Dispatcher verifies VPMSUMD before selecting this kernel. +/// Runtime dispatch selects this kernel only when VPMSUMD is available. #[inline] -pub fn crc16_ibm_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } #[inline] -pub fn crc16_ibm_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_2way( @@ -852,7 +902,7 @@ pub fn crc16_ibm_vpmsum_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_4way( @@ -867,7 +917,7 @@ pub fn crc16_ibm_vpmsum_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpmsum_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. unsafe { crc16_width32_vpmsum_8way( diff --git a/src/checksum/crc16/riscv64.rs b/src/checksum/crc16/riscv64.rs index f042f18b..4c8430ab 100644 --- a/src/checksum/crc16/riscv64.rs +++ b/src/checksum/crc16/riscv64.rs @@ -8,17 +8,14 @@ //! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required //! CPU features are available before executing the accelerated path (the //! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitXor, BitXorAssign}, }; +use crate::checksum::common::{low_u16, low_u32}; + use super::keys::{ CRC16_CCITT_KEYS_REFLECTED, CRC16_CCITT_STREAM_REFLECTED, CRC16_IBM_KEYS_REFLECTED, CRC16_IBM_STREAM_REFLECTED, }; @@ -70,11 +67,11 @@ impl Simd { // Zbc carryless multiply primitives #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_lo(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -91,11 +88,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_hi(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -112,18 +109,14 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } + fn mul64(a: u64, b: u64) -> Self { + Self { + hi: Self::clmul_hi(a, b), + lo: Self::clmul_lo(a, b), } } } @@ -132,34 +125,20 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0usize; - while i < 8 { - let lo = u64::from_le(block[i * 2]); - let hi = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(hi, lo)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } #[inline] fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { let mut hi = [0u64; 8]; let mut lo = [0u64; 8]; - - let mut i = 0usize; - while i < 8 { - lo[i] = u64::from_le(block[i * 2]); - hi[i] = u64::from_le(block[i * 2 + 1]); - i = i.strict_add(1); + for ((hi_lane, lo_lane), &[low, high]) in hi.iter_mut().zip(&mut lo).zip(block.as_chunks::<2>().0) { + *hi_lane = u64::from_le(high); + *lo_lane = u64::from_le(low); } (hi, lo) @@ -168,277 +147,249 @@ fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { // ZBC (scalar carryless multiply) backend #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) - } +fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { data_to_xor ^ fold_16_zbc(x, coeff) } +fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { + data_to_xor ^ fold_16_zbc(x, coeff) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = Simd::mul64(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = Simd::mul64(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Simd::mul64(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Simd::mul64(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Simd::mul64(x.low_64(), mu); - let l = Simd::mul64(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = Simd::mul64(x.low_64(), mu); + let l = Simd::mul64(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut res = x[7]; - res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); - res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); - res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); - res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); - res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); - res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); - res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); - - barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); + res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); + res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); + res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); + res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); + res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); + res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); + + barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block(first); - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = (keys[4], keys[3]); - for block in rest { - let chunk = load_block(block); - x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); - } +fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block(first); + x[0] ^= Simd::new(0, state as u64); - finalize_lanes_width32_reflected_zbc(x, keys) + let coeff_128b = (keys[4], keys[3]); + for block in rest { + let chunk = load_block(block); + x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected_zbc(x, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let chunk = load_block(block); - x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); - } +fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { + let chunk = load_block(block); + x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } - - let even = blocks.len() & !1usize; - let coeff_256 = fold_256b; - let coeff_128 = (keys[4], keys[3]); + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); + let even = blocks.len() & !1usize; + let coeff_256 = fold_256b; + let coeff_128 = (keys[4], keys[3]); - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); - let mut i: usize = 2; - while i < even { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); - s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); - s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); - s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); - s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); - s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); - s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); - s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); - - if even != blocks.len() { - fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); - } + let mut i: usize = 2; + while i < even { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - finalize_lanes_width32_reflected_zbc(s1, keys) + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); + s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); + s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); + s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); + s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); + s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); + s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); + s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); + + if even != blocks.len() { + fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected_zbc(s1, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_4way( +fn update_simd_zbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } + debug_assert!(!blocks.is_empty()); - let aligned = blocks.len().strict_div(4).strict_mul(4); + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let coeff_512 = fold_512b; - let coeff_128 = (keys[4], keys[3]); - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; + let aligned = blocks.len().strict_div(4).strict_mul(4); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); + let coeff_512 = fold_512b; + let coeff_128 = (keys[4], keys[3]); + let c384 = combine[0]; + let c256 = combine[1]; + let c128 = combine[2]; - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); - s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); - s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); - s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); - s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); - s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); - s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); - s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); - - s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); - s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); - s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); - s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); - s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); - s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); - s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); - s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); - - s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); - s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); - s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); - s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); - s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); - s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); - s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); - s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected_zbc(&mut s3, block, coeff_128); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - finalize_lanes_width32_reflected_zbc(s3, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); + s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); + s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); + s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); + s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); + s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); + s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); + s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); + + s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); + s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); + s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); + s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); + s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); + s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); + s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); + s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); + + s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); + s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); + s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); + s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); + s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); + s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); + s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); + s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); + + for block in &blocks[aligned..] { + fold_block_128_reflected_zbc(&mut s3, block, coeff_128); } + + finalize_lanes_width32_reflected_zbc(s3, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc16_width32_zbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc16_width32_zbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -447,25 +398,25 @@ unsafe fn crc16_width32_zbc(mut state: u16, data: &[u8], keys: &[u64; 23], porta state = portable(state, left); let state32 = update_simd_zbc(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc16_width32_zbc_2way( +fn crc16_width32_zbc_2way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -473,25 +424,25 @@ unsafe fn crc16_width32_zbc_2way( } state = portable(state, left); let state32 = update_simd_zbc_2way(state as u32, middle, stream.fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc16_width32_zbc_4way( +fn crc16_width32_zbc_4way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -499,7 +450,7 @@ unsafe fn crc16_width32_zbc_4way( } state = portable(state, left); let state32 = update_simd_zbc_4way(state as u32, middle, stream.fold_512b, &stream.combine_4way, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -513,10 +464,7 @@ unsafe fn crc16_width32_zbc_4way( /// Requires RISC-V `v` + `zvbc`. #[inline] #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn mul64_zvbc(a: u64, b: u64) -> Simd { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -543,92 +491,68 @@ unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_reflected_zvbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC + V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { data_to_xor ^ fold_16_zvbc(x, coeff) } +fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC + V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = mul64_zvbc(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = mul64_zvbc(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = mul64_zvbc(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = mul64_zvbc(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu); - let l = mul64_zvbc(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = mul64_zvbc(x.low_64(), mu); + let l = mul64_zvbc(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); - - barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) - } +fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { + let mut acc = Simd::new(hi[7], lo[7]); + acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); + acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); + acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); + acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); + acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); + acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); + acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); + + barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( +fn fold_block_128_zvbc( x_hi: &mut [u64; 8], x_lo: &mut [u64; 8], chunk_hi: &[u64; 8], @@ -636,15 +560,12 @@ unsafe fn fold_block_128_zvbc( coeff_low: u64, coeff_high: u64, ) { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { let mut offset = 0usize; while offset < 8 { - let remaining = 8 - offset; + let remaining = 8usize.strict_sub(offset); let vl: usize; asm!( "vsetvli {vl}, {avl}, e64, m1, ta, ma", @@ -684,174 +605,162 @@ unsafe fn fold_block_128_zvbc( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split(first); - x_lo[0] ^= state as u64; +fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let (mut x_hi, mut x_lo) = load_block_split(first); + x_lo[0] ^= state as u64; - let coeff_low = keys[3]; - let coeff_high = keys[4]; + let coeff_low = keys[3]; + let coeff_high = keys[4]; - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } - - fold_tail_zvbc(x_hi, x_lo, keys) + for block in rest { + let (chunk_hi, chunk_lo) = load_block_split(block); + fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); } + + fold_tail_zvbc(x_hi, x_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } - - let even = blocks.len() & !1usize; + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; + let even = blocks.len() & !1usize; - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + let coeff_256_low = fold_256b.1; + let coeff_256_high = fold_256b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let mut i: usize = 2; - while i < even { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split(&blocks[even]); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + let mut i: usize = 2; + while i < even { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); + i = i.strict_add(2); + } - fold_tail_zvbc(s0_hi, s0_lo, keys) + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); + + if even != blocks.len() { + let (tail_hi, tail_lo) = load_block_split(&blocks[even]); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( +fn update_simd_zvbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; - - let mut i: usize = 4; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split(block); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512_low = fold_512b.1; + let coeff_512_high = fold_512b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; + + let c384_low = combine[0].1; + let c384_high = combine[0].0; + let c256_low = combine[1].1; + let c256_high = combine[1].0; + let c128_low = combine[2].1; + let c128_high = combine[2].0; + + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); + let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); + + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; + + let mut i: usize = 4; + while i < aligned { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); + let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); + i = i.strict_add(4); + } - fold_tail_zvbc(s0_hi, s0_lo, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); + + for block in &blocks[aligned..] { + let (tail_hi, tail_lo) = load_block_split(block); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc16_width32_zvbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc16_width32_zvbc(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -860,25 +769,25 @@ unsafe fn crc16_width32_zvbc(mut state: u16, data: &[u8], keys: &[u64; 23], port state = portable(state, left); let state32 = update_simd_zvbc(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc16_width32_zvbc_2way( +fn crc16_width32_zvbc_2way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -887,25 +796,25 @@ unsafe fn crc16_width32_zvbc_2way( state = portable(state, left); let state32 = update_simd_zvbc_2way(state as u32, middle, stream.fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc16_width32_zvbc_4way( +fn crc16_width32_zvbc_4way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -914,7 +823,7 @@ unsafe fn crc16_width32_zvbc_4way( state = portable(state, left); let state32 = update_simd_zvbc_4way(state as u32, middle, stream.fold_512b, &stream.combine_4way, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -922,12 +831,8 @@ unsafe fn crc16_width32_zvbc_4way( // Public Safe Kernels /// CRC-16/CCITT Zbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zbc before selecting this kernel. #[inline] -pub fn crc16_ccitt_zbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc( @@ -940,12 +845,8 @@ pub fn crc16_ccitt_zbc_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/CCITT Zvbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zvbc before selecting this kernel. #[inline] -pub fn crc16_ccitt_zvbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zvbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc( @@ -958,29 +859,21 @@ pub fn crc16_ccitt_zvbc_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM Zbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zbc before selecting this kernel. #[inline] -pub fn crc16_ibm_zbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM Zvbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zvbc before selecting this kernel. #[inline] -pub fn crc16_ibm_zvbc_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zvbc_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } #[inline] -pub fn crc16_ccitt_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_2way( @@ -994,7 +887,7 @@ pub fn crc16_ccitt_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_4way( @@ -1008,7 +901,7 @@ pub fn crc16_ccitt_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_2way( @@ -1022,7 +915,7 @@ pub fn crc16_ccitt_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_4way( @@ -1036,7 +929,7 @@ pub fn crc16_ccitt_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_2way( @@ -1050,7 +943,7 @@ pub fn crc16_ibm_zbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc16_width32_zbc_4way( @@ -1064,7 +957,7 @@ pub fn crc16_ibm_zbc_4way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_2way( @@ -1078,7 +971,7 @@ pub fn crc16_ibm_zvbc_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_zvbc_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. unsafe { crc16_width32_zvbc_4way( diff --git a/src/checksum/crc16/s390x.rs b/src/checksum/crc16/s390x.rs index fac679a1..8bc41a39 100644 --- a/src/checksum/crc16/s390x.rs +++ b/src/checksum/crc16/s390x.rs @@ -8,18 +8,15 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitAnd, BitXor, BitXorAssign}, simd::i64x2, }; +use crate::checksum::common::{low_u16, low_u32}; + use super::keys::{ CRC16_CCITT_KEYS_REFLECTED, CRC16_CCITT_STREAM_REFLECTED, CRC16_IBM_KEYS_REFLECTED, CRC16_IBM_STREAM_REFLECTED, }; @@ -79,8 +76,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees: // 1. VECTOR target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -98,76 +98,66 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } /// Fold 16 bytes (reflected width32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - // Like VPMSUMD: VGFM performs a per-lane carryless multiply and XORs the lane products. - // The folding primitive needs cross terms, so swap coefficient lanes. - Self(Self::vgfm(self.0, coeff.swap_lanes().0)) - } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { data_to_xor ^ self.fold_16(coeff) } + fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { + data_to_xor ^ self.fold_16(coeff) } /// Fold 16 bytes down to the "width32" reduction state (reflected mode). #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR + VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let clmul = Self::mul64(self.low_64(), low); - let shifted = Self::new(0, self.high_64()); - let mut state = clmul ^ shifted; - - let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); - let masked = state & mask2; - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Self::mul64(shifted_high, high); - state = clmul ^ masked; - - state - } + fn fold_width32_reflected(self, high: u64, low: u64) -> Self { + let clmul = Self::mul64(self.low_64(), low); + let shifted = Self::new(0, self.high_64()); + let mut state = clmul ^ shifted; + + let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); + let masked = state & mask2; + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Self::mul64(shifted_high, high); + state = clmul ^ masked; + + state } /// Barrett reduction for reflected width32; returns the updated CRC state. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let t1 = Self::mul64(self.low_64(), mu); - let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 - } + fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { + let t1 = Self::mul64(self.low_64(), mu); + let l = Self::mul64(t1.low_64(), poly); + low_u32((self ^ l).high_64()) } } @@ -175,239 +165,215 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut res = x[7]; - res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); - res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); - res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); - res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); - res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); - res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); - res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); - - res = res.fold_width32_reflected(keys[6], keys[5]); - res.barrett_width32_reflected(keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); + res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); + res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); + res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); + res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); + res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); + res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); + + res = res.fold_width32_reflected(keys[6], keys[5]); + res.barrett_width32_reflected(keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut x = load_block(first); - - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = Simd::new(keys[4], keys[3]); - for block in rest { - let chunk = load_block(block); - x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); - } +fn update_simd_width32_reflected(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block(first); - finalize_lanes_width32_reflected(x, keys) + x[0] ^= Simd::new(0, state as u64); + + let coeff_128b = Simd::new(keys[4], keys[3]); + for block in rest { + let chunk = load_block(block); + x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected(x, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128_reflected(x: &mut [Simd; 8], block: &Block, coeff: Simd) { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let chunk = load_block(block); - x[0] = x[0].fold_16_reflected(coeff, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff, chunk[7]); - } +fn fold_block_128_reflected(x: &mut [Simd; 8], block: &Block, coeff: Simd) { + let chunk = load_block(block); + x[0] = x[0].fold_16_reflected(coeff, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_2way( - state: u32, - blocks: &[Block], - fold_256b: (u64, u64), - keys: &[u64; 23], -) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_width32_reflected_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected(state, first, rest, keys); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected(state, first, rest, keys); + } - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 2; - let even = blocks.len() & !1usize; - while i < even { - fold_block_128_reflected(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } - - // Merge: A·s0 ⊕ s1. - let mut combined = s1; - combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); - combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); - combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); - combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); - combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); - combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); - combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); - combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); - - if even != blocks.len() { - fold_block_128_reflected(&mut combined, &blocks[even], coeff_128); - } + let mut i: usize = 2; + let even = blocks.len() & !1usize; + while i < even { + fold_block_128_reflected(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - finalize_lanes_width32_reflected(combined, keys) + // Merge: A·s0 ⊕ s1. + let mut combined = s1; + combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); + combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); + combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); + combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); + combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); + combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); + combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); + combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); + + if even != blocks.len() { + fold_block_128_reflected(&mut combined, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected(combined, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_4way( +fn update_simd_width32_reflected_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected(state, first, rest, keys); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected(state, first, rest, keys); + } - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); + let aligned = blocks.len().strict_div(4).strict_mul(4); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut acc = s3; - acc[0] = s2[0].fold_16_reflected(c128, acc[0]); - acc[1] = s2[1].fold_16_reflected(c128, acc[1]); - acc[2] = s2[2].fold_16_reflected(c128, acc[2]); - acc[3] = s2[3].fold_16_reflected(c128, acc[3]); - acc[4] = s2[4].fold_16_reflected(c128, acc[4]); - acc[5] = s2[5].fold_16_reflected(c128, acc[5]); - acc[6] = s2[6].fold_16_reflected(c128, acc[6]); - acc[7] = s2[7].fold_16_reflected(c128, acc[7]); - - acc[0] = s1[0].fold_16_reflected(c256, acc[0]); - acc[1] = s1[1].fold_16_reflected(c256, acc[1]); - acc[2] = s1[2].fold_16_reflected(c256, acc[2]); - acc[3] = s1[3].fold_16_reflected(c256, acc[3]); - acc[4] = s1[4].fold_16_reflected(c256, acc[4]); - acc[5] = s1[5].fold_16_reflected(c256, acc[5]); - acc[6] = s1[6].fold_16_reflected(c256, acc[6]); - acc[7] = s1[7].fold_16_reflected(c256, acc[7]); - - acc[0] = s0[0].fold_16_reflected(c384, acc[0]); - acc[1] = s0[1].fold_16_reflected(c384, acc[1]); - acc[2] = s0[2].fold_16_reflected(c384, acc[2]); - acc[3] = s0[3].fold_16_reflected(c384, acc[3]); - acc[4] = s0[4].fold_16_reflected(c384, acc[4]); - acc[5] = s0[5].fold_16_reflected(c384, acc[5]); - acc[6] = s0[6].fold_16_reflected(c384, acc[6]); - acc[7] = s0[7].fold_16_reflected(c384, acc[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected(&mut acc, block, coeff_128); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - finalize_lanes_width32_reflected(acc, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut acc = s3; + acc[0] = s2[0].fold_16_reflected(c128, acc[0]); + acc[1] = s2[1].fold_16_reflected(c128, acc[1]); + acc[2] = s2[2].fold_16_reflected(c128, acc[2]); + acc[3] = s2[3].fold_16_reflected(c128, acc[3]); + acc[4] = s2[4].fold_16_reflected(c128, acc[4]); + acc[5] = s2[5].fold_16_reflected(c128, acc[5]); + acc[6] = s2[6].fold_16_reflected(c128, acc[6]); + acc[7] = s2[7].fold_16_reflected(c128, acc[7]); + + acc[0] = s1[0].fold_16_reflected(c256, acc[0]); + acc[1] = s1[1].fold_16_reflected(c256, acc[1]); + acc[2] = s1[2].fold_16_reflected(c256, acc[2]); + acc[3] = s1[3].fold_16_reflected(c256, acc[3]); + acc[4] = s1[4].fold_16_reflected(c256, acc[4]); + acc[5] = s1[5].fold_16_reflected(c256, acc[5]); + acc[6] = s1[6].fold_16_reflected(c256, acc[6]); + acc[7] = s1[7].fold_16_reflected(c256, acc[7]); + + acc[0] = s0[0].fold_16_reflected(c384, acc[0]); + acc[1] = s0[1].fold_16_reflected(c384, acc[1]); + acc[2] = s0[2].fold_16_reflected(c384, acc[2]); + acc[3] = s0[3].fold_16_reflected(c384, acc[3]); + acc[4] = s0[4].fold_16_reflected(c384, acc[4]); + acc[5] = s0[5].fold_16_reflected(c384, acc[5]); + acc[6] = s0[6].fold_16_reflected(c384, acc[6]); + acc[7] = s0[7].fold_16_reflected(c384, acc[7]); + + for block in &blocks[aligned..] { + fold_block_128_reflected(&mut acc, block, coeff_128); } + + finalize_lanes_width32_reflected(acc, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc16_width32_vgfm(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc16_width32_vgfm(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -416,23 +382,25 @@ unsafe fn crc16_width32_vgfm(mut state: u16, data: &[u8], keys: &[u64; 23], port state = portable(state, left); let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc16_width32_vgfm_2way( +fn crc16_width32_vgfm_2way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -441,23 +409,25 @@ unsafe fn crc16_width32_vgfm_2way( state = portable(state, left); let state32 = update_simd_width32_reflected_2way(state as u32, middle, stream.fold_256b, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc16_width32_vgfm_4way( +fn crc16_width32_vgfm_4way( mut state: u16, data: &[u8], keys: &[u64; 23], stream: &super::keys::Width32StreamConstants, portable: fn(u16, &[u8]) -> u16, ) -> u16 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -467,7 +437,7 @@ unsafe fn crc16_width32_vgfm_4way( state = portable(state, left); let state32 = update_simd_width32_reflected_4way(state as u32, middle, stream.fold_512b, &stream.combine_4way, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } @@ -475,12 +445,8 @@ unsafe fn crc16_width32_vgfm_4way( // Public Safe Kernels /// CRC-16/CCITT VGFM kernel. -/// -/// # Safety -/// -/// Dispatcher verifies VECTOR facility before selecting this kernel. #[inline] -pub fn crc16_ccitt_vgfm_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vgfm_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm( @@ -493,7 +459,7 @@ pub fn crc16_ccitt_vgfm_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_2way( @@ -507,7 +473,7 @@ pub fn crc16_ccitt_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ccitt_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_4way( @@ -521,18 +487,14 @@ pub fn crc16_ccitt_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { } /// CRC-16/IBM VGFM kernel. -/// -/// # Safety -/// -/// Dispatcher verifies VECTOR facility before selecting this kernel. #[inline] -pub fn crc16_ibm_vgfm_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vgfm_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } #[inline] -pub fn crc16_ibm_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_2way( @@ -546,7 +508,7 @@ pub fn crc16_ibm_vgfm_2way_safe(crc: u16, data: &[u8]) -> u16 { } #[inline] -pub fn crc16_ibm_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vgfm_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc16_width32_vgfm_4way( diff --git a/src/checksum/crc16/x86_64.rs b/src/checksum/crc16/x86_64.rs index 95b270c7..6b7c3665 100644 --- a/src/checksum/crc16/x86_64.rs +++ b/src/checksum/crc16/x86_64.rs @@ -6,10 +6,9 @@ //! //! # Safety //! -//! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure SSSE3 + PCLMULQDQ -//! are available before executing these kernels (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! The baseline kernels require SSE2, SSSE3, and PCLMULQDQ. The wide kernels +//! additionally require AVX-512F/VL/BW/DQ and VPCLMULQDQ. The private safe +//! wrappers are installed only through capability-gated dispatcher tables. use core::{ arch::x86_64::*, @@ -25,6 +24,12 @@ use super::keys::{ #[derive(Copy, Clone)] struct Simd128(__m128i); +#[inline] +const fn low_u16(value: u32) -> u16 { + let [low0, low1, ..] = value.to_le_bytes(); + u16::from_le_bytes([low0, low1]) +} + impl BitXor for Simd128 { type Output = Self; @@ -43,44 +48,87 @@ impl BitXorAssign for Simd128 { } impl Simd128 { + /// Loads 16 bytes without requiring alignment. + /// + /// # Safety + /// + /// `ptr` must be valid to read 16 initialized bytes. The source may be + /// unaligned because the bytes are copied into aligned local storage. + #[inline] + unsafe fn load_unaligned(ptr: *const u8) -> Self { + let mut value = core::mem::MaybeUninit::::uninit(); + + // SAFETY: The caller guarantees a readable 16-byte source. `value` is an + // aligned, non-overlapping 16-byte destination, and every bit pattern is + // valid for the integer vector inside `Simd128`. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 16); + value.assume_init() + } + } + + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_right_8(self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_srli_si128::<8>(self.0)) } + /// Shifts the vector left by 12 bytes, filling the low bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_left_12(self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_slli_si128::<12>(self.0)) } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn and(self, mask: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. Self(_mm_and_si128(self.0, mask.0)) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. let h = _mm_clmulepi64_si128::<0x10>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x01>(self.0, coeff.0); Self(_mm_xor_si128(_mm_xor_si128(h, l), data_to_xor.0)) } - /// Fold 16 bytes down to the "width32" reduction state (reflected mode). + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -105,6 +153,11 @@ impl Simd128 { } } + /// Applies Barrett reduction and returns the low width-32 state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -116,11 +169,16 @@ impl Simd128 { let xorred = _mm_xor_si128(self.0, clmul2); let hi = _mm_srli_si128::<8>(xorred); - _mm_cvtsi128_si64(hi) as u32 + _mm_cvtsi128_si32(hi).cast_unsigned() } } } +/// Combines eight folded lanes and applies width-32 Barrett reduction. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> u32 { @@ -140,6 +198,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> } } +/// Folds one or more 128-byte blocks into a width-32 CRC state. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected( @@ -172,6 +235,11 @@ unsafe fn update_simd_width32_reflected( // PCLMULQDQ multi-stream (2/4/7/8-way, 128B blocks) +/// Folds one 128-byte block into eight parallel lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_block_128_reflected(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff: Simd128) { @@ -188,6 +256,11 @@ unsafe fn fold_block_128_reflected(x: &mut [Simd128; 8], chunk: &[Simd128; 8], c } } +/// Folds 128-byte blocks through two parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_2way( @@ -222,7 +295,7 @@ unsafe fn update_simd_width32_reflected_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -268,6 +341,11 @@ unsafe fn update_simd_width32_reflected_2way( } } +/// Folds 128-byte blocks through four parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_4way( @@ -308,7 +386,7 @@ unsafe fn update_simd_width32_reflected_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -332,7 +410,7 @@ unsafe fn update_simd_width32_reflected_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { fold_block_128_reflected(&mut s0, &blocks[i], coeff_512); fold_block_128_reflected(&mut s1, &blocks[i.strict_add(1)], coeff_512); @@ -378,6 +456,11 @@ unsafe fn update_simd_width32_reflected_4way( } } +/// Folds 128-byte blocks through seven parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_7way( @@ -400,7 +483,7 @@ unsafe fn update_simd_width32_reflected_7way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let coeff_896 = Simd128::new(fold_896b.0, fold_896b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -504,6 +587,11 @@ unsafe fn update_simd_width32_reflected_7way( } } +/// Folds 128-byte blocks through eight parallel PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_8way( @@ -526,7 +614,7 @@ unsafe fn update_simd_width32_reflected_8way( return update_simd_width32_reflected(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let coeff_1024 = Simd128::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -642,6 +730,11 @@ unsafe fn update_simd_width32_reflected_8way( } } +/// Updates a CRC-16 value with a selected multi-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc16_width32_pclmul_stream( @@ -668,11 +761,16 @@ unsafe fn crc16_width32_pclmul_stream( 2 => update_simd_width32_reflected_2way(state as u32, middle, stream.fold_256b, keys), _ => update_simd_width32_reflected(state as u32, first, rest, keys), }; - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// Updates a CRC-16 value with the single-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn crc16_width32_pclmul_small( @@ -694,26 +792,31 @@ unsafe fn crc16_width32_pclmul_small( let coeff_16b = Simd128::new(keys[2], keys[1]); - let mut x0 = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let mut x0 = Simd128::load_unaligned(buf); x0 ^= Simd128::new(0, state as u64); buf = buf.add(16); len = len.strict_sub(16); while len >= 16 { - let chunk = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let chunk = Simd128::load_unaligned(buf); x0 = x0.fold_16_reflected(coeff_16b, chunk); buf = buf.add(16); len = len.strict_sub(16); } let x0 = x0.fold_width32_reflected(keys[6], keys[5]); - state = x0.barrett_width32_reflected(keys[8], keys[7]) as u16; + state = low_u16(x0.barrett_width32_reflected(keys[8], keys[7])); let tail = core::slice::from_raw_parts(buf, len); portable(state, tail) } } +/// Updates a CRC-16 value with the baseline PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc16_width32_pclmul(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { @@ -727,29 +830,74 @@ unsafe fn crc16_width32_pclmul(mut state: u16, data: &[u8], keys: &[u64; 23], po state = portable(state, left); let state32 = update_simd_width32_reflected(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } // AVX-512 VPCLMULQDQ Tier +/// Loads 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 64 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_512(ptr: *const u8) -> __m512i { + let mut value = core::mem::MaybeUninit::<__m512i>::uninit(); + + // SAFETY: The caller guarantees a readable 64-byte source. `value` is an + // aligned, non-overlapping 64-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 64); + value.assume_init() + } +} + +/// Stores 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 64 bytes and must not overlap `value`. +#[inline] +unsafe fn store_unaligned_512(ptr: *mut u8, value: __m512i) { + // SAFETY: The caller guarantees a writable, non-overlapping 64-byte + // destination. `value` provides exactly 64 initialized source bytes. + unsafe { core::ptr::copy_nonoverlapping(core::ptr::from_ref(&value).cast::(), ptr, 64) } +} + +/// Multiplies the high lane of each 128-bit element in `a` by the low lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul10_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x10) } +/// Multiplies the low lane of each 128-bit element in `a` by the high lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul01_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x01) } +/// Folds four reflected 16-byte lanes and XORs their supplied input lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512i) -> __m512i { @@ -758,11 +906,14 @@ unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512 unsafe { _mm512_ternarylogic_epi64(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff), data, 0x96) } } +/// Broadcasts a pair of 64-bit coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_set_epi64( high.cast_signed(), low.cast_signed(), @@ -775,14 +926,22 @@ unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { ) } +/// Places the width-32 CRC state in the low 32 bits of lane zero. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn state_mask_lane0(state: u32) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, state as i64) } +/// Folds one or more 128-byte blocks into a width-32 CRC state with VPCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul( @@ -795,25 +954,25 @@ unsafe fn update_simd_width32_reflected_vpclmul( // attribute. Pointer arithmetic: first has 8 Simd128 = 128 bytes, so ptr.add(64) is within // bounds. Each chunk in rest is likewise 128 bytes. unsafe { - let ptr = first.as_ptr() as *const u8; - let mut x0 = _mm512_loadu_si512(ptr as *const __m512i); - let mut x1 = _mm512_loadu_si512(ptr.add(64) as *const __m512i); + let ptr = first.as_ptr().cast::(); + let mut x0 = load_unaligned_512(ptr); + let mut x1 = load_unaligned_512(ptr.add(64)); x0 = _mm512_xor_si512(x0, state_mask_lane0(state)); let coeff_128b = broadcast_coeff_128b(keys[4], keys[3]); for chunk in rest { - let ptr = chunk.as_ptr() as *const u8; - let y0 = _mm512_loadu_si512(ptr as *const __m512i); - let y1 = _mm512_loadu_si512(ptr.add(64) as *const __m512i); + let ptr = chunk.as_ptr().cast::(); + let y0 = load_unaligned_512(ptr); + let y1 = load_unaligned_512(ptr.add(64)); x0 = fold_16_reflected_vpclmul(x0, coeff_128b, y0); x1 = fold_16_reflected_vpclmul(x1, coeff_128b, y1); } let mut lanes0 = [Simd128(_mm_setzero_si128()); 4]; let mut lanes1 = [Simd128(_mm_setzero_si128()); 4]; - _mm512_storeu_si512(lanes0.as_mut_ptr() as *mut __m512i, x0); - _mm512_storeu_si512(lanes1.as_mut_ptr() as *mut __m512i, x1); + store_unaligned_512(lanes0.as_mut_ptr().cast::(), x0); + store_unaligned_512(lanes1.as_mut_ptr().cast::(), x1); let x = [ lanes0[0], lanes0[1], lanes0[2], lanes0[3], lanes1[0], lanes1[1], lanes1[2], lanes1[3], @@ -823,6 +982,11 @@ unsafe fn update_simd_width32_reflected_vpclmul( } } +/// Broadcasts one pair of folding coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { @@ -831,20 +995,27 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { unsafe { broadcast_coeff_128b(pair.0, pair.1) } } +/// Loads a 128-byte block as two unaligned 512-bit vectors. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn load_128b_block(block: &[Simd128; 8]) -> (__m512i, __m512i) { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] // attribute. block has 8 Simd128 = 128 bytes, so ptr.add(64) is within bounds. unsafe { - let ptr = block.as_ptr() as *const u8; - ( - _mm512_loadu_si512(ptr as *const __m512i), - _mm512_loadu_si512(ptr.add(64) as *const __m512i), - ) + let ptr = block.as_ptr().cast::(); + (load_unaligned_512(ptr), load_unaligned_512(ptr.add(64))) } } +/// Combines two four-lane VPCLMULQDQ states and applies width-32 reduction. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, SSE2, and PCLMULQDQ. #[inline] #[target_feature(enable = "avx512f")] unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> u32 { @@ -871,6 +1042,11 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> } } +/// Folds 128-byte blocks through two parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_2way( @@ -906,7 +1082,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -962,6 +1138,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_2way( } } +/// Folds 128-byte blocks through four parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_4way( @@ -1003,7 +1184,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -1049,7 +1230,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { let (y0, y1) = load_128b_block(&blocks[i]); x0_0 = fold_16_reflected_vpclmul(x0_0, coeff_512, y0); @@ -1087,6 +1268,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_4way( } } +/// Folds 128-byte blocks through seven parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_7way( @@ -1110,7 +1296,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_7way( return update_simd_width32_reflected_vpclmul(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let (mut x0_0, mut x1_0) = load_128b_block(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block(&blocks[1]); @@ -1194,6 +1380,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_7way( } } +/// Folds 128-byte blocks through eight parallel VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_8way( @@ -1217,7 +1408,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_8way( return update_simd_width32_reflected_vpclmul(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let (mut x0_0, mut x1_0) = load_128b_block(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block(&blocks[1]); @@ -1309,6 +1500,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_8way( } } +/// Updates a CRC-16 value with a selected multi-stream VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc16_width32_vpclmul_stream( @@ -1341,11 +1537,16 @@ unsafe fn crc16_width32_vpclmul_stream( 2 => update_simd_width32_reflected_vpclmul_2way(state as u32, middle, stream.fold_256b, keys), _ => update_simd_width32_reflected_vpclmul(state as u32, first, rest, keys), }; - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } +/// Updates a CRC-16 value with the baseline VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc16_width32_vpclmul(mut state: u16, data: &[u8], keys: &[u64; 23], portable: fn(u16, &[u8]) -> u16) -> u16 { @@ -1359,20 +1560,19 @@ unsafe fn crc16_width32_vpclmul(mut state: u16, data: &[u8], keys: &[u64; 23], p state = portable(state, left); let state32 = update_simd_width32_reflected_vpclmul(state as u32, first, rest, keys); - state = state32 as u16; + state = low_u16(state32); portable(state, right) } } -// Public Safe Kernels (matching CRC-64 pure fn(u16, &[u8]) -> u16 signature) +// Private safe kernel adapters matching the dispatcher's function signature. /// CRC-16/CCITT PCLMULQDQ kernel. /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ccitt_pclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul( @@ -1388,11 +1588,10 @@ pub fn crc16_ccitt_pclmul_safe(crc: u16, data: &[u8]) -> u16 { /// /// Optimized for inputs smaller than a folding block (128 bytes). /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ccitt_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_small( @@ -1406,7 +1605,7 @@ pub fn crc16_ccitt_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1422,7 +1621,7 @@ pub fn crc16_ccitt_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1438,7 +1637,7 @@ pub fn crc16_ccitt_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1454,7 +1653,7 @@ pub fn crc16_ccitt_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT PCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ccitt_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1470,11 +1669,10 @@ pub fn crc16_ccitt_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (AVX-512). /// -/// # Safety -/// -/// Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying VPCLMULQDQ +/// and AVX-512 support. #[inline] -pub fn crc16_ccitt_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul( @@ -1488,7 +1686,7 @@ pub fn crc16_ccitt_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1504,7 +1702,7 @@ pub fn crc16_ccitt_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1520,7 +1718,7 @@ pub fn crc16_ccitt_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1536,7 +1734,7 @@ pub fn crc16_ccitt_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/CCITT VPCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ccitt_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ccitt_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1552,11 +1750,10 @@ pub fn crc16_ccitt_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel. /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ibm_pclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } @@ -1565,18 +1762,17 @@ pub fn crc16_ibm_pclmul_safe(crc: u16, data: &[u8]) -> u16 { /// /// Optimized for inputs smaller than a folding block (128 bytes). /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc16_ibm_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_small_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_small(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM PCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1592,7 +1788,7 @@ pub fn crc16_ibm_pclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1608,7 +1804,7 @@ pub fn crc16_ibm_pclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1624,7 +1820,7 @@ pub fn crc16_ibm_pclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM PCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ibm_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc16_width32_pclmul_stream( @@ -1640,18 +1836,17 @@ pub fn crc16_ibm_pclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (AVX-512). /// -/// # Safety -/// -/// Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying VPCLMULQDQ +/// and AVX-512 support. #[inline] -pub fn crc16_ibm_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul(crc, data, &CRC16_IBM_KEYS_REFLECTED, super::portable::crc16_ibm_slice8) } } /// CRC-16/IBM VPCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1667,7 +1862,7 @@ pub fn crc16_ibm_vpclmul_2way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1683,7 +1878,7 @@ pub fn crc16_ibm_vpclmul_4way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1699,7 +1894,7 @@ pub fn crc16_ibm_vpclmul_7way_safe(crc: u16, data: &[u8]) -> u16 { /// CRC-16/IBM VPCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc16_ibm_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { +pub(super) fn crc16_ibm_vpclmul_8way_safe(crc: u16, data: &[u8]) -> u16 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc16_width32_vpclmul_stream( @@ -1725,14 +1920,21 @@ mod tests { const OFFSETS: &[usize] = &[0, 1, 7, 15]; const STATES: &[u16] = &[0, 0x1d0f, 0xa5a5, u16::MAX]; + fn data() -> Vec { + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(59).wrapping_add(high) + }) + .collect() + } + fn assert_kernel(name: &str, kernel: fn(u16, &[u8]) -> u16, portable: fn(u16, &[u8]) -> u16) { - let input: Vec = (0..4111) - .map(|i| (i as u8).wrapping_mul(59).wrapping_add((i >> 8) as u8)) - .collect(); + let input = data(); for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), portable(state, slice), diff --git a/src/checksum/crc24/aarch64.rs b/src/checksum/crc24/aarch64.rs index 294e96a7..360923a9 100644 --- a/src/checksum/crc24/aarch64.rs +++ b/src/checksum/crc24/aarch64.rs @@ -6,10 +6,8 @@ //! //! # Safety //! -//! Uses `unsafe` for ARM SIMD intrinsics. Callers must ensure PMULL is -//! available before executing these kernels (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! Uses `unsafe` for ARM SIMD intrinsics. Callers must establish NEON and AES +//! (PMULL) support before executing the accelerated kernels. use core::{ arch::aarch64::*, @@ -45,33 +43,57 @@ impl BitXorAssign for Simd { } impl Simd { + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn new(high: u64, low: u64) -> Self { Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high))) } + /// Loads 16 bytes from `ptr` without requiring alignment. + /// + /// # Safety + /// + /// The current CPU must support NEON, and `ptr` must address at least 16 initialized readable + /// bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn load(ptr: *const u8) -> Self { - // SAFETY: Caller guarantees: - // 1. NEON target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + // SAFETY: The caller guarantees NEON support and 16 initialized readable bytes at `ptr`. unsafe { Self(vld1q_u8(ptr)) } } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn and(self, mask: Self) -> Self { Self(vandq_u8(self.0, mask.0)) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_right_8(self) -> Self { Self(vextq_u8(self.0, vdupq_n_u8(0), 8)) } + /// Moves the low 32-bit lane to the high 32-bit lane and clears the rest. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn shift_left_12(self) -> Self { @@ -80,13 +102,22 @@ impl Simd { Self(vreinterpretq_u8_u32(result)) } - /// Reverse bits within each byte (u8::reverse_bits), lane-wise. + /// Reverses the bits within each byte lane. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn bitrev_bytes(self) -> Self { Self(vrbitq_u8(self.0)) } + /// Multiplies the low polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul00(self, other: Self) -> Self { @@ -95,6 +126,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's high polynomial lane by the other vector's low lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul01(self, other: Self) -> Self { @@ -103,6 +139,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies this vector's low polynomial lane by the other vector's high lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul10(self, other: Self) -> Self { @@ -111,6 +152,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Multiplies the high polynomial lanes. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul11(self, other: Self) -> Self { @@ -119,6 +165,11 @@ impl Simd { Self(vreinterpretq_u8_p128(vmull_p64(a, b))) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { @@ -132,6 +183,11 @@ impl Simd { } } + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -156,6 +212,11 @@ impl Simd { } } + /// Applies Barrett reduction and returns the low CRC-24 state in a `u32`. + /// + /// # Safety + /// + /// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -176,6 +237,11 @@ impl Simd { // 8-lane width32 update (128B blocks) +/// Reduces eight folded SIMD lanes to a reflected CRC-24 state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { @@ -197,6 +263,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// Reverses the bits within every byte of a 128-byte block. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn bitrev_block(block: &[Simd; 8]) -> [Simd; 8] { @@ -217,6 +288,11 @@ unsafe fn bitrev_block(block: &[Simd; 8]) -> [Simd; 8] { } } +/// Bit-reverses and folds one 128-byte block into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_block_128_width32_reflected_bitrev_bytes(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { @@ -244,6 +320,11 @@ unsafe fn fold_block_128_width32_reflected_bitrev_bytes(x: &mut [Simd; 8], chunk } } +/// Bit-reverses and folds 128-byte blocks through two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( state: u32, @@ -274,7 +355,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -321,6 +402,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( } } +/// Bit-reverses and folds 128-byte blocks through three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( state: u32, @@ -359,7 +445,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B let mut i: usize = 3; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -381,7 +467,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( } // Handle remaining triplets. - let triple_aligned = (blocks.len() / 3) * 3; + let triple_aligned = blocks.len().strict_div(3).strict_mul(3); while i < triple_aligned { fold_block_128_width32_reflected_bitrev_bytes(&mut s0, &blocks[i], coeff_384b); fold_block_128_width32_reflected_bitrev_bytes(&mut s1, &blocks[i.strict_add(1)], coeff_384b); @@ -419,6 +505,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_3way( } } +/// Bit-reverses and folds a reflected sequence of 128-byte blocks. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_width32_reflected_bitrev_bytes( @@ -471,6 +562,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes( // Single-stream kernel entry points +/// Computes reflected CRC-24 with single-lane PMULL folding for small buffers. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull_small(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -508,6 +604,11 @@ unsafe fn crc24_width32_pmull_small(mut state: u32, data: &[u8], keys: &[u64; 23 } } +/// Computes reflected CRC-24 with PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -527,6 +628,11 @@ unsafe fn crc24_width32_pmull(mut state: u32, data: &[u8], keys: &[u64; 23]) -> } } +/// Computes reflected CRC-24 with two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull_2way(mut state: u32, data: &[u8], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { @@ -549,6 +655,11 @@ unsafe fn crc24_width32_pmull_2way(mut state: u32, data: &[u8], fold_256b: (u64, } } +/// Computes reflected CRC-24 with three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support AES (PMULL) and NEON. #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc24_width32_pmull_3way( @@ -573,17 +684,13 @@ unsafe fn crc24_width32_pmull_3way( } } -// Public Safe Kernel +// Safe kernel wrappers. /// CRC-24/OPENPGP PMULL kernel. -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } @@ -591,27 +698,19 @@ pub fn crc24_openpgp_pmull_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PMULL small-buffer kernel. /// /// Optimized for inputs smaller than a folding block (128 bytes). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull_small(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; from_reflected_state(state) } /// CRC-24/OPENPGP PMULL kernel (2-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull_2way( state, @@ -624,14 +723,10 @@ pub fn crc24_openpgp_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { } /// CRC-24/OPENPGP PMULL kernel (3-way striping). -/// -/// # Safety -/// -/// Dispatcher verifies PMULL before selecting this kernel. #[inline] -pub fn crc24_openpgp_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); - // SAFETY: Dispatcher verifies PMULL before selecting this kernel. + // SAFETY: All callers establish PMULL support before invoking this private wrapper. state = unsafe { crc24_width32_pmull_3way( state, @@ -657,8 +752,11 @@ mod tests { const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; fn data() -> Vec { - (0..4111) - .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(31).wrapping_add(high) + }) .collect() } @@ -678,7 +776,7 @@ mod tests { for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), super::super::portable::crc24_openpgp_slice8(state, slice), diff --git a/src/checksum/crc24/config.rs b/src/checksum/crc24/config.rs index ac14fdbe..462048d2 100644 --- a/src/checksum/crc24/config.rs +++ b/src/checksum/crc24/config.rs @@ -82,8 +82,16 @@ fn parse_force_env() -> Crc24Force { #[inline] #[must_use] -#[allow(unused_variables)] fn clamp_force_to_caps(requested: Crc24Force, caps: Caps) -> Crc24Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc24Force::Auto | Crc24Force::Reference | Crc24Force::Portable => requested, Crc24Force::Clmul => { @@ -158,7 +166,7 @@ fn config(caps: Caps) -> Crc24Config { /// platform capabilities. #[inline] #[must_use] -pub fn get() -> Crc24Config { +pub(super) fn get() -> Crc24Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc24/kernels.rs b/src/checksum/crc24/kernels.rs index 12c4700a..6d7e022e 100644 --- a/src/checksum/crc24/kernels.rs +++ b/src/checksum/crc24/kernels.rs @@ -24,22 +24,22 @@ //! state back to the MSB-first OpenPGP representation. /// Reference (bitwise) kernel name. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; /// Portable slice-by-8 kernel name. -pub const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; +pub(in crate::checksum) const PORTABLE_SLICE8: &str = kernels::PORTABLE_SLICE8; // Kernel Name Tables and Functions (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP PCLMUL kernel. - pub const OPENPGP_PCLMUL: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_PCLMUL: [Crc24Fn; 5] = [ arch::crc24_openpgp_pclmul_safe, arch::crc24_openpgp_pclmul_2way_safe, arch::crc24_openpgp_pclmul_4way_safe, @@ -48,10 +48,10 @@ pub mod x86_64 { ]; /// OpenPGP PCLMUL small-buffer kernel. - pub const OPENPGP_PCLMUL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pclmul_small_safe; + pub(in crate::checksum) const OPENPGP_PCLMUL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pclmul_small_safe; /// OpenPGP VPCLMUL kernel. - pub const OPENPGP_VPCLMUL: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_VPCLMUL: [Crc24Fn; 5] = [ arch::crc24_openpgp_vpclmul_safe, arch::crc24_openpgp_vpclmul_2way_safe, arch::crc24_openpgp_vpclmul_4way_safe, @@ -61,12 +61,12 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)]. - pub const OPENPGP_PMULL: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_PMULL: [Crc24Fn; 5] = [ arch::crc24_openpgp_pmull_safe, arch::crc24_openpgp_pmull_2way_safe, arch::crc24_openpgp_pmull_3way_safe, @@ -75,16 +75,16 @@ pub mod aarch64 { ]; /// OpenPGP PMULL small-buffer kernel. - pub const OPENPGP_PMULL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pmull_small_safe; + pub(in crate::checksum) const OPENPGP_PMULL_SMALL_KERNEL: Crc24Fn = arch::crc24_openpgp_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const OPENPGP_VPMSUM: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_VPMSUM: [Crc24Fn; 5] = [ arch::crc24_openpgp_vpmsum_safe, arch::crc24_openpgp_vpmsum_2way_safe, arch::crc24_openpgp_vpmsum_4way_safe, @@ -94,12 +94,12 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc24Fn; /// OpenPGP VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const OPENPGP_VGFM: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_VGFM: [Crc24Fn; 5] = [ arch::crc24_openpgp_vgfm_safe, arch::crc24_openpgp_vgfm_2way_safe, arch::crc24_openpgp_vgfm_4way_safe, @@ -109,31 +109,12 @@ pub mod s390x { } #[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { +pub(in crate::checksum) mod riscv64 { use super::super::riscv64 as arch; use crate::checksum::dispatchers::Crc24Fn; - /// Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", // dup for index consistency - "riscv64/zbc-4way", // dup for index consistency - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", // dup for index consistency - "riscv64/zvbc-4way", // dup for index consistency - ]; - /// OpenPGP Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const OPENPGP_ZBC: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_ZBC: [Crc24Fn; 5] = [ arch::crc24_openpgp_zbc_safe, arch::crc24_openpgp_zbc_2way_safe, arch::crc24_openpgp_zbc_4way_safe, @@ -142,7 +123,7 @@ pub mod riscv64 { ]; /// OpenPGP Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const OPENPGP_ZVBC: [Crc24Fn; 5] = [ + pub(in crate::checksum) const OPENPGP_ZVBC: [Crc24Fn; 5] = [ arch::crc24_openpgp_zvbc_safe, arch::crc24_openpgp_zvbc_2way_safe, arch::crc24_openpgp_zvbc_4way_safe, diff --git a/src/checksum/crc24/keys.rs b/src/checksum/crc24/keys.rs index 5587d653..0055f527 100644 --- a/src/checksum/crc24/keys.rs +++ b/src/checksum/crc24/keys.rs @@ -21,43 +21,75 @@ use crate::checksum::common::tables::CRC24_OPENPGP_POLY; /// CRC-24/OPENPGP polynomial in reflected CRC-32 form (LSB-first). /// /// This is `reverse_bits(poly32)` where `poly32 = CRC24_OPENPGP_POLY << 8`. -pub(crate) const CRC24_OPENPGP_POLY_REFLECTED: u32 = (CRC24_OPENPGP_POLY << 8).reverse_bits(); +pub(super) const CRC24_OPENPGP_POLY_REFLECTED: u32 = (CRC24_OPENPGP_POLY << 8).reverse_bits(); /// Key schedule for CRC-24/OPENPGP in the width32 folding strategy. #[rustfmt::skip] -pub(crate) const CRC24_OPENPGP_KEYS_REFLECTED: [u64; 23] = build_keys(CRC24_OPENPGP_POLY_REFLECTED); +pub(super) const CRC24_OPENPGP_KEYS_REFLECTED: [u64; 23] = build_keys(CRC24_OPENPGP_POLY_REFLECTED); /// Multi-stream folding constants for CRC-24/OPENPGP in the width32 strategy. /// /// These constants enable multi-way striping (2-way / 3-way / 4-way / 7-way / 8-way) /// for carryless-multiply kernels. #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64/aarch64/power/s390x/riscv64 stream widths). -pub(crate) struct Crc24StreamConstants { - pub fold_256b: (u64, u64), - pub fold_384b: (u64, u64), - pub fold_512b: (u64, u64), - pub fold_896b: (u64, u64), - pub fold_1024b: (u64, u64), - pub combine_4way: [(u64, u64); 3], - pub combine_7way: [(u64, u64); 6], - pub combine_8way: [(u64, u64); 7], +pub(super) struct Crc24StreamConstants { + pub(super) fold_256b: (u64, u64), + #[cfg(target_arch = "aarch64")] + pub(super) fold_384b: (u64, u64), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) fold_512b: (u64, u64), + #[cfg(target_arch = "x86_64")] + pub(super) fold_896b: (u64, u64), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) fold_1024b: (u64, u64), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] + pub(super) combine_4way: [(u64, u64); 3], + #[cfg(target_arch = "x86_64")] + pub(super) combine_7way: [(u64, u64); 6], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] + pub(super) combine_8way: [(u64, u64); 7], } impl Crc24StreamConstants { #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + const fn new(reflected_poly: u32) -> Self { Self { fold_256b: fold16_coeff_for_bytes(reflected_poly, 256), + #[cfg(target_arch = "aarch64")] fold_384b: fold16_coeff_for_bytes(reflected_poly, 384), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] fold_512b: fold16_coeff_for_bytes(reflected_poly, 512), + #[cfg(target_arch = "x86_64")] fold_896b: fold16_coeff_for_bytes(reflected_poly, 896), + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] fold_1024b: fold16_coeff_for_bytes(reflected_poly, 1024), + #[cfg(any( + target_arch = "x86_64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "riscv64" + ))] combine_4way: [ fold16_coeff_for_bytes(reflected_poly, 384), fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(target_arch = "x86_64")] combine_7way: [ fold16_coeff_for_bytes(reflected_poly, 768), fold16_coeff_for_bytes(reflected_poly, 640), @@ -66,6 +98,7 @@ impl Crc24StreamConstants { fold16_coeff_for_bytes(reflected_poly, 256), fold16_coeff_for_bytes(reflected_poly, 128), ], + #[cfg(any(target_arch = "x86_64", target_arch = "powerpc64"))] combine_8way: [ fold16_coeff_for_bytes(reflected_poly, 896), fold16_coeff_for_bytes(reflected_poly, 768), @@ -79,7 +112,7 @@ impl Crc24StreamConstants { } } -pub(crate) const CRC24_OPENPGP_STREAM_REFLECTED: Crc24StreamConstants = +pub(super) const CRC24_OPENPGP_STREAM_REFLECTED: Crc24StreamConstants = Crc24StreamConstants::new(CRC24_OPENPGP_POLY_REFLECTED); // Constant Generation (compile-time) @@ -112,16 +145,16 @@ const fn reduce128(hi: u64, lo: u64, poly: u32) -> u32 { let poly_full: u128 = (1u128.strict_shl(32)) | (poly as u128); let mut val: u128 = (hi as u128).strict_shl(64) | (lo as u128); - let mut bit: i32 = 127; + let mut bit: u32 = 127; while bit >= 32 { - let b = bit as u32; - if ((val.strict_shr(b)) & 1) != 0 { - val ^= poly_full.strict_shl(b.strict_sub(32)); + if ((val.strict_shr(bit)) & 1) != 0 { + val ^= poly_full.strict_shl(bit.strict_sub(32)); } bit = bit.strict_sub(1); } - val as u32 + let [b0, b1, b2, b3, ..] = val.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } /// Compute x^n mod (x^width + poly) in GF(2) where `poly` is the normal CRC polynomial diff --git a/src/checksum/crc24/mod.rs b/src/checksum/crc24/mod.rs index 6dc423bd..40c94eff 100644 --- a/src/checksum/crc24/mod.rs +++ b/src/checksum/crc24/mod.rs @@ -3,25 +3,15 @@ //! This module provides: //! - [`Crc24OpenPgp`] - CRC-24/OPENPGP (RFC 4880) //! -//! # Quick Start -//! -//! ```rust -//! use rscrypto::checksum::{Checksum, ChecksumCombine, Crc24OpenPgp}; -//! -//! let data = b"123456789"; -//! assert_eq!(Crc24OpenPgp::checksum(data), 0x21CF02); -//! -//! let (a, b) = data.split_at(4); -//! let combined = Crc24OpenPgp::combine( -//! Crc24OpenPgp::checksum(a), -//! Crc24OpenPgp::checksum(b), -//! b.len(), -//! ); -//! assert_eq!(combined, Crc24OpenPgp::checksum(data)); -//! ``` - pub(crate) mod config; pub(crate) mod kernels; +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" +))] pub(crate) mod keys; pub(crate) mod portable; #[cfg(any( @@ -33,7 +23,6 @@ pub(crate) mod portable; ))] mod reflected; -#[allow(unused_imports)] pub use config::{Crc24Config, Crc24Force}; #[cfg(any(test, feature = "std"))] @@ -42,10 +31,6 @@ use crate::checksum::common::{ combine::{Gf2Matrix24, combine_crc24, generate_shift8_matrix_24}, tables::{CRC24_OPENPGP_POLY, generate_crc24_tables_8}, }; -// Re-export traits for test modules (`use super::*`). -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; - #[cfg(target_arch = "aarch64")] mod aarch64; #[cfg(target_arch = "powerpc64")] @@ -61,7 +46,7 @@ mod x86_64; mod kernel_tables { use super::*; - pub static OPENPGP_TABLES_8: [[u32; 256]; 8] = generate_crc24_tables_8(CRC24_OPENPGP_POLY); + pub(super) static OPENPGP_TABLES_8: [[u32; 256]; 8] = generate_crc24_tables_8(CRC24_OPENPGP_POLY); } // Reference Kernel Wrapper @@ -331,6 +316,7 @@ impl crate::traits::ChecksumCombine for Crc24OpenPgp { #[cfg(feature = "alloc")] impl Crc24OpenPgp { + /// Creates a buffering wrapper that coalesces short updates before CRC-24/OpenPGP dispatch. #[must_use] pub fn buffered() -> BufferedCrc24OpenPgp { BufferedCrc24OpenPgp::new() @@ -373,6 +359,7 @@ mod tests { extern crate std; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; #[test] fn test_vectors_crc24_openpgp() { @@ -429,6 +416,7 @@ mod cross_check { use super::*; use crate::checksum::common::tests::{STREAMING_CHUNK_SIZES, TEST_LENGTHS, generate_test_data}; + use crate::traits::{Checksum, ChecksumCombine}; // CRC-24/OPENPGP Cross-Check Tests diff --git a/src/checksum/crc24/portable.rs b/src/checksum/crc24/portable.rs index 0afe0baf..b818e597 100644 --- a/src/checksum/crc24/portable.rs +++ b/src/checksum/crc24/portable.rs @@ -5,7 +5,7 @@ use crate::checksum::common::portable; /// CRC-24/OPENPGP slice-by-8 computation. #[inline] -pub fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { portable::slice8_24(crc, data, &kernel_tables::OPENPGP_TABLES_8) } @@ -15,8 +15,7 @@ pub fn crc24_openpgp_slice8(crc: u32, data: &[u8]) -> u32 { /// /// Uses one 256-entry table rather than the slice-by-8 table set. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by byte cast, table is [u32; 256] -pub fn crc24_openpgp_bytewise(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc24_openpgp_bytewise(crc: u32, data: &[u8]) -> u32 { const MASK24: u32 = 0x00FF_FFFF; let mut state = (crc & MASK24) << 8; for &byte in data { diff --git a/src/checksum/crc24/power.rs b/src/checksum/crc24/power.rs index 0e00b74f..17447a4a 100644 --- a/src/checksum/crc24/power.rs +++ b/src/checksum/crc24/power.rs @@ -9,10 +9,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -20,6 +17,8 @@ use core::{ simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::{ keys::{CRC24_OPENPGP_KEYS_REFLECTED, CRC24_OPENPGP_STREAM_REFLECTED}, reflected::{crc24_reflected_update_bitrev_bytes, from_reflected_state, to_reflected_state}, @@ -87,11 +86,13 @@ impl Simd { } /// Normalize a loaded vector to little-endian lane encoding. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { - // SAFETY: All intrinsics and asm require altivec+vsx+power8-vector, ensured by this function's - // #[target_feature] attribute. #[cfg(target_endian = "little")] { self @@ -117,6 +118,9 @@ impl Simd { Self::new(bitrev_bytes_u64(self.high_64()), bitrev_bytes_u64(self.low_64())) } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -140,6 +144,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -157,6 +164,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -170,6 +180,9 @@ impl Simd { unsafe { Self(Self::vpmsumd(self.0, coeff.swap_lanes().0)) } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -183,6 +196,9 @@ impl Simd { unsafe { data_to_xor ^ self.fold_16(coeff) } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -208,6 +224,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -221,11 +240,14 @@ impl Simd { unsafe { let t1 = Self::mul64(self.low_64(), mu); let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 + low_u32((self ^ l).high_64()) } } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -251,6 +273,9 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u3 } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -295,6 +320,9 @@ unsafe fn update_simd_width32_reflected_bitrev( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -317,6 +345,9 @@ unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd; 8], chunk: &[Simd; 8], } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -340,6 +371,9 @@ unsafe fn normalize_block_le_bitrev(mut block: [Simd; 8]) -> [Simd; 8] { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -400,6 +434,9 @@ unsafe fn update_simd_width32_reflected_bitrev_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -426,7 +463,7 @@ unsafe fn update_simd_width32_reflected_bitrev_4way( return update_simd_width32_reflected_bitrev(state, first, rest, keys); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -487,6 +524,9 @@ unsafe fn update_simd_width32_reflected_bitrev_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -513,7 +553,7 @@ unsafe fn update_simd_width32_reflected_bitrev_8way( return update_simd_width32_reflected_bitrev(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(keys[4], keys[3]); @@ -622,6 +662,9 @@ unsafe fn update_simd_width32_reflected_bitrev_8way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -645,6 +688,9 @@ unsafe fn crc24_width32_vpmsum_bitrev(mut state: u32, data: &[u8], keys: &[u64; } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -668,6 +714,9 @@ unsafe fn crc24_width32_vpmsum_bitrev_2way(mut state: u32, data: &[u8], keys: &[ } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -697,6 +746,9 @@ unsafe fn crc24_width32_vpmsum_bitrev_4way(mut state: u32, data: &[u8], keys: &[ } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -730,11 +782,9 @@ unsafe fn crc24_width32_vpmsum_bitrev_8way(mut state: u32, data: &[u8], keys: &[ /// CRC-24/OPENPGP VPMSUMD kernel. /// -/// # Safety -/// -/// Dispatcher verifies VPMSUMD before selecting this kernel. +/// Runtime dispatch selects this kernel only when VPMSUMD is available. #[inline] -pub fn crc24_openpgp_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -742,7 +792,7 @@ pub fn crc24_openpgp_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -750,7 +800,7 @@ pub fn crc24_openpgp_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -758,7 +808,7 @@ pub fn crc24_openpgp_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPMSUMD before selecting this kernel. state = unsafe { crc24_width32_vpmsum_bitrev_8way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; diff --git a/src/checksum/crc24/reflected.rs b/src/checksum/crc24/reflected.rs index ae059085..8540e29f 100644 --- a/src/checksum/crc24/reflected.rs +++ b/src/checksum/crc24/reflected.rs @@ -9,7 +9,6 @@ // SAFETY: All array indexing in this module uses bounded indices: // - Table generation iterates i: 0..256 into a [u32; 256] // - Table lookups use `& 0xFF` which produces indices 0..255 -#![allow(clippy::indexing_slicing)] use super::keys::CRC24_OPENPGP_POLY_REFLECTED; @@ -42,10 +41,13 @@ const fn crc32_reflected_table_entry(poly: u32, index: u8) -> u32 { #[must_use] const fn generate_crc32_reflected_table(poly: u32) -> [u32; 256] { let mut table = [0u32; 256]; - let mut i: u16 = 0; - while i < 256 { - table[i as usize] = crc32_reflected_table_entry(poly, i as u8); - i = i.strict_add(1); + let mut index = 0u8; + loop { + table[index as usize] = crc32_reflected_table_entry(poly, index); + if index == u8::MAX { + break; + } + index = index.strict_add(1); } table } diff --git a/src/checksum/crc24/riscv64.rs b/src/checksum/crc24/riscv64.rs index 5e1bd1a5..fcaec847 100644 --- a/src/checksum/crc24/riscv64.rs +++ b/src/checksum/crc24/riscv64.rs @@ -9,17 +9,14 @@ //! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required //! CPU features are available before executing the accelerated path (the //! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitXor, BitXorAssign}, }; +use crate::checksum::common::low_u32; + use super::{ keys::{CRC24_OPENPGP_KEYS_REFLECTED, CRC24_OPENPGP_STREAM_REFLECTED}, reflected::{crc24_reflected_update_bitrev_bytes, from_reflected_state, to_reflected_state}, @@ -72,11 +69,11 @@ impl Simd { // Zbc carryless multiply primitives #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_lo(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -93,11 +90,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. + fn clmul_hi(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -114,18 +111,14 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } + fn mul64(a: u64, b: u64) -> Self { + Self { + hi: Self::clmul_hi(a, b), + lo: Self::clmul_lo(a, b), } } } @@ -143,34 +136,23 @@ const fn bitrev_bytes_u64(mut x: u64) -> u64 { #[inline(always)] fn load_block_bitrev(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0usize; - while i < 8 { - let lo = bitrev_bytes_u64(u64::from_le(block[i * 2])); - let hi = bitrev_bytes_u64(u64::from_le(block[i * 2 + 1])); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(hi, lo)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new( + bitrev_bytes_u64(u64::from_le(high)), + bitrev_bytes_u64(u64::from_le(low)), + ); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } #[inline] fn load_block_split_bitrev(block: &Block) -> ([u64; 8], [u64; 8]) { let mut hi = [0u64; 8]; let mut lo = [0u64; 8]; - - let mut i = 0usize; - while i < 8 { - lo[i] = bitrev_bytes_u64(u64::from_le(block[i * 2])); - hi[i] = bitrev_bytes_u64(u64::from_le(block[i * 2 + 1])); - i = i.strict_add(1); + for ((hi_lane, lo_lane), &[low, high]) in hi.iter_mut().zip(&mut lo).zip(block.as_chunks::<2>().0) { + *hi_lane = bitrev_bytes_u64(u64::from_le(high)); + *lo_lane = bitrev_bytes_u64(u64::from_le(low)); } (hi, lo) @@ -179,277 +161,249 @@ fn load_block_split_bitrev(block: &Block) -> ([u64; 8], [u64; 8]) { // ZBC (scalar carryless multiply) backend #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) - } +fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { data_to_xor ^ fold_16_zbc(x, coeff) } +fn fold_16_reflected_zbc(x: Simd, coeff: (u64, u64), data_to_xor: Simd) -> Simd { + data_to_xor ^ fold_16_zbc(x, coeff) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. ZBC + ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = Simd::mul64(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = Simd::mul64(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Simd::mul64(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Simd::mul64(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Simd::mul64(x.low_64(), mu); - let l = Simd::mul64(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = Simd::mul64(x.low_64(), mu); + let l = Simd::mul64(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut res = x[7]; - res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); - res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); - res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); - res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); - res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); - res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); - res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); - - barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected_zbc(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = fold_16_reflected_zbc(x[0], (keys[10], keys[9]), res); + res = fold_16_reflected_zbc(x[1], (keys[12], keys[11]), res); + res = fold_16_reflected_zbc(x[2], (keys[14], keys[13]), res); + res = fold_16_reflected_zbc(x[3], (keys[16], keys[15]), res); + res = fold_16_reflected_zbc(x[4], (keys[18], keys[17]), res); + res = fold_16_reflected_zbc(x[5], (keys[20], keys[19]), res); + res = fold_16_reflected_zbc(x[6], (keys[2], keys[1]), res); + + barrett_width32_reflected_zbc(fold_width32_reflected_zbc(res, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block_bitrev(first); - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = (keys[4], keys[3]); - for block in rest { - let chunk = load_block_bitrev(block); - x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); - } +fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block_bitrev(first); + x[0] ^= Simd::new(0, state as u64); - finalize_lanes_width32_reflected_zbc(x, keys) + let coeff_128b = (keys[4], keys[3]); + for block in rest { + let chunk = load_block_bitrev(block); + x[0] = fold_16_reflected_zbc(x[0], coeff_128b, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff_128b, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff_128b, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff_128b, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff_128b, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff_128b, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff_128b, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected_zbc(x, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let chunk = load_block_bitrev(block); - x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); - x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); - x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); - x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); - x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); - x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); - x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); - x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); - } +fn fold_block_128_reflected_zbc(x: &mut [Simd; 8], block: &Block, coeff: (u64, u64)) { + let chunk = load_block_bitrev(block); + x[0] = fold_16_reflected_zbc(x[0], coeff, chunk[0]); + x[1] = fold_16_reflected_zbc(x[1], coeff, chunk[1]); + x[2] = fold_16_reflected_zbc(x[2], coeff, chunk[2]); + x[3] = fold_16_reflected_zbc(x[3], coeff, chunk[3]); + x[4] = fold_16_reflected_zbc(x[4], coeff, chunk[4]); + x[5] = fold_16_reflected_zbc(x[5], coeff, chunk[5]); + x[6] = fold_16_reflected_zbc(x[6], coeff, chunk[6]); + x[7] = fold_16_reflected_zbc(x[7], coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let even = blocks.len() & !1usize; - let coeff_256 = fold_256b; - let coeff_128 = (keys[4], keys[3]); + let even = blocks.len() & !1usize; + let coeff_256 = fold_256b; + let coeff_128 = (keys[4], keys[3]); - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 2; - while i < even { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } + let mut i: usize = 2; + while i < even { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); - s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); - s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); - s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); - s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); - s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); - s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); - s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); - - if even != blocks.len() { - fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); - } + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + s1[0] = fold_16_reflected_zbc(s0[0], coeff_128, s1[0]); + s1[1] = fold_16_reflected_zbc(s0[1], coeff_128, s1[1]); + s1[2] = fold_16_reflected_zbc(s0[2], coeff_128, s1[2]); + s1[3] = fold_16_reflected_zbc(s0[3], coeff_128, s1[3]); + s1[4] = fold_16_reflected_zbc(s0[4], coeff_128, s1[4]); + s1[5] = fold_16_reflected_zbc(s0[5], coeff_128, s1[5]); + s1[6] = fold_16_reflected_zbc(s0[6], coeff_128, s1[6]); + s1[7] = fold_16_reflected_zbc(s0[7], coeff_128, s1[7]); - finalize_lanes_width32_reflected_zbc(s1, keys) + if even != blocks.len() { + fold_block_128_reflected_zbc(&mut s1, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected_zbc(s1, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_4way( +fn update_simd_zbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, keys); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, keys); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); + let aligned = blocks.len().strict_div(4).strict_mul(4); - let coeff_512 = fold_512b; - let coeff_128 = (keys[4], keys[3]); - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; + let coeff_512 = fold_512b; + let coeff_128 = (keys[4], keys[3]); + let c384 = combine[0]; + let c256 = combine[1]; + let c128 = combine[2]; - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); - let mut s2 = load_block_bitrev(&blocks[2]); - let mut s3 = load_block_bitrev(&blocks[3]); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); + let mut s2 = load_block_bitrev(&blocks[2]); + let mut s3 = load_block_bitrev(&blocks[3]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected_zbc(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected_zbc(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected_zbc(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected_zbc(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); - s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); - s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); - s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); - s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); - s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); - s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); - s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); - - s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); - s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); - s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); - s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); - s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); - s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); - s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); - s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); - - s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); - s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); - s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); - s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); - s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); - s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); - s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); - s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected_zbc(&mut s3, block, coeff_128); - } + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + s3[0] = fold_16_reflected_zbc(s2[0], c128, s3[0]); + s3[1] = fold_16_reflected_zbc(s2[1], c128, s3[1]); + s3[2] = fold_16_reflected_zbc(s2[2], c128, s3[2]); + s3[3] = fold_16_reflected_zbc(s2[3], c128, s3[3]); + s3[4] = fold_16_reflected_zbc(s2[4], c128, s3[4]); + s3[5] = fold_16_reflected_zbc(s2[5], c128, s3[5]); + s3[6] = fold_16_reflected_zbc(s2[6], c128, s3[6]); + s3[7] = fold_16_reflected_zbc(s2[7], c128, s3[7]); + + s3[0] = fold_16_reflected_zbc(s1[0], c256, s3[0]); + s3[1] = fold_16_reflected_zbc(s1[1], c256, s3[1]); + s3[2] = fold_16_reflected_zbc(s1[2], c256, s3[2]); + s3[3] = fold_16_reflected_zbc(s1[3], c256, s3[3]); + s3[4] = fold_16_reflected_zbc(s1[4], c256, s3[4]); + s3[5] = fold_16_reflected_zbc(s1[5], c256, s3[5]); + s3[6] = fold_16_reflected_zbc(s1[6], c256, s3[6]); + s3[7] = fold_16_reflected_zbc(s1[7], c256, s3[7]); + + s3[0] = fold_16_reflected_zbc(s0[0], c384, s3[0]); + s3[1] = fold_16_reflected_zbc(s0[1], c384, s3[1]); + s3[2] = fold_16_reflected_zbc(s0[2], c384, s3[2]); + s3[3] = fold_16_reflected_zbc(s0[3], c384, s3[3]); + s3[4] = fold_16_reflected_zbc(s0[4], c384, s3[4]); + s3[5] = fold_16_reflected_zbc(s0[5], c384, s3[5]); + s3[6] = fold_16_reflected_zbc(s0[6], c384, s3[6]); + s3[7] = fold_16_reflected_zbc(s0[7], c384, s3[7]); - finalize_lanes_width32_reflected_zbc(s3, keys) + for block in &blocks[aligned..] { + fold_block_128_reflected_zbc(&mut s3, block, coeff_128); } + + finalize_lanes_width32_reflected_zbc(s3, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc24_width32_zbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -463,13 +417,13 @@ unsafe fn crc24_width32_zbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u3 } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc24_width32_zbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -483,13 +437,13 @@ unsafe fn crc24_width32_zbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc24_width32_zbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. ZBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -517,10 +471,7 @@ unsafe fn crc24_width32_zbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) /// Requires RISC-V `v` + `zvbc`. #[inline] #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn mul64_zvbc(a: u64, b: u64) -> Simd { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -547,81 +498,68 @@ unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) - } +fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Caller guarantees: - // 1. V + ZVBC + V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = mul64_zvbc(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = mul64_zvbc(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = mul64_zvbc(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = mul64_zvbc(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu); - let l = mul64_zvbc(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_width32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = mul64_zvbc(x.low_64(), mu); + let l = mul64_zvbc(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); - - barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) - } +fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], keys: &[u64; 23]) -> u32 { + let mut acc = Simd::new(hi[7], lo[7]); + acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), (keys[10], keys[9])); + acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), (keys[12], keys[11])); + acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), (keys[14], keys[13])); + acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), (keys[16], keys[15])); + acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), (keys[18], keys[17])); + acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), (keys[20], keys[19])); + acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), (keys[2], keys[1])); + + barrett_width32_reflected_zvbc(fold_width32_reflected_zvbc(acc, keys[6], keys[5]), keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( +fn fold_block_128_zvbc( x_hi: &mut [u64; 8], x_lo: &mut [u64; 8], chunk_hi: &[u64; 8], @@ -629,15 +567,12 @@ unsafe fn fold_block_128_zvbc( coeff_low: u64, coeff_high: u64, ) { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { let mut offset = 0usize; while offset < 8 { - let remaining = 8 - offset; + let remaining = 8usize.strict_sub(offset); let vl: usize; asm!( "vsetvli {vl}, {avl}, e64, m1, ta, ma", @@ -677,174 +612,162 @@ unsafe fn fold_block_128_zvbc( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split_bitrev(first); - x_lo[0] ^= state as u64; - - let coeff_low = keys[3]; - let coeff_high = keys[4]; +fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let (mut x_hi, mut x_lo) = load_block_split_bitrev(first); + x_lo[0] ^= state as u64; - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split_bitrev(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } + let coeff_low = keys[3]; + let coeff_high = keys[4]; - fold_tail_zvbc(x_hi, x_lo, keys) + for block in rest { + let (chunk_hi, chunk_lo) = load_block_split_bitrev(block); + fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); } + + fold_tail_zvbc(x_hi, x_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); +fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23]) -> u32 { + debug_assert!(!blocks.is_empty()); - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } - let even = blocks.len() & !1usize; + let even = blocks.len() & !1usize; - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; + let coeff_256_low = fold_256b.1; + let coeff_256_high = fold_256b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; - let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); + let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; - let mut i: usize = 2; - while i < even { - let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } + let mut i: usize = 2; + while i < even { + let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); + i = i.strict_add(2); + } - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split_bitrev(&blocks[even]); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - fold_tail_zvbc(s0_hi, s0_lo, keys) + if even != blocks.len() { + let (tail_hi, tail_lo) = load_block_split_bitrev(&blocks[even]); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( +fn update_simd_zvbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, keys); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = keys[3]; - let coeff_128_high = keys[4]; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split_bitrev(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split_bitrev(&blocks[3]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state as u64; - - let mut i: usize = 4; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split_bitrev(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split_bitrev(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split_bitrev(block); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + debug_assert!(!blocks.is_empty()); - fold_tail_zvbc(s0_hi, s0_lo, keys) + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, keys); + } + + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512_low = fold_512b.1; + let coeff_512_high = fold_512b.0; + let coeff_128_low = keys[3]; + let coeff_128_high = keys[4]; + + let c384_low = combine[0].1; + let c384_high = combine[0].0; + let c256_low = combine[1].1; + let c256_high = combine[1].0; + let c128_low = combine[2].1; + let c128_high = combine[2].0; + + let (mut s0_hi, mut s0_lo) = load_block_split_bitrev(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split_bitrev(&blocks[1]); + let (mut s2_hi, mut s2_lo) = load_block_split_bitrev(&blocks[2]); + let (mut s3_hi, mut s3_lo) = load_block_split_bitrev(&blocks[3]); + + // Inject CRC into stream 0. + s0_lo[0] ^= state as u64; + + let mut i: usize = 4; + while i < aligned { + let (b0_hi, b0_lo) = load_block_split_bitrev(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split_bitrev(&blocks[i.strict_add(1)]); + let (b2_hi, b2_lo) = load_block_split_bitrev(&blocks[i.strict_add(2)]); + let (b3_hi, b3_lo) = load_block_split_bitrev(&blocks[i.strict_add(3)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); + i = i.strict_add(4); + } + + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &s3_hi, &s3_lo, c128_low, c128_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &s2_hi, &s2_lo, c256_low, c256_high); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, c384_low, c384_high); + + for block in &blocks[aligned..] { + let (tail_hi, tail_lo) = load_block_split_bitrev(block); + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(s0_hi, s0_lo, keys) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc24_width32_zvbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zvbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -858,13 +781,13 @@ unsafe fn crc24_width32_zvbc(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc24_width32_zvbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zvbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -878,13 +801,13 @@ unsafe fn crc24_width32_zvbc_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc24_width32_zvbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. V + ZVBC target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc24_width32_zvbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -906,12 +829,8 @@ unsafe fn crc24_width32_zvbc_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) // Public Safe Kernels /// CRC-24/OPENPGP Zbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zbc before selecting this kernel. #[inline] -pub fn crc24_openpgp_zbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zbc_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zbc before selecting this kernel. state = unsafe { crc24_width32_zbc(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -919,7 +838,7 @@ pub fn crc24_openpgp_zbc_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zbc before selecting this kernel. state = unsafe { crc24_width32_zbc_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -927,7 +846,7 @@ pub fn crc24_openpgp_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zbc before selecting this kernel. state = unsafe { crc24_width32_zbc_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -935,12 +854,8 @@ pub fn crc24_openpgp_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { } /// CRC-24/OPENPGP Zvbc kernel. -/// -/// # Safety -/// -/// Dispatcher verifies Zvbc before selecting this kernel. #[inline] -pub fn crc24_openpgp_zvbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zvbc_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. state = unsafe { crc24_width32_zvbc(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -948,7 +863,7 @@ pub fn crc24_openpgp_zvbc_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. state = unsafe { crc24_width32_zvbc_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -956,7 +871,7 @@ pub fn crc24_openpgp_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies Zvbc before selecting this kernel. state = unsafe { crc24_width32_zvbc_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; diff --git a/src/checksum/crc24/s390x.rs b/src/checksum/crc24/s390x.rs index f8b35ca8..9ebe45d3 100644 --- a/src/checksum/crc24/s390x.rs +++ b/src/checksum/crc24/s390x.rs @@ -9,18 +9,15 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, - mem::MaybeUninit, ops::{BitAnd, BitXor, BitXorAssign}, simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::{ keys::{CRC24_OPENPGP_KEYS_REFLECTED, CRC24_OPENPGP_STREAM_REFLECTED}, reflected::{crc24_reflected_update_bitrev_bytes, from_reflected_state, to_reflected_state}, @@ -90,8 +87,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees: // 1. VECTOR target features are available (dispatch check). // 2. All SIMD operations are pure register computations after loads. @@ -109,68 +109,62 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR + VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { data_to_xor ^ self.fold_16(coeff) } + fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { + data_to_xor ^ self.fold_16(coeff) } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees: - // 1. VECTOR + VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let clmul = Self::mul64(self.low_64(), low); - let shifted = Self::new(0, self.high_64()); - let mut state = clmul ^ shifted; - - let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); - let masked = state & mask2; - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Self::mul64(shifted_high, high); - state = clmul ^ masked; - - state - } + fn fold_width32_reflected(self, high: u64, low: u64) -> Self { + let clmul = Self::mul64(self.low_64(), low); + let shifted = Self::new(0, self.high_64()); + let mut state = clmul ^ shifted; + + let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); + let masked = state & mask2; + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Self::mul64(shifted_high, high); + state = clmul ^ masked; + + state } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let t1 = Self::mul64(self.low_64(), mu); - let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 - } + fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { + let t1 = Self::mul64(self.low_64(), mu); + let l = Self::mul64(t1.low_64(), poly); + low_u32((self ^ l).high_64()) } } @@ -178,238 +172,222 @@ impl Simd { #[inline(always)] fn load_block_bitrev(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0usize; - while i < 8 { - let low = bitrev_bytes_u64(u64::from_le(block[i * 2])); - let high = bitrev_bytes_u64(u64::from_le(block[i * 2 + 1])); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new( + bitrev_bytes_u64(u64::from_le(high)), + bitrev_bytes_u64(u64::from_le(low)), + ); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut res = x[7]; - res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); - res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); - res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); - res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); - res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); - res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); - res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); - - res = res.fold_width32_reflected(keys[6], keys[5]); - res.barrett_width32_reflected(keys[8], keys[7]) - } +fn finalize_lanes_width32_reflected(x: [Simd; 8], keys: &[u64; 23]) -> u32 { + let mut res = x[7]; + res = x[0].fold_16_reflected(Simd::new(keys[10], keys[9]), res); + res = x[1].fold_16_reflected(Simd::new(keys[12], keys[11]), res); + res = x[2].fold_16_reflected(Simd::new(keys[14], keys[13]), res); + res = x[3].fold_16_reflected(Simd::new(keys[16], keys[15]), res); + res = x[4].fold_16_reflected(Simd::new(keys[18], keys[17]), res); + res = x[5].fold_16_reflected(Simd::new(keys[20], keys[19]), res); + res = x[6].fold_16_reflected(Simd::new(keys[2], keys[1]), res); + + res = res.fold_width32_reflected(keys[6], keys[5]); + res.barrett_width32_reflected(keys[8], keys[7]) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_bitrev(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let mut x = load_block_bitrev(first); - x[0] ^= Simd::new(0, state as u64); - - let coeff_128b = Simd::new(keys[4], keys[3]); - for block in rest { - let chunk = load_block_bitrev(block); - x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); - } +fn update_simd_width32_reflected_bitrev(state: u32, first: &Block, rest: &[Block], keys: &[u64; 23]) -> u32 { + let mut x = load_block_bitrev(first); + x[0] ^= Simd::new(0, state as u64); - finalize_lanes_width32_reflected(x, keys) + let coeff_128b = Simd::new(keys[4], keys[3]); + for block in rest { + let chunk = load_block_bitrev(block); + x[0] = x[0].fold_16_reflected(coeff_128b, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff_128b, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff_128b, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff_128b, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff_128b, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff_128b, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff_128b, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff_128b, chunk[7]); } + + finalize_lanes_width32_reflected(x, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd; 8], block: &Block, coeff: Simd) { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - let chunk = load_block_bitrev(block); - x[0] = x[0].fold_16_reflected(coeff, chunk[0]); - x[1] = x[1].fold_16_reflected(coeff, chunk[1]); - x[2] = x[2].fold_16_reflected(coeff, chunk[2]); - x[3] = x[3].fold_16_reflected(coeff, chunk[3]); - x[4] = x[4].fold_16_reflected(coeff, chunk[4]); - x[5] = x[5].fold_16_reflected(coeff, chunk[5]); - x[6] = x[6].fold_16_reflected(coeff, chunk[6]); - x[7] = x[7].fold_16_reflected(coeff, chunk[7]); - } +fn fold_block_128_reflected_bitrev(x: &mut [Simd; 8], block: &Block, coeff: Simd) { + let chunk = load_block_bitrev(block); + x[0] = x[0].fold_16_reflected(coeff, chunk[0]); + x[1] = x[1].fold_16_reflected(coeff, chunk[1]); + x[2] = x[2].fold_16_reflected(coeff, chunk[2]); + x[3] = x[3].fold_16_reflected(coeff, chunk[3]); + x[4] = x[4].fold_16_reflected(coeff, chunk[4]); + x[5] = x[5].fold_16_reflected(coeff, chunk[5]); + x[6] = x[6].fold_16_reflected(coeff, chunk[6]); + x[7] = x[7].fold_16_reflected(coeff, chunk[7]); } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_bitrev_2way( +fn update_simd_width32_reflected_bitrev_2way( state: u32, blocks: &[Block], fold_256b: (u64, u64), keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected_bitrev(state, first, rest, keys); - } + debug_assert!(!blocks.is_empty()); - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected_bitrev(state, first, rest, keys); + } - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); - s0[0] ^= Simd::new(0, state as u64); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); - let mut i: usize = 2; - let even = blocks.len() & !1usize; - while i < even { - fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_256); - fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_256); - i = i.strict_add(2); - } + s0[0] ^= Simd::new(0, state as u64); - // Merge: A·s0 ⊕ s1 (A = shift by 128B). - let mut combined = s1; - combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); - combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); - combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); - combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); - combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); - combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); - combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); - combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); - - if even != blocks.len() { - fold_block_128_reflected_bitrev(&mut combined, &blocks[even], coeff_128); - } + let mut i: usize = 2; + let even = blocks.len() & !1usize; + while i < even { + fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_256); + fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_256); + i = i.strict_add(2); + } - finalize_lanes_width32_reflected(combined, keys) + // Merge: A·s0 ⊕ s1 (A = shift by 128B). + let mut combined = s1; + combined[0] = s0[0].fold_16_reflected(coeff_128, combined[0]); + combined[1] = s0[1].fold_16_reflected(coeff_128, combined[1]); + combined[2] = s0[2].fold_16_reflected(coeff_128, combined[2]); + combined[3] = s0[3].fold_16_reflected(coeff_128, combined[3]); + combined[4] = s0[4].fold_16_reflected(coeff_128, combined[4]); + combined[5] = s0[5].fold_16_reflected(coeff_128, combined[5]); + combined[6] = s0[6].fold_16_reflected(coeff_128, combined[6]); + combined[7] = s0[7].fold_16_reflected(coeff_128, combined[7]); + + if even != blocks.len() { + fold_block_128_reflected_bitrev(&mut combined, &blocks[even], coeff_128); } + + finalize_lanes_width32_reflected(combined, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_width32_reflected_bitrev_4way( +fn update_simd_width32_reflected_bitrev_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], keys: &[u64; 23], ) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_width32_reflected_bitrev(state, first, rest, keys); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_width32_reflected_bitrev(state, first, rest, keys); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); + let aligned = blocks.len().strict_div(4).strict_mul(4); - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(keys[4], keys[3]); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(keys[4], keys[3]); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); - let mut s0 = load_block_bitrev(&blocks[0]); - let mut s1 = load_block_bitrev(&blocks[1]); - let mut s2 = load_block_bitrev(&blocks[2]); - let mut s3 = load_block_bitrev(&blocks[3]); + let mut s0 = load_block_bitrev(&blocks[0]); + let mut s1 = load_block_bitrev(&blocks[1]); + let mut s2 = load_block_bitrev(&blocks[2]); + let mut s3 = load_block_bitrev(&blocks[3]); - s0[0] ^= Simd::new(0, state as u64); + s0[0] ^= Simd::new(0, state as u64); - let mut i: usize = 4; - while i < aligned { - fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_512); - fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_512); - fold_block_128_reflected_bitrev(&mut s2, &blocks[i.strict_add(2)], coeff_512); - fold_block_128_reflected_bitrev(&mut s3, &blocks[i.strict_add(3)], coeff_512); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut acc = s3; - acc[0] = s2[0].fold_16_reflected(c128, acc[0]); - acc[1] = s2[1].fold_16_reflected(c128, acc[1]); - acc[2] = s2[2].fold_16_reflected(c128, acc[2]); - acc[3] = s2[3].fold_16_reflected(c128, acc[3]); - acc[4] = s2[4].fold_16_reflected(c128, acc[4]); - acc[5] = s2[5].fold_16_reflected(c128, acc[5]); - acc[6] = s2[6].fold_16_reflected(c128, acc[6]); - acc[7] = s2[7].fold_16_reflected(c128, acc[7]); - - acc[0] = s1[0].fold_16_reflected(c256, acc[0]); - acc[1] = s1[1].fold_16_reflected(c256, acc[1]); - acc[2] = s1[2].fold_16_reflected(c256, acc[2]); - acc[3] = s1[3].fold_16_reflected(c256, acc[3]); - acc[4] = s1[4].fold_16_reflected(c256, acc[4]); - acc[5] = s1[5].fold_16_reflected(c256, acc[5]); - acc[6] = s1[6].fold_16_reflected(c256, acc[6]); - acc[7] = s1[7].fold_16_reflected(c256, acc[7]); - - acc[0] = s0[0].fold_16_reflected(c384, acc[0]); - acc[1] = s0[1].fold_16_reflected(c384, acc[1]); - acc[2] = s0[2].fold_16_reflected(c384, acc[2]); - acc[3] = s0[3].fold_16_reflected(c384, acc[3]); - acc[4] = s0[4].fold_16_reflected(c384, acc[4]); - acc[5] = s0[5].fold_16_reflected(c384, acc[5]); - acc[6] = s0[6].fold_16_reflected(c384, acc[6]); - acc[7] = s0[7].fold_16_reflected(c384, acc[7]); - - for block in &blocks[aligned..] { - fold_block_128_reflected_bitrev(&mut acc, block, coeff_128); - } + let mut i: usize = 4; + while i < aligned { + fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_512); + fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128_reflected_bitrev(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128_reflected_bitrev(&mut s3, &blocks[i.strict_add(3)], coeff_512); + i = i.strict_add(4); + } - finalize_lanes_width32_reflected(acc, keys) + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut acc = s3; + acc[0] = s2[0].fold_16_reflected(c128, acc[0]); + acc[1] = s2[1].fold_16_reflected(c128, acc[1]); + acc[2] = s2[2].fold_16_reflected(c128, acc[2]); + acc[3] = s2[3].fold_16_reflected(c128, acc[3]); + acc[4] = s2[4].fold_16_reflected(c128, acc[4]); + acc[5] = s2[5].fold_16_reflected(c128, acc[5]); + acc[6] = s2[6].fold_16_reflected(c128, acc[6]); + acc[7] = s2[7].fold_16_reflected(c128, acc[7]); + + acc[0] = s1[0].fold_16_reflected(c256, acc[0]); + acc[1] = s1[1].fold_16_reflected(c256, acc[1]); + acc[2] = s1[2].fold_16_reflected(c256, acc[2]); + acc[3] = s1[3].fold_16_reflected(c256, acc[3]); + acc[4] = s1[4].fold_16_reflected(c256, acc[4]); + acc[5] = s1[5].fold_16_reflected(c256, acc[5]); + acc[6] = s1[6].fold_16_reflected(c256, acc[6]); + acc[7] = s1[7].fold_16_reflected(c256, acc[7]); + + acc[0] = s0[0].fold_16_reflected(c384, acc[0]); + acc[1] = s0[1].fold_16_reflected(c384, acc[1]); + acc[2] = s0[2].fold_16_reflected(c384, acc[2]); + acc[3] = s0[3].fold_16_reflected(c384, acc[3]); + acc[4] = s0[4].fold_16_reflected(c384, acc[4]); + acc[5] = s0[5].fold_16_reflected(c384, acc[5]); + acc[6] = s0[6].fold_16_reflected(c384, acc[6]); + acc[7] = s0[7].fold_16_reflected(c384, acc[7]); + + for block in &blocks[aligned..] { + fold_block_128_reflected_bitrev(&mut acc, block, coeff_128); } + + finalize_lanes_width32_reflected(acc, keys) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc24_width32_vgfm_bitrev(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc24_width32_vgfm_bitrev(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -423,11 +401,13 @@ unsafe fn crc24_width32_vgfm_bitrev(mut state: u32, data: &[u8], keys: &[u64; 23 } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc24_width32_vgfm_bitrev_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc24_width32_vgfm_bitrev_2way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -441,11 +421,13 @@ unsafe fn crc24_width32_vgfm_bitrev_2way(mut state: u32, data: &[u8], keys: &[u6 } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc24_width32_vgfm_bitrev_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { - // SAFETY: Caller guarantees: - // 1. VECTOR target features are available (dispatch check). - // 2. All SIMD operations are pure register computations after loads. +fn crc24_width32_vgfm_bitrev_4way(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -467,12 +449,8 @@ unsafe fn crc24_width32_vgfm_bitrev_4way(mut state: u32, data: &[u8], keys: &[u6 // Public Safe Kernel /// CRC-24/OPENPGP VGFM kernel. -/// -/// # Safety -/// -/// Dispatcher verifies VECTOR facility before selecting this kernel. #[inline] -pub fn crc24_openpgp_vgfm_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vgfm_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. state = unsafe { crc24_width32_vgfm_bitrev(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -480,7 +458,7 @@ pub fn crc24_openpgp_vgfm_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. state = unsafe { crc24_width32_vgfm_bitrev_2way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -488,7 +466,7 @@ pub fn crc24_openpgp_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc24_openpgp_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. state = unsafe { crc24_width32_vgfm_bitrev_4way(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; diff --git a/src/checksum/crc24/x86_64.rs b/src/checksum/crc24/x86_64.rs index 934e4929..e964e39a 100644 --- a/src/checksum/crc24/x86_64.rs +++ b/src/checksum/crc24/x86_64.rs @@ -9,11 +9,9 @@ //! //! # Safety //! -//! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure SSSE3 + -//! PCLMULQDQ are available before executing these kernels (the dispatcher does -//! this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] +//! The baseline kernels require SSE2, SSSE3, and PCLMULQDQ. The wide kernels +//! additionally require AVX-512F/VL/BW/DQ and VPCLMULQDQ. The private safe +//! wrappers are installed only through capability-gated dispatcher tables. use core::{ arch::x86_64::*, @@ -49,40 +47,77 @@ impl BitXorAssign for Simd128 { } impl Simd128 { + /// Loads 16 bytes without requiring alignment. + /// + /// # Safety + /// + /// `ptr` must be valid to read 16 initialized bytes. The source may be + /// unaligned because the bytes are copied into aligned local storage. + #[inline] + unsafe fn load_unaligned(ptr: *const u8) -> Self { + let mut value = core::mem::MaybeUninit::::uninit(); + + // SAFETY: The caller guarantees a readable 16-byte source. `value` is an + // aligned, non-overlapping 16-byte destination, and every bit pattern is + // valid for the integer vector inside `Simd128`. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 16); + value.assume_init() + } + } + + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_right_8(self) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_srli_si128::<8>(self.0)) } + /// Shifts the vector left by 12 bytes, filling the low bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_left_12(self) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_slli_si128::<12>(self.0)) } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn and(self, mask: Self) -> Self { - // SAFETY: All intrinsics require SSE2, ensured by this function's #[target_feature] attribute. Self(_mm_and_si128(self.0, mask.0)) } - /// Reverse bits within each byte (u8::reverse_bits), lane-wise. + /// Reverses the bits within each byte. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and SSSE3. #[inline] #[target_feature(enable = "sse2", enable = "ssse3")] unsafe fn bitrev_bytes(self) -> Self { - // SAFETY: All intrinsics require SSE2+SSSE3, ensured by this function's #[target_feature] - // attribute. let mask = _mm_set1_epi8(0x0f); let lo = _mm_and_si128(self.0, mask); let hi = _mm_and_si128(_mm_srli_epi16::<4>(self.0), mask); @@ -95,17 +130,24 @@ impl Simd128 { Self(_mm_or_si128(lo_shift, hi_rev)) } + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16_reflected(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: All intrinsics require SSE2+PCLMULQDQ, ensured by this function's #[target_feature] - // attribute. let h = _mm_clmulepi64_si128::<0x10>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x01>(self.0, coeff.0); Self(_mm_xor_si128(_mm_xor_si128(h, l), data_to_xor.0)) } - /// Fold 16 bytes down to the "width32" reduction state (reflected mode). + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_width32_reflected(self, high: u64, low: u64) -> Self { @@ -131,6 +173,11 @@ impl Simd128 { } } + /// Applies Barrett reduction and returns the width-32 state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett_width32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -143,13 +190,18 @@ impl Simd128 { let xorred = _mm_xor_si128(self.0, clmul2); let hi = _mm_srli_si128::<8>(xorred); - _mm_cvtsi128_si64(hi) as u32 + _mm_cvtsi128_si32(hi).cast_unsigned() } } } // 8-lane width32 update (128B blocks) +/// Combines eight folded lanes and applies width-32 Barrett reduction. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> u32 { @@ -170,6 +222,11 @@ unsafe fn finalize_lanes_width32_reflected(x: [Simd128; 8], keys: &[u64; 23]) -> } } +/// Bit-reverses and folds one or more 128-byte blocks into a width-32 CRC state. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes( @@ -221,6 +278,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes( // PCLMULQDQ multi-stream (2/4/7/8-way, 128B blocks) +/// Bit-reverses and folds one 128-byte block into eight parallel lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff: Simd128) { @@ -247,6 +309,11 @@ unsafe fn fold_block_128_reflected_bitrev(x: &mut [Simd128; 8], chunk: &[Simd128 } } +/// Bit-reverses and folds 128-byte blocks through two PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( @@ -301,7 +368,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -346,6 +413,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_2way( } } +/// Bit-reverses and folds 128-byte blocks through four PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( @@ -398,7 +470,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -422,7 +494,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { fold_block_128_reflected_bitrev(&mut s0, &blocks[i], coeff_512); fold_block_128_reflected_bitrev(&mut s1, &blocks[i.strict_add(1)], coeff_512); @@ -467,6 +539,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_4way( } } +/// Bit-reverses and folds 128-byte blocks through seven PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_7way( @@ -490,7 +567,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_7way( return update_simd_width32_reflected_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let coeff_896 = Simd128::new(fold_896b.0, fold_896b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -604,6 +681,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_7way( } } +/// Bit-reverses and folds 128-byte blocks through eight PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_width32_reflected_bitrev_bytes_8way( @@ -627,7 +709,7 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_8way( return update_simd_width32_reflected_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let coeff_1024 = Simd128::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd128::new(keys[4], keys[3]); @@ -753,6 +835,11 @@ unsafe fn update_simd_width32_reflected_bitrev_bytes_8way( } } +/// Updates CRC-24 with a selected multi-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc24_width32_pclmul_stream( @@ -788,6 +875,11 @@ unsafe fn crc24_width32_pclmul_stream( // Single-stream kernel entry points +/// Updates CRC-24 with the single-stream PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc24_width32_pclmul_small(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -804,13 +896,13 @@ unsafe fn crc24_width32_pclmul_small(mut state: u32, data: &[u8], keys: &[u64; 2 let coeff_16b = Simd128::new(keys[2], keys[1]); - let mut x0 = Simd128(_mm_loadu_si128(buf as *const __m128i)).bitrev_bytes(); + let mut x0 = Simd128::load_unaligned(buf).bitrev_bytes(); x0 ^= Simd128::new(0, state as u64); buf = buf.add(16); len = len.strict_sub(16); while len >= 16 { - let chunk = Simd128(_mm_loadu_si128(buf as *const __m128i)).bitrev_bytes(); + let chunk = Simd128::load_unaligned(buf).bitrev_bytes(); x0 = x0.fold_16_reflected(coeff_16b, chunk); buf = buf.add(16); len = len.strict_sub(16); @@ -824,6 +916,11 @@ unsafe fn crc24_width32_pclmul_small(mut state: u32, data: &[u8], keys: &[u64; 2 } } +/// Updates CRC-24 with the baseline PCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn crc24_width32_pclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -845,22 +942,67 @@ unsafe fn crc24_width32_pclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) -> // AVX-512 VPCLMULQDQ Tier +/// Loads 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 64 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_512(ptr: *const u8) -> __m512i { + let mut value = core::mem::MaybeUninit::<__m512i>::uninit(); + + // SAFETY: The caller guarantees a readable 64-byte source. `value` is an + // aligned, non-overlapping 64-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 64); + value.assume_init() + } +} + +/// Stores 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to write 64 bytes and must not overlap `value`. +#[inline] +unsafe fn store_unaligned_512(ptr: *mut u8, value: __m512i) { + // SAFETY: The caller guarantees a writable, non-overlapping 64-byte + // destination. `value` provides exactly 64 initialized source bytes. + unsafe { core::ptr::copy_nonoverlapping(core::ptr::from_ref(&value).cast::(), ptr, 64) } +} + +/// Multiplies the high lane of each 128-bit element in `a` by the low lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul10_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: All intrinsics require AVX512F+AVX512VL+AVX512BW+AVX512DQ+VPCLMULQDQ, ensured by this - // function's #[target_feature] attribute. _mm512_clmulepi64_epi128(a, b, 0x10) } +/// Multiplies the low lane of each 128-bit element in `a` by the high lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul01_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: All intrinsics require AVX512F+AVX512VL+AVX512BW+AVX512DQ+VPCLMULQDQ, ensured by this - // function's #[target_feature] attribute. _mm512_clmulepi64_epi128(a, b, 0x01) } +/// Folds four reflected 16-byte lanes and XORs their supplied input lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, AVX-512VL, AVX-512BW, AVX-512DQ, and +/// VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512i) -> __m512i { @@ -870,10 +1012,14 @@ unsafe fn fold_16_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512 unsafe { _mm512_ternarylogic_epi64(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff), data, 0x96) } } +/// Broadcasts a pair of 64-bit coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { - // SAFETY: All intrinsics require AVX512F, ensured by this function's #[target_feature] attribute. _mm512_set_epi64( high.cast_signed(), low.cast_signed(), @@ -886,18 +1032,25 @@ unsafe fn broadcast_coeff_128b(high: u64, low: u64) -> __m512i { ) } +/// Places the width-32 CRC state in the low 32 bits of lane zero. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn state_mask_lane0(state: u32) -> __m512i { - // SAFETY: All intrinsics require AVX512F, ensured by this function's #[target_feature] attribute. _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, state as i64) } +/// Reverses the bits within each byte of a 512-bit vector. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and AVX-512BW. #[inline] #[target_feature(enable = "avx512f,avx512bw")] unsafe fn bitrev_bytes_vpclmul(x: __m512i) -> __m512i { - // SAFETY: All intrinsics require AVX512F+AVX512BW, ensured by this function's #[target_feature] - // attribute. let mask = _mm512_set1_epi8(0x0f); let lo = _mm512_and_si512(x, mask); let hi = _mm512_and_si512(_mm512_srli_epi16(x, 4), mask); @@ -909,6 +1062,11 @@ unsafe fn bitrev_bytes_vpclmul(x: __m512i) -> __m512i { _mm512_or_si512(_mm512_slli_epi16(lo_rev, 4), hi_rev) } +/// Bit-reverses and folds 128-byte blocks into a width-32 VPCLMULQDQ state. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( @@ -921,9 +1079,9 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( // AVX512F+AVX512VL+AVX512BW+AVX512DQ+VPCLMULQDQ+SSSE3+PCLMULQDQ+SSE2, ensured by this function's // #[target_feature] attribute. Pointer arithmetic stays within bounds via first/rest structure. unsafe { - let ptr = first.as_ptr() as *const u8; - let mut x0 = _mm512_loadu_si512(ptr as *const __m512i); - let mut x1 = _mm512_loadu_si512(ptr.add(64) as *const __m512i); + let ptr = first.as_ptr().cast::(); + let mut x0 = load_unaligned_512(ptr); + let mut x1 = load_unaligned_512(ptr.add(64)); x0 = bitrev_bytes_vpclmul(x0); x1 = bitrev_bytes_vpclmul(x1); @@ -932,17 +1090,17 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( let coeff_128b = broadcast_coeff_128b(keys[4], keys[3]); for chunk in rest { - let ptr = chunk.as_ptr() as *const u8; - let y0 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr as *const __m512i)); - let y1 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr.add(64) as *const __m512i)); + let ptr = chunk.as_ptr().cast::(); + let y0 = bitrev_bytes_vpclmul(load_unaligned_512(ptr)); + let y1 = bitrev_bytes_vpclmul(load_unaligned_512(ptr.add(64))); x0 = fold_16_reflected_vpclmul(x0, coeff_128b, y0); x1 = fold_16_reflected_vpclmul(x1, coeff_128b, y1); } let mut lanes0 = [Simd128(_mm_setzero_si128()); 4]; let mut lanes1 = [Simd128(_mm_setzero_si128()); 4]; - _mm512_storeu_si512(lanes0.as_mut_ptr() as *mut __m512i, x0); - _mm512_storeu_si512(lanes1.as_mut_ptr() as *mut __m512i, x1); + store_unaligned_512(lanes0.as_mut_ptr().cast::(), x0); + store_unaligned_512(lanes1.as_mut_ptr().cast::(), x1); let x = [ lanes0[0], lanes0[1], lanes0[2], lanes0[3], lanes1[0], lanes1[1], lanes1[2], lanes1[3], @@ -952,6 +1110,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes( } } +/// Broadcasts one pair of folding coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. #[inline] #[target_feature(enable = "avx512f")] unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { @@ -959,6 +1122,11 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { unsafe { broadcast_coeff_128b(pair.0, pair.1) } } +/// Loads, bit-reverses, and returns one 128-byte block as two vectors. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and AVX-512BW. #[inline] #[target_feature(enable = "avx512f")] unsafe fn load_128b_block_bitrev(block: &[Simd128; 8]) -> (__m512i, __m512i) { @@ -966,13 +1134,18 @@ unsafe fn load_128b_block_bitrev(block: &[Simd128; 8]) -> (__m512i, __m512i) { // #[target_feature] attribute. Pointer arithmetic stays within bounds: block is 128 bytes, // loading at offsets 0 and 64. unsafe { - let ptr = block.as_ptr() as *const u8; - let y0 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr as *const __m512i)); - let y1 = bitrev_bytes_vpclmul(_mm512_loadu_si512(ptr.add(64) as *const __m512i)); + let ptr = block.as_ptr().cast::(); + let y0 = bitrev_bytes_vpclmul(load_unaligned_512(ptr)); + let y1 = bitrev_bytes_vpclmul(load_unaligned_512(ptr.add(64))); (y0, y1) } } +/// Combines two four-lane VPCLMULQDQ states and applies width-32 reduction. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F, SSE2, and PCLMULQDQ. #[inline] #[target_feature(enable = "avx512f")] unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> u32 { @@ -999,6 +1172,11 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, keys: &[u64; 23]) -> } } +/// Bit-reverses and folds 128-byte blocks through two VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_2way( @@ -1035,7 +1213,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -1091,6 +1269,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_2way( } } +/// Bit-reverses and folds 128-byte blocks through four VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( @@ -1133,7 +1316,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); while i.strict_add(DOUBLE_GROUP) <= aligned { let prefetch_idx = i.strict_add(LARGE_BLOCK_DISTANCE / BLOCK_SIZE); @@ -1179,7 +1362,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( } // Handle remaining quads. - let quad_aligned = (blocks.len() / 4) * 4; + let quad_aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < quad_aligned { let (y0, y1) = load_128b_block_bitrev(&blocks[i]); x0_0 = fold_16_reflected_vpclmul(x0_0, coeff_512, y0); @@ -1217,6 +1400,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_4way( } } +/// Bit-reverses and folds 128-byte blocks through seven VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_7way( @@ -1241,7 +1429,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_7way( return update_simd_width32_reflected_vpclmul_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let (mut x0_0, mut x1_0) = load_128b_block_bitrev(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block_bitrev(&blocks[1]); @@ -1325,6 +1513,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_7way( } } +/// Bit-reverses and folds 128-byte blocks through eight VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_8way( @@ -1349,7 +1542,7 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_8way( return update_simd_width32_reflected_vpclmul_bitrev_bytes(state, first, rest, keys); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let (mut x0_0, mut x1_0) = load_128b_block_bitrev(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block_bitrev(&blocks[1]); @@ -1441,6 +1634,11 @@ unsafe fn update_simd_width32_reflected_vpclmul_bitrev_bytes_8way( } } +/// Updates CRC-24 with a selected multi-stream VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc24_width32_vpclmul_stream( @@ -1491,6 +1689,11 @@ unsafe fn crc24_width32_vpclmul_stream( } } +/// Updates CRC-24 with the baseline VPCLMULQDQ kernel. +/// +/// # Safety +/// +/// The current CPU must support all target features enabled on this function. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn crc24_width32_vpclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) -> u32 { @@ -1511,15 +1714,14 @@ unsafe fn crc24_width32_vpclmul(mut state: u32, data: &[u8], keys: &[u64; 23]) - } } -// Public Safe Kernels +// Private safe kernel adapters matching the dispatcher's function signature. /// CRC-24/OPENPGP PCLMULQDQ kernel. /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc24_openpgp_pclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { crc24_width32_pclmul(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -1530,11 +1732,10 @@ pub fn crc24_openpgp_pclmul_safe(crc: u32, data: &[u8]) -> u32 { /// /// Optimized for inputs smaller than a folding block (128 bytes). /// -/// # Safety -/// -/// Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying SSSE3 and +/// PCLMULQDQ support. #[inline] -pub fn crc24_openpgp_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { crc24_width32_pclmul_small(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -1543,7 +1744,7 @@ pub fn crc24_openpgp_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1560,7 +1761,7 @@ pub fn crc24_openpgp_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1577,7 +1778,7 @@ pub fn crc24_openpgp_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1594,7 +1795,7 @@ pub fn crc24_openpgp_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP PCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc24_openpgp_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. state = unsafe { @@ -1611,11 +1812,10 @@ pub fn crc24_openpgp_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (AVX-512). /// -/// # Safety -/// -/// Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. +/// The dispatcher selects this private kernel only after verifying VPCLMULQDQ +/// and AVX-512 support. #[inline] -pub fn crc24_openpgp_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { crc24_width32_vpclmul(state, data, &CRC24_OPENPGP_KEYS_REFLECTED) }; @@ -1624,7 +1824,7 @@ pub fn crc24_openpgp_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (2-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1641,7 +1841,7 @@ pub fn crc24_openpgp_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (4-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1658,7 +1858,7 @@ pub fn crc24_openpgp_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (7-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1675,7 +1875,7 @@ pub fn crc24_openpgp_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { /// CRC-24/OPENPGP VPCLMULQDQ kernel (8-way multi-stream). #[inline] -pub fn crc24_openpgp_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc24_openpgp_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { let mut state = to_reflected_state(crc); // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. state = unsafe { @@ -1702,14 +1902,21 @@ mod tests { const OFFSETS: &[usize] = &[0, 1, 7, 15]; const STATES: &[u32] = &[0, 0x00b7_04ce, 0x005a_a5a5, 0x00ff_ffff]; + fn data() -> Vec { + (0u16..4111) + .map(|i| { + let [low, high] = i.to_le_bytes(); + low.wrapping_mul(61).wrapping_add(high) + }) + .collect() + } + fn assert_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32) { - let input: Vec = (0..4111) - .map(|i| (i as u8).wrapping_mul(61).wrapping_add((i >> 8) as u8)) - .collect(); + let input = data(); for &state in STATES { for &offset in OFFSETS { for &len in LENS { - let slice = &input[offset..offset + len]; + let slice = &input[offset..offset.strict_add(len)]; assert_eq!( kernel(state, slice), super::super::portable::crc24_openpgp_slice8(state, slice), diff --git a/src/checksum/crc32/aarch64.rs b/src/checksum/crc32/aarch64.rs index 19703708..f594b125 100644 --- a/src/checksum/crc32/aarch64.rs +++ b/src/checksum/crc32/aarch64.rs @@ -10,12 +10,9 @@ //! Uses `unsafe` for aarch64 intrinsics. Callers must ensure required target //! features are available before selecting a kernel (the dispatcher does this). -#![allow(unsafe_code)] -#![cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] // SAFETY: This module is intrinsics-heavy and uses tight, invariant-driven indexing // (e.g. fixed-size lanes and chunked processing) where bounds are proven by // control flow; Clippy cannot always see these invariants. -#![allow(clippy::indexing_slicing)] // This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. use core::{arch::aarch64::*, ptr}; @@ -28,11 +25,64 @@ use crate::checksum::common::{ const CRC32_SHIFT8_MATRIX: Gf2Matrix32 = generate_shift8_matrix_32(CRC32_IEEE_POLY); const CRC32C_SHIFT8_MATRIX: Gf2Matrix32 = generate_shift8_matrix_32(CRC32C_POLY); +/// Read an integer from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read two initialized bytes from one live allocation. +#[inline(always)] +unsafe fn load_u16(source: *const u8) -> u16 { + // SAFETY: The caller provides the validity and initialization contract; `read_unaligned` does + // not require `source` to satisfy `u16` alignment. + unsafe { ptr::read_unaligned(source.cast()) } +} + +/// Read an integer from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read four initialized bytes from one live allocation. +#[inline(always)] +unsafe fn load_u32(source: *const u8) -> u32 { + // SAFETY: The caller provides the validity and initialization contract; `read_unaligned` does + // not require `source` to satisfy `u32` alignment. + unsafe { ptr::read_unaligned(source.cast()) } +} + +/// Read an integer from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read eight initialized bytes from one live allocation. +#[inline(always)] +unsafe fn load_u64(source: *const u8) -> u64 { + // SAFETY: The caller provides the validity and initialization contract; `read_unaligned` does + // not require `source` to satisfy `u64` alignment. + unsafe { ptr::read_unaligned(source.cast()) } +} + +/// Load two `u64` lanes from an unaligned byte pointer. +/// +/// # Safety +/// +/// `source` must remain valid to read 16 initialized bytes from one live allocation. The caller +/// must also ensure the AArch64 NEON feature is available. +#[inline(always)] +unsafe fn load_u64x2(source: *const u8) -> uint64x2_t { + // SAFETY: The caller provides the memory and target-feature contracts. `vld1q_u64` implements + // its load with `read_unaligned`, so `source` need not satisfy `u64` alignment. + unsafe { vld1q_u64(source.cast()) } +} + // Hardware CRC extension (CRC-only) /// CRC-32 (IEEE) update using ARMv8 CRC extension. /// /// `crc` is the current state (pre-inverted). +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32_armv8(crc: u32, data: &[u8]) -> u32 { @@ -53,41 +103,41 @@ unsafe fn crc32_armv8(crc: u32, data: &[u8]) -> u32 { } while len >= 64 { - state = __crc32d(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(24) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(32) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(40) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(48) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(56) as *const u64)); + state = __crc32d(state, load_u64(buf)); + state = __crc32d(state, load_u64(buf.add(8))); + state = __crc32d(state, load_u64(buf.add(16))); + state = __crc32d(state, load_u64(buf.add(24))); + state = __crc32d(state, load_u64(buf.add(32))); + state = __crc32d(state, load_u64(buf.add(40))); + state = __crc32d(state, load_u64(buf.add(48))); + state = __crc32d(state, load_u64(buf.add(56))); buf = buf.add(64); len = len.strict_sub(64); } while len >= 32 { - state = __crc32d(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32d(state, ptr::read_unaligned(buf.add(24) as *const u64)); + state = __crc32d(state, load_u64(buf)); + state = __crc32d(state, load_u64(buf.add(8))); + state = __crc32d(state, load_u64(buf.add(16))); + state = __crc32d(state, load_u64(buf.add(24))); buf = buf.add(32); len = len.strict_sub(32); } while len >= 8 { - state = __crc32d(state, ptr::read_unaligned(buf as *const u64)); + state = __crc32d(state, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } if len >= 4 { - state = __crc32w(state, ptr::read_unaligned(buf as *const u32)); + state = __crc32w(state, load_u32(buf)); buf = buf.add(4); len = len.strict_sub(4); } if len >= 2 { - state = __crc32h(state, ptr::read_unaligned(buf as *const u16)); + state = __crc32h(state, load_u16(buf)); buf = buf.add(2); len = len.strict_sub(2); } @@ -103,6 +153,10 @@ unsafe fn crc32_armv8(crc: u32, data: &[u8]) -> u32 { /// CRC-32C (Castagnoli) update using ARMv8 CRC extension. /// /// `crc` is the current state (pre-inverted). +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32c_armv8(crc: u32, data: &[u8]) -> u32 { @@ -123,41 +177,41 @@ unsafe fn crc32c_armv8(crc: u32, data: &[u8]) -> u32 { } while len >= 64 { - state = __crc32cd(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(24) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(32) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(40) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(48) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(56) as *const u64)); + state = __crc32cd(state, load_u64(buf)); + state = __crc32cd(state, load_u64(buf.add(8))); + state = __crc32cd(state, load_u64(buf.add(16))); + state = __crc32cd(state, load_u64(buf.add(24))); + state = __crc32cd(state, load_u64(buf.add(32))); + state = __crc32cd(state, load_u64(buf.add(40))); + state = __crc32cd(state, load_u64(buf.add(48))); + state = __crc32cd(state, load_u64(buf.add(56))); buf = buf.add(64); len = len.strict_sub(64); } while len >= 32 { - state = __crc32cd(state, ptr::read_unaligned(buf as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(8) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(16) as *const u64)); - state = __crc32cd(state, ptr::read_unaligned(buf.add(24) as *const u64)); + state = __crc32cd(state, load_u64(buf)); + state = __crc32cd(state, load_u64(buf.add(8))); + state = __crc32cd(state, load_u64(buf.add(16))); + state = __crc32cd(state, load_u64(buf.add(24))); buf = buf.add(32); len = len.strict_sub(32); } while len >= 8 { - state = __crc32cd(state, ptr::read_unaligned(buf as *const u64)); + state = __crc32cd(state, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } if len >= 4 { - state = __crc32cw(state, ptr::read_unaligned(buf as *const u32)); + state = __crc32cw(state, load_u32(buf)); buf = buf.add(4); len = len.strict_sub(4); } if len >= 2 { - state = __crc32ch(state, ptr::read_unaligned(buf as *const u16)); + state = __crc32ch(state, load_u16(buf)); buf = buf.add(2); len = len.strict_sub(2); } @@ -172,20 +226,26 @@ unsafe fn crc32c_armv8(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 ARMv8 CRC extension kernel. #[inline] -pub fn crc32_armv8_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_armv8_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32_armv8(crc, data) } } /// Safe wrapper for CRC-32C ARMv8 CRC extension kernel. #[inline] -pub fn crc32c_armv8_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_armv8_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32c_armv8(crc, data) } } // Hardware CRC extension (multi-stream wrappers) +/// Update CRC-32 (IEEE) through `N` independent ARMv8 CRC streams. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available and instantiate `N` as two +/// or three. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { @@ -201,7 +261,7 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { return crc32_armv8(crc, data); } - let chunk_len = len / N; + let chunk_len = len.strict_div(N); let mut lanes = [!0u32; N]; let mut i: usize = 0; @@ -210,10 +270,7 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { let mut lane_idx: usize = 0; while lane_idx < N { let base = lane_idx.strict_mul(chunk_len).strict_add(i); - lanes[lane_idx] = __crc32d( - lanes[lane_idx], - ptr::read_unaligned(data.as_ptr().add(base) as *const u64), - ); + lanes[lane_idx] = __crc32d(lanes[lane_idx], load_u64(data.as_ptr().add(base))); lane_idx = lane_idx.strict_add(1); } i = i.strict_add(8); @@ -230,7 +287,7 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { let tail_start = chunk_len.strict_mul(N); if tail_start < len { - lanes[N - 1] = crc32_armv8(lanes[N - 1], data.get_unchecked(tail_start..)); + lanes[N.strict_sub(1)] = crc32_armv8(lanes[N.strict_sub(1)], data.get_unchecked(tail_start..)); } let mut data_crc_final: u32 = 0; @@ -257,6 +314,12 @@ unsafe fn crc32_armv8_nway(crc: u32, data: &[u8]) -> u32 { } } +/// Update CRC-32C (Castagnoli) through `N` independent ARMv8 CRC streams. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC target feature is available and instantiate `N` as two +/// or three. #[inline] #[target_feature(enable = "crc")] unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { @@ -272,7 +335,7 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { return crc32c_armv8(crc, data); } - let chunk_len = len / N; + let chunk_len = len.strict_div(N); let mut lanes = [!0u32; N]; let mut i: usize = 0; @@ -281,10 +344,7 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { let mut lane_idx: usize = 0; while lane_idx < N { let base = lane_idx.strict_mul(chunk_len).strict_add(i); - lanes[lane_idx] = __crc32cd( - lanes[lane_idx], - ptr::read_unaligned(data.as_ptr().add(base) as *const u64), - ); + lanes[lane_idx] = __crc32cd(lanes[lane_idx], load_u64(data.as_ptr().add(base))); lane_idx = lane_idx.strict_add(1); } i = i.strict_add(8); @@ -301,7 +361,7 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { let tail_start = chunk_len.strict_mul(N); if tail_start < len { - lanes[N - 1] = crc32c_armv8(lanes[N - 1], data.get_unchecked(tail_start..)); + lanes[N.strict_sub(1)] = crc32c_armv8(lanes[N.strict_sub(1)], data.get_unchecked(tail_start..)); } let mut data_crc_final: u32 = 0; @@ -329,25 +389,25 @@ unsafe fn crc32c_armv8_nway(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32_armv8_nway::<2>(crc, data) } } #[inline] -pub fn crc32_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32_armv8_nway::<3>(crc, data) } } #[inline] -pub fn crc32c_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_armv8_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32c_armv8_nway::<2>(crc, data) } } #[inline] -pub fn crc32c_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_armv8_3way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC extension before selecting this kernel. unsafe { crc32c_armv8_nway::<3>(crc, data) } } @@ -391,7 +451,7 @@ fn crc32_sve2_pmull_nway( let tail_start = chunk_len.strict_mul(N); if tail_start < len { - lanes[N - 1] = update(lanes[N - 1], &data[tail_start..]); + lanes[N.strict_sub(1)] = update(lanes[N.strict_sub(1)], &data[tail_start..]); } // Combine the N independent CRCs with precomputed matrices to avoid @@ -422,51 +482,55 @@ fn crc32_sve2_pmull_nway( } #[inline] -pub fn crc32_iso_hdlc_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32_iso_hdlc_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32_iso_hdlc_pmull_v12e_v1_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32_iso_hdlc_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32_iso_hdlc_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32_iso_hdlc_pmull_v12e_v1_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32c_iscsi_sve2_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32c_iscsi_pmull_v12e_v1_safe, CRC32C_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32c_iscsi_sve2_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32c_iscsi_pmull_v12e_v1_safe, CRC32C_SHIFT8_MATRIX) } // PMULL tier multi-stream wrappers (2/3-way) #[inline] -pub fn crc32_iso_hdlc_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32_iso_hdlc_pmull_v9s3x2e_s3_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32_iso_hdlc_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32_iso_hdlc_pmull_v9s3x2e_s3_safe, CRC32_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32c_iscsi_pmull_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32c_iscsi_pmull_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } // EOR3 tier multi-stream wrappers (2/3-way) #[inline] -pub fn crc32_iso_hdlc_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>( crc, data, @@ -476,7 +540,7 @@ pub fn crc32_iso_hdlc_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_iso_hdlc_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>( crc, data, @@ -486,19 +550,19 @@ pub fn crc32_iso_hdlc_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_iscsi_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_eor3_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<2>(crc, data, crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } #[inline] -pub fn crc32c_iscsi_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_eor3_3way_safe(crc: u32, data: &[u8]) -> u32 { crc32_sve2_pmull_nway::<3>(crc, data, crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe, CRC32C_SHIFT8_MATRIX) } // Small-buffer wrappers (selected for len < fold block) #[inline] -pub fn crc32_iso_hdlc_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { if data.len() <= 256 { return crc32_armv8_safe(crc, data); } @@ -506,7 +570,7 @@ pub fn crc32_iso_hdlc_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_iscsi_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { if data.len() <= 256 { return crc32c_armv8_safe(crc, data); } @@ -514,12 +578,14 @@ pub fn crc32c_iscsi_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_iso_hdlc_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32_iso_hdlc_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { crc32_iso_hdlc_pmull_v12e_v1_safe(crc, data) } #[inline] -pub fn crc32c_iscsi_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc32c_iscsi_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { crc32c_iscsi_pmull_v12e_v1_safe(crc, data) } @@ -533,7 +599,7 @@ pub fn crc32c_iscsi_sve2_pmull_small_safe(crc: u32, data: &[u8]) -> u32 { /// /// # Safety /// -/// Caller must verify that both CRC and AES (PMLL) target features are +/// Caller must verify that both CRC and AES (PMULL) target features are /// available at runtime (via capability detection). #[inline] #[target_feature(enable = "crc,aes")] @@ -546,7 +612,7 @@ pub(crate) unsafe fn crc32_ieee_fusion_inline(crc: u32, data: &[u8]) -> u32 { /// /// # Safety /// -/// Caller must verify that both CRC and AES (PMLL) target features are +/// Caller must verify that both CRC and AES (PMULL) target features are /// available at runtime (via capability detection). #[inline] #[target_feature(enable = "crc,aes")] @@ -585,31 +651,47 @@ pub(crate) unsafe fn crc32c_iscsi_hwcrc_inline(crc: u32, data: &[u8]) -> u32 { // PMULL helpers (fusion kernels) +/// Carry-less multiply the low lanes of two vectors. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_lo(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. - // vmull_p64, vgetq_lane_u64, vreinterpretq_u64_p128 are safe with target_feature on the fn. let result = vmull_p64(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0)); vreinterpretq_u64_p128(result) } +/// Carry-less multiply the high lanes of two vectors. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_hi(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. let result = vmull_p64(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1)); vreinterpretq_u64_p128(result) } +/// Carry-less multiply two scalar CRC values. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_scalar(a: u32, b: u32) -> uint64x2_t { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. let result = vmull_p64(a as u64, b as u64); vreinterpretq_u64_p128(result) } +/// Carry-less multiply the low lanes, then XOR the product with `c`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_lo_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { @@ -617,6 +699,11 @@ unsafe fn clmul_lo_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint6 unsafe { veorq_u64(clmul_lo(a, b), c) } } +/// Carry-less multiply the high lanes, then XOR the product with `c`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 AES/PMULL target feature is available. #[inline] #[target_feature(enable = "aes")] unsafe fn clmul_hi_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { @@ -626,6 +713,12 @@ unsafe fn clmul_hi_and_xor(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint6 // Fusion: CRC-32C (iSCSI) - PMULL v12e_v1 +/// Update CRC-32C with the 12-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -641,7 +734,7 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -650,18 +743,18 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: let end = buf.add(len); let limit = buf.add(len.strict_sub(192)); - let mut x0 = vld1q_u64(buf as *const u64); - let mut x1 = vld1q_u64(buf.add(16) as *const u64); - let mut x2 = vld1q_u64(buf.add(32) as *const u64); - let mut x3 = vld1q_u64(buf.add(48) as *const u64); - let mut x4 = vld1q_u64(buf.add(64) as *const u64); - let mut x5 = vld1q_u64(buf.add(80) as *const u64); - let mut x6 = vld1q_u64(buf.add(96) as *const u64); - let mut x7 = vld1q_u64(buf.add(112) as *const u64); - let mut x8 = vld1q_u64(buf.add(128) as *const u64); - let mut x9 = vld1q_u64(buf.add(144) as *const u64); - let mut x10 = vld1q_u64(buf.add(160) as *const u64); - let mut x11 = vld1q_u64(buf.add(176) as *const u64); + let mut x0 = load_u64x2(buf); + let mut x1 = load_u64x2(buf.add(16)); + let mut x2 = load_u64x2(buf.add(32)); + let mut x3 = load_u64x2(buf.add(48)); + let mut x4 = load_u64x2(buf.add(64)); + let mut x5 = load_u64x2(buf.add(80)); + let mut x6 = load_u64x2(buf.add(96)); + let mut x7 = load_u64x2(buf.add(112)); + let mut x8 = load_u64x2(buf.add(128)); + let mut x9 = load_u64x2(buf.add(144)); + let mut x10 = load_u64x2(buf.add(160)); + let mut x11 = load_u64x2(buf.add(176)); let k_vals: [u64; 2] = [0xa87ab8a8, 0xab7aff2a]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -671,29 +764,29 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: buf = buf.add(192); while buf <= limit { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); - let y1 = clmul_lo_and_xor(x1, k, vld1q_u64(buf.add(16) as *const u64)); + let y1 = clmul_lo_and_xor(x1, k, load_u64x2(buf.add(16))); x1 = clmul_hi_and_xor(x1, k, y1); - let y2 = clmul_lo_and_xor(x2, k, vld1q_u64(buf.add(32) as *const u64)); + let y2 = clmul_lo_and_xor(x2, k, load_u64x2(buf.add(32))); x2 = clmul_hi_and_xor(x2, k, y2); - let y3 = clmul_lo_and_xor(x3, k, vld1q_u64(buf.add(48) as *const u64)); + let y3 = clmul_lo_and_xor(x3, k, load_u64x2(buf.add(48))); x3 = clmul_hi_and_xor(x3, k, y3); - let y4 = clmul_lo_and_xor(x4, k, vld1q_u64(buf.add(64) as *const u64)); + let y4 = clmul_lo_and_xor(x4, k, load_u64x2(buf.add(64))); x4 = clmul_hi_and_xor(x4, k, y4); - let y5 = clmul_lo_and_xor(x5, k, vld1q_u64(buf.add(80) as *const u64)); + let y5 = clmul_lo_and_xor(x5, k, load_u64x2(buf.add(80))); x5 = clmul_hi_and_xor(x5, k, y5); - let y6 = clmul_lo_and_xor(x6, k, vld1q_u64(buf.add(96) as *const u64)); + let y6 = clmul_lo_and_xor(x6, k, load_u64x2(buf.add(96))); x6 = clmul_hi_and_xor(x6, k, y6); - let y7 = clmul_lo_and_xor(x7, k, vld1q_u64(buf.add(112) as *const u64)); + let y7 = clmul_lo_and_xor(x7, k, load_u64x2(buf.add(112))); x7 = clmul_hi_and_xor(x7, k, y7); - let y8 = clmul_lo_and_xor(x8, k, vld1q_u64(buf.add(128) as *const u64)); + let y8 = clmul_lo_and_xor(x8, k, load_u64x2(buf.add(128))); x8 = clmul_hi_and_xor(x8, k, y8); - let y9 = clmul_lo_and_xor(x9, k, vld1q_u64(buf.add(144) as *const u64)); + let y9 = clmul_lo_and_xor(x9, k, load_u64x2(buf.add(144))); x9 = clmul_hi_and_xor(x9, k, y9); - let y10 = clmul_lo_and_xor(x10, k, vld1q_u64(buf.add(160) as *const u64)); + let y10 = clmul_lo_and_xor(x10, k, load_u64x2(buf.add(160))); x10 = clmul_hi_and_xor(x10, k, y10); - let y11 = clmul_lo_and_xor(x11, k, vld1q_u64(buf.add(176) as *const u64)); + let y11 = clmul_lo_and_xor(x11, k, load_u64x2(buf.add(176))); x11 = clmul_hi_and_xor(x11, k, y11); buf = buf.add(192); } @@ -735,11 +828,11 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: crc0 = __crc32cd(0, vgetq_lane_u64(x0, 0)); crc0 = __crc32cd(crc0, vgetq_lane_u64(x0, 1)); - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 16 { - let mut x0 = vld1q_u64(buf as *const u64); + let mut x0 = load_u64x2(buf); let k_vals: [u64; 2] = [0xf20c0dfe, 0x493c7d27]; let k = vld1q_u64(k_vals.as_ptr()); @@ -750,7 +843,7 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: len = len.strict_sub(16); while len >= 16 { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); buf = buf.add(16); len = len.strict_sub(16); @@ -761,7 +854,7 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: } while len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -778,13 +871,19 @@ unsafe fn crc32c_iscsi_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: /// Safe wrapper for CRC-32C fusion kernel (CRC+PMULL v12e_v1). #[inline] -pub fn crc32c_iscsi_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32c_iscsi_pmull_v12e_v1(crc, data.as_ptr(), data.len()) } } // Fusion: CRC-32 (ISO-HDLC / IEEE) - PMULL v12e_v1 +/// Update CRC-32 (IEEE) with the 12-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -800,7 +899,7 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -809,18 +908,18 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le let end = buf.add(len); let limit = buf.add(len.strict_sub(192)); - let mut x0 = vld1q_u64(buf as *const u64); - let mut x1 = vld1q_u64(buf.add(16) as *const u64); - let mut x2 = vld1q_u64(buf.add(32) as *const u64); - let mut x3 = vld1q_u64(buf.add(48) as *const u64); - let mut x4 = vld1q_u64(buf.add(64) as *const u64); - let mut x5 = vld1q_u64(buf.add(80) as *const u64); - let mut x6 = vld1q_u64(buf.add(96) as *const u64); - let mut x7 = vld1q_u64(buf.add(112) as *const u64); - let mut x8 = vld1q_u64(buf.add(128) as *const u64); - let mut x9 = vld1q_u64(buf.add(144) as *const u64); - let mut x10 = vld1q_u64(buf.add(160) as *const u64); - let mut x11 = vld1q_u64(buf.add(176) as *const u64); + let mut x0 = load_u64x2(buf); + let mut x1 = load_u64x2(buf.add(16)); + let mut x2 = load_u64x2(buf.add(32)); + let mut x3 = load_u64x2(buf.add(48)); + let mut x4 = load_u64x2(buf.add(64)); + let mut x5 = load_u64x2(buf.add(80)); + let mut x6 = load_u64x2(buf.add(96)); + let mut x7 = load_u64x2(buf.add(112)); + let mut x8 = load_u64x2(buf.add(128)); + let mut x9 = load_u64x2(buf.add(144)); + let mut x10 = load_u64x2(buf.add(160)); + let mut x11 = load_u64x2(buf.add(176)); let k_vals: [u64; 2] = [0x596c8d81, 0xf5e48c85]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -830,29 +929,29 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le buf = buf.add(192); while buf <= limit { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); - let y1 = clmul_lo_and_xor(x1, k, vld1q_u64(buf.add(16) as *const u64)); + let y1 = clmul_lo_and_xor(x1, k, load_u64x2(buf.add(16))); x1 = clmul_hi_and_xor(x1, k, y1); - let y2 = clmul_lo_and_xor(x2, k, vld1q_u64(buf.add(32) as *const u64)); + let y2 = clmul_lo_and_xor(x2, k, load_u64x2(buf.add(32))); x2 = clmul_hi_and_xor(x2, k, y2); - let y3 = clmul_lo_and_xor(x3, k, vld1q_u64(buf.add(48) as *const u64)); + let y3 = clmul_lo_and_xor(x3, k, load_u64x2(buf.add(48))); x3 = clmul_hi_and_xor(x3, k, y3); - let y4 = clmul_lo_and_xor(x4, k, vld1q_u64(buf.add(64) as *const u64)); + let y4 = clmul_lo_and_xor(x4, k, load_u64x2(buf.add(64))); x4 = clmul_hi_and_xor(x4, k, y4); - let y5 = clmul_lo_and_xor(x5, k, vld1q_u64(buf.add(80) as *const u64)); + let y5 = clmul_lo_and_xor(x5, k, load_u64x2(buf.add(80))); x5 = clmul_hi_and_xor(x5, k, y5); - let y6 = clmul_lo_and_xor(x6, k, vld1q_u64(buf.add(96) as *const u64)); + let y6 = clmul_lo_and_xor(x6, k, load_u64x2(buf.add(96))); x6 = clmul_hi_and_xor(x6, k, y6); - let y7 = clmul_lo_and_xor(x7, k, vld1q_u64(buf.add(112) as *const u64)); + let y7 = clmul_lo_and_xor(x7, k, load_u64x2(buf.add(112))); x7 = clmul_hi_and_xor(x7, k, y7); - let y8 = clmul_lo_and_xor(x8, k, vld1q_u64(buf.add(128) as *const u64)); + let y8 = clmul_lo_and_xor(x8, k, load_u64x2(buf.add(128))); x8 = clmul_hi_and_xor(x8, k, y8); - let y9 = clmul_lo_and_xor(x9, k, vld1q_u64(buf.add(144) as *const u64)); + let y9 = clmul_lo_and_xor(x9, k, load_u64x2(buf.add(144))); x9 = clmul_hi_and_xor(x9, k, y9); - let y10 = clmul_lo_and_xor(x10, k, vld1q_u64(buf.add(160) as *const u64)); + let y10 = clmul_lo_and_xor(x10, k, load_u64x2(buf.add(160))); x10 = clmul_hi_and_xor(x10, k, y10); - let y11 = clmul_lo_and_xor(x11, k, vld1q_u64(buf.add(176) as *const u64)); + let y11 = clmul_lo_and_xor(x11, k, load_u64x2(buf.add(176))); x11 = clmul_hi_and_xor(x11, k, y11); buf = buf.add(192); } @@ -894,11 +993,11 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le crc0 = __crc32d(0, vgetq_lane_u64(x0, 0)); crc0 = __crc32d(crc0, vgetq_lane_u64(x0, 1)); - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 16 { - let mut x0 = vld1q_u64(buf as *const u64); + let mut x0 = load_u64x2(buf); let k_vals: [u64; 2] = [0xae689191, 0xccaa009e]; let k = vld1q_u64(k_vals.as_ptr()); @@ -909,7 +1008,7 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le len = len.strict_sub(16); while len >= 16 { - let y0 = clmul_lo_and_xor(x0, k, vld1q_u64(buf as *const u64)); + let y0 = clmul_lo_and_xor(x0, k, load_u64x2(buf)); x0 = clmul_hi_and_xor(x0, k, y0); buf = buf.add(16); len = len.strict_sub(16); @@ -920,7 +1019,7 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le } while len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -937,19 +1036,30 @@ unsafe fn crc32_iso_hdlc_pmull_v12e_v1(mut crc0: u32, mut buf: *const u8, mut le /// Safe wrapper for CRC-32 fusion kernel (CRC+PMULL v12e_v1). #[inline] -pub fn crc32_iso_hdlc_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_v12e_v1_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32_iso_hdlc_pmull_v12e_v1(crc, data.as_ptr(), data.len()) } } // Fusion: PMULL v9s3x2e_s3 (no EOR3) +/// XOR three vector-register values. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 NEON target feature is available. #[inline] unsafe fn xor3_u64x2(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { - // SAFETY: Caller guarantees NEON is available. veorq_u64 operates on registers. + // SAFETY: The caller establishes NEON support; both intrinsics operate only on registers. unsafe { veorq_u64(veorq_u64(a, b), c) } } +/// Update CRC-32C with the three-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -966,7 +1076,7 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -980,15 +1090,15 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x7e908048, 0xc96cfdc0]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1014,25 +1124,22 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = xor3_u64x2(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = xor3_u64x2(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = xor3_u64x2(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = xor3_u64x2(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = xor3_u64x2(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = xor3_u64x2(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = xor3_u64x2(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = xor3_u64x2(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = xor3_u64x2(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32cd( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + x0 = xor3_u64x2(x0, y0, load_u64x2(buf2)); + x1 = xor3_u64x2(x1, y1, load_u64x2(buf2.add(16))); + x2 = xor3_u64x2(x2, y2, load_u64x2(buf2.add(32))); + x3 = xor3_u64x2(x3, y3, load_u64x2(buf2.add(48))); + x4 = xor3_u64x2(x4, y4, load_u64x2(buf2.add(64))); + x5 = xor3_u64x2(x5, y5, load_u64x2(buf2.add(80))); + x6 = xor3_u64x2(x6, y6, load_u64x2(buf2.add(96))); + x7 = xor3_u64x2(x7, y7, load_u64x2(buf2.add(112))); + x8 = xor3_u64x2(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1084,15 +1191,12 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l x0 = clmul_hi(x0, k); x0 = xor3_u64x2(x0, y0, x4); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32cd( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); let vc1 = crc_shift_iscsi(crc1, klen.strict_add(blk.strict_mul(144))); @@ -1103,7 +1207,7 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l crc0 = __crc32cd(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { @@ -1112,9 +1216,9 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l let mut crc2 = 0u32; loop { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1128,13 +1232,13 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32cd(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1151,11 +1255,17 @@ unsafe fn crc32c_iscsi_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut l /// Safe wrapper for CRC-32C fusion kernel (CRC+PMULL, no EOR3). #[inline] -pub fn crc32c_iscsi_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32c_iscsi_pmull_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } +/// Update CRC-32 (IEEE) with the three-stream PMULL fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. When `len` +/// is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -1172,7 +1282,7 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1186,15 +1296,15 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x26b70c3d, 0x3f41287a]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1220,25 +1330,22 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = xor3_u64x2(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = xor3_u64x2(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = xor3_u64x2(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = xor3_u64x2(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = xor3_u64x2(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = xor3_u64x2(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = xor3_u64x2(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = xor3_u64x2(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = xor3_u64x2(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32d( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + x0 = xor3_u64x2(x0, y0, load_u64x2(buf2)); + x1 = xor3_u64x2(x1, y1, load_u64x2(buf2.add(16))); + x2 = xor3_u64x2(x2, y2, load_u64x2(buf2.add(32))); + x3 = xor3_u64x2(x3, y3, load_u64x2(buf2.add(48))); + x4 = xor3_u64x2(x4, y4, load_u64x2(buf2.add(64))); + x5 = xor3_u64x2(x5, y5, load_u64x2(buf2.add(80))); + x6 = xor3_u64x2(x6, y6, load_u64x2(buf2.add(96))); + x7 = xor3_u64x2(x7, y7, load_u64x2(buf2.add(112))); + x8 = xor3_u64x2(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1290,15 +1397,12 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut x0 = clmul_hi(x0, k); x0 = xor3_u64x2(x0, y0, x4); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)) as *const u64)); - crc2 = __crc32d( - crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)) as *const u64), - ); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); let vc1 = crc_shift_iso_hdlc(crc1, klen.strict_add(blk.strict_mul(144))); @@ -1309,7 +1413,7 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut crc0 = __crc32d(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { @@ -1318,9 +1422,9 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut let mut crc2 = 0u32; loop { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1334,13 +1438,13 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32d(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1357,13 +1461,19 @@ unsafe fn crc32_iso_hdlc_pmull_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut /// Safe wrapper for CRC-32 fusion kernel (CRC+PMULL, no EOR3). #[inline] -pub fn crc32_iso_hdlc_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_iso_hdlc_pmull_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL before selecting this kernel. unsafe { crc32_iso_hdlc_pmull_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } // Fusion: EOR3 variants (CRC + PMULL + SHA3) +/// Update CRC-32C with the three-stream PMULL/EOR3 fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC, AES/PMULL, and SHA3/EOR3 target features are available. +/// When `len` is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes,sha3")] unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -1380,7 +1490,7 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1394,15 +1504,15 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x7e908048, 0xc96cfdc0]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1428,22 +1538,22 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = veor3q_u64(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = veor3q_u64(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = veor3q_u64(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = veor3q_u64(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = veor3q_u64(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = veor3q_u64(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = veor3q_u64(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = veor3q_u64(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = veor3q_u64(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + x0 = veor3q_u64(x0, y0, load_u64x2(buf2)); + x1 = veor3q_u64(x1, y1, load_u64x2(buf2.add(16))); + x2 = veor3q_u64(x2, y2, load_u64x2(buf2.add(32))); + x3 = veor3q_u64(x3, y3, load_u64x2(buf2.add(48))); + x4 = veor3q_u64(x4, y4, load_u64x2(buf2.add(64))); + x5 = veor3q_u64(x5, y5, load_u64x2(buf2.add(80))); + x6 = veor3q_u64(x6, y6, load_u64x2(buf2.add(96))); + x7 = veor3q_u64(x7, y7, load_u64x2(buf2.add(112))); + x8 = veor3q_u64(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1495,15 +1605,15 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, x0 = clmul_hi(x0, k); x0 = veor3q_u64(x0, y0, x4); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32cd(crc0, load_u64(buf.add(8))); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); - let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2) + blk.strict_mul(144)); - let vc1 = crc_shift_iscsi(crc1, klen + blk.strict_mul(144)); + let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); + let vc1 = crc_shift_iscsi(crc1, klen.strict_add(blk.strict_mul(144))); let vc2 = crc_shift_iscsi(crc2, blk.strict_mul(144)); let vc = vgetq_lane_u64(veor3q_u64(vc0, vc1, vc2), 0); @@ -1511,18 +1621,18 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, crc0 = __crc32cd(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { - let klen = ((len.strict_sub(8)) / 24).strict_mul(8); + let klen = len.strict_sub(8).strict_div(24).strict_mul(8); let mut crc1 = 0u32; let mut crc2 = 0u32; loop { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32cd(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32cd(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); + crc1 = __crc32cd(crc1, load_u64(buf.add(klen))); + crc2 = __crc32cd(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1530,19 +1640,19 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, } } - let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2) + 8); - let vc1 = crc_shift_iscsi(crc1, klen + 8); + let vc0 = crc_shift_iscsi(crc0, klen.strict_mul(2).strict_add(8)); + let vc1 = crc_shift_iscsi(crc1, klen.strict_add(8)); let vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0); buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32cd(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32cd(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32cd(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1557,24 +1667,32 @@ unsafe fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, } // unsafe } +/// Return the CRC-32C shift factor for `nbytes`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "aes")] unsafe fn crc_shift_iscsi(crc: u32, nbytes: usize) -> uint64x2_t { - // SAFETY: Caller guarantees AES (PMULL) is available. clmul_scalar operates on registers. + // SAFETY: The caller establishes CRC and AES/PMULL support; both callees operate on registers. unsafe { clmul_scalar(crc, xnmodp_crc32_iscsi((nbytes.strict_mul(8).strict_sub(33)) as u64)) } } +/// Compute `x^n mod p` for the reflected CRC-32C polynomial. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn xnmodp_crc32_iscsi(mut n: u64) -> u32 { - // SAFETY: Caller guarantees CRC+AES target features are available (dispatch check). - // All CRC and NEON intrinsics operate on scalar/register values; no memory access. let mut stack = !1u64; let mut acc: u32; let mut low: u32; while n > 191 { - stack = (stack << 1) + (n & 1); + stack = stack.strict_shl(1) | (n & 1); n = (n >> 1).strict_sub(16); } stack = !stack; @@ -1602,11 +1720,17 @@ unsafe fn xnmodp_crc32_iscsi(mut n: u64) -> u32 { /// Safe wrapper for CRC-32C fusion kernel (CRC+PMULL+EOR3 v9s3x2e_s3). #[inline] -pub fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL + SHA3/EOR3 before selecting this kernel. unsafe { crc32c_iscsi_pmull_eor3_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } +/// Update CRC-32 (IEEE) with the three-stream PMULL/EOR3 fusion kernel. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC, AES/PMULL, and SHA3/EOR3 target features are available. +/// When `len` is nonzero, `buf` must be valid to read `len` initialized bytes from one allocation. #[inline] #[target_feature(enable = "crc,aes,sha3")] unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -1623,7 +1747,7 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 } if (buf as usize & 8) != 0 && len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1637,15 +1761,15 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = vld1q_u64(buf2 as *const u64); - let mut x1 = vld1q_u64(buf2.add(16) as *const u64); - let mut x2 = vld1q_u64(buf2.add(32) as *const u64); - let mut x3 = vld1q_u64(buf2.add(48) as *const u64); - let mut x4 = vld1q_u64(buf2.add(64) as *const u64); - let mut x5 = vld1q_u64(buf2.add(80) as *const u64); - let mut x6 = vld1q_u64(buf2.add(96) as *const u64); - let mut x7 = vld1q_u64(buf2.add(112) as *const u64); - let mut x8 = vld1q_u64(buf2.add(128) as *const u64); + let mut x0 = load_u64x2(buf2); + let mut x1 = load_u64x2(buf2.add(16)); + let mut x2 = load_u64x2(buf2.add(32)); + let mut x3 = load_u64x2(buf2.add(48)); + let mut x4 = load_u64x2(buf2.add(64)); + let mut x5 = load_u64x2(buf2.add(80)); + let mut x6 = load_u64x2(buf2.add(96)); + let mut x7 = load_u64x2(buf2.add(112)); + let mut x8 = load_u64x2(buf2.add(128)); let k_vals: [u64; 2] = [0x26b70c3d, 0x3f41287a]; let mut k = vld1q_u64(k_vals.as_ptr()); @@ -1671,22 +1795,22 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 let y8 = clmul_lo(x8, k); x8 = clmul_hi(x8, k); - x0 = veor3q_u64(x0, y0, vld1q_u64(buf2 as *const u64)); - x1 = veor3q_u64(x1, y1, vld1q_u64(buf2.add(16) as *const u64)); - x2 = veor3q_u64(x2, y2, vld1q_u64(buf2.add(32) as *const u64)); - x3 = veor3q_u64(x3, y3, vld1q_u64(buf2.add(48) as *const u64)); - x4 = veor3q_u64(x4, y4, vld1q_u64(buf2.add(64) as *const u64)); - x5 = veor3q_u64(x5, y5, vld1q_u64(buf2.add(80) as *const u64)); - x6 = veor3q_u64(x6, y6, vld1q_u64(buf2.add(96) as *const u64)); - x7 = veor3q_u64(x7, y7, vld1q_u64(buf2.add(112) as *const u64)); - x8 = veor3q_u64(x8, y8, vld1q_u64(buf2.add(128) as *const u64)); - - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + x0 = veor3q_u64(x0, y0, load_u64x2(buf2)); + x1 = veor3q_u64(x1, y1, load_u64x2(buf2.add(16))); + x2 = veor3q_u64(x2, y2, load_u64x2(buf2.add(32))); + x3 = veor3q_u64(x3, y3, load_u64x2(buf2.add(48))); + x4 = veor3q_u64(x4, y4, load_u64x2(buf2.add(64))); + x5 = veor3q_u64(x5, y5, load_u64x2(buf2.add(80))); + x6 = veor3q_u64(x6, y6, load_u64x2(buf2.add(96))); + x7 = veor3q_u64(x7, y7, load_u64x2(buf2.add(112))); + x8 = veor3q_u64(x8, y8, load_u64x2(buf2.add(128))); + + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); buf = buf.add(16); buf2 = buf2.add(144); @@ -1738,15 +1862,15 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 x0 = clmul_hi(x0, k); x0 = veor3q_u64(x0, y0, x4); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = __crc32d(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); + crc0 = __crc32d(crc0, load_u64(buf.add(8))); + crc1 = __crc32d(crc1, load_u64(buf.add(klen.strict_add(8)))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2).strict_add(8)))); - let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2) + blk.strict_mul(144)); - let vc1 = crc_shift_iso_hdlc(crc1, klen + blk.strict_mul(144)); + let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2).strict_add(blk.strict_mul(144))); + let vc1 = crc_shift_iso_hdlc(crc1, klen.strict_add(blk.strict_mul(144))); let vc2 = crc_shift_iso_hdlc(crc2, blk.strict_mul(144)); let vc = vgetq_lane_u64(veor3q_u64(vc0, vc1, vc2), 0); @@ -1754,18 +1878,18 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 crc0 = __crc32d(crc0, vc ^ vgetq_lane_u64(x0, 1)); buf = buf2; - len = end.offset_from(buf) as usize; + len = end.offset_from_unsigned(buf); } if len >= 32 { - let klen = ((len.strict_sub(8)) / 24).strict_mul(8); + let klen = len.strict_sub(8).strict_div(24).strict_mul(8); let mut crc1 = 0u32; let mut crc2 = 0u32; loop { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = __crc32d(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = __crc32d(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); + crc1 = __crc32d(crc1, load_u64(buf.add(klen))); + crc2 = __crc32d(crc2, load_u64(buf.add(klen.strict_mul(2)))); buf = buf.add(8); len = len.strict_sub(24); if len < 32 { @@ -1773,19 +1897,19 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 } } - let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2) + 8); - let vc1 = crc_shift_iso_hdlc(crc1, klen + 8); + let vc0 = crc_shift_iso_hdlc(crc0, klen.strict_mul(2).strict_add(8)); + let vc1 = crc_shift_iso_hdlc(crc1, klen.strict_add(8)); let vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0); buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = __crc32d(crc0, load_u64(buf) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = __crc32d(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = __crc32d(crc0, load_u64(buf)); buf = buf.add(8); len = len.strict_sub(8); } @@ -1800,24 +1924,32 @@ unsafe fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(mut crc0: u32, mut buf: *const u8 } // unsafe } +/// Return the CRC-32 (IEEE) shift factor for `nbytes`. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "aes")] unsafe fn crc_shift_iso_hdlc(crc: u32, nbytes: usize) -> uint64x2_t { - // SAFETY: Caller guarantees AES (PMULL) is available. clmul_scalar operates on registers. + // SAFETY: The caller establishes CRC and AES/PMULL support; both callees operate on registers. unsafe { clmul_scalar(crc, xnmodp_iso_hdlc((nbytes.strict_mul(8).strict_sub(33)) as u64)) } } +/// Compute `x^n mod p` for the reflected CRC-32 (IEEE) polynomial. +/// +/// # Safety +/// +/// The caller must ensure the AArch64 CRC and AES/PMULL target features are available. #[inline] #[target_feature(enable = "crc,aes")] unsafe fn xnmodp_iso_hdlc(mut n: u64) -> u32 { - // SAFETY: Caller guarantees CRC+AES target features are available (dispatch check). - // All CRC and NEON intrinsics operate on scalar/register values; no memory access. let mut stack = !1u64; let mut acc: u32; let mut low: u32; while n > 191 { - stack = (stack << 1) + (n & 1); + stack = stack.strict_shl(1) | (n & 1); n = (n >> 1).strict_sub(16); } stack = !stack; @@ -1845,7 +1977,7 @@ unsafe fn xnmodp_iso_hdlc(mut n: u64) -> u32 { /// Safe wrapper for CRC-32 fusion kernel (CRC+PMULL+EOR3 v9s3x2e_s3). #[inline] -pub fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies CRC + PMULL + SHA3/EOR3 before selecting this kernel. unsafe { crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3(crc, data.as_ptr(), data.len()) } } @@ -1866,25 +1998,34 @@ mod tests { fn make_data(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + .map(|i| { + let [low, high, ..] = i.to_le_bytes(); + low.wrapping_mul(31).wrapping_add(high) + }) .collect() } fn assert_crc32_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32, lens: &[usize]) { for &len in lens { - let data = make_data(len); - let expected = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let got = kernel(!0, &data) ^ !0; - assert_eq!(got, expected, "{name} len={len}"); + for offset in 0usize..16 { + let storage = make_data(len.strict_add(offset)); + let data = &storage[offset..]; + let expected = super::super::portable::crc32_slice16_ieee(!0, data) ^ !0; + let got = kernel(!0, data) ^ !0; + assert_eq!(got, expected, "{name} len={len} offset={offset}"); + } } } fn assert_crc32c_kernel(name: &str, kernel: fn(u32, &[u8]) -> u32, lens: &[usize]) { for &len in lens { - let data = make_data(len); - let expected = super::super::portable::crc32c_slice16(!0, &data) ^ !0; - let got = kernel(!0, &data) ^ !0; - assert_eq!(got, expected, "{name} len={len}"); + for offset in 0usize..16 { + let storage = make_data(len.strict_add(offset)); + let data = &storage[offset..]; + let expected = super::super::portable::crc32c_slice16(!0, data) ^ !0; + let got = kernel(!0, data) ^ !0; + assert_eq!(got, expected, "{name} len={len} offset={offset}"); + } } } diff --git a/src/checksum/crc32/clmul.rs b/src/checksum/crc32/clmul.rs index 6e67c644..23143cf1 100644 --- a/src/checksum/crc32/clmul.rs +++ b/src/checksum/crc32/clmul.rs @@ -35,16 +35,17 @@ const fn reduce128_crc32(hi: u64, lo: u64, poly: u32) -> u32 { let poly_full: u128 = (1u128 << 32) | (poly as u128); let mut val: u128 = ((hi as u128) << 64) | (lo as u128); - let mut bit: i32 = 127; - while bit >= 32 { - let b = bit as u32; + let mut bit = 128u32; + while bit > 32 { + bit = bit.strict_sub(1); + let b = bit; if ((val >> b) & 1) != 0 { val ^= poly_full << b.strict_sub(32); } - bit = bit.strict_sub(1); } - val as u32 + let bytes = val.to_le_bytes(); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) } /// Compute `x^n mod (x^32 + poly)` in GF(2), where `poly` is the **normal** CRC polynomial @@ -114,12 +115,12 @@ const fn compute_mu33(poly: u64) -> u64 { let mut i: u32 = 1; while i <= k { let p_i = (poly >> i) & 1; - let q_j = (inv >> (k - i)) & 1; + let q_j = (inv >> k.strict_sub(i)) & 1; s ^= p_i & q_j; i = i.strict_add(1); } - inv |= s << k; + inv |= s.strict_shl(k); k = k.strict_add(1); } @@ -160,7 +161,7 @@ pub(super) struct Crc32ClmulConstants { impl Crc32ClmulConstants { #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + pub(super) const fn new(reflected_poly: u32) -> Self { let poly = reciprocal_poly_crc32(reflected_poly); let mu = compute_mu33(poly); @@ -185,27 +186,30 @@ impl Crc32ClmulConstants { /// Multi-stream folding constants for CRC-32 CLMUL kernels. #[derive(Clone, Copy, Debug)] -#[allow(dead_code)] // Field subsets vary by architecture (x86_64/aarch64/power vs s390x stream widths). pub(super) struct Crc32StreamConstants { pub fold_256b: (u64, u64), pub fold_512b: (u64, u64), + #[cfg(target_arch = "powerpc64")] pub fold_1024b: (u64, u64), pub combine_4way: [(u64, u64); 3], + #[cfg(target_arch = "powerpc64")] pub combine_8way: [(u64, u64); 7], } impl Crc32StreamConstants { #[must_use] - pub const fn new(reflected_poly: u32) -> Self { + pub(super) const fn new(reflected_poly: u32) -> Self { Self { fold_256b: fold16_coeff_for_bytes_crc32(reflected_poly, 256), fold_512b: fold16_coeff_for_bytes_crc32(reflected_poly, 512), + #[cfg(target_arch = "powerpc64")] fold_1024b: fold16_coeff_for_bytes_crc32(reflected_poly, 1024), combine_4way: [ fold16_coeff_for_bytes_crc32(reflected_poly, 384), fold16_coeff_for_bytes_crc32(reflected_poly, 256), fold16_coeff_for_bytes_crc32(reflected_poly, 128), ], + #[cfg(target_arch = "powerpc64")] combine_8way: [ fold16_coeff_for_bytes_crc32(reflected_poly, 896), fold16_coeff_for_bytes_crc32(reflected_poly, 768), diff --git a/src/checksum/crc32/config.rs b/src/checksum/crc32/config.rs index dd6e986f..64008fc8 100644 --- a/src/checksum/crc32/config.rs +++ b/src/checksum/crc32/config.rs @@ -123,8 +123,16 @@ fn parse_force_env() -> Crc32Force { #[inline] #[must_use] -#[allow(unused_variables)] fn clamp_force_to_caps(requested: Crc32Force, caps: Caps) -> Crc32Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc32Force::Auto | Crc32Force::Reference | Crc32Force::Portable => requested, Crc32Force::Hwcrc => { @@ -267,7 +275,7 @@ fn config(caps: Caps) -> Crc32Config { /// platform capabilities. #[inline] #[must_use] -pub fn get() -> Crc32Config { +pub(super) fn get() -> Crc32Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc32/kernels.rs b/src/checksum/crc32/kernels.rs index 12f64d59..c173cae9 100644 --- a/src/checksum/crc32/kernels.rs +++ b/src/checksum/crc32/kernels.rs @@ -14,21 +14,21 @@ //! - Tier 4 (Wide): VPCLMUL (x86_64), PMULL+EOR3/SVE2 (aarch64), Zvbc (riscv64) /// Portable fallback kernel name. -pub use kernels::PORTABLE_SLICE16 as PORTABLE; +pub(in crate::checksum) use kernels::PORTABLE_SLICE16 as PORTABLE; /// Reference (bitwise) kernel name - always available for force mode. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; // Kernel Name Tables (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) PCLMUL kernels. - pub const CRC32_PCLMUL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_PCLMUL: [Crc32Fn; 5] = [ arch::crc32_ieee_pclmul_safe, arch::crc32_ieee_pclmul_2way_safe, arch::crc32_ieee_pclmul_4way_safe, @@ -36,10 +36,10 @@ pub mod x86_64 { arch::crc32_ieee_pclmul_8way_safe, ]; /// CRC-32 (IEEE) PCLMUL small-buffer kernel. - pub const CRC32_PCLMUL_SMALL_KERNEL: Crc32Fn = arch::crc32_ieee_pclmul_small_safe; + pub(in crate::checksum) const CRC32_PCLMUL_SMALL_KERNEL: Crc32Fn = arch::crc32_ieee_pclmul_small_safe; /// CRC-32 (IEEE) VPCLMUL kernels. - pub const CRC32_VPCLMUL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_VPCLMUL: [Crc32Fn; 5] = [ arch::crc32_ieee_vpclmul_safe, arch::crc32_ieee_vpclmul_2way_safe, arch::crc32_ieee_vpclmul_4way_safe, @@ -47,7 +47,7 @@ pub mod x86_64 { arch::crc32_ieee_vpclmul_8way_safe, ]; /// CRC-32C SSE4.2 kernel function array. - pub const CRC32C_HWCRC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_HWCRC: [Crc32Fn; 5] = [ arch::crc32c_sse42_safe, arch::crc32c_sse42_2way_safe, arch::crc32c_sse42_4way_safe, @@ -56,7 +56,7 @@ pub mod x86_64 { ]; /// CRC-32C fusion (SSE4.2 + PCLMULQDQ) kernels. - pub const CRC32C_FUSION_SSE: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_FUSION_SSE: [Crc32Fn; 5] = [ arch::crc32c_iscsi_sse_v4s3x3_safe, arch::crc32c_iscsi_sse_v4s3x3_2way_safe, arch::crc32c_iscsi_sse_v4s3x3_4way_safe, @@ -65,7 +65,7 @@ pub mod x86_64 { ]; /// CRC-32C fusion (AVX-512 + VPCLMULQDQ) kernels. - pub const CRC32C_FUSION_VPCLMUL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_FUSION_VPCLMUL: [Crc32Fn; 5] = [ arch::crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe, arch::crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe, arch::crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe, @@ -75,14 +75,13 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { - #![cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) CRC-extension kernel function array. - pub const CRC32_HWCRC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_HWCRC: [Crc32Fn; 5] = [ arch::crc32_armv8_safe, arch::crc32_armv8_2way_safe, arch::crc32_armv8_3way_safe, @@ -91,7 +90,7 @@ pub mod aarch64 { ]; /// CRC-32C (Castagnoli) CRC-extension kernel function array. - pub const CRC32C_HWCRC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_HWCRC: [Crc32Fn; 5] = [ arch::crc32c_armv8_safe, arch::crc32c_armv8_2way_safe, arch::crc32c_armv8_3way_safe, @@ -100,7 +99,7 @@ pub mod aarch64 { ]; /// CRC-32 (IEEE) PMULL fusion kernels. - pub const CRC32_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_PMULL: [Crc32Fn; 5] = [ arch::crc32_iso_hdlc_pmull_v9s3x2e_s3_safe, arch::crc32_iso_hdlc_pmull_2way_safe, arch::crc32_iso_hdlc_pmull_3way_safe, @@ -108,10 +107,10 @@ pub mod aarch64 { arch::crc32_iso_hdlc_pmull_3way_safe, // dup for index consistency ]; /// CRC-32 (IEEE) PMULL small-buffer kernel. - pub const CRC32_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_pmull_small_safe; + pub(in crate::checksum) const CRC32_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_pmull_small_safe; /// CRC-32 (IEEE) PMULL+EOR3 fusion kernels. - pub const CRC32_PMULL_EOR3: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_PMULL_EOR3: [Crc32Fn; 5] = [ arch::crc32_iso_hdlc_pmull_eor3_v9s3x2e_s3_safe, arch::crc32_iso_hdlc_pmull_eor3_2way_safe, arch::crc32_iso_hdlc_pmull_eor3_3way_safe, @@ -120,7 +119,7 @@ pub mod aarch64 { ]; /// CRC-32C (Castagnoli) PMULL fusion kernels. - pub const CRC32C_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_PMULL: [Crc32Fn; 5] = [ arch::crc32c_iscsi_pmull_v9s3x2e_s3_safe, arch::crc32c_iscsi_pmull_2way_safe, arch::crc32c_iscsi_pmull_3way_safe, @@ -128,10 +127,10 @@ pub mod aarch64 { arch::crc32c_iscsi_pmull_3way_safe, // dup for index consistency ]; /// CRC-32C PMULL small-buffer kernel. - pub const CRC32C_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_pmull_small_safe; + pub(in crate::checksum) const CRC32C_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_pmull_small_safe; /// CRC-32C (Castagnoli) PMULL+EOR3 fusion kernels. - pub const CRC32C_PMULL_EOR3: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_PMULL_EOR3: [Crc32Fn; 5] = [ arch::crc32c_iscsi_pmull_eor3_v9s3x2e_s3_safe, arch::crc32c_iscsi_pmull_eor3_2way_safe, arch::crc32c_iscsi_pmull_eor3_3way_safe, @@ -139,36 +138,40 @@ pub mod aarch64 { arch::crc32c_iscsi_pmull_eor3_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// CRC-32 (IEEE) "SVE2 PMULL" tier kernels (2/3-way striping). - pub const CRC32_SVE2_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_SVE2_PMULL: [Crc32Fn; 5] = [ arch::crc32_iso_hdlc_pmull_v12e_v1_safe, arch::crc32_iso_hdlc_sve2_pmull_2way_safe, arch::crc32_iso_hdlc_sve2_pmull_3way_safe, arch::crc32_iso_hdlc_sve2_pmull_3way_safe, arch::crc32_iso_hdlc_sve2_pmull_3way_safe, ]; + #[cfg(feature = "std")] /// CRC-32 "SVE2 PMULL" small-buffer kernel. - pub const CRC32_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_sve2_pmull_small_safe; + pub(in crate::checksum) const CRC32_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32_iso_hdlc_sve2_pmull_small_safe; + #[cfg(feature = "std")] /// CRC-32C (Castagnoli) "SVE2 PMULL" tier kernels (2/3-way striping). - pub const CRC32C_SVE2_PMULL: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_SVE2_PMULL: [Crc32Fn; 5] = [ arch::crc32c_iscsi_pmull_v12e_v1_safe, arch::crc32c_iscsi_sve2_pmull_2way_safe, arch::crc32c_iscsi_sve2_pmull_3way_safe, arch::crc32c_iscsi_sve2_pmull_3way_safe, arch::crc32c_iscsi_sve2_pmull_3way_safe, ]; + #[cfg(feature = "std")] /// CRC-32C "SVE2 PMULL" small-buffer kernel. - pub const CRC32C_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_sve2_pmull_small_safe; + pub(in crate::checksum) const CRC32C_SVE2_PMULL_SMALL_KERNEL: Crc32Fn = arch::crc32c_iscsi_sve2_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const CRC32_VPMSUM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_VPMSUM: [Crc32Fn; 5] = [ arch::crc32_ieee_vpmsum_safe, arch::crc32_ieee_vpmsum_2way_safe, arch::crc32_ieee_vpmsum_4way_safe, @@ -177,7 +180,7 @@ pub mod power { ]; /// CRC-32C VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)]. - pub const CRC32C_VPMSUM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_VPMSUM: [Crc32Fn; 5] = [ arch::crc32c_vpmsum_safe, arch::crc32c_vpmsum_2way_safe, arch::crc32c_vpmsum_4way_safe, @@ -187,12 +190,12 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc32Fn; /// CRC-32 (IEEE) VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_VGFM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_VGFM: [Crc32Fn; 5] = [ arch::crc32_ieee_vgfm_safe, arch::crc32_ieee_vgfm_2way_safe, arch::crc32_ieee_vgfm_4way_safe, @@ -201,7 +204,7 @@ pub mod s390x { ]; /// CRC-32C VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_VGFM: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_VGFM: [Crc32Fn; 5] = [ arch::crc32c_vgfm_safe, arch::crc32c_vgfm_2way_safe, arch::crc32c_vgfm_4way_safe, @@ -211,31 +214,12 @@ pub mod s390x { } #[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { +pub(in crate::checksum) mod riscv64 { use super::super::riscv64 as arch; use crate::checksum::dispatchers::Crc32Fn; - /// Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - /// CRC-32 (IEEE) Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_ZBC: [Crc32Fn; 5] = [ arch::crc32_ieee_zbc_safe, arch::crc32_ieee_zbc_2way_safe, arch::crc32_ieee_zbc_4way_safe, @@ -244,7 +228,7 @@ pub mod riscv64 { ]; /// CRC-32 (IEEE) Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32_ZVBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32_ZVBC: [Crc32Fn; 5] = [ arch::crc32_ieee_zvbc_safe, arch::crc32_ieee_zvbc_2way_safe, arch::crc32_ieee_zvbc_4way_safe, @@ -252,26 +236,8 @@ pub mod riscv64 { arch::crc32_ieee_zvbc_4way_safe, // dup for index consistency ]; - /// CRC-32C Zbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - "riscv64/zbc-4way", - ]; - - /// CRC-32C Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - /// CRC-32C Zbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_ZBC: [Crc32Fn; 5] = [ arch::crc32c_zbc_safe, arch::crc32c_zbc_2way_safe, arch::crc32c_zbc_4way_safe, @@ -280,7 +246,7 @@ pub mod riscv64 { ]; /// CRC-32C Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)]. - pub const CRC32C_ZVBC: [Crc32Fn; 5] = [ + pub(in crate::checksum) const CRC32C_ZVBC: [Crc32Fn; 5] = [ arch::crc32c_zvbc_safe, arch::crc32c_zvbc_2way_safe, arch::crc32c_zvbc_4way_safe, diff --git a/src/checksum/crc32/mod.rs b/src/checksum/crc32/mod.rs index 9a1a0ca3..5ba8b47d 100644 --- a/src/checksum/crc32/mod.rs +++ b/src/checksum/crc32/mod.rs @@ -14,29 +14,6 @@ //! - riscv64: ZVBC (RVV vector CLMUL) / Zbc folding (CRC-32 and CRC-32C) //! - wasm32/wasm64: portable only (no CRC32/CLMUL instructions) //! -//! # Quick Start -//! -//! ```rust -//! use rscrypto::checksum::{Checksum, ChecksumCombine, Crc32, Crc32C}; -//! -//! let data = b"123456789"; -//! -//! // One-shot -//! assert_eq!(Crc32::checksum(data), 0xCBF4_3926); -//! assert_eq!(Crc32C::checksum(data), 0xE306_9283); -//! -//! // Streaming -//! let mut hasher = Crc32::new(); -//! hasher.update(b"1234"); -//! hasher.update(b"56789"); -//! assert_eq!(hasher.finalize(), Crc32::checksum(data)); -//! -//! // Combine: crc(A || B) == combine(crc(A), crc(B), len(B)) -//! let (a, b) = data.split_at(4); -//! let combined = Crc32::combine(Crc32::checksum(a), Crc32::checksum(b), b.len()); -//! assert_eq!(combined, Crc32::checksum(data)); -//! ``` - #[cfg(any(target_arch = "powerpc64", target_arch = "s390x", target_arch = "riscv64"))] mod clmul; pub(crate) mod config; @@ -58,7 +35,6 @@ mod s390x; #[cfg(target_arch = "riscv64")] mod riscv64; -#[allow(unused_imports)] pub use config::{Crc32Config, Crc32Force}; #[cfg(any(test, feature = "std"))] @@ -69,16 +45,14 @@ use crate::checksum::common::{ }; #[cfg(feature = "diag")] use crate::checksum::diag::{Crc32Polynomial, Crc32SelectionDiag}; -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; // Kernel Tables (compile-time) /// Portable kernel tables (pre-computed at compile time). mod kernel_tables { use super::*; - pub static IEEE_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32_IEEE_POLY); - pub static CRC32C_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32C_POLY); + pub(super) static IEEE_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32_IEEE_POLY); + pub(super) static CRC32C_TABLES_16: [[u32; 256]; 16] = generate_crc32_tables_16(CRC32C_POLY); } /// Block size for CRC-32 folding operations. @@ -87,8 +61,7 @@ pub(crate) const CRC32_FOLD_BLOCK_BYTES: usize = 128; // Portable Kernel Wrappers -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc32_portable(crc: u32, data: &[u8]) -> u32 { const THRESHOLD: usize = 64; if data.len() < THRESHOLD { @@ -98,8 +71,7 @@ fn crc32_portable(crc: u32, data: &[u8]) -> u32 { } } -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc32c_portable(crc: u32, data: &[u8]) -> u32 { const THRESHOLD: usize = 64; if data.len() < THRESHOLD { @@ -791,6 +763,7 @@ impl crate::traits::ChecksumCombine for Crc32 { #[cfg(feature = "alloc")] impl Crc32 { + /// Creates a buffering wrapper that coalesces short updates before CRC-32/IEEE dispatch. #[must_use] pub fn buffered() -> BufferedCrc32 { BufferedCrc32::new() @@ -934,6 +907,7 @@ impl crate::traits::ChecksumCombine for Crc32C { #[cfg(feature = "alloc")] impl Crc32C { + /// Creates a buffering wrapper that coalesces short updates before CRC-32C dispatch. #[must_use] pub fn buffered() -> BufferedCrc32C { BufferedCrc32C::new() @@ -991,9 +965,18 @@ mod tests { use alloc::{string::String, vec::Vec}; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; const TEST_DATA: &[u8] = b"123456789"; + fn patterned_data(len: usize, multiplier: u8, addend: u8) -> Vec { + (0u8..=u8::MAX) + .cycle() + .take(len) + .map(|byte| byte.wrapping_mul(multiplier).wrapping_add(addend)) + .collect() + } + #[test] fn test_crc32_test_vectors() { assert_eq!(Crc32::checksum(TEST_DATA), 0xCBF4_3926); @@ -1063,10 +1046,7 @@ mod tests { #[test] fn test_crc32_various_lengths_streaming_matches_oneshot() { - let mut data = [0u8; 512]; - for (i, b) in data.iter_mut().enumerate() { - *b = (i as u8).wrapping_mul(17).wrapping_add(i as u8); - } + let data = patterned_data(512, 18, 0); for &len in &[0usize, 1, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 400, 512] { let slice = &data[..len]; @@ -1098,7 +1078,7 @@ mod tests { #[cfg(feature = "alloc")] #[test] fn test_buffered_crc32_matches_unbuffered() { - let data: Vec = (0..2048).map(|i| (i as u8).wrapping_mul(31)).collect(); + let data = patterned_data(2048, 31, 0); let expected = Crc32::checksum(&data); let mut buffered = BufferedCrc32::new(); @@ -1111,7 +1091,7 @@ mod tests { #[cfg(feature = "alloc")] #[test] fn test_buffered_crc32c_matches_unbuffered() { - let data: Vec = (0..2048).map(|i| (i as u8).wrapping_mul(29).wrapping_add(7)).collect(); + let data = patterned_data(2048, 29, 7); let expected = Crc32C::checksum(&data); let mut buffered = BufferedCrc32C::new(); @@ -1135,7 +1115,7 @@ mod tests { for &threshold in &crc32_thresholds { let size = threshold + 256; - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(13)).collect(); + let data = patterned_data(size, 13, 0); let oneshot32 = Crc32::checksum(&data); @@ -1148,7 +1128,7 @@ mod tests { // Same thresholds for CRC32C for &threshold in &crc32_thresholds { let size = threshold + 256; - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(13)).collect(); + let data = patterned_data(size, 13, 0); let oneshot32c = Crc32C::checksum(&data); @@ -1173,7 +1153,7 @@ mod tests { } let len = 64 * 1024; - let data: Vec = (0..len).map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)).collect(); + let data = patterned_data(len, 31, 7); let expected = portable::crc32_slice16_ieee(!0, &data) ^ !0; let got = Crc32::checksum(&data); assert_eq!(got, expected); @@ -1234,7 +1214,7 @@ mod tests { } let len = 64 * 1024; - let data: Vec = (0..len).map(|i| (i as u8).wrapping_mul(29).wrapping_add(7)).collect(); + let data = patterned_data(len, 29, 7); let expected = portable::crc32c_slice16(!0, &data) ^ !0; let got = Crc32C::checksum(&data); assert_eq!(got, expected); diff --git a/src/checksum/crc32/portable.rs b/src/checksum/crc32/portable.rs index 20cada34..be43345e 100644 --- a/src/checksum/crc32/portable.rs +++ b/src/checksum/crc32/portable.rs @@ -16,7 +16,7 @@ pub(crate) const BYTEWISE_KERNEL_NAME: &str = "portable/bytewise"; /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { crc32_bytewise(crc, data, &kernel_tables::IEEE_TABLES_16[0]) } @@ -24,19 +24,19 @@ pub fn crc32_bytewise_ieee(crc: u32, data: &[u8]) -> u32 { /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc32c_bytewise(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_bytewise(crc: u32, data: &[u8]) -> u32 { crc32_bytewise(crc, data, &kernel_tables::CRC32C_TABLES_16[0]) } /// CRC-32 (IEEE) slice-by-16 computation. #[inline] -pub fn crc32_slice16_ieee(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32_slice16_ieee(crc: u32, data: &[u8]) -> u32 { crc32_slice16(crc, data, &kernel_tables::IEEE_TABLES_16) } /// CRC-32C (Castagnoli) slice-by-16 computation. #[inline] -pub fn crc32c_slice16(crc: u32, data: &[u8]) -> u32 { +pub(in crate::checksum) fn crc32c_slice16(crc: u32, data: &[u8]) -> u32 { crc32_slice16(crc, data, &kernel_tables::CRC32C_TABLES_16) } @@ -44,8 +44,7 @@ pub fn crc32c_slice16(crc: u32, data: &[u8]) -> u32 { /// Update CRC-32 state using a byte-at-a-time lookup table. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by mask, table is [u32; 256] -pub fn crc32_bytewise(mut crc: u32, data: &[u8], table: &[u32; 256]) -> u32 { +fn crc32_bytewise(mut crc: u32, data: &[u8], table: &[u32; 256]) -> u32 { for &b in data { let index = ((crc ^ (b as u32)) & 0xFF) as usize; crc = table[index] ^ (crc >> 8); @@ -55,6 +54,6 @@ pub fn crc32_bytewise(mut crc: u32, data: &[u8], table: &[u32; 256]) -> u32 { /// Update CRC-32 state using slice-by-16 algorithm. #[inline] -pub fn crc32_slice16(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { +fn crc32_slice16(crc: u32, data: &[u8], tables: &[[u32; 256]; 16]) -> u32 { portable::slice16_32(crc, data, tables) } diff --git a/src/checksum/crc32/power.rs b/src/checksum/crc32/power.rs index f827b352..8000cfd2 100644 --- a/src/checksum/crc32/power.rs +++ b/src/checksum/crc32/power.rs @@ -8,10 +8,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -19,6 +16,8 @@ use core::{ simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::clmul::{Crc32ClmulConstants, Crc32StreamConstants}; #[repr(transparent)] @@ -78,6 +77,10 @@ impl Simd { /// On `powerpc64le` this is a no-op. On big-endian `powerpc64`, we byte-swap /// each 64-bit lane so the folding algorithm sees the same lane values as on /// little-endian platforms. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { @@ -103,6 +106,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -125,6 +131,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -144,6 +153,10 @@ impl Simd { /// Fold 16 bytes (reflected CRC32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -162,6 +175,10 @@ impl Simd { } /// Fold 16B → CRC32 width (reflected), returning an intermediate 128-bit state. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -190,6 +207,10 @@ impl Simd { } /// Barrett reduction for reflected CRC32; returns the updated (pre-inverted) CRC. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -204,13 +225,16 @@ impl Simd { // Mirror the x86 reduction scheme (2 multiplies + xor, extract hi32). let t1 = Self::mul64(self.low_64(), mu); let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 + low_u32((self ^ l).high_64()) } } } // Folding helpers +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline(always)] unsafe fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { // SAFETY: All target-feature-gated calls require power8-crypto, ensured by callers' @@ -239,6 +263,9 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -261,6 +288,9 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -293,6 +323,9 @@ unsafe fn update_simd(state: u32, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -348,7 +381,7 @@ unsafe fn update_simd_2way( let mut i = 2usize; while i < even { fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); i = i.strict_add(2); } @@ -371,6 +404,9 @@ unsafe fn update_simd_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -396,7 +432,7 @@ unsafe fn update_simd_4way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -450,9 +486,9 @@ unsafe fn update_simd_4way( let mut i = 4usize; while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); i = i.strict_add(4); } @@ -503,6 +539,9 @@ unsafe fn update_simd_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -528,7 +567,7 @@ unsafe fn update_simd_8way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -568,13 +607,13 @@ unsafe fn update_simd_8way( let mut idx = 8usize; while idx < aligned { fold_block_128(&mut s0, &blocks[idx], coeff_1024); - fold_block_128(&mut s1, &blocks[idx + 1], coeff_1024); - fold_block_128(&mut s2, &blocks[idx + 2], coeff_1024); - fold_block_128(&mut s3, &blocks[idx + 3], coeff_1024); - fold_block_128(&mut s4, &blocks[idx + 4], coeff_1024); - fold_block_128(&mut s5, &blocks[idx + 5], coeff_1024); - fold_block_128(&mut s6, &blocks[idx + 6], coeff_1024); - fold_block_128(&mut s7, &blocks[idx + 7], coeff_1024); + fold_block_128(&mut s1, &blocks[idx.strict_add(1)], coeff_1024); + fold_block_128(&mut s2, &blocks[idx.strict_add(2)], coeff_1024); + fold_block_128(&mut s3, &blocks[idx.strict_add(3)], coeff_1024); + fold_block_128(&mut s4, &blocks[idx.strict_add(4)], coeff_1024); + fold_block_128(&mut s5, &blocks[idx.strict_add(5)], coeff_1024); + fold_block_128(&mut s6, &blocks[idx.strict_add(6)], coeff_1024); + fold_block_128(&mut s7, &blocks[idx.strict_add(7)], coeff_1024); idx = idx.strict_add(8); } @@ -611,6 +650,9 @@ unsafe fn update_simd_8way( // Public kernels (IEEE + CRC32C) +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -635,6 +677,9 @@ unsafe fn crc32_fold_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -658,6 +703,9 @@ unsafe fn crc32c_fold_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -693,6 +741,9 @@ unsafe fn crc32_fold_kernel_nway( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -731,13 +782,13 @@ unsafe fn crc32c_fold_kernel_nway( // Safe wrappers (dispatcher entrypoints) #[inline] -pub fn crc32_ieee_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel_nway::<2>( @@ -750,7 +801,7 @@ pub fn crc32_ieee_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel_nway::<4>( @@ -763,7 +814,7 @@ pub fn crc32_ieee_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32_fold_kernel_nway::<8>( @@ -776,25 +827,25 @@ pub fn crc32_ieee_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel_nway::<2>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel_nway::<4>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vpmsum_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc32c_fold_kernel_nway::<8>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } diff --git a/src/checksum/crc32/riscv64.rs b/src/checksum/crc32/riscv64.rs index ad20d05b..c8e22de2 100644 --- a/src/checksum/crc32/riscv64.rs +++ b/src/checksum/crc32/riscv64.rs @@ -9,16 +9,14 @@ //! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required //! CPU features are available before executing the accelerated path (the //! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, ops::{BitXor, BitXorAssign}, }; +use crate::checksum::common::low_u32; + use super::clmul::{Crc32ClmulConstants, Crc32StreamConstants}; type Block = [u64; 16]; // 128 bytes (8×16B lanes) @@ -68,8 +66,11 @@ impl Simd { // Zbc carryless multiply primitives #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { + fn clmul_lo(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -86,8 +87,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { + fn clmul_hi(a: u64, b: u64) -> u64 { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -104,15 +108,14 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } + fn mul64(a: u64, b: u64) -> Self { + Self { + hi: Self::clmul_hi(a, b), + lo: Self::clmul_lo(a, b), } } } @@ -122,13 +125,8 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { let mut out = [Simd::new(0, 0); 8]; - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - out[i] = Simd::new(high, low); - i = i.strict_add(1); + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } out @@ -137,256 +135,243 @@ fn load_block(block: &Block) -> [Simd; 8] { // Zbc folding helpers (scalar) #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - // Reflected CRC32 fold primitive is cross-term: low×high ⊕ high×low. - Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) - } +fn fold_16_zbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + // Reflected CRC32 fold primitive is cross-term: low×high ⊕ high×low. + Simd::mul64(x.low_64(), coeff_high) ^ Simd::mul64(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_width_crc32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = Simd::mul64(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width_crc32_reflected_zbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = Simd::mul64(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Simd::mul64(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Simd::mul64(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn barrett_crc32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Simd::mul64(x.low_64(), mu); - let l = Simd::mul64(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_crc32_reflected_zbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = Simd::mul64(x.low_64(), mu); + let l = Simd::mul64(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_tail_zbc(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = x[7]; - acc ^= fold_16_zbc(x[0], consts.tail_fold_16b[0]); - acc ^= fold_16_zbc(x[1], consts.tail_fold_16b[1]); - acc ^= fold_16_zbc(x[2], consts.tail_fold_16b[2]); - acc ^= fold_16_zbc(x[3], consts.tail_fold_16b[3]); - acc ^= fold_16_zbc(x[4], consts.tail_fold_16b[4]); - acc ^= fold_16_zbc(x[5], consts.tail_fold_16b[5]); - acc ^= fold_16_zbc(x[6], consts.tail_fold_16b[6]); - - let (fold_width_high, fold_width_low) = consts.fold_width; - barrett_crc32_reflected_zbc( - fold_width_crc32_reflected_zbc(acc, fold_width_high, fold_width_low), - consts.poly, - consts.mu, - ) - } +fn fold_tail_zbc(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { + let mut acc = x[7]; + acc ^= fold_16_zbc(x[0], consts.tail_fold_16b[0]); + acc ^= fold_16_zbc(x[1], consts.tail_fold_16b[1]); + acc ^= fold_16_zbc(x[2], consts.tail_fold_16b[2]); + acc ^= fold_16_zbc(x[3], consts.tail_fold_16b[3]); + acc ^= fold_16_zbc(x[4], consts.tail_fold_16b[4]); + acc ^= fold_16_zbc(x[5], consts.tail_fold_16b[5]); + acc ^= fold_16_zbc(x[6], consts.tail_fold_16b[6]); + + let (fold_width_high, fold_width_low) = consts.fold_width; + barrett_crc32_reflected_zbc( + fold_width_crc32_reflected_zbc(acc, fold_width_high, fold_width_low), + consts.poly, + consts.mu, + ) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn fold_block_128_zbc(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: (u64, u64)) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - x[0] = chunk[0] ^ fold_16_zbc(x[0], coeff); - x[1] = chunk[1] ^ fold_16_zbc(x[1], coeff); - x[2] = chunk[2] ^ fold_16_zbc(x[2], coeff); - x[3] = chunk[3] ^ fold_16_zbc(x[3], coeff); - x[4] = chunk[4] ^ fold_16_zbc(x[4], coeff); - x[5] = chunk[5] ^ fold_16_zbc(x[5], coeff); - x[6] = chunk[6] ^ fold_16_zbc(x[6], coeff); - x[7] = chunk[7] ^ fold_16_zbc(x[7], coeff); - } +fn fold_block_128_zbc(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: (u64, u64)) { + x[0] = chunk[0] ^ fold_16_zbc(x[0], coeff); + x[1] = chunk[1] ^ fold_16_zbc(x[1], coeff); + x[2] = chunk[2] ^ fold_16_zbc(x[2], coeff); + x[3] = chunk[3] ^ fold_16_zbc(x[3], coeff); + x[4] = chunk[4] ^ fold_16_zbc(x[4], coeff); + x[5] = chunk[5] ^ fold_16_zbc(x[5], coeff); + x[6] = chunk[6] ^ fold_16_zbc(x[6], coeff); + x[7] = chunk[7] ^ fold_16_zbc(x[7], coeff); } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block(first); - x[0].lo ^= state as u64; - - let coeff = consts.fold_128b; - for block in rest { - let chunk = load_block(block); - fold_block_128_zbc(&mut x, &chunk, coeff); - } - - fold_tail_zbc(x, consts) +fn update_simd_zbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { + let mut x = load_block(first); + x[0].lo ^= state as u64; + + let coeff = consts.fold_128b; + for block in rest { + let chunk = load_block(block); + fold_block_128_zbc(&mut x, &chunk, coeff); } + + fold_tail_zbc(x, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_2way( - state: u32, - blocks: &[Block], - fold_256b: (u64, u64), - consts: &Crc32ClmulConstants, -) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; +fn update_simd_zbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { + debug_assert!(!blocks.is_empty()); + + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, consts); + } - let coeff_256 = fold_256b; - let coeff_128 = consts.fold_128b; + let even = blocks.len() & !1usize; - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - s0[0].lo ^= state as u64; + let coeff_256 = fold_256b; + let coeff_128 = consts.fold_128b; - let mut i = 2usize; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128_zbc(&mut s0, &b0, coeff_256); - fold_block_128_zbc(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + s0[0].lo ^= state as u64; - let mut combined = s1; - combined[0] ^= fold_16_zbc(s0[0], coeff_128); - combined[1] ^= fold_16_zbc(s0[1], coeff_128); - combined[2] ^= fold_16_zbc(s0[2], coeff_128); - combined[3] ^= fold_16_zbc(s0[3], coeff_128); - combined[4] ^= fold_16_zbc(s0[4], coeff_128); - combined[5] ^= fold_16_zbc(s0[5], coeff_128); - combined[6] ^= fold_16_zbc(s0[6], coeff_128); - combined[7] ^= fold_16_zbc(s0[7], coeff_128); - - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128_zbc(&mut combined, &tail, coeff_128); - } + let mut i = 2usize; + while i < even { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + fold_block_128_zbc(&mut s0, &b0, coeff_256); + fold_block_128_zbc(&mut s1, &b1, coeff_256); + i = i.strict_add(2); + } - fold_tail_zbc(combined, consts) + let mut combined = s1; + combined[0] ^= fold_16_zbc(s0[0], coeff_128); + combined[1] ^= fold_16_zbc(s0[1], coeff_128); + combined[2] ^= fold_16_zbc(s0[2], coeff_128); + combined[3] ^= fold_16_zbc(s0[3], coeff_128); + combined[4] ^= fold_16_zbc(s0[4], coeff_128); + combined[5] ^= fold_16_zbc(s0[5], coeff_128); + combined[6] ^= fold_16_zbc(s0[6], coeff_128); + combined[7] ^= fold_16_zbc(s0[7], coeff_128); + + if even != blocks.len() { + let tail = load_block(&blocks[even]); + fold_block_128_zbc(&mut combined, &tail, coeff_128); } + + fold_tail_zbc(combined, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn update_simd_zbc_4way( +fn update_simd_zbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], consts: &Crc32ClmulConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zbc(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zbc(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = fold_512b; - let coeff_128 = consts.fold_128b; - - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - s0[0].lo ^= state as u64; - - let mut i = 4usize; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128_zbc(&mut s0, &b0, coeff_512); - fold_block_128_zbc(&mut s1, &b1, coeff_512); - fold_block_128_zbc(&mut s2, &b2, coeff_512); - fold_block_128_zbc(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512 = fold_512b; + let coeff_128 = consts.fold_128b; + + let c384 = combine[0]; + let c256 = combine[1]; + let c128 = combine[2]; + + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); + s0[0].lo ^= state as u64; + + let mut i = 4usize; + while i < aligned { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + let b2 = load_block(&blocks[i.strict_add(2)]); + let b3 = load_block(&blocks[i.strict_add(3)]); + fold_block_128_zbc(&mut s0, &b0, coeff_512); + fold_block_128_zbc(&mut s1, &b1, coeff_512); + fold_block_128_zbc(&mut s2, &b2, coeff_512); + fold_block_128_zbc(&mut s3, &b3, coeff_512); + i = i.strict_add(4); + } - let mut combined = s3; - combined[0] ^= fold_16_zbc(s2[0], c128); - combined[1] ^= fold_16_zbc(s2[1], c128); - combined[2] ^= fold_16_zbc(s2[2], c128); - combined[3] ^= fold_16_zbc(s2[3], c128); - combined[4] ^= fold_16_zbc(s2[4], c128); - combined[5] ^= fold_16_zbc(s2[5], c128); - combined[6] ^= fold_16_zbc(s2[6], c128); - combined[7] ^= fold_16_zbc(s2[7], c128); - - combined[0] ^= fold_16_zbc(s1[0], c256); - combined[1] ^= fold_16_zbc(s1[1], c256); - combined[2] ^= fold_16_zbc(s1[2], c256); - combined[3] ^= fold_16_zbc(s1[3], c256); - combined[4] ^= fold_16_zbc(s1[4], c256); - combined[5] ^= fold_16_zbc(s1[5], c256); - combined[6] ^= fold_16_zbc(s1[6], c256); - combined[7] ^= fold_16_zbc(s1[7], c256); - - combined[0] ^= fold_16_zbc(s0[0], c384); - combined[1] ^= fold_16_zbc(s0[1], c384); - combined[2] ^= fold_16_zbc(s0[2], c384); - combined[3] ^= fold_16_zbc(s0[3], c384); - combined[4] ^= fold_16_zbc(s0[4], c384); - combined[5] ^= fold_16_zbc(s0[5], c384); - combined[6] ^= fold_16_zbc(s0[6], c384); - combined[7] ^= fold_16_zbc(s0[7], c384); - - if aligned != blocks.len() { - let tail_blocks = &blocks[aligned..]; - let Some((first, rest)) = tail_blocks.split_first() else { - return fold_tail_zbc(combined, consts); - }; - let mut x = combined; - let first = load_block(first); - fold_block_128_zbc(&mut x, &first, coeff_128); - for b in rest { - let chunk = load_block(b); - fold_block_128_zbc(&mut x, &chunk, coeff_128); - } - return fold_tail_zbc(x, consts); + let mut combined = s3; + combined[0] ^= fold_16_zbc(s2[0], c128); + combined[1] ^= fold_16_zbc(s2[1], c128); + combined[2] ^= fold_16_zbc(s2[2], c128); + combined[3] ^= fold_16_zbc(s2[3], c128); + combined[4] ^= fold_16_zbc(s2[4], c128); + combined[5] ^= fold_16_zbc(s2[5], c128); + combined[6] ^= fold_16_zbc(s2[6], c128); + combined[7] ^= fold_16_zbc(s2[7], c128); + + combined[0] ^= fold_16_zbc(s1[0], c256); + combined[1] ^= fold_16_zbc(s1[1], c256); + combined[2] ^= fold_16_zbc(s1[2], c256); + combined[3] ^= fold_16_zbc(s1[3], c256); + combined[4] ^= fold_16_zbc(s1[4], c256); + combined[5] ^= fold_16_zbc(s1[5], c256); + combined[6] ^= fold_16_zbc(s1[6], c256); + combined[7] ^= fold_16_zbc(s1[7], c256); + + combined[0] ^= fold_16_zbc(s0[0], c384); + combined[1] ^= fold_16_zbc(s0[1], c384); + combined[2] ^= fold_16_zbc(s0[2], c384); + combined[3] ^= fold_16_zbc(s0[3], c384); + combined[4] ^= fold_16_zbc(s0[4], c384); + combined[5] ^= fold_16_zbc(s0[5], c384); + combined[6] ^= fold_16_zbc(s0[6], c384); + combined[7] ^= fold_16_zbc(s0[7], c384); + + if aligned != blocks.len() { + let tail_blocks = &blocks[aligned..]; + let Some((first, rest)) = tail_blocks.split_first() else { + return fold_tail_zbc(combined, consts); + }; + let mut x = combined; + let first = load_block(first); + fold_block_128_zbc(&mut x, &first, coeff_128); + for b in rest { + let chunk = load_block(b); + fold_block_128_zbc(&mut x, &chunk, coeff_128); } - - fold_tail_zbc(combined, consts) + return fold_tail_zbc(x, consts); } + + fold_tail_zbc(combined, consts) } // ZVBC (vector carryless multiply) backend @@ -395,12 +380,9 @@ unsafe fn update_simd_zbc_4way( fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { let mut hi = [0u64; 8]; let mut lo = [0u64; 8]; - - let mut i = 0; - while i < 8 { - lo[i] = u64::from_le(block[i * 2]); - hi[i] = u64::from_le(block[i * 2 + 1]); - i = i.strict_add(1); + for ((hi_lane, lo_lane), &[low, high]) in hi.iter_mut().zip(&mut lo).zip(block.as_chunks::<2>().0) { + *hi_lane = u64::from_le(high); + *lo_lane = u64::from_le(low); } (hi, lo) @@ -413,7 +395,7 @@ fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { /// Requires RISC-V `v` + `zvbc`. #[inline] #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { +fn mul64_zvbc(a: u64, b: u64) -> Simd { // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size // buffers and register-only operands are established before this block. unsafe { @@ -440,74 +422,73 @@ unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) - } +fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { + let (coeff_high, coeff_low) = coeff; + mul64_zvbc(x.low_64(), coeff_high) ^ mul64_zvbc(x.high_64(), coeff_low) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_width_crc32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let clmul = mul64_zvbc(x.low_64(), low); - let shifted = Simd::new(0, x.high_64()); - let mut state = clmul ^ shifted; +fn fold_width_crc32_reflected_zvbc(x: Simd, high: u64, low: u64) -> Simd { + let clmul = mul64_zvbc(x.low_64(), low); + let shifted = Simd::new(0, x.high_64()); + let mut state = clmul ^ shifted; - let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = mul64_zvbc(shifted_high, high); - state = clmul ^ masked; + let masked = Simd::new(state.high_64(), state.low_64() & 0xFFFF_FFFF_0000_0000); + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = mul64_zvbc(shifted_high, high); + state = clmul ^ masked; - state - } + state } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_crc32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu); - let l = mul64_zvbc(t1.low_64(), poly); - (x ^ l).high_64() as u32 - } +fn barrett_crc32_reflected_zvbc(x: Simd, poly: u64, mu: u64) -> u32 { + let t1 = mul64_zvbc(x.low_64(), mu); + let l = mul64_zvbc(t1.low_64(), poly); + low_u32((x ^ l).high_64()) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), consts.tail_fold_16b[0]); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), consts.tail_fold_16b[1]); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), consts.tail_fold_16b[2]); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), consts.tail_fold_16b[3]); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), consts.tail_fold_16b[4]); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), consts.tail_fold_16b[5]); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), consts.tail_fold_16b[6]); - - let (fold_width_high, fold_width_low) = consts.fold_width; - barrett_crc32_reflected_zvbc( - fold_width_crc32_reflected_zvbc(acc, fold_width_high, fold_width_low), - consts.poly, - consts.mu, - ) - } +fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], consts: &Crc32ClmulConstants) -> u32 { + let mut acc = Simd::new(hi[7], lo[7]); + acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), consts.tail_fold_16b[0]); + acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), consts.tail_fold_16b[1]); + acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), consts.tail_fold_16b[2]); + acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), consts.tail_fold_16b[3]); + acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), consts.tail_fold_16b[4]); + acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), consts.tail_fold_16b[5]); + acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), consts.tail_fold_16b[6]); + + let (fold_width_high, fold_width_low) = consts.fold_width; + barrett_crc32_reflected_zvbc( + fold_width_crc32_reflected_zvbc(acc, fold_width_high, fold_width_low), + consts.poly, + consts.mu, + ) } #[inline] +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( +fn fold_block_128_zvbc( x_hi: &mut [u64; 8], x_lo: &mut [u64; 8], chunk_hi: &[u64; 8], @@ -520,7 +501,7 @@ unsafe fn fold_block_128_zvbc( unsafe { let mut offset = 0usize; while offset < 8 { - let remaining = 8 - offset; + let remaining = 8usize.strict_sub(offset); let vl: usize; asm!( "vsetvli {vl}, {avl}, e64, m1, ta, ma", @@ -560,188 +541,183 @@ unsafe fn fold_block_128_zvbc( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split(first); - x_lo[0] ^= state as u64; - - let coeff_low = consts.fold_128b.1; - let coeff_high = consts.fold_128b.0; +fn update_simd_zvbc(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { + let (mut x_hi, mut x_lo) = load_block_split(first); + x_lo[0] ^= state as u64; - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } + let coeff_low = consts.fold_128b.1; + let coeff_high = consts.fold_128b.0; - fold_tail_zvbc(x_hi, x_lo, consts) + for block in rest { + let (chunk_hi, chunk_lo) = load_block_split(block); + fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); } + + fold_tail_zvbc(x_hi, x_lo, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way( - state: u32, - blocks: &[Block], - fold_256b: (u64, u64), - consts: &Crc32ClmulConstants, -) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; +fn update_simd_zvbc_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { + debug_assert!(!blocks.is_empty()); + + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, consts); + } - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; + let even = blocks.len() & !1usize; - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - s0_lo[0] ^= state as u64; + let coeff_256_low = fold_256b.1; + let coeff_256_high = fold_256b.0; + let coeff_128_low = consts.fold_128b.1; + let coeff_128_high = consts.fold_128b.0; - let mut i = 2usize; - while i < even { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + s0_lo[0] ^= state as u64; - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). Use the same trick as CRC64: - // fold each stream into the current combined value using `chunk = combined`. - let mut combined_hi = s1_hi; - let mut combined_lo = s1_lo; + let mut i = 2usize; + while i < even { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); + i = i.strict_add(2); + } + // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). Use the same trick as CRC64: + // fold each stream into the current combined value using `chunk = combined`. + let mut combined_hi = s1_hi; + let mut combined_lo = s1_lo; + + fold_block_128_zvbc( + &mut s0_hi, + &mut s0_lo, + &combined_hi, + &combined_lo, + coeff_128_low, + coeff_128_high, + ); + combined_hi = s0_hi; + combined_lo = s0_lo; + + if even != blocks.len() { + let (tail_hi, tail_lo) = load_block_split(&blocks[even]); fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &combined_hi, - &combined_lo, + &mut combined_hi, + &mut combined_lo, + &tail_hi, + &tail_lo, coeff_128_low, coeff_128_high, ); - combined_hi = s0_hi; - combined_lo = s0_lo; - - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split(&blocks[even]); - fold_block_128_zvbc( - &mut combined_hi, - &mut combined_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } - - fold_tail_zvbc(combined_hi, combined_lo, consts) } + + fold_tail_zvbc(combined_hi, combined_lo, consts) } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( +fn update_simd_zvbc_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), combine: &[(u64, u64); 3], consts: &Crc32ClmulConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); + debug_assert!(!blocks.is_empty()); - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd_zvbc(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); - s0_lo[0] ^= state as u64; - - let mut i = 4usize; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512_low = fold_512b.1; + let coeff_512_high = fold_512b.0; + let coeff_128_low = consts.fold_128b.1; + let coeff_128_high = consts.fold_128b.0; + + let c384_low = combine[0].1; + let c384_high = combine[0].0; + let c256_low = combine[1].1; + let c256_high = combine[1].0; + let c128_low = combine[2].1; + let c128_high = combine[2].0; + + let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); + let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); + let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); + let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); + s0_lo[0] ^= state as u64; + + let mut i = 4usize; + while i < aligned { + let (b0_hi, b0_lo) = load_block_split(&blocks[i]); + let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); + let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); + let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); + fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); + i = i.strict_add(4); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut combined_hi = s3_hi; - let mut combined_lo = s3_lo; - - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &combined_hi, &combined_lo, c128_low, c128_high); - combined_hi = s2_hi; - combined_lo = s2_lo; - - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &combined_hi, &combined_lo, c256_low, c256_high); - combined_hi = s1_hi; - combined_lo = s1_lo; - - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &combined_hi, &combined_lo, c384_low, c384_high); - combined_hi = s0_hi; - combined_lo = s0_lo; - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split(block); - fold_block_128_zvbc( - &mut combined_hi, - &mut combined_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut combined_hi = s3_hi; + let mut combined_lo = s3_lo; + + fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &combined_hi, &combined_lo, c128_low, c128_high); + combined_hi = s2_hi; + combined_lo = s2_lo; + + fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &combined_hi, &combined_lo, c256_low, c256_high); + combined_hi = s1_hi; + combined_lo = s1_lo; + + fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &combined_hi, &combined_lo, c384_low, c384_high); + combined_hi = s0_hi; + combined_lo = s0_lo; - fold_tail_zvbc(combined_hi, combined_lo, consts) + for block in &blocks[aligned..] { + let (tail_hi, tail_lo) = load_block_split(block); + fold_block_128_zvbc( + &mut combined_hi, + &mut combined_lo, + &tail_hi, + &tail_lo, + coeff_128_low, + coeff_128_high, + ); } + + fold_tail_zvbc(combined_hi, combined_lo, consts) } // Public kernels (IEEE + CRC32C) +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -769,15 +745,18 @@ unsafe fn crc32_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32_zbc_nway( +fn crc32_zbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -804,10 +783,13 @@ unsafe fn crc32_zbc_nway( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32c_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32c_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -832,15 +814,18 @@ unsafe fn crc32c_zbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "zbc")] -unsafe fn crc32c_zbc_nway( +fn crc32c_zbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -867,10 +852,13 @@ unsafe fn crc32c_zbc_nway( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -895,15 +883,18 @@ unsafe fn crc32_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32_zvbc_nway( +fn crc32_zvbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -930,10 +921,13 @@ unsafe fn crc32_zvbc_nway( } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32c_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +fn crc32c_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -958,15 +952,18 @@ unsafe fn crc32c_zvbc(mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants } } +/// # Safety +/// +/// Requires the RISC-V extensions enabled by the target-feature attribute. #[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc32c_zvbc_nway( +fn crc32c_zvbc_nway( mut state: u32, bytes: &[u8], consts: &Crc32ClmulConstants, stream: &Crc32StreamConstants, ) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { debug_assert!(N == 2 || N == 4); let (left, middle, right) = bytes.align_to::(); @@ -996,15 +993,14 @@ unsafe fn crc32c_zvbc_nway( // Safe wrappers (dispatcher entrypoints) #[inline] -pub fn crc32_ieee_zbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_zbc_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies ZBC before selecting this kernel. unsafe { crc32_zbc(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32_zbc_nway::<2>( crc, @@ -1016,9 +1012,8 @@ pub fn crc32_ieee_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32_zbc_nway::<4>( crc, @@ -1030,15 +1025,14 @@ pub fn crc32_ieee_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_zvbc_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_zvbc_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies V+ZVBC before selecting this kernel. unsafe { crc32_zvbc(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32_zvbc_nway::<2>( crc, @@ -1050,9 +1044,8 @@ pub fn crc32_ieee_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32_ieee_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32_zvbc_nway::<4>( crc, @@ -1064,44 +1057,38 @@ pub fn crc32_ieee_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_zbc_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zbc_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32c_zbc(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32c_zbc_nway::<2>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } #[inline] -pub fn crc32c_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies Zbc before selecting this kernel. unsafe { crc32c_zbc_nway::<4>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } #[inline] -pub fn crc32c_zvbc_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zvbc_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32c_zvbc(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zvbc_2way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32c_zvbc_nway::<2>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } #[inline] -pub fn crc32c_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. +pub(super) fn crc32c_zvbc_4way_safe(crc: u32, data: &[u8]) -> u32 { + // SAFETY: Dispatcher verifies V and Zvbc before selecting this kernel. unsafe { crc32c_zvbc_nway::<4>(crc, data, &super::clmul::CRC32C_CLMUL, &super::clmul::CRC32C_STREAM) } } diff --git a/src/checksum/crc32/s390x.rs b/src/checksum/crc32/s390x.rs index 9c3fed1d..0a3754b9 100644 --- a/src/checksum/crc32/s390x.rs +++ b/src/checksum/crc32/s390x.rs @@ -8,19 +8,15 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] -// This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. use core::{ arch::asm, - mem::MaybeUninit, ops::{BitAnd, BitXor, BitXorAssign}, simd::i64x2, }; +use crate::checksum::common::low_u32; + use super::clmul::{Crc32ClmulConstants, Crc32StreamConstants}; type Block = [u64; 16]; // 128 bytes (8×16B lanes) @@ -78,8 +74,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees the s390x vector facility is available // (verified by dispatch). The VGFM instruction operates on pure register // values with no memory access. @@ -97,63 +96,57 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } /// Fold 16 bytes (reflected CRC32 folding primitive): /// `self.low ⊗ coeff.high ⊕ self.high ⊗ coeff.low`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values after lane swap. - unsafe { - // Like VPMSUMD: VGFM performs a per-lane carryless multiply and XORs the lane products. - // The CRC32 folding primitive needs cross terms, so swap coefficient lanes. - Self(Self::vgfm(self.0, coeff.swap_lanes().0)) - } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.swap_lanes().0)) } /// Fold 16B → CRC32 width (reflected), returning an intermediate 128-bit state. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_width_crc32_reflected(self, high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // All operations are pure register computations via mul64/vgfm. - unsafe { - let clmul = Self::mul64(self.low_64(), low); - let shifted = Self::new(0, self.high_64()); - let mut state = clmul ^ shifted; - - let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); - let masked = state & mask2; - let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); - let clmul = Self::mul64(shifted_high, high); - state = clmul ^ masked; - - state - } + fn fold_width_crc32_reflected(self, high: u64, low: u64) -> Self { + let clmul = Self::mul64(self.low_64(), low); + let shifted = Self::new(0, self.high_64()); + let mut state = clmul ^ shifted; + + let mask2 = Self::new(0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_0000_0000); + let masked = state & mask2; + let shifted_high = (state.low_64() & 0xFFFF_FFFF).strict_shl(32); + let clmul = Self::mul64(shifted_high, high); + state = clmul ^ masked; + + state } /// Barrett reduction for reflected CRC32; returns the updated (pre-inverted) CRC. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett_crc32_reflected(self, poly: u64, mu: u64) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available. - // mul64 and XOR are pure register computations. - unsafe { - let t1 = Self::mul64(self.low_64(), mu); - let l = Self::mul64(t1.low_64(), poly); - (self ^ l).high_64() as u32 - } + fn barrett_crc32_reflected(self, poly: u64, mu: u64) -> u32 { + let t1 = Self::mul64(self.low_64(), mu); + let l = Self::mul64(t1.low_64(), poly); + low_u32((self ^ l).high_64()) } } @@ -161,149 +154,133 @@ impl Simd { #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers -#[inline(always)] -unsafe fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 / fold_width / barrett calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); - let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); - let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); - let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); - let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); - let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); - let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); - - let mut acc = x[7]; - acc ^= x[0].fold_16(c0); - acc ^= x[1].fold_16(c1); - acc ^= x[2].fold_16(c2); - acc ^= x[3].fold_16(c3); - acc ^= x[4].fold_16(c4); - acc ^= x[5].fold_16(c5); - acc ^= x[6].fold_16(c6); - - let (fold_width_high, fold_width_low) = consts.fold_width; - let state = acc.fold_width_crc32_reflected(fold_width_high, fold_width_low); - state.barrett_crc32_reflected(consts.poly, consts.mu) - } +#[inline] +/// # Safety +/// +/// Requires the s390x vector facility. +#[target_feature(enable = "vector")] +fn fold_tail(x: [Simd; 8], consts: &Crc32ClmulConstants) -> u32 { + let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); + let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); + let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); + let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); + let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); + let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); + let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); + + let mut acc = x[7]; + acc ^= x[0].fold_16(c0); + acc ^= x[1].fold_16(c1); + acc ^= x[2].fold_16(c2); + acc ^= x[3].fold_16(c3); + acc ^= x[4].fold_16(c4); + acc ^= x[5].fold_16(c5); + acc ^= x[6].fold_16(c6); + + let (fold_width_high, fold_width_low) = consts.fold_width; + let state = acc.fold_width_crc32_reflected(fold_width_high, fold_width_low); + state.barrett_crc32_reflected(consts.poly, consts.mu) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - x[0] = chunk[0] ^ x[0].fold_16(coeff); - x[1] = chunk[1] ^ x[1].fold_16(coeff); - x[2] = chunk[2] ^ x[2].fold_16(coeff); - x[3] = chunk[3] ^ x[3].fold_16(coeff); - x[4] = chunk[4] ^ x[4].fold_16(coeff); - x[5] = chunk[5] ^ x[5].fold_16(coeff); - x[6] = chunk[6] ^ x[6].fold_16(coeff); - x[7] = chunk[7] ^ x[7].fold_16(coeff); - } +fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { + x[0] = chunk[0] ^ x[0].fold_16(coeff); + x[1] = chunk[1] ^ x[1].fold_16(coeff); + x[2] = chunk[2] ^ x[2].fold_16(coeff); + x[3] = chunk[3] ^ x[3].fold_16(coeff); + x[4] = chunk[4] ^ x[4].fold_16(coeff); + x[5] = chunk[5] ^ x[5].fold_16(coeff); + x[6] = chunk[6] ^ x[6].fold_16(coeff); + x[7] = chunk[7] ^ x[7].fold_16(coeff); } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Block iteration is bounded by the slice length; no out-of-bounds access. - unsafe { - let mut x = load_block(first); +fn update_simd(state: u32, first: &Block, rest: &[Block], consts: &Crc32ClmulConstants) -> u32 { + let mut x = load_block(first); - // XOR initial CRC into the first 16-byte lane (low 32 bits). - x[0] ^= Simd::new(0, state as u64); + // XOR initial CRC into the first 16-byte lane (low 32 bits). + x[0] ^= Simd::new(0, state as u64); - let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - for block in rest { - let chunk = load_block(block); - fold_block_128(&mut x, &chunk, coeff); - } - - fold_tail(x, consts) + let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + for block in rest { + let chunk = load_block(block); + fold_block_128(&mut x, &chunk, coeff); } + + fold_tail(x, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { +fn update_simd_2way(state: u32, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc32ClmulConstants) -> u32 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 2 and is bounded by `even` (≤ blocks.len()), - // so `blocks[i]` and `blocks[i + 1]` are always in-bounds. - unsafe { - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let even = blocks.len() & !1usize; - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - s0[0] ^= Simd::new(0, state as u64); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let mut i = 2usize; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128(&mut s0, &b0, coeff_256); - fold_block_128(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + s0[0] ^= Simd::new(0, state as u64); - let mut combined = s1; - combined[0] ^= s0[0].fold_16(coeff_128); - combined[1] ^= s0[1].fold_16(coeff_128); - combined[2] ^= s0[2].fold_16(coeff_128); - combined[3] ^= s0[3].fold_16(coeff_128); - combined[4] ^= s0[4].fold_16(coeff_128); - combined[5] ^= s0[5].fold_16(coeff_128); - combined[6] ^= s0[6].fold_16(coeff_128); - combined[7] ^= s0[7].fold_16(coeff_128); - - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128(&mut combined, &tail, coeff_128); - } + let mut i = 2usize; + while i < even { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + fold_block_128(&mut s0, &b0, coeff_256); + fold_block_128(&mut s1, &b1, coeff_256); + i = i.strict_add(2); + } - fold_tail(combined, consts) + let mut combined = s1; + combined[0] ^= s0[0].fold_16(coeff_128); + combined[1] ^= s0[1].fold_16(coeff_128); + combined[2] ^= s0[2].fold_16(coeff_128); + combined[3] ^= s0[3].fold_16(coeff_128); + combined[4] ^= s0[4].fold_16(coeff_128); + combined[5] ^= s0[5].fold_16(coeff_128); + combined[6] ^= s0[6].fold_16(coeff_128); + combined[7] ^= s0[7].fold_16(coeff_128); + + if even != blocks.len() { + let tail = load_block(&blocks[even]); + fold_block_128(&mut combined, &tail, coeff_128); } + + fold_tail(combined, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_4way( +fn update_simd_4way( state: u32, blocks: &[Block], fold_512b: (u64, u64), @@ -312,99 +289,96 @@ unsafe fn update_simd_4way( ) -> u32 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 4 and is bounded by `aligned` (≤ blocks.len()), - // so `blocks[i..i+3]` are always in-bounds. Tail processing uses split_first. - unsafe { - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - s0[0] ^= Simd::new(0, state as u64); - - let mut i = 4usize; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128(&mut s0, &b0, coeff_512); - fold_block_128(&mut s1, &b1, coeff_512); - fold_block_128(&mut s2, &b2, coeff_512); - fold_block_128(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); + + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); + s0[0] ^= Simd::new(0, state as u64); + + let mut i = 4usize; + while i < aligned { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + let b2 = load_block(&blocks[i.strict_add(2)]); + let b3 = load_block(&blocks[i.strict_add(3)]); + fold_block_128(&mut s0, &b0, coeff_512); + fold_block_128(&mut s1, &b1, coeff_512); + fold_block_128(&mut s2, &b2, coeff_512); + fold_block_128(&mut s3, &b3, coeff_512); + i = i.strict_add(4); + } - let mut combined = s3; - combined[0] ^= s2[0].fold_16(c128); - combined[1] ^= s2[1].fold_16(c128); - combined[2] ^= s2[2].fold_16(c128); - combined[3] ^= s2[3].fold_16(c128); - combined[4] ^= s2[4].fold_16(c128); - combined[5] ^= s2[5].fold_16(c128); - combined[6] ^= s2[6].fold_16(c128); - combined[7] ^= s2[7].fold_16(c128); - - combined[0] ^= s1[0].fold_16(c256); - combined[1] ^= s1[1].fold_16(c256); - combined[2] ^= s1[2].fold_16(c256); - combined[3] ^= s1[3].fold_16(c256); - combined[4] ^= s1[4].fold_16(c256); - combined[5] ^= s1[5].fold_16(c256); - combined[6] ^= s1[6].fold_16(c256); - combined[7] ^= s1[7].fold_16(c256); - - combined[0] ^= s0[0].fold_16(c384); - combined[1] ^= s0[1].fold_16(c384); - combined[2] ^= s0[2].fold_16(c384); - combined[3] ^= s0[3].fold_16(c384); - combined[4] ^= s0[4].fold_16(c384); - combined[5] ^= s0[5].fold_16(c384); - combined[6] ^= s0[6].fold_16(c384); - combined[7] ^= s0[7].fold_16(c384); - - if aligned != blocks.len() { - let tail_blocks = &blocks[aligned..]; - let Some((first, rest)) = tail_blocks.split_first() else { - return fold_tail(combined, consts); - }; - let mut x = combined; - let first = load_block(first); - fold_block_128(&mut x, &first, coeff_128); - for b in rest { - let chunk = load_block(b); - fold_block_128(&mut x, &chunk, coeff_128); - } - return fold_tail(x, consts); + let mut combined = s3; + combined[0] ^= s2[0].fold_16(c128); + combined[1] ^= s2[1].fold_16(c128); + combined[2] ^= s2[2].fold_16(c128); + combined[3] ^= s2[3].fold_16(c128); + combined[4] ^= s2[4].fold_16(c128); + combined[5] ^= s2[5].fold_16(c128); + combined[6] ^= s2[6].fold_16(c128); + combined[7] ^= s2[7].fold_16(c128); + + combined[0] ^= s1[0].fold_16(c256); + combined[1] ^= s1[1].fold_16(c256); + combined[2] ^= s1[2].fold_16(c256); + combined[3] ^= s1[3].fold_16(c256); + combined[4] ^= s1[4].fold_16(c256); + combined[5] ^= s1[5].fold_16(c256); + combined[6] ^= s1[6].fold_16(c256); + combined[7] ^= s1[7].fold_16(c256); + + combined[0] ^= s0[0].fold_16(c384); + combined[1] ^= s0[1].fold_16(c384); + combined[2] ^= s0[2].fold_16(c384); + combined[3] ^= s0[3].fold_16(c384); + combined[4] ^= s0[4].fold_16(c384); + combined[5] ^= s0[5].fold_16(c384); + combined[6] ^= s0[6].fold_16(c384); + combined[7] ^= s0[7].fold_16(c384); + + if aligned != blocks.len() { + let tail_blocks = &blocks[aligned..]; + let Some((first, rest)) = tail_blocks.split_first() else { + return fold_tail(combined, consts); + }; + let mut x = combined; + let first = load_block(first); + fold_block_128(&mut x, &first, coeff_128); + for b in rest { + let chunk = load_block(b); + fold_block_128(&mut x, &chunk, coeff_128); } - - fold_tail(combined, consts) + return fold_tail(x, consts); } + + fold_tail(combined, consts) } // Public kernels (IEEE + CRC32C) #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd operates on aligned Block data. +fn crc32_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -418,10 +392,13 @@ unsafe fn crc32_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32c_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd operates on aligned Block data. +fn crc32c_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> u32 { + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); let Some((first, rest)) = middle.split_first() else { @@ -435,16 +412,19 @@ unsafe fn crc32c_kernel(crc: u32, data: &[u8], consts: &Crc32ClmulConstants) -> } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32_kernel_nway( +fn crc32_kernel_nway( crc: u32, data: &[u8], stream: &Crc32StreamConstants, consts: &Crc32ClmulConstants, ) -> u32 { debug_assert!(N == 2 || N == 4); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd_* operate on aligned Block data. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -461,16 +441,19 @@ unsafe fn crc32_kernel_nway( } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc32c_kernel_nway( +fn crc32c_kernel_nway( crc: u32, data: &[u8], stream: &Crc32StreamConstants, consts: &Crc32ClmulConstants, ) -> u32 { debug_assert!(N == 2 || N == 4); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices; update_simd_* operate on aligned Block data. + // SAFETY: Every bit pattern is valid for Block; align_to returns + // non-overlapping subslices of the original allocation. unsafe { let (left, middle, right) = data.align_to::(); if middle.is_empty() { @@ -489,13 +472,13 @@ unsafe fn crc32c_kernel_nway( // Safe wrappers (dispatcher entrypoints) #[inline] -pub fn crc32_ieee_vgfm_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vgfm_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32_kernel(crc, data, &super::clmul::CRC32_IEEE_CLMUL) } } #[inline] -pub fn crc32_ieee_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32_kernel_nway::<2>( @@ -508,7 +491,7 @@ pub fn crc32_ieee_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32_kernel_nway::<4>( @@ -521,19 +504,19 @@ pub fn crc32_ieee_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_vgfm_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vgfm_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32c_kernel(crc, data, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vgfm_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32c_kernel_nway::<2>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } #[inline] -pub fn crc32c_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_vgfm_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies vector support before selecting this kernel. unsafe { crc32c_kernel_nway::<4>(crc, data, &super::clmul::CRC32C_STREAM, &super::clmul::CRC32C_CLMUL) } } diff --git a/src/checksum/crc32/x86_64.rs b/src/checksum/crc32/x86_64.rs index ea31b094..27654cce 100644 --- a/src/checksum/crc32/x86_64.rs +++ b/src/checksum/crc32/x86_64.rs @@ -4,11 +4,9 @@ //! //! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure SSE4.2 is //! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] // SAFETY: This module is intrinsics-heavy and uses tight, invariant-driven indexing // (e.g. fixed-size lanes and chunked block processing) where bounds are proven by // control flow; Clippy cannot always see these invariants. -#![allow(clippy::indexing_slicing)] use core::{ arch::x86_64::*, @@ -70,10 +68,13 @@ fn pow_shift8_crc32c(len_bytes: usize) -> Gf2Matrix32 { /// CRC-32C update using SSE4.2 `crc32` instruction. /// /// `crc` is the current state (pre-inverted). +/// +/// # Safety +/// +/// The current CPU must support SSE4.2. #[inline] #[target_feature(enable = "sse4.2")] unsafe fn crc32c_sse42(crc: u32, data: &[u8]) -> u32 { - // SAFETY: SSE4.2 intrinsics are available via this function's #[target_feature] attribute. let mut state64 = crc as u64; let (chunks8, tail8) = data.as_chunks::<8>(); @@ -81,7 +82,8 @@ unsafe fn crc32c_sse42(crc: u32, data: &[u8]) -> u32 { state64 = _mm_crc32_u64(state64, u64::from_le_bytes(*chunk)); } - let mut state = state64 as u32; + let [state0, state1, state2, state3, ..] = state64.to_le_bytes(); + let mut state = u32::from_le_bytes([state0, state1, state2, state3]); let (chunks4, tail4) = tail8.as_chunks::<4>(); for chunk in chunks4 { @@ -102,18 +104,25 @@ unsafe fn crc32c_sse42(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32C SSE4.2 kernel. #[inline] -pub fn crc32c_sse42_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42(crc, data) } } // CRC-32C HWCRC Multi-stream Kernels (SSE4.2) +/// Computes CRC-32C with the private 2-, 4-, 7-, or 8-lane SSE4.2 kernels. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2. #[inline] #[target_feature(enable = "sse4.2")] unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { - // SAFETY: SSE4.2 intrinsics are available via this function's #[target_feature] attribute. - // Pointer arithmetic and get_unchecked are bounded by `chunk_len` and `len` derived from `data`. + // SAFETY: The target-feature contract permits SSE4.2 intrinsics. For every lane, `base` starts + // within `N * chunk_len`; the eight-byte path additionally proves `i + 8 <= chunk_len`, and the + // byte path proves `i < chunk_len`. The tail starts at `N * chunk_len <= len`. The shared slice + // supplies initialized readable bytes, and `read_unaligned` imposes no alignment requirement. unsafe { debug_assert!(N == 2 || N == 4 || N == 7 || N == 8); @@ -123,7 +132,7 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { } // Use even-sized chunking (leave tail on the last stream). - let chunk_len = len / N; + let chunk_len = len.strict_div(N); let mut lanes = [!0u32; N]; // Process a striped prefix where all lanes have full `chunk_len`. @@ -134,10 +143,12 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { let mut lane_idx: usize = 0; while lane_idx < N { let base = lane_idx.strict_mul(chunk_len).strict_add(i); - lanes[lane_idx] = _mm_crc32_u64( + let next = _mm_crc32_u64( lanes[lane_idx] as u64, - ptr::read_unaligned(data.as_ptr().add(base) as *const u64), - ) as u32; + ptr::read_unaligned(data.as_ptr().add(base).cast::()), + ); + let [b0, b1, b2, b3, _, _, _, _] = next.to_le_bytes(); + lanes[lane_idx] = u32::from_le_bytes([b0, b1, b2, b3]); lane_idx = lane_idx.strict_add(1); } i = i.strict_add(8); @@ -154,8 +165,9 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { // Finish the tail on the last lane (remaining bytes after N*chunk_len). let tail_start = chunk_len.strict_mul(N); + let last_lane = N.strict_sub(1); if tail_start < len { - lanes[N - 1] = crc32c_sse42(lanes[N - 1], data.get_unchecked(tail_start..)); + lanes[last_lane] = crc32c_sse42(lanes[last_lane], data.get_unchecked(tail_start..)); } // Compute CRC(data) under the standard initial state, then append it to `crc`. @@ -188,25 +200,25 @@ unsafe fn crc32c_sse42_nway(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32c_sse42_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<2>(crc, data) } } #[inline] -pub fn crc32c_sse42_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<4>(crc, data) } } #[inline] -pub fn crc32c_sse42_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_7way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<7>(crc, data) } } #[inline] -pub fn crc32c_sse42_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_sse42_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 before selecting this kernel. unsafe { crc32c_sse42_nway::<8>(crc, data) } } @@ -216,31 +228,47 @@ pub fn crc32c_sse42_8way_safe(crc: u32, data: &[u8]) -> u32 { // Derived from Corsix `fast-crc32` generator output (v4s3x3 / v3x2 families). // These kernels fuse hardware CRC instructions with carryless multiply folding. +/// Multiplies the low 64-bit lanes of two vectors over GF(2). +/// +/// # Safety +/// +/// The current CPU must support PCLMULQDQ. #[inline] #[target_feature(enable = "pclmulqdq")] unsafe fn clmul_lo_sse(a: __m128i, b: __m128i) -> __m128i { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. _mm_clmulepi64_si128::<0x00>(a, b) } +/// Multiplies the high 64-bit lanes of two vectors over GF(2). +/// +/// # Safety +/// +/// The current CPU must support PCLMULQDQ. #[inline] #[target_feature(enable = "pclmulqdq")] unsafe fn clmul_hi_sse(a: __m128i, b: __m128i) -> __m128i { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. _mm_clmulepi64_si128::<0x11>(a, b) } +/// Multiplies two 32-bit scalars over GF(2) and returns the 64-bit product. +/// +/// # Safety +/// +/// The current CPU must support PCLMULQDQ. #[inline] #[target_feature(enable = "pclmulqdq")] unsafe fn clmul_scalar_sse(a: u32, b: u32) -> __m128i { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. _mm_clmulepi64_si128::<0x00>(_mm_cvtsi32_si128(a.cast_signed()), _mm_cvtsi32_si128(b.cast_signed())) } +/// Extracts the low lane for index zero and the high lane otherwise. +/// +/// # Safety +/// +/// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn mm_extract_epi64(val: __m128i, idx: i32) -> u64 { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. if idx == 0 { _mm_cvtsi128_si64(val).cast_unsigned() } else { @@ -248,14 +276,24 @@ unsafe fn mm_extract_epi64(val: __m128i, idx: i32) -> u64 { } } +/// Updates a CRC-32C state with one 64-bit word. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2. #[inline] #[target_feature(enable = "sse4.2")] unsafe fn mm_crc32c_u64(crc: u32, val: u64) -> u32 { - // SAFETY: SSE4.2 intrinsics are available via this function's #[target_feature] attribute. - _mm_crc32_u64(crc as u64, val) as u32 + let result = _mm_crc32_u64(u64::from(crc), val); + let [b0, b1, b2, b3, ..] = result.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } -// x^n mod P (iSCSI / CRC32C), in O(log n) time. +/// Computes x^n modulo the reflected CRC-32C polynomial in O(log n) time. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2 and PCLMULQDQ. #[target_feature(enable = "sse4.2,pclmulqdq")] unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 { // SAFETY: SSE4.2/PCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -266,7 +304,7 @@ unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 { let mut low: u32; while n > 191 { - stack = (stack << 1) + (n & 1); + stack = (stack << 1) | (n & 1); n = (n >> 1).strict_sub(16); } stack = !stack; @@ -293,6 +331,12 @@ unsafe fn xnmodp_iscsi_sse(mut n: u64) -> u32 { } } +/// Computes the PCLMULQDQ coefficient that shifts `crc` by `nbytes`. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2 and PCLMULQDQ, and `nbytes` must be at +/// least five so the x-power exponent is nonnegative. #[inline] #[target_feature(enable = "sse4.2,pclmulqdq")] unsafe fn crc_shift_iscsi_sse(crc: u32, nbytes: usize) -> __m128i { @@ -304,6 +348,31 @@ unsafe fn crc_shift_iscsi_sse(crc: u32, nbytes: usize) -> __m128i { } } +/// Loads 16 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 16 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_128(ptr: *const u8) -> __m128i { + let mut value = core::mem::MaybeUninit::<__m128i>::uninit(); + + // SAFETY: The caller guarantees a readable 16-byte source. `value` is an + // aligned, non-overlapping 16-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 16); + value.assume_init() + } +} + +/// Updates CRC-32C with the SSE4.2/PCLMULQDQ v4s3x3 fusion kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2 and PCLMULQDQ. `buf` must point to +/// `len` initialized readable bytes in one allocation. #[inline] #[target_feature(enable = "sse4.2,pclmulqdq")] unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -319,22 +388,22 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us // Handle 8-byte alignment. if (buf as usize & 8) != 0 && len >= 8 { - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } if len >= 144 { - let blk = (len.strict_sub(8)) / 136; + let blk = len.strict_sub(8).strict_div(136); let klen = blk.strict_mul(24); let buf2_start = buf; let mut crc1 = 0u32; let mut crc2 = 0u32; - let mut x0 = _mm_loadu_si128(buf2_start as *const __m128i); - let mut x1 = _mm_loadu_si128(buf2_start.add(16) as *const __m128i); - let mut x2 = _mm_loadu_si128(buf2_start.add(32) as *const __m128i); - let mut x3 = _mm_loadu_si128(buf2_start.add(48) as *const __m128i); + let mut x0 = load_unaligned_128(buf2_start); + let mut x1 = load_unaligned_128(buf2_start.add(16)); + let mut x2 = load_unaligned_128(buf2_start.add(32)); + let mut x3 = load_unaligned_128(buf2_start.add(48)); // iSCSI-specific folding constant. let mut k = _mm_setr_epi32(0x740eef02u32.cast_signed(), 0, 0x9e4addf8u32.cast_signed(), 0); @@ -357,26 +426,29 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us let mut y3 = clmul_lo_sse(x3, k); x3 = clmul_hi_sse(x3, k); - y0 = _mm_xor_si128(y0, _mm_loadu_si128(buf2 as *const __m128i)); + y0 = _mm_xor_si128(y0, load_unaligned_128(buf2)); x0 = _mm_xor_si128(x0, y0); - y1 = _mm_xor_si128(y1, _mm_loadu_si128(buf2.add(16) as *const __m128i)); + y1 = _mm_xor_si128(y1, load_unaligned_128(buf2.add(16))); x1 = _mm_xor_si128(x1, y1); - y2 = _mm_xor_si128(y2, _mm_loadu_si128(buf2.add(32) as *const __m128i)); + y2 = _mm_xor_si128(y2, load_unaligned_128(buf2.add(32))); x2 = _mm_xor_si128(x2, y2); - y3 = _mm_xor_si128(y3, _mm_loadu_si128(buf2.add(48) as *const __m128i)); + y3 = _mm_xor_si128(y3, load_unaligned_128(buf2.add(48))); x3 = _mm_xor_si128(x3, y3); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 16) as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen).cast::())); + crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)).cast::())); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)).cast::())); + crc2 = mm_crc32c_u64( + crc2, + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)).cast::()), + ); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(16)).cast::())); crc2 = mm_crc32c_u64( crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2) + 16) as *const u64), + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(16)).cast::()), ); buf = buf.add(24); @@ -405,29 +477,35 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us x0 = _mm_xor_si128(x0, y0); // Final scalar chunk. - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 8) as *const u64)); - crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2) + 8) as *const u64)); - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16) as *const u64)); - crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen + 16) as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen).cast::())); + crc2 = mm_crc32c_u64(crc2, ptr::read_unaligned(buf.add(klen.strict_mul(2)).cast::())); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(8).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(8)).cast::())); crc2 = mm_crc32c_u64( crc2, - ptr::read_unaligned(buf.add(klen.strict_mul(2) + 16) as *const u64), + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(8)).cast::()), + ); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.add(16).cast::())); + crc1 = mm_crc32c_u64(crc1, ptr::read_unaligned(buf.add(klen.strict_add(16)).cast::())); + crc2 = mm_crc32c_u64( + crc2, + ptr::read_unaligned(buf.add(klen.strict_mul(2).strict_add(16)).cast::()), ); buf = buf.add(24); - let vc0 = crc_shift_iscsi_sse(crc0, klen.strict_mul(2) + 8); - let vc1 = crc_shift_iscsi_sse(crc1, klen + 8); + let vc0 = crc_shift_iscsi_sse(crc0, klen.strict_mul(2).strict_add(8)); + let vc1 = crc_shift_iscsi_sse(crc1, klen.strict_add(8)); let mut vc = mm_extract_epi64(_mm_xor_si128(vc0, vc1), 0); // Reduce 128 bits to 32 bits, and multiply by x^32. let x0_low = mm_extract_epi64(x0, 0); let x0_high = mm_extract_epi64(x0, 1); let x0_combined = mm_extract_epi64( - crc_shift_iscsi_sse(mm_crc32c_u64(mm_crc32c_u64(0, x0_low), x0_high), klen.strict_mul(3) + 8), + crc_shift_iscsi_sse( + mm_crc32c_u64(mm_crc32c_u64(0, x0_low), x0_high), + klen.strict_mul(3).strict_add(8), + ), 0, ); vc ^= x0_combined; @@ -435,13 +513,13 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us // Final 8 bytes. buf = buf.add(klen.strict_mul(2)); crc0 = crc2; - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64) ^ vc); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::()) ^ vc); buf = buf.add(8); len = len.strict_sub(8); } while len >= 8 { - crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf as *const u64)); + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } @@ -458,7 +536,7 @@ unsafe fn crc32c_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: us /// Safe wrapper for CRC-32C fusion kernel (SSE4.2 + PCLMULQDQ). #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSE4.2 + PCLMULQDQ before selecting this kernel. unsafe { crc32c_iscsi_sse_v4s3x3(crc, data.as_ptr(), data.len()) } } @@ -515,41 +593,72 @@ fn crc32c_fusion_nway(crc: u32, data: &[u8], update: fn(u32, &[u } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<2>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_4way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<4>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_7way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<7>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } #[inline] -pub fn crc32c_iscsi_sse_v4s3x3_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_sse_v4s3x3_8way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<8>(crc, data, crc32c_iscsi_sse_v4s3x3_safe) } +/// Multiplies the low 64-bit lanes of each 128-bit element over GF(2). +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul_lo_avx512_vpclmulqdq(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0) } +/// Multiplies the high 64-bit lanes of each 128-bit element over GF(2). +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul_hi_avx512_vpclmulqdq(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 17) } +/// Loads 64 bytes without requiring alignment. +/// +/// # Safety +/// +/// `ptr` must be valid to read 64 initialized bytes. The source may be +/// unaligned because the bytes are copied into aligned local storage. +#[inline] +unsafe fn load_unaligned_512(ptr: *const u8) -> __m512i { + let mut value = core::mem::MaybeUninit::<__m512i>::uninit(); + + // SAFETY: The caller guarantees a readable 64-byte source. `value` is an + // aligned, non-overlapping 64-byte destination, and every bit pattern is + // valid for an integer vector. + unsafe { + core::ptr::copy_nonoverlapping(ptr, value.as_mut_ptr().cast::(), 64); + value.assume_init() + } +} + +/// Updates CRC-32C with the AVX-512/VPCLMULQDQ v3x2 fusion kernel. +/// +/// # Safety +/// +/// The current CPU must support SSE4.2, AVX-512F/VL/BW/DQ, and VPCLMULQDQ. +/// `buf` must point to `len` initialized readable bytes in one allocation. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,sse4.2")] unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 { @@ -565,16 +674,16 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, // Align to 64-byte boundary (cache line). while (buf as usize & 56) != 0 && len >= 8 { - crc0 = _mm_crc32_u64(crc0 as u64, ptr::read_unaligned(buf as *const u64)) as u32; + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } if len >= 384 { // Load three 512-bit vectors (192 bytes). - let mut x0 = _mm512_loadu_si512(buf as *const __m512i); - let mut x1 = _mm512_loadu_si512(buf.add(64) as *const __m512i); - let mut x2 = _mm512_loadu_si512(buf.add(128) as *const __m512i); + let mut x0 = load_unaligned_512(buf); + let mut x1 = load_unaligned_512(buf.add(64)); + let mut x2 = load_unaligned_512(buf.add(128)); // Broadcast folding constant to each 128-bit lane. let k_128 = _mm_setr_epi32(0xa87ab8a8u32.cast_signed(), 0, 0xab7aff2au32.cast_signed(), 0); @@ -592,9 +701,9 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, let mut y2 = clmul_lo_avx512_vpclmulqdq(x2, k); x2 = clmul_hi_avx512_vpclmulqdq(x2, k); - x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf.add(192) as *const __m512i), 0x96); - x1 = _mm512_ternarylogic_epi64(x1, y1, _mm512_loadu_si512(buf.add(256) as *const __m512i), 0x96); - x2 = _mm512_ternarylogic_epi64(x2, y2, _mm512_loadu_si512(buf.add(320) as *const __m512i), 0x96); + x0 = _mm512_ternarylogic_epi64(x0, y0, load_unaligned_512(buf.add(192)), 0x96); + x1 = _mm512_ternarylogic_epi64(x1, y1, load_unaligned_512(buf.add(256)), 0x96); + x2 = _mm512_ternarylogic_epi64(x2, y2, load_unaligned_512(buf.add(320)), 0x96); buf = buf.add(384); len = len.strict_sub(384); @@ -608,9 +717,9 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, y2 = clmul_lo_avx512_vpclmulqdq(x2, k); x2 = clmul_hi_avx512_vpclmulqdq(x2, k); - x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf as *const __m512i), 0x96); - x1 = _mm512_ternarylogic_epi64(x1, y1, _mm512_loadu_si512(buf.add(64) as *const __m512i), 0x96); - x2 = _mm512_ternarylogic_epi64(x2, y2, _mm512_loadu_si512(buf.add(128) as *const __m512i), 0x96); + x0 = _mm512_ternarylogic_epi64(x0, y0, load_unaligned_512(buf), 0x96); + x1 = _mm512_ternarylogic_epi64(x1, y1, load_unaligned_512(buf.add(64)), 0x96); + x2 = _mm512_ternarylogic_epi64(x2, y2, load_unaligned_512(buf.add(128)), 0x96); // Second folding step. y0 = clmul_lo_avx512_vpclmulqdq(x0, k); @@ -620,9 +729,9 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, y2 = clmul_lo_avx512_vpclmulqdq(x2, k); x2 = clmul_hi_avx512_vpclmulqdq(x2, k); - x0 = _mm512_ternarylogic_epi64(x0, y0, _mm512_loadu_si512(buf.add(192) as *const __m512i), 0x96); - x1 = _mm512_ternarylogic_epi64(x1, y1, _mm512_loadu_si512(buf.add(256) as *const __m512i), 0x96); - x2 = _mm512_ternarylogic_epi64(x2, y2, _mm512_loadu_si512(buf.add(320) as *const __m512i), 0x96); + x0 = _mm512_ternarylogic_epi64(x0, y0, load_unaligned_512(buf.add(192)), 0x96); + x1 = _mm512_ternarylogic_epi64(x1, y1, load_unaligned_512(buf.add(256)), 0x96); + x2 = _mm512_ternarylogic_epi64(x2, y2, load_unaligned_512(buf.add(320)), 0x96); buf = buf.add(384); len = len.strict_sub(384); @@ -673,12 +782,12 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, let mut z0 = _mm_ternarylogic_epi64(lane0, lane1, lane2, 0x96); z0 = _mm_xor_si128(z0, lane3); - crc0 = _mm_crc32_u64(0, mm_extract_epi64(z0, 0)) as u32; - crc0 = _mm_crc32_u64(crc0 as u64, mm_extract_epi64(z0, 1)) as u32; + crc0 = mm_crc32c_u64(0, mm_extract_epi64(z0, 0)); + crc0 = mm_crc32c_u64(crc0, mm_extract_epi64(z0, 1)); } while len >= 8 { - crc0 = _mm_crc32_u64(crc0 as u64, ptr::read_unaligned(buf as *const u64)) as u32; + crc0 = mm_crc32c_u64(crc0, ptr::read_unaligned(buf.cast::())); buf = buf.add(8); len = len.strict_sub(8); } @@ -695,28 +804,28 @@ unsafe fn crc32c_iscsi_avx512_vpclmulqdq_v3x2(mut crc0: u32, mut buf: *const u8, /// Safe wrapper for CRC-32C fusion kernel (AVX-512 VPCLMULQDQ v3x2). #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32c_iscsi_avx512_vpclmulqdq_v3x2(crc, data.as_ptr(), data.len()) } } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<2>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<4>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_7way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<7>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } #[inline] -pub fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32c_iscsi_avx512_vpclmulqdq_v3x2_8way_safe(crc: u32, data: &[u8]) -> u32 { crc32c_fusion_nway::<8>(crc, data, crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe) } @@ -790,7 +899,8 @@ const fn reduce128_crc32(hi: u64, lo: u64, poly: u32) -> u32 { bit = bit.strict_sub(1); } - val as u32 + let [b0, b1, b2, b3, ..] = val.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) } /// Compute x^n mod (x^32 + poly) in GF(2) (poly is the normal CRC polynomial without the x^32 @@ -922,45 +1032,68 @@ impl BitXorAssign for Simd128 { } impl Simd128 { + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } + /// Shifts the vector right by eight bytes, filling the high bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_right_8(self) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_srli_si128::<8>(self.0)) } + /// Shifts the vector left by 12 bytes, filling the low bytes with zero. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn shift_left_12(self) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_slli_si128::<12>(self.0)) } + /// Computes the bitwise AND of two vectors. + /// + /// # Safety + /// + /// The current CPU must support SSE2. #[inline] #[target_feature(enable = "sse2")] unsafe fn and(self, mask: Self) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_and_si128(self.0, mask.0)) } - /// Fold 16 bytes using the reflected CRC32 folding primitive. + /// Folds one reflected 16-byte lane and XORs the supplied input lane. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16(self, coeff: Self, data_to_xor: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. let h = _mm_clmulepi64_si128::<0x10>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x01>(self.0, coeff.0); Self(_mm_xor_si128(_mm_xor_si128(h, l), data_to_xor.0)) } - /// Fold 16 bytes down to CRC32 width (reflected mode). + /// Folds a reflected CRC state from 128 bits to the width-32 reduction state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_width_crc32_reflected(self, high: u64, low: u64) -> Self { @@ -985,7 +1118,11 @@ impl Simd128 { } } - /// Barrett reduction for reflected CRC32; returns the updated (pre-inverted) CRC. + /// Applies reflected Barrett reduction and returns the width-32 CRC state. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett_crc32_reflected(self, poly: u64, mu: u64) -> u32 { @@ -997,11 +1134,16 @@ impl Simd128 { let xorred = _mm_xor_si128(self.0, clmul2); let hi = _mm_srli_si128::<8>(xorred); - _mm_cvtsi128_si64(hi) as u32 + _mm_cvtsi128_si32(hi).cast_unsigned() } } } +/// Combines eight folded lanes and applies width-32 Barrett reduction. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn finalize_lanes_crc32_ieee_reflected(x: [Simd128; 8]) -> u32 { @@ -1025,6 +1167,11 @@ unsafe fn finalize_lanes_crc32_ieee_reflected(x: [Simd128; 8]) -> u32 { } } +/// Folds one or more 128-byte blocks into a reflected CRC-32 state. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_reflected(state: u32, first: &[Simd128; 8], rest: &[[Simd128; 8]]) -> u32 { @@ -1091,7 +1238,7 @@ pub(crate) unsafe fn crc32_ieee_pclmul(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 (IEEE) PCLMUL kernel. #[inline] -pub fn crc32_ieee_pclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul(crc, data) } } @@ -1122,13 +1269,13 @@ unsafe fn crc32_ieee_pclmul_small(crc: u32, data: &[u8]) -> u32 { let coeff_16b = Simd128::new(keys[2], keys[1]); // Load the first lane and inject the initial CRC state (low 32 bits). - let mut x0 = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let mut x0 = Simd128(load_unaligned_128(buf)); x0 ^= Simd128::new(0, crc as u64); buf = buf.add(16); len = len.strict_sub(16); while len >= 16 { - let chunk = Simd128(_mm_loadu_si128(buf as *const __m128i)); + let chunk = Simd128(load_unaligned_128(buf)); x0 = x0.fold_16(coeff_16b, chunk); buf = buf.add(16); len = len.strict_sub(16); @@ -1144,13 +1291,18 @@ unsafe fn crc32_ieee_pclmul_small(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 (IEEE) "pclmul-small" kernel. #[inline] -pub fn crc32_ieee_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_small_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies PCLMULQDQ (directly or via VPCLMUL-ready) before selecting this path. unsafe { crc32_ieee_pclmul_small(crc, data) } } // CRC-32 (IEEE) Multi-stream Folding (SSSE3 + PCLMULQDQ) +/// Folds one 128-byte block into eight PCLMULQDQ lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_block_128_crc32(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff: Simd128) { @@ -1176,6 +1328,12 @@ unsafe fn fold_block_128_crc32(x: &mut [Simd128; 8], chunk: &[Simd128; 8], coeff } } +/// Folds 128-byte blocks through two PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. `blocks` must +/// contain at least two blocks. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_2way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1199,7 +1357,7 @@ unsafe fn update_simd_crc32_ieee_2way(state: u32, blocks: &[[Simd128; 8]]) -> u3 const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= aligned { @@ -1247,6 +1405,11 @@ unsafe fn update_simd_crc32_ieee_2way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } } +/// Folds 128-byte blocks through four PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1278,7 +1441,7 @@ unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u3 const DOUBLE_GROUP: usize = 8; // 2 × 4-way = 8 blocks = 1KB let mut i: usize = 4; - let double_aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let double_aligned = blocks.len().strict_sub(blocks.len().strict_rem(DOUBLE_GROUP)); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= double_aligned { @@ -1304,7 +1467,7 @@ unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } // Handle remaining 4-block groups - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(4)); while i < aligned { fold_block_128_crc32(&mut s0, &blocks[i], coeff_512b); fold_block_128_crc32(&mut s1, &blocks[i.strict_add(1)], coeff_512b); @@ -1328,6 +1491,11 @@ unsafe fn update_simd_crc32_ieee_4way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } } +/// Folds 128-byte blocks through seven PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_7way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1341,7 +1509,7 @@ unsafe fn update_simd_crc32_ieee_7way(state: u32, blocks: &[[Simd128; 8]]) -> u3 return update_simd_crc32_ieee_reflected(state, first, rest); } - let aligned = (blocks.len() / 7) * 7; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(7)); let coeff_896b = Simd128::new(CRC32_IEEE_STREAM.fold_896b.0, CRC32_IEEE_STREAM.fold_896b.1); let coeff_128b = Simd128::new(CRC32_IEEE_KEYS_REFLECTED[4], CRC32_IEEE_KEYS_REFLECTED[3]); @@ -1403,6 +1571,11 @@ unsafe fn update_simd_crc32_ieee_7way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } } +/// Folds 128-byte blocks through eight PCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] unsafe fn update_simd_crc32_ieee_8way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1416,7 +1589,7 @@ unsafe fn update_simd_crc32_ieee_8way(state: u32, blocks: &[[Simd128; 8]]) -> u3 return update_simd_crc32_ieee_reflected(state, first, rest); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len().strict_sub(blocks.len().strict_rem(8)); let coeff_1024b = Simd128::new(CRC32_IEEE_STREAM.fold_1024b.0, CRC32_IEEE_STREAM.fold_1024b.1); let coeff_128b = Simd128::new(CRC32_IEEE_KEYS_REFLECTED[4], CRC32_IEEE_KEYS_REFLECTED[3]); @@ -1483,6 +1656,10 @@ unsafe fn update_simd_crc32_ieee_8way(state: u32, blocks: &[[Simd128; 8]]) -> u3 } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (2-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_2way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1504,12 +1681,16 @@ pub(crate) unsafe fn crc32_ieee_pclmul_2way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_2way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (4-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_4way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1527,12 +1708,16 @@ pub(crate) unsafe fn crc32_ieee_pclmul_4way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_4way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (7-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_7way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1550,12 +1735,16 @@ pub(crate) unsafe fn crc32_ieee_pclmul_7way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_7way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using PCLMULQDQ folding (8-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "ssse3", enable = "pclmulqdq")] pub(crate) unsafe fn crc32_ieee_pclmul_8way(crc: u32, data: &[u8]) -> u32 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. @@ -1573,27 +1762,38 @@ pub(crate) unsafe fn crc32_ieee_pclmul_8way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_pclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies SSSE3 + PCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_pclmul_8way(crc, data) } } +/// Multiplies the high lane of each 128-bit element in `a` by the low lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul10_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x10) } +/// Multiplies the low lane of each 128-bit element in `a` by the high lane in `b`. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn clmul01_vpclmul(a: __m512i, b: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_clmulepi64_epi128(a, b, 0x01) } +/// Folds four reflected 16-byte lanes and XORs their supplied input lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_16_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i, data: __m512i) -> __m512i { @@ -1602,6 +1802,12 @@ unsafe fn fold_16_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i, data: unsafe { _mm512_ternarylogic_epi64(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff), data, 0x96) } } +/// Folds a leading 128-byte block and the remaining blocks with VPCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_reflected_vpclmul(state: u32, first: &[Simd128; 8], rest: &[[Simd128; 8]]) -> u32 { @@ -1625,8 +1831,8 @@ unsafe fn update_simd_crc32_ieee_reflected_vpclmul(state: u32, first: &[Simd128; // Load the first 128-byte block as 2×512-bit registers (8×16B lanes). let base = first.as_ptr().cast::(); - let mut x0 = _mm512_loadu_si512(base.cast::<__m512i>()); - let mut x1 = _mm512_loadu_si512(base.add(64).cast::<__m512i>()); + let mut x0 = load_unaligned_512(base); + let mut x1 = load_unaligned_512(base.add(64)); // Inject CRC into the first lane (low 32 bits). let injected = _mm512_setr_epi64(state as i64, 0, 0, 0, 0, 0, 0, 0); @@ -1634,8 +1840,8 @@ unsafe fn update_simd_crc32_ieee_reflected_vpclmul(state: u32, first: &[Simd128; for block in rest { let ptr = block.as_ptr().cast::(); - let y0 = _mm512_loadu_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_loadu_si512(ptr.add(64).cast::<__m512i>()); + let y0 = load_unaligned_512(ptr); + let y1 = load_unaligned_512(ptr.add(64)); x0 = fold_16_crc32_reflected_vpclmul(x0, coeff, y0); x1 = fold_16_crc32_reflected_vpclmul(x1, coeff, y1); } @@ -1695,18 +1901,21 @@ pub(crate) unsafe fn crc32_ieee_vpclmul(crc: u32, data: &[u8]) -> u32 { /// Safe wrapper for CRC-32 (IEEE) VPCLMUL kernel. #[inline] -pub fn crc32_ieee_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul(crc, data) } } // CRC-32 (IEEE) Multi-stream Folding (AVX-512 VPCLMULQDQ) +/// Broadcasts one pair of folding coefficients across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. // Each 128-bit lane expects: low64 = pair.1, high64 = pair.0. _mm512_setr_epi64( pair.1.cast_signed(), @@ -1720,6 +1929,11 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { ) } +/// Folds four reflected lanes without injecting another data vector. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn fold_only_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i) -> __m512i { @@ -1728,6 +1942,11 @@ unsafe fn fold_only_crc32_reflected_vpclmul(state: __m512i, coeff: __m512i) -> _ unsafe { _mm512_xor_si512(clmul10_vpclmul(state, coeff), clmul01_vpclmul(state, coeff)) } } +/// Loads one 128-byte block as two unaligned 512-bit vectors. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq")] unsafe fn load_128b_block(block: &[Simd128; 8]) -> (__m512i, __m512i) { @@ -1735,12 +1954,18 @@ unsafe fn load_128b_block(block: &[Simd128; 8]) -> (__m512i, __m512i) { // attribute. Pointer arithmetic: block is 128 bytes, so ptr+64 is within bounds. unsafe { let ptr = block.as_ptr().cast::(); - let y0 = _mm512_loadu_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_loadu_si512(ptr.add(64).cast::<__m512i>()); + let y0 = load_unaligned_512(ptr); + let y1 = load_unaligned_512(ptr.add(64)); (y0, y1) } } +/// Combines two four-lane VPCLMULQDQ states and applies CRC-32 reduction. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F/VL/BW/DQ, VPCLMULQDQ, SSE2, and +/// PCLMULQDQ. #[inline] #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,sse2,pclmulqdq")] unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i) -> u32 { @@ -1771,6 +1996,13 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i) -> u32 { } } +/// Folds 128-byte blocks through two VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. +/// `blocks` must contain at least two blocks. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_2way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -1843,6 +2075,12 @@ unsafe fn update_simd_crc32_ieee_vpclmul_2way(state: u32, blocks: &[[Simd128; 8] } } +/// Folds 128-byte blocks through four VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_4way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -1957,6 +2195,12 @@ unsafe fn update_simd_crc32_ieee_vpclmul_4way(state: u32, blocks: &[[Simd128; 8] } } +/// Folds 128-byte blocks through seven VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_7way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2056,6 +2300,12 @@ unsafe fn update_simd_crc32_ieee_vpclmul_7way(state: u32, blocks: &[[Simd128; 8] } } +/// Folds 128-byte blocks through eight VPCLMULQDQ streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] unsafe fn update_simd_crc32_ieee_vpclmul_8way(state: u32, blocks: &[[Simd128; 8]]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2164,6 +2414,11 @@ unsafe fn update_simd_crc32_ieee_vpclmul_8way(state: u32, blocks: &[[Simd128; 8] } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (2-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_2way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2181,12 +2436,17 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_2way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_2way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_2way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (4-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_4way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2204,12 +2464,17 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_4way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_4way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_4way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (7-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_7way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2227,12 +2492,17 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_7way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_7way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_7way(crc, data) } } /// CRC-32 (IEEE / ISO-HDLC) update using AVX-512 VPCLMULQDQ folding (8-way multi-stream). +/// +/// # Safety +/// +/// The current CPU must support SSE2, SSSE3, PCLMULQDQ, AVX-512F/VL/BW/DQ, +/// and VPCLMULQDQ. #[target_feature(enable = "avx512f,avx512vl,avx512bw,avx512dq,vpclmulqdq,ssse3,pclmulqdq,sse2")] pub(crate) unsafe fn crc32_ieee_vpclmul_8way(crc: u32, data: &[u8]) -> u32 { // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] @@ -2250,7 +2520,7 @@ pub(crate) unsafe fn crc32_ieee_vpclmul_8way(crc: u32, data: &[u8]) -> u32 { } #[inline] -pub fn crc32_ieee_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { +pub(super) fn crc32_ieee_vpclmul_8way_safe(crc: u32, data: &[u8]) -> u32 { // SAFETY: Dispatcher verifies AVX-512 VPCLMULQDQ before selecting this kernel. unsafe { crc32_ieee_vpclmul_8way(crc, data) } } @@ -2263,6 +2533,14 @@ mod tests { use super::*; + fn deterministic_data(len: usize, multiplier: u8, increment: u8) -> Vec { + (0u8..=u8::MAX) + .cycle() + .take(len) + .map(|byte| byte.wrapping_mul(multiplier).wrapping_add(increment)) + .collect() + } + fn vpclmul_available_for_tests() -> bool { !cfg!(miri) && std::arch::is_x86_feature_detected!("avx512f") @@ -2298,17 +2576,21 @@ mod tests { return; } - for len in [ - 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024, - ] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(4096usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 129, 255, 256, 1024, 4096, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_pclmul_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } } - - let portable = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let pclmul = crc32_ieee_pclmul_safe(!0, &data) ^ !0; - assert_eq!(pclmul, portable, "len={len}"); } } @@ -2318,15 +2600,21 @@ mod tests { return; } - for len in [0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); - } + let data_len = 127usize.strict_add(15); + let data = deterministic_data(data_len, 31, 7); - let portable = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let pclmul_small = crc32_ieee_pclmul_small_safe(!0, &data) ^ !0; - assert_eq!(pclmul_small, portable, "len={len}"); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[0usize, 1, 2, 3, 4, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_pclmul_small_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } + } } } @@ -2336,17 +2624,21 @@ mod tests { return; } - for len in [ - 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024, 4096, - ] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(4096usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 129, 255, 256, 1024, 4096, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_vpclmul_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } } - - let portable = super::super::portable::crc32_slice16_ieee(!0, &data) ^ !0; - let vpclmul = crc32_ieee_vpclmul_safe(!0, &data) ^ !0; - assert_eq!(vpclmul, portable, "len={len}"); } } @@ -2356,34 +2648,36 @@ mod tests { return; } - for &init in &[!0u32, 0x1234_5678, 0xDEAD_BEEF] { - for len in [0usize, 1, 15, 16, 127, 128, 255, 256, 1024, 4096, 16384] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(16384usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 15, 16, 127, 128, 129, 255, 256, 257, 511, 512, 513, 895, 896, 897, 1023, 1024, 1025, 1791, 1792, + 1793, 2047, 2048, 2049, 4095, 4096, 4097, 16384, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_pclmul_2way_safe(init, input), + expected, + "2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_pclmul_4way_safe(init, input), + expected, + "4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_pclmul_7way_safe(init, input), + expected, + "7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_pclmul_8way_safe(init, input), + expected, + "8way init={init:#x} offset={offset} len={len}" + ); } - - let expected = super::super::portable::crc32_slice16_ieee(init, &data); - assert_eq!( - crc32_ieee_pclmul_2way_safe(init, &data), - expected, - "2way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_pclmul_4way_safe(init, &data), - expected, - "4way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_pclmul_7way_safe(init, &data), - expected, - "7way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_pclmul_8way_safe(init, &data), - expected, - "8way init={init:#x} len={len}" - ); } } } @@ -2394,34 +2688,63 @@ mod tests { return; } - for &init in &[!0u32, 0x1234_5678, 0xDEAD_BEEF] { - for len in [0usize, 1, 15, 16, 127, 128, 255, 256, 1024, 4096, 16384] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(31).wrapping_add(7)); + let data = deterministic_data(16384usize.strict_add(15), 31, 7); + for &init in &[!0u32, 0, 0x1234_5678, 0xDEAD_BEEF] { + for &offset in &[0usize, 1, 3, 7, 15] { + for &len in &[ + 0usize, 1, 15, 16, 127, 128, 129, 255, 256, 257, 511, 512, 513, 895, 896, 897, 1023, 1024, 1025, 1791, 1792, + 1793, 2047, 2048, 2049, 4095, 4096, 4097, 16384, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32_slice16_ieee(init, input); + assert_eq!( + crc32_ieee_vpclmul_2way_safe(init, input), + expected, + "2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_vpclmul_4way_safe(init, input), + expected, + "4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_vpclmul_7way_safe(init, input), + expected, + "7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32_ieee_vpclmul_8way_safe(init, input), + expected, + "8way init={init:#x} offset={offset} len={len}" + ); } + } + } + } - let expected = super::super::portable::crc32_slice16_ieee(init, &data); - assert_eq!( - crc32_ieee_vpclmul_2way_safe(init, &data), - expected, - "2way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_vpclmul_4way_safe(init, &data), - expected, - "4way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_vpclmul_7way_safe(init, &data), - expected, - "7way init={init:#x} len={len}" - ); - assert_eq!( - crc32_ieee_vpclmul_8way_safe(init, &data), - expected, - "8way init={init:#x} len={len}" - ); + #[test] + fn test_crc32c_sse42_matches_portable() { + if !std::arch::is_x86_feature_detected!("sse4.2") { + return; + } + + let data: Vec = (0u8..=u8::MAX) + .cycle() + .take(4096usize.strict_add(7)) + .map(|byte| byte.wrapping_mul(17).wrapping_add(3)) + .collect(); + + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for &offset in &[0usize, 1, 3, 7] { + for &len in &[0usize, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 31, 32, 127, 128, 4096] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_sse42_safe(init, input), + expected, + "init={init:#x} offset={offset} len={len}" + ); + } } } } @@ -2432,34 +2755,38 @@ mod tests { return; } - for &init in &[!0u32, 0x0123_4567, 0x89AB_CDEF] { - for len in [0usize, 1, 7, 8, 15, 16, 127, 128, 255, 256, 2048, 8192] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(17).wrapping_add(3)); + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for len in [ + 0usize, 1, 7, 8, 15, 16, 127, 128, 255, 256, 511, 512, 513, 1023, 1024, 1025, 1791, 1792, 1793, 2047, 2048, + 2049, 4095, 4096, 4097, 8192, + ] { + for &offset in &[0usize, 1, 3, 7] { + let data_len = offset.strict_add(len); + let data = deterministic_data(data_len, 17, 3); + + let input = &data[offset..]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_sse42_2way_safe(init, input), + expected, + "2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_sse42_4way_safe(init, input), + expected, + "4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_sse42_7way_safe(init, input), + expected, + "7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_sse42_8way_safe(init, input), + expected, + "8way init={init:#x} offset={offset} len={len}" + ); } - - let expected = super::super::portable::crc32c_slice16(init, &data); - assert_eq!( - crc32c_sse42_2way_safe(init, &data), - expected, - "2way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_sse42_4way_safe(init, &data), - expected, - "4way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_sse42_7way_safe(init, &data), - expected, - "7way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_sse42_8way_safe(init, &data), - expected, - "8way init={init:#x} len={len}" - ); } } } @@ -2470,34 +2797,86 @@ mod tests { return; } - for &init in &[!0u32, 0x0123_4567, 0x89AB_CDEF] { - for len in [0usize, 1, 7, 8, 15, 16, 127, 128, 255, 256, 2048, 8192, 65536] { - let mut data = Vec::with_capacity(len); - for i in 0..len { - data.push((i as u8).wrapping_mul(19).wrapping_add(5)); + let data = deterministic_data(65536usize.strict_add(7), 19, 5); + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for &offset in &[0usize, 1, 3, 7] { + for &len in &[ + 0usize, 1, 7, 8, 15, 16, 63, 64, 127, 128, 135, 136, 143, 144, 145, 255, 256, 271, 272, 273, 383, 384, 385, + 511, 512, 513, 1023, 1024, 1025, 2048, 8192, 65536, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_safe(init, input), + expected, + "fusion/1way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_2way_safe(init, input), + expected, + "fusion/2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_4way_safe(init, input), + expected, + "fusion/4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_7way_safe(init, input), + expected, + "fusion/7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_sse_v4s3x3_8way_safe(init, input), + expected, + "fusion/8way init={init:#x} offset={offset} len={len}" + ); } + } + } + } - let expected = super::super::portable::crc32c_slice16(init, &data); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_2way_safe(init, &data), - expected, - "fusion/2way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_4way_safe(init, &data), - expected, - "fusion/4way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_7way_safe(init, &data), - expected, - "fusion/7way init={init:#x} len={len}" - ); - assert_eq!( - crc32c_iscsi_sse_v4s3x3_8way_safe(init, &data), - expected, - "fusion/8way init={init:#x} len={len}" - ); + #[test] + fn test_crc32c_fusion_vpclmul_multistream_matches_portable() { + if !(vpclmul_available_for_tests() && std::arch::is_x86_feature_detected!("sse4.2")) { + return; + } + + let data = deterministic_data(8192usize.strict_add(7), 19, 5); + for &init in &[!0u32, 0, 0x0123_4567, 0x89AB_CDEF] { + for &offset in &[0usize, 1, 3, 7] { + for &len in &[ + 0usize, 1, 7, 8, 15, 16, 63, 64, 127, 128, 191, 192, 383, 384, 385, 511, 512, 513, 1023, 1024, 1025, 2048, + 8192, + ] { + let input = &data[offset..offset.strict_add(len)]; + let expected = super::super::portable::crc32c_slice16(init, input); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_safe(init, input), + expected, + "fusion-vpclmul/1way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_2way_safe(init, input), + expected, + "fusion-vpclmul/2way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_4way_safe(init, input), + expected, + "fusion-vpclmul/4way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_7way_safe(init, input), + expected, + "fusion-vpclmul/7way init={init:#x} offset={offset} len={len}" + ); + assert_eq!( + crc32c_iscsi_avx512_vpclmulqdq_v3x2_8way_safe(init, input), + expected, + "fusion-vpclmul/8way init={init:#x} offset={offset} len={len}" + ); + } } } } diff --git a/src/checksum/crc64/aarch64.rs b/src/checksum/crc64/aarch64.rs index 9e9a0c25..94f9cd49 100644 --- a/src/checksum/crc64/aarch64.rs +++ b/src/checksum/crc64/aarch64.rs @@ -5,13 +5,8 @@ //! //! # Safety //! -//! Uses `unsafe` for ARM SIMD intrinsics. Callers must ensure PMULL is -//! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] -// This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. +//! Uses `unsafe` for ARM SIMD intrinsics. Callers must establish NEON, AES +//! (PMULL), and SHA3 (EOR3) support as required by each accelerated path. use core::{ arch::aarch64::*, @@ -24,50 +19,71 @@ use crate::checksum::common::clmul::{CRC64_NVME_STREAM, CRC64_XZ_STREAM, Crc64Cl #[derive(Copy, Clone, Debug)] struct Simd(uint8x16_t); -#[allow(non_camel_case_types)] -type poly64_t = u64; - impl Simd { + /// Multiplies two carry-less 64-bit polynomials into one 128-bit vector. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn from_mul(a: poly64_t, b: poly64_t) -> Self { - // SAFETY: Caller guarantees NEON+AES (PMULL) is available. Intrinsics operate on registers. + unsafe fn from_mul(a: u64, b: u64) -> Self { let mul = vmull_p64(a, b); Self(vreinterpretq_u8_p128(mul)) } + /// Returns the vector's two 64-bit polynomial lanes in low-to-high order. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES. #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn into_poly64s(self) -> [poly64_t; 2] { - // SAFETY: Caller guarantees NEON+AES is available. Intrinsics operate on registers. + unsafe fn into_poly64s(self) -> [u64; 2] { let x = vreinterpretq_p64_u8(self.0); [vgetq_lane_p64(x, 0), vgetq_lane_p64(x, 1)] } + /// Returns the high 64-bit polynomial lane. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES. #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn high_64(self) -> poly64_t { - // SAFETY: Caller guarantees NEON+AES is available. Intrinsics operate on registers. + unsafe fn high_64(self) -> u64 { let x = vreinterpretq_p64_u8(self.0); vgetq_lane_p64(x, 1) } + /// Returns the low 64-bit polynomial lane. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES. #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn low_64(self) -> poly64_t { - // SAFETY: Caller guarantees NEON+AES is available. Intrinsics operate on registers. + unsafe fn low_64(self) -> u64 { let x = vreinterpretq_p64_u8(self.0); vgetq_lane_p64(x, 0) } + /// Creates a vector from its high and low 64-bit lanes. + /// + /// # Safety + /// + /// The current CPU must support NEON. #[inline] #[target_feature(enable = "neon")] unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: Caller guarantees NEON is available. Intrinsics operate on registers. Self(vcombine_u8(vcreate_u8(low), vcreate_u8(high))) } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] unsafe fn fold_16(self, coeff: Self) -> Self { @@ -86,9 +102,13 @@ impl Simd { /// /// Equivalent to `self.fold_16(Simd::new(high, low))` but avoids repeatedly /// extracting the coefficient lanes inside hot loops. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] - unsafe fn fold_16_pair(self, coeff_low: poly64_t, coeff_high: poly64_t) -> Self { + unsafe fn fold_16_pair(self, coeff_low: u64, coeff_high: u64) -> Self { // SAFETY: Caller guarantees NEON+AES (PMULL) is available. All operations are register // computations. unsafe { @@ -100,6 +120,10 @@ impl Simd { } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] unsafe fn fold_8(self, coeff: u64) -> Self { @@ -114,6 +138,10 @@ impl Simd { } /// Barrett reduction to finalize the CRC. + /// + /// # Safety + /// + /// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "neon", enable = "aes")] unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { @@ -145,6 +173,11 @@ impl BitXorAssign for Simd { } } +/// Folds a sequence of 128-byte blocks and reduces it to a CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: Caller guarantees NEON+AES (PMULL) target features are available (dispatch check). @@ -167,6 +200,11 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } // unsafe } +/// Folds a sequence of 128-byte blocks with EOR3 and reduces it to a CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: Caller guarantees NEON+AES+SHA3 (PMULL+EOR3) target features are available (dispatch @@ -188,6 +226,11 @@ unsafe fn update_simd_eor3(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], co } // unsafe } +/// Reduces eight folded SIMD lanes to a CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[inline(always)] unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: Caller guarantees NEON+AES (PMULL) target features are available (dispatch check). @@ -218,6 +261,11 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { // PMULL multi-stream (2-way/3-way, 128B blocks) +/// Folds one 128-byte block into the current SIMD state. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[inline] #[target_feature(enable = "aes", enable = "neon")] unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u64, coeff_high: u64) { @@ -239,6 +287,10 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u64, c /// /// This reduces the XOR dependency chain: instead of `chunk ^ (h ^ l)` (2 XORs), /// we use `veor3(chunk, h, l)` (1 instruction). +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[inline] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn fold_block_128_eor3(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u64, coeff_high: u64) { @@ -260,9 +312,13 @@ unsafe fn fold_block_128_eor3(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff_low: u /// /// Computes: `data ^ pmull(coeff_low, x.low) ^ pmull(coeff_high, x.high)` /// using a single 3-way XOR instruction. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[inline] #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] -unsafe fn fold_lane_eor3(x: Simd, data: Simd, coeff_low: poly64_t, coeff_high: poly64_t) -> Simd { +unsafe fn fold_lane_eor3(x: Simd, data: Simd, coeff_low: u64, coeff_high: u64) -> Simd { // SAFETY: Caller guarantees NEON+AES+SHA3 (PMULL+EOR3) is available. All operations are register // computations. unsafe { @@ -274,6 +330,11 @@ unsafe fn fold_lane_eor3(x: Simd, data: Simd, coeff_low: poly64_t, coeff_high: p } } +/// Folds 128-byte blocks through two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_2way( state: u64, @@ -304,7 +365,7 @@ unsafe fn update_simd_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= aligned { @@ -353,6 +414,11 @@ unsafe fn update_simd_2way( } // unsafe } +/// Folds 128-byte blocks through three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn update_simd_3way( state: u64, @@ -392,7 +458,7 @@ unsafe fn update_simd_3way( const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B let mut i: usize = 3; - let double_aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let double_aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= double_aligned { @@ -416,7 +482,7 @@ unsafe fn update_simd_3way( } // Handle remaining 3-block groups - let aligned = (blocks.len() / 3) * 3; + let aligned = blocks.len().strict_div(3).strict_mul(3); while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_384_low, coeff_384_high); fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_384_low, coeff_384_high); @@ -459,6 +525,10 @@ unsafe fn update_simd_3way( /// /// Maintains two independent fold streams and uses EOR3 when merging each /// polynomial product with input data. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3_2way( state: u64, @@ -490,7 +560,7 @@ unsafe fn update_simd_eor3_2way( const DOUBLE_GROUP: usize = 4; // 2 × 2-way = 4 blocks = 512B let mut i: usize = 2; - let aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= aligned { @@ -546,6 +616,10 @@ unsafe fn update_simd_eor3_2way( /// /// Maintains three independent fold streams and uses EOR3 when merging each /// polynomial product with input data. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn update_simd_eor3_3way( state: u64, @@ -586,7 +660,7 @@ unsafe fn update_simd_eor3_3way( const DOUBLE_GROUP: usize = 6; // 2 × 3-way = 6 blocks = 768B let mut i: usize = 3; - let double_aligned = (blocks.len() / DOUBLE_GROUP) * DOUBLE_GROUP; + let double_aligned = blocks.len().strict_div(DOUBLE_GROUP).strict_mul(DOUBLE_GROUP); // Double-unrolled loop while i.strict_add(DOUBLE_GROUP) <= double_aligned { @@ -610,7 +684,7 @@ unsafe fn update_simd_eor3_3way( } // Handle remaining 3-block groups - let aligned = (blocks.len() / 3) * 3; + let aligned = blocks.len().strict_div(3).strict_mul(3); while i < aligned { fold_block_128_eor3(&mut s0, &blocks[i], coeff_384_low, coeff_384_high); fold_block_128_eor3(&mut s1, &blocks[i.strict_add(1)], coeff_384_low, coeff_384_high); @@ -650,6 +724,11 @@ unsafe fn update_simd_eor3_3way( } // unsafe } +/// Computes CRC-64 with two EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc64_pmull_eor3_2way( mut state: u64, @@ -671,6 +750,11 @@ unsafe fn crc64_pmull_eor3_2way( } } +/// Computes CRC-64 with three EOR3-assisted PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc64_pmull_eor3_3way( mut state: u64, @@ -693,6 +777,11 @@ unsafe fn crc64_pmull_eor3_3way( } } +/// Computes CRC-64 with two independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_2way( mut state: u64, @@ -714,6 +803,11 @@ unsafe fn crc64_pmull_2way( } } +/// Computes CRC-64 with three independent PMULL streams. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_3way( mut state: u64, @@ -736,6 +830,11 @@ unsafe fn crc64_pmull_3way( } } +/// Computes CRC-64 with PMULL folding. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8]) -> u64 { // SAFETY: Caller guarantees NEON+AES (PMULL) is available. align_to produces valid sub-slices. @@ -754,6 +853,10 @@ unsafe fn crc64_pmull(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants /// PMULL+EOR3 path: uses EOR3 to combine three XOR operands. /// /// Available on ARMv8.2+ with SHA3 extension (Apple M1+, AWS Graviton3+). +/// +/// # Safety +/// +/// The current CPU must support NEON, AES (PMULL), and SHA3 (EOR3). #[target_feature(enable = "aes", enable = "neon", enable = "sha3")] unsafe fn crc64_pmull_eor3( mut state: u64, @@ -778,6 +881,10 @@ unsafe fn crc64_pmull_eor3( /// /// This avoids constructing the full 128-byte fold state when only complete /// 16-byte lanes are available. +/// +/// # Safety +/// +/// The current CPU must support NEON and AES (PMULL). #[target_feature(enable = "aes", enable = "neon")] unsafe fn crc64_pmull_small( mut state: u64, @@ -864,6 +971,7 @@ pub(crate) unsafe fn crc64_xz_pmull_small(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_xz_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -879,6 +987,7 @@ pub(crate) unsafe fn crc64_xz_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_xz_sve2_pmull_3way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -934,6 +1043,7 @@ pub(crate) unsafe fn crc64_nvme_pmull_small(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_nvme_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -949,6 +1059,7 @@ pub(crate) unsafe fn crc64_nvme_sve2_pmull_2way(crc: u64, data: &[u8]) -> u64 { /// /// Requires PMULL (crypto/aes). Caller must verify via /// `crate::platform::caps().has(aarch64::SVE2_PMULL)` and `PMULL_READY` before selecting. +#[cfg(any(feature = "std", test))] #[target_feature(enable = "aes", enable = "neon")] pub(crate) unsafe fn crc64_nvme_sve2_pmull_3way(crc: u64, data: &[u8]) -> u64 { // SAFETY: the caller guarantees `SVE2_PMULL` and `PMULL_READY` before selecting this kernel. @@ -1103,122 +1214,130 @@ pub(crate) unsafe fn crc64_nvme_pmull_eor3_3way(crc: u64, data: &[u8]) -> u64 { /// Safe wrapper for CRC-64-XZ PMULL kernel. #[inline] -pub fn crc64_xz_pmull_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL small-buffer kernel. #[inline] -pub fn crc64_xz_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull_small(crc, data) } } /// Safe wrapper for CRC-64-XZ tuned "SVE2 PMULL" tier (single-stream). #[inline] -pub fn crc64_xz_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_xz_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { crc64_xz_pmull_safe(crc, data) } /// Safe wrapper for CRC-64-XZ tuned "SVE2 PMULL" tier (small-buffer). #[inline] -pub fn crc64_xz_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_xz_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { crc64_xz_pmull_small_safe(crc, data) } /// Safe wrapper for CRC-64-XZ tuned SVE2 2-way PMULL kernel. #[inline] -pub fn crc64_xz_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_xz_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_xz_sve2_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ tuned SVE2 3-way PMULL kernel. #[inline] -pub fn crc64_xz_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_xz_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_xz_sve2_pmull_3way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL kernel. #[inline] -pub fn crc64_nvme_pmull_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL small-buffer kernel. #[inline] -pub fn crc64_nvme_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull_small(crc, data) } } /// Safe wrapper for CRC-64-NVME tuned "SVE2 PMULL" tier (single-stream). #[inline] -pub fn crc64_nvme_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_nvme_sve2_pmull_safe(crc: u64, data: &[u8]) -> u64 { crc64_nvme_pmull_safe(crc, data) } /// Safe wrapper for CRC-64-NVME tuned "SVE2 PMULL" tier (small-buffer). #[inline] -pub fn crc64_nvme_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(feature = "std")] +pub(super) fn crc64_nvme_sve2_pmull_small_safe(crc: u64, data: &[u8]) -> u64 { crc64_nvme_pmull_small_safe(crc, data) } /// Safe wrapper for CRC-64-NVME tuned SVE2 2-way PMULL kernel. #[inline] -pub fn crc64_nvme_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_nvme_sve2_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_nvme_sve2_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME tuned SVE2 3-way PMULL kernel. #[inline] -pub fn crc64_nvme_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +#[cfg(any(feature = "std", test))] +pub(super) fn crc64_nvme_sve2_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies SVE2_PMULL + PMULL before selecting this kernel. unsafe { crc64_nvme_sve2_pmull_3way(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL+EOR3 kernel. #[inline] -pub fn crc64_xz_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_xz_pmull_eor3(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL+EOR3 kernel. #[inline] -pub fn crc64_nvme_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_eor3_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_nvme_pmull_eor3(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL+EOR3 2-way kernel. #[inline] -pub fn crc64_xz_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_xz_pmull_eor3_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL+EOR3 3-way kernel. #[inline] -pub fn crc64_xz_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_xz_pmull_eor3_3way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL+EOR3 2-way kernel. #[inline] -pub fn crc64_nvme_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_eor3_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_nvme_pmull_eor3_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL+EOR3 3-way kernel. #[inline] -pub fn crc64_nvme_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_eor3_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL_EOR3_READY before selecting this kernel. unsafe { crc64_nvme_pmull_eor3_3way(crc, data) } } @@ -1317,28 +1436,28 @@ pub(crate) unsafe fn crc64_nvme_pmull_3way(crc: u64, data: &[u8]) -> u64 { /// Safe wrapper for CRC-64-XZ PMULL 2-way kernel. #[inline] -pub fn crc64_xz_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ PMULL 3-way kernel. #[inline] -pub fn crc64_xz_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_xz_pmull_3way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL 2-way kernel. #[inline] -pub fn crc64_nvme_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME PMULL 3-way kernel. #[inline] -pub fn crc64_nvme_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pmull_3way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PMULL (crypto/aes) before selecting this kernel. unsafe { crc64_nvme_pmull_3way(crc, data) } } @@ -1356,7 +1475,11 @@ mod tests { fn make_data(len: usize) -> alloc::vec::Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 3) as u8)) + .map(|i| { + let [low, high, ..] = i.to_le_bytes(); + let shifted = low.strict_shr(3) | high.strict_shl(5); + low.wrapping_mul(17).wrapping_add(shifted) + }) .collect() } @@ -1588,7 +1711,6 @@ mod tests { return; } - // SAFETY: We just checked AES+SHA3 is available. let crc = crc64_xz_pmull_eor3_safe(!0, TEST_DATA) ^ !0; assert_eq!(crc, 0x995D_C9BB_DF19_39FA); } @@ -1599,7 +1721,6 @@ mod tests { return; } - // SAFETY: We just checked AES+SHA3 is available. let crc = crc64_nvme_pmull_eor3_safe(!0, TEST_DATA) ^ !0; assert_eq!(crc, 0xAE8B_1486_0A79_9888); } diff --git a/src/checksum/crc64/config.rs b/src/checksum/crc64/config.rs index 3fa7e4a3..c685a664 100644 --- a/src/checksum/crc64/config.rs +++ b/src/checksum/crc64/config.rs @@ -35,10 +35,6 @@ pub enum Crc64Force { Vpmsum, /// Force s390x VGFM (if supported). Vgfm, - /// Force riscv64 Zbc carryless multiply (if supported). - Zbc, - /// Force riscv64 Zvbc (vector carryless multiply) folding (if supported). - Zvbc, } impl Crc64Force { @@ -56,8 +52,6 @@ impl Crc64Force { Self::Sve2Pmull => "sve2-pmull", Self::Vpmsum => "vpmsum", Self::Vgfm => "vgfm", - Self::Zbc => "zbc", - Self::Zvbc => "zvbc", } } } @@ -111,20 +105,20 @@ fn parse_force_env() -> Crc64Force { if value.eq_ignore_ascii_case("vgfm") || value.eq_ignore_ascii_case("gfmsum") { return Crc64Force::Vgfm; } - if value.eq_ignore_ascii_case("zbc") { - return Crc64Force::Zbc; - } - if value.eq_ignore_ascii_case("zvbc") || value.eq_ignore_ascii_case("vclmul") { - return Crc64Force::Zvbc; - } - Crc64Force::Auto } #[inline] #[must_use] -#[allow(unused_variables)] // `caps` only used on x86_64/aarch64 fn clamp_force_to_caps(requested: Crc64Force, caps: Caps) -> Crc64Force { + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "x86_64" + )))] + let _ = caps; + match requested { Crc64Force::Auto | Crc64Force::Reference | Crc64Force::Portable => requested, Crc64Force::Pclmul => { @@ -191,24 +185,6 @@ fn clamp_force_to_caps(requested: Crc64Force, caps: Caps) -> Crc64Force { } Crc64Force::Auto } - Crc64Force::Zbc => { - #[cfg(target_arch = "riscv64")] - { - if caps.has(crate::platform::caps::riscv::ZBC) { - return Crc64Force::Zbc; - } - } - Crc64Force::Auto - } - Crc64Force::Zvbc => { - #[cfg(target_arch = "riscv64")] - { - if caps.has(crate::platform::caps::riscv::ZVBC) { - return Crc64Force::Zvbc; - } - } - Crc64Force::Auto - } } } @@ -249,7 +225,7 @@ fn config(caps: Caps) -> Crc64Config { /// platform capabilities. #[inline] #[must_use] -pub fn get() -> Crc64Config { +pub(super) fn get() -> Crc64Config { #[cfg(feature = "std")] { use std::sync::OnceLock; diff --git a/src/checksum/crc64/kernels.rs b/src/checksum/crc64/kernels.rs index 317b5cd8..3890646e 100644 --- a/src/checksum/crc64/kernels.rs +++ b/src/checksum/crc64/kernels.rs @@ -20,23 +20,23 @@ //! - CRC64-specific dispatch functions generated via macro /// Portable fallback kernel name. -pub use kernels::PORTABLE_SLICE16 as PORTABLE; +pub(in crate::checksum) use kernels::PORTABLE_SLICE16 as PORTABLE; /// Reference (bitwise) kernel name - always available for force mode. -pub use kernels::REFERENCE; +pub(in crate::checksum) use kernels::REFERENCE; use crate::checksum::common::kernels; // Kernel Name Tables (per architecture) #[cfg(target_arch = "x86_64")] -pub mod x86_64 { +pub(in crate::checksum) mod x86_64 { use super::super::x86_64 as arch; use crate::checksum::dispatchers::Crc64Fn; // CRC64-XZ Kernel Function Arrays /// XZ PCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const XZ_PCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_PCLMUL: [Crc64Fn; 5] = [ arch::crc64_xz_pclmul_safe, arch::crc64_xz_pclmul_2way_safe, arch::crc64_xz_pclmul_4way_safe, @@ -44,10 +44,10 @@ pub mod x86_64 { arch::crc64_xz_pclmul_8way_safe, ]; /// XZ PCLMUL small buffer kernel. - pub const XZ_PCLMUL_SMALL: Crc64Fn = arch::crc64_xz_pclmul_small_safe; + pub(in crate::checksum) const XZ_PCLMUL_SMALL: Crc64Fn = arch::crc64_xz_pclmul_small_safe; /// XZ VPCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const XZ_VPCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_VPCLMUL: [Crc64Fn; 5] = [ arch::crc64_xz_vpclmul_safe, arch::crc64_xz_vpclmul_2way_safe, arch::crc64_xz_vpclmul_4way_safe, @@ -55,12 +55,12 @@ pub mod x86_64 { arch::crc64_xz_vpclmul_8way_safe, ]; /// XZ VPCLMUL 4×512-bit kernel. - pub const XZ_VPCLMUL_4X512: Crc64Fn = arch::crc64_xz_vpclmul_4x512_safe; + pub(in crate::checksum) const XZ_VPCLMUL_4X512: Crc64Fn = arch::crc64_xz_vpclmul_4x512_safe; // CRC64-NVME Kernel Function Arrays /// NVME PCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const NVME_PCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_PCLMUL: [Crc64Fn; 5] = [ arch::crc64_nvme_pclmul_safe, arch::crc64_nvme_pclmul_2way_safe, arch::crc64_nvme_pclmul_4way_safe, @@ -68,10 +68,10 @@ pub mod x86_64 { arch::crc64_nvme_pclmul_8way_safe, ]; /// NVME PCLMUL small buffer kernel. - pub const NVME_PCLMUL_SMALL: Crc64Fn = arch::crc64_nvme_pclmul_small_safe; + pub(in crate::checksum) const NVME_PCLMUL_SMALL: Crc64Fn = arch::crc64_nvme_pclmul_small_safe; /// NVME VPCLMUL kernels: [1-way, 2-way, 4-way, 7-way, 8-way] - pub const NVME_VPCLMUL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_VPCLMUL: [Crc64Fn; 5] = [ arch::crc64_nvme_vpclmul_safe, arch::crc64_nvme_vpclmul_2way_safe, arch::crc64_nvme_vpclmul_4way_safe, @@ -81,9 +81,7 @@ pub mod x86_64 { } #[cfg(target_arch = "aarch64")] -pub mod aarch64 { - #![cfg_attr(not(any(test, feature = "std")), allow(dead_code))] - #![cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))] +pub(in crate::checksum) mod aarch64 { use super::super::aarch64 as arch; use crate::checksum::dispatchers::Crc64Fn; @@ -92,7 +90,7 @@ pub mod aarch64 { // Note: aarch64 only supports up to 3-way, slots 3-4 are duplicates /// XZ PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const XZ_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_PMULL: [Crc64Fn; 5] = [ arch::crc64_xz_pmull_safe, arch::crc64_xz_pmull_2way_safe, arch::crc64_xz_pmull_3way_safe, @@ -100,10 +98,10 @@ pub mod aarch64 { arch::crc64_xz_pmull_3way_safe, // dup for index consistency ]; /// XZ PMULL small buffer kernel. - pub const XZ_PMULL_SMALL: Crc64Fn = arch::crc64_xz_pmull_small_safe; + pub(in crate::checksum) const XZ_PMULL_SMALL: Crc64Fn = arch::crc64_xz_pmull_small_safe; /// XZ PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const XZ_PMULL_EOR3: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_PMULL_EOR3: [Crc64Fn; 5] = [ arch::crc64_xz_pmull_eor3_safe, arch::crc64_xz_pmull_eor3_2way_safe, arch::crc64_xz_pmull_eor3_3way_safe, @@ -111,21 +109,23 @@ pub mod aarch64 { arch::crc64_xz_pmull_eor3_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// XZ SVE2 PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const XZ_SVE2_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_SVE2_PMULL: [Crc64Fn; 5] = [ arch::crc64_xz_sve2_pmull_safe, arch::crc64_xz_sve2_pmull_2way_safe, arch::crc64_xz_sve2_pmull_3way_safe, arch::crc64_xz_sve2_pmull_3way_safe, // dup for index consistency arch::crc64_xz_sve2_pmull_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// XZ SVE2 PMULL small buffer kernel. - pub const XZ_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_xz_sve2_pmull_small_safe; + pub(in crate::checksum) const XZ_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_xz_sve2_pmull_small_safe; // CRC64-NVME Kernel Function Arrays /// NVME PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const NVME_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_PMULL: [Crc64Fn; 5] = [ arch::crc64_nvme_pmull_safe, arch::crc64_nvme_pmull_2way_safe, arch::crc64_nvme_pmull_3way_safe, @@ -133,10 +133,10 @@ pub mod aarch64 { arch::crc64_nvme_pmull_3way_safe, // dup for index consistency ]; /// NVME PMULL small buffer kernel. - pub const NVME_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_pmull_small_safe; + pub(in crate::checksum) const NVME_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_pmull_small_safe; /// NVME PMULL+EOR3 kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const NVME_PMULL_EOR3: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_PMULL_EOR3: [Crc64Fn; 5] = [ arch::crc64_nvme_pmull_eor3_safe, arch::crc64_nvme_pmull_eor3_2way_safe, arch::crc64_nvme_pmull_eor3_3way_safe, @@ -144,27 +144,29 @@ pub mod aarch64 { arch::crc64_nvme_pmull_eor3_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// NVME SVE2 PMULL kernels: [1-way, 2-way, 3-way, 3-way(dup), 3-way(dup)] - pub const NVME_SVE2_PMULL: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_SVE2_PMULL: [Crc64Fn; 5] = [ arch::crc64_nvme_sve2_pmull_safe, arch::crc64_nvme_sve2_pmull_2way_safe, arch::crc64_nvme_sve2_pmull_3way_safe, arch::crc64_nvme_sve2_pmull_3way_safe, // dup for index consistency arch::crc64_nvme_sve2_pmull_3way_safe, // dup for index consistency ]; + #[cfg(feature = "std")] /// NVME SVE2 PMULL small buffer kernel. - pub const NVME_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_sve2_pmull_small_safe; + pub(in crate::checksum) const NVME_SVE2_PMULL_SMALL: Crc64Fn = arch::crc64_nvme_sve2_pmull_small_safe; } #[cfg(target_arch = "powerpc64")] -pub mod power { +pub(in crate::checksum) mod power { use super::super::power as arch; use crate::checksum::dispatchers::Crc64Fn; // CRC64-XZ Kernel Function Arrays /// XZ VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const XZ_VPMSUM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_VPMSUM: [Crc64Fn; 5] = [ arch::crc64_xz_vpmsum_safe, arch::crc64_xz_vpmsum_2way_safe, arch::crc64_xz_vpmsum_4way_safe, @@ -175,7 +177,7 @@ pub mod power { // CRC64-NVME Kernel Function Arrays /// NVME VPMSUM kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const NVME_VPMSUM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_VPMSUM: [Crc64Fn; 5] = [ arch::crc64_nvme_vpmsum_safe, arch::crc64_nvme_vpmsum_2way_safe, arch::crc64_nvme_vpmsum_4way_safe, @@ -185,7 +187,7 @@ pub mod power { } #[cfg(target_arch = "s390x")] -pub mod s390x { +pub(in crate::checksum) mod s390x { use super::super::s390x as arch; use crate::checksum::dispatchers::Crc64Fn; @@ -193,7 +195,7 @@ pub mod s390x { // Note: s390x only supports up to 4-way, slots 3-4 are duplicates /// XZ VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const XZ_VGFM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const XZ_VGFM: [Crc64Fn; 5] = [ arch::crc64_xz_vgfm_safe, arch::crc64_xz_vgfm_2way_safe, arch::crc64_xz_vgfm_4way_safe, @@ -204,7 +206,7 @@ pub mod s390x { // CRC64-NVME Kernel Function Arrays /// NVME VGFM kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const NVME_VGFM: [Crc64Fn; 5] = [ + pub(in crate::checksum) const NVME_VGFM: [Crc64Fn; 5] = [ arch::crc64_nvme_vgfm_safe, arch::crc64_nvme_vgfm_2way_safe, arch::crc64_nvme_vgfm_4way_safe, @@ -212,69 +214,3 @@ pub mod s390x { arch::crc64_nvme_vgfm_4way_safe, // dup for index consistency ]; } - -#[cfg(target_arch = "riscv64")] -#[allow(dead_code)] -pub mod riscv64 { - use super::super::riscv64 as arch; - use crate::checksum::dispatchers::Crc64Fn; - - /// Zbc kernel names: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const ZBC_NAMES: &[&str] = &[ - "riscv64/zbc", - "riscv64/zbc-2way", - "riscv64/zbc-4way", - "riscv64/zbc-8way", - "riscv64/zbc-8way", - ]; - - /// Zvbc kernel names: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const ZVBC_NAMES: &[&str] = &[ - "riscv64/zvbc", - "riscv64/zvbc-2way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - "riscv64/zvbc-4way", - ]; - - // CRC64-XZ Kernel Function Arrays - // Note: riscv64 Zbc exposes up to 8-way; slot 4 is duplicated for index consistency. - - /// XZ Zbc kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const XZ_ZBC: [Crc64Fn; 5] = [ - arch::crc64_xz_zbc_safe, - arch::crc64_xz_zbc_2way_safe, - arch::crc64_xz_zbc_4way_safe, - arch::crc64_xz_zbc_8way_safe, - arch::crc64_xz_zbc_8way_safe, // dup for index consistency - ]; - - /// XZ Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const XZ_ZVBC: [Crc64Fn; 5] = [ - arch::crc64_xz_zvbc_safe, - arch::crc64_xz_zvbc_2way_safe, - arch::crc64_xz_zvbc_4way_safe, - arch::crc64_xz_zvbc_4way_safe, // dup for index consistency - arch::crc64_xz_zvbc_4way_safe, // dup for index consistency - ]; - - // CRC64-NVME Kernel Function Arrays - - /// NVME Zbc kernels: [1-way, 2-way, 4-way, 8-way, 8-way(dup)] - pub const NVME_ZBC: [Crc64Fn; 5] = [ - arch::crc64_nvme_zbc_safe, - arch::crc64_nvme_zbc_2way_safe, - arch::crc64_nvme_zbc_4way_safe, - arch::crc64_nvme_zbc_8way_safe, - arch::crc64_nvme_zbc_8way_safe, // dup for index consistency - ]; - - /// NVME Zvbc kernels: [1-way, 2-way, 4-way, 4-way(dup), 4-way(dup)] - pub const NVME_ZVBC: [Crc64Fn; 5] = [ - arch::crc64_nvme_zvbc_safe, - arch::crc64_nvme_zvbc_2way_safe, - arch::crc64_nvme_zvbc_4way_safe, - arch::crc64_nvme_zvbc_4way_safe, // dup for index consistency - arch::crc64_nvme_zvbc_4way_safe, // dup for index consistency - ]; -} diff --git a/src/checksum/crc64/mod.rs b/src/checksum/crc64/mod.rs index 4954d8cc..b50fb167 100644 --- a/src/checksum/crc64/mod.rs +++ b/src/checksum/crc64/mod.rs @@ -10,7 +10,6 @@ //! - aarch64: PMULL folding //! - Power: VPMSUMD folding //! - s390x: VGFM folding -//! - riscv64: ZVBC (RVV vector CLMUL) / Zbc folding //! - wasm32/wasm64: portable only (no CLMUL) pub(crate) mod config; @@ -33,11 +32,7 @@ mod power; #[cfg(target_arch = "s390x")] mod s390x; -#[cfg(target_arch = "riscv64")] -mod riscv64; - // Re-export config types for public API (Crc64Force only used internally on SIMD archs) -#[allow(unused_imports)] pub use config::{Crc64Config, Crc64Force}; #[cfg(any(test, feature = "std"))] @@ -47,10 +42,6 @@ use crate::checksum::common::tables::generate_crc64_tables_8; use crate::checksum::common::tables::{CRC64_NVME_POLY, CRC64_XZ_POLY, generate_crc64_tables_16}; #[cfg(feature = "diag")] use crate::checksum::diag::{Crc64Polynomial, Crc64SelectionDiag}; -// Re-export traits for test module (`use super::*`). -#[allow(unused_imports)] -pub(super) use crate::traits::{Checksum, ChecksumCombine}; - // Kernel Name Introspection /// Get the name of the CRC-64/XZ kernel that would be selected for a given buffer length. @@ -206,23 +197,21 @@ pub(crate) fn diag_crc64_nvme(len: usize) -> Crc64SelectionDiag { mod kernel_tables { use super::*; #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - pub static XZ_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_XZ_POLY); + pub(super) static XZ_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_XZ_POLY); #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - pub static NVME_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_NVME_POLY); - pub static XZ_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_XZ_POLY); - pub static NVME_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_NVME_POLY); + pub(super) static NVME_TABLES_8: [[u64; 256]; 8] = generate_crc64_tables_8(CRC64_NVME_POLY); + pub(super) static XZ_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_XZ_POLY); + pub(super) static NVME_TABLES_16: [[u64; 256]; 16] = generate_crc64_tables_16(CRC64_NVME_POLY); } /// CRC-64-XZ portable kernel wrapper. -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc64_xz_portable(crc: u64, data: &[u8]) -> u64 { portable::crc64_slice16_xz(crc, data) } /// CRC-64-NVME portable kernel wrapper. -#[cfg(any(test, feature = "std"))] -#[cfg_attr(all(test, not(feature = "std")), allow(dead_code))] +#[cfg(feature = "std")] fn crc64_nvme_portable(crc: u64, data: &[u8]) -> u64 { portable::crc64_slice16_nvme(crc, data) } @@ -849,6 +838,9 @@ impl crate::traits::ChecksumCombine for Crc64Nvme { #[cfg(feature = "alloc")] impl Crc64Nvme { + /// Buffer many short updates before dispatching to the active CRC-64/NVME kernel. + /// + /// For large contiguous buffers, use [`Crc64Nvme`] directly. #[must_use] pub fn buffered() -> BufferedCrc64Nvme { BufferedCrc64Nvme::new() @@ -964,9 +956,18 @@ mod tests { use alloc::vec::Vec; use super::*; + use crate::traits::{Checksum, ChecksumCombine}; const TEST_DATA: &[u8] = b"123456789"; + fn patterned_data(len: usize, multiplier: u8) -> Vec { + (0u8..=u8::MAX) + .cycle() + .take(len) + .map(|byte| byte.wrapping_mul(multiplier)) + .collect() + } + #[test] fn test_crc64_xz_checksum() { // Standard test vector for CRC-64-XZ (ECMA-182) @@ -1087,11 +1088,7 @@ mod tests { /// regardless of whether the portable slice-by-8 or SIMD path is selected. #[test] fn test_crc64_various_lengths() { - // Generate predictable test data - let mut data = [0u8; 512]; - for (i, byte) in data.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(17).wrapping_add(i as u8); - } + let data = patterned_data(512, 18); // Test lengths around key thresholds: // - 0-15: always portable (below 16B lane minimum) @@ -1136,7 +1133,7 @@ mod tests { // Generate test data larger than threshold let size = threshold + 128; - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(31)).collect(); + let data = patterned_data(size, 31); let oneshot = Crc64::checksum(&data); @@ -1237,7 +1234,7 @@ mod tests { let len = 4096usize; // Execute both variants to ensure the selected tier doesn't trap. - let data: Vec = (0..len).map(|i| (i as u8).wrapping_mul(13)).collect(); + let data = patterned_data(len, 13); let ours_xz = Crc64::checksum(&data); let ours_nvme = Crc64Nvme::checksum(&data); @@ -1335,11 +1332,7 @@ mod tests { #[cfg(feature = "alloc")] #[test] fn test_buffered_crc64_xz_mixed_sizes() { - // Generate test data - let mut data = [0u8; 1024]; - for (i, byte) in data.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(13); - } + let data = patterned_data(1024, 13); let expected = Crc64::checksum(&data); let mut buffered = BufferedCrc64::new(); diff --git a/src/checksum/crc64/portable.rs b/src/checksum/crc64/portable.rs index 1d9bb895..d2ccccc9 100644 --- a/src/checksum/crc64/portable.rs +++ b/src/checksum/crc64/portable.rs @@ -9,30 +9,28 @@ use crate::checksum::common::portable; // Polynomial-specific wrappers /// CRC-64-XZ slice-by-8 computation. -#[cfg_attr(miri, allow(dead_code))] #[inline] #[cfg(all(test, any(target_arch = "x86_64", target_arch = "aarch64")))] -pub fn crc64_slice8_xz(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_slice8_xz(crc: u64, data: &[u8]) -> u64 { crc64_slice8(crc, data, &kernel_tables::XZ_TABLES_8) } /// CRC-64-NVME slice-by-8 computation. -#[cfg_attr(miri, allow(dead_code))] #[inline] #[cfg(all(test, any(target_arch = "x86_64", target_arch = "aarch64")))] -pub fn crc64_slice8_nvme(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_slice8_nvme(crc: u64, data: &[u8]) -> u64 { crc64_slice8(crc, data, &kernel_tables::NVME_TABLES_8) } /// CRC-64-XZ slice-by-16 computation. #[inline] -pub fn crc64_slice16_xz(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_slice16_xz(crc: u64, data: &[u8]) -> u64 { crc64_slice16(crc, data, &kernel_tables::XZ_TABLES_16) } /// CRC-64-NVME slice-by-16 computation. #[inline] -pub fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { crc64_slice16(crc, data, &kernel_tables::NVME_TABLES_16) } @@ -42,7 +40,7 @@ pub fn crc64_slice16_nvme(crc: u64, data: &[u8]) -> u64 { /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { crc64_bytewise(crc, data, &kernel_tables::XZ_TABLES_16[0]) } @@ -50,13 +48,12 @@ pub fn crc64_xz_bytewise(crc: u64, data: &[u8]) -> u64 { /// /// Uses one 256-entry table rather than the slice-by-16 table set. #[inline(always)] -pub fn crc64_nvme_bytewise(crc: u64, data: &[u8]) -> u64 { +pub(in crate::checksum) fn crc64_nvme_bytewise(crc: u64, data: &[u8]) -> u64 { crc64_bytewise(crc, data, &kernel_tables::NVME_TABLES_16[0]) } /// Update CRC-64 state using a byte-at-a-time lookup table. #[inline(always)] -#[allow(clippy::indexing_slicing)] // index is 0..=255 by mask, table is [u64; 256] fn crc64_bytewise(mut crc: u64, data: &[u8], table: &[u64; 256]) -> u64 { for &b in data { let index = ((crc ^ (b as u64)) & 0xFF) as usize; @@ -76,7 +73,7 @@ fn crc64_bytewise(mut crc: u64, data: &[u8], table: &[u64; 256]) -> u64 { /// * `tables` - 8 lookup tables (256 entries each) #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] #[inline] -pub fn crc64_slice8(crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { +pub(super) fn crc64_slice8(crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { portable::slice8_64(crc, data, tables) } @@ -88,7 +85,7 @@ pub fn crc64_slice8(crc: u64, data: &[u8], tables: &[[u64; 256]; 8]) -> u64 { /// * `data` - Input data /// * `tables` - 16 lookup tables (256 entries each) #[inline] -pub fn crc64_slice16(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { +pub(super) fn crc64_slice16(crc: u64, data: &[u8], tables: &[[u64; 256]; 16]) -> u64 { portable::slice16_64(crc, data, tables) } diff --git a/src/checksum/crc64/power.rs b/src/checksum/crc64/power.rs index 7451295b..8f51ee3a 100644 --- a/src/checksum/crc64/power.rs +++ b/src/checksum/crc64/power.rs @@ -8,10 +8,7 @@ //! Uses `unsafe` for Power SIMD + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::asm, @@ -66,6 +63,10 @@ impl Simd { /// On `powerpc64le` this is a no-op. On big-endian `powerpc64`, we byte-swap /// each 64-bit lane so the folding algorithm sees the same lane values as on /// little-endian platforms. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn to_le(self) -> Self { @@ -90,6 +91,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -113,6 +117,9 @@ impl Simd { } } + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -131,6 +138,10 @@ impl Simd { } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -145,6 +156,10 @@ impl Simd { } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -162,6 +177,10 @@ impl Simd { } /// Barrett reduction to finalize the CRC. + /// + /// # Safety + /// + /// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -226,6 +245,9 @@ const NVME_COMBINE_8WAY: [(u64, u64); 7] = [ // Folding helpers +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline(always)] unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: POWER8 vector crypto intrinsics/asm are available via this function's #[target_feature] @@ -253,6 +275,9 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[inline] #[target_feature( enable = "altivec", @@ -275,6 +300,9 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -308,6 +336,9 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -363,7 +394,7 @@ unsafe fn update_simd_2way( let mut i = 2; while i < even { fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); i = i.strict_add(2); } @@ -387,6 +418,9 @@ unsafe fn update_simd_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -412,7 +446,7 @@ unsafe fn update_simd_4way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = blocks.len() & !3_usize; let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -466,9 +500,9 @@ unsafe fn update_simd_4way( let mut i = 4; while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); i = i.strict_add(4); } @@ -509,6 +543,9 @@ unsafe fn update_simd_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -534,7 +571,7 @@ unsafe fn update_simd_8way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = blocks.len() & !7_usize; let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -633,13 +670,13 @@ unsafe fn update_simd_8way( let mut i = 8; while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_1024); - fold_block_128(&mut s1, &blocks[i + 1], coeff_1024); - fold_block_128(&mut s2, &blocks[i + 2], coeff_1024); - fold_block_128(&mut s3, &blocks[i + 3], coeff_1024); - fold_block_128(&mut s4, &blocks[i + 4], coeff_1024); - fold_block_128(&mut s5, &blocks[i + 5], coeff_1024); - fold_block_128(&mut s6, &blocks[i + 6], coeff_1024); - fold_block_128(&mut s7, &blocks[i + 7], coeff_1024); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_1024); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_1024); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_1024); + fold_block_128(&mut s4, &blocks[i.strict_add(4)], coeff_1024); + fold_block_128(&mut s5, &blocks[i.strict_add(5)], coeff_1024); + fold_block_128(&mut s6, &blocks[i.strict_add(6)], coeff_1024); + fold_block_128(&mut s7, &blocks[i.strict_add(7)], coeff_1024); i = i.strict_add(8); } @@ -718,6 +755,9 @@ unsafe fn update_simd_8way( // Public kernels (XZ + NVME) +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -739,6 +779,9 @@ unsafe fn crc64_vpmsum(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstant } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -772,6 +815,9 @@ unsafe fn crc64_vpmsum_2way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -800,6 +846,9 @@ unsafe fn crc64_vpmsum_4way( } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, POWER8 vector, and POWER8 crypto instructions are available. #[target_feature( enable = "altivec", enable = "vsx", @@ -1041,49 +1090,49 @@ pub(crate) unsafe fn crc64_nvme_vpmsum_8way(crc: u64, data: &[u8]) -> u64 { // Safe wrappers #[inline] -pub fn crc64_xz_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum(crc, data) } } #[inline] -pub fn crc64_xz_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum_2way(crc, data) } } #[inline] -pub fn crc64_xz_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum_4way(crc, data) } } #[inline] -pub fn crc64_xz_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_xz_vpmsum_8way(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_2way(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_4way(crc, data) } } #[inline] -pub fn crc64_nvme_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpmsum_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPMSUM-ready CPU features before selecting this kernel. unsafe { crc64_nvme_vpmsum_8way(crc, data) } } diff --git a/src/checksum/crc64/riscv64.rs b/src/checksum/crc64/riscv64.rs deleted file mode 100644 index 011ad1d5..00000000 --- a/src/checksum/crc64/riscv64.rs +++ /dev/null @@ -1,1547 +0,0 @@ -//! riscv64 hardware-accelerated CRC-64 kernels (XZ + NVME). -//! -//! This is a scalar Zbc (`clmul`/`clmulh`) implementation of the Intel/TiKV -//! folding algorithm for CRC-64/XZ and CRC-64/NVME. -//! -//! # Safety -//! -//! Uses `unsafe` for RISC-V inline assembly. Callers must ensure the required -//! CPU features are available before executing the accelerated path (the -//! dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] - -use core::{ - arch::asm, - ops::{BitXor, BitXorAssign}, -}; - -use crate::checksum::common::{ - clmul::{Crc64ClmulConstants, fold16_coeff_for_bytes}, - tables::{CRC64_NVME_POLY, CRC64_XZ_POLY}, -}; - -type Block = [u64; 16]; // 128 bytes (8×16B lanes) - -#[derive(Copy, Clone, Debug)] -struct Simd { - hi: u64, - lo: u64, -} - -impl BitXor for Simd { - type Output = Self; - - #[inline] - fn bitxor(self, other: Self) -> Self { - Self { - hi: self.hi ^ other.hi, - lo: self.lo ^ other.lo, - } - } -} - -impl BitXorAssign for Simd { - #[inline] - fn bitxor_assign(&mut self, other: Self) { - self.hi ^= other.hi; - self.lo ^= other.lo; - } -} - -impl Simd { - #[inline] - const fn new(high: u64, low: u64) -> Self { - Self { hi: high, lo: low } - } - - #[inline] - const fn low_64(self) -> u64 { - self.lo - } - - #[inline] - const fn high_64(self) -> u64 { - self.hi - } - - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn clmul_lo(a: u64, b: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let out: u64; - asm!( - "clmul {out}, {a}, {b}", - out = lateout(reg) out, - a = in(reg) a, - b = in(reg) b, - options(nomem, nostack, pure) - ); - out - } - } - - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn clmul_hi(a: u64, b: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let out: u64; - asm!( - "clmulh {out}, {a}, {b}", - out = lateout(reg) out, - a = in(reg) a, - b = in(reg) b, - options(nomem, nostack, pure) - ); - out - } - } - - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - Self { - hi: Self::clmul_hi(a, b), - lo: Self::clmul_lo(a, b), - } - } - } - - /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn fold_16(self, coeff: (u64, u64)) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - Self::mul64(self.low_64(), coeff_low) ^ Self::mul64(self.high_64(), coeff_high) - } - } - - /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn fold_8(self, coeff: u64) -> Self { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let prod = Self::mul64(self.low_64(), coeff); - prod ^ Self::new(0, self.high_64()) - } - } - - /// Barrett reduction to finalize the CRC. - #[inline] - #[target_feature(enable = "zbc")] - unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = Self::clmul_lo(self.low_64(), mu); - let l = Self::mul64(t1, poly); - (self ^ l).high_64() ^ t1 - } - } -} - -// Multi-stream coefficients (compile-time) - -// 2-way: update step shifts by 2×128B = 256B. -const XZ_FOLD_256B: (u64, u64) = fold16_coeff_for_bytes(CRC64_XZ_POLY, 256); -const NVME_FOLD_256B: (u64, u64) = fold16_coeff_for_bytes(CRC64_NVME_POLY, 256); - -// 4-way: update step shifts by 4×128B = 512B, combine shifts by 3/2/1 blocks. -const XZ_FOLD_512B: (u64, u64) = fold16_coeff_for_bytes(CRC64_XZ_POLY, 512); -const NVME_FOLD_512B: (u64, u64) = fold16_coeff_for_bytes(CRC64_NVME_POLY, 512); -const XZ_COMBINE_4WAY: [(u64, u64); 3] = [ - fold16_coeff_for_bytes(CRC64_XZ_POLY, 384), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 256), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 128), -]; -const NVME_COMBINE_4WAY: [(u64, u64); 3] = [ - fold16_coeff_for_bytes(CRC64_NVME_POLY, 384), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 256), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 128), -]; - -// 8-way: update step shifts by 8×128B = 1024B, combine shifts by 7/6/5/4/3/2/1 blocks. -const XZ_FOLD_1024B: (u64, u64) = fold16_coeff_for_bytes(CRC64_XZ_POLY, 1024); -const NVME_FOLD_1024B: (u64, u64) = fold16_coeff_for_bytes(CRC64_NVME_POLY, 1024); -const XZ_COMBINE_8WAY: [(u64, u64); 7] = [ - fold16_coeff_for_bytes(CRC64_XZ_POLY, 896), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 768), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 640), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 512), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 384), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 256), - fold16_coeff_for_bytes(CRC64_XZ_POLY, 128), -]; -const NVME_COMBINE_8WAY: [(u64, u64); 7] = [ - fold16_coeff_for_bytes(CRC64_NVME_POLY, 896), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 768), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 640), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 512), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 384), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 256), - fold16_coeff_for_bytes(CRC64_NVME_POLY, 128), -]; - -// Load helpers - -#[inline(always)] -fn load_block(block: &Block) -> [Simd; 8] { - let mut out = [Simd::new(0, 0); 8]; - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - out[i] = Simd::new(high, low); - i = i.strict_add(1); - } - - out -} - -// Folding helpers - -#[inline] -#[target_feature(enable = "zbc")] -unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = x[7]; - acc ^= x[0].fold_16(consts.tail_fold_16b[0]); - acc ^= x[1].fold_16(consts.tail_fold_16b[1]); - acc ^= x[2].fold_16(consts.tail_fold_16b[2]); - acc ^= x[3].fold_16(consts.tail_fold_16b[3]); - acc ^= x[4].fold_16(consts.tail_fold_16b[4]); - acc ^= x[5].fold_16(consts.tail_fold_16b[5]); - acc ^= x[6].fold_16(consts.tail_fold_16b[6]); - - acc.fold_8(consts.fold_8b).barrett(consts.poly, consts.mu) - } -} - -#[inline] -#[target_feature(enable = "zbc")] -unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: (u64, u64)) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - x[0] = chunk[0] ^ x[0].fold_16(coeff); - x[1] = chunk[1] ^ x[1].fold_16(coeff); - x[2] = chunk[2] ^ x[2].fold_16(coeff); - x[3] = chunk[3] ^ x[3].fold_16(coeff); - x[4] = chunk[4] ^ x[4].fold_16(coeff); - x[5] = chunk[5] ^ x[5].fold_16(coeff); - x[6] = chunk[6] ^ x[6].fold_16(coeff); - x[7] = chunk[7] ^ x[7].fold_16(coeff); - } -} - -#[inline] -#[target_feature(enable = "zbc")] -unsafe fn xor_fold_stream(combined: &mut [Simd; 8], stream: &[Simd; 8], coeff: (u64, u64)) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut lane = 0usize; - while lane < 8 { - combined[lane] ^= stream[lane].fold_16(coeff); - lane = lane.strict_add(1); - } - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut x = load_block(first); - - // XOR the initial CRC into the first lane. - x[0] ^= Simd::new(0, state); - - let coeff = consts.fold_128b; - for block in rest { - let chunk = load_block(block); - fold_block_128(&mut x, &chunk, coeff); - } - - fold_tail(x, consts) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd_2way(state: u64, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; - - let coeff_256 = fold_256b; - let coeff_128 = consts.fold_128b; - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); - - let mut i = 2; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128(&mut s0, &b0, coeff_256); - fold_block_128(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } - - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). - let mut combined = s1; - combined[0] ^= s0[0].fold_16(coeff_128); - combined[1] ^= s0[1].fold_16(coeff_128); - combined[2] ^= s0[2].fold_16(coeff_128); - combined[3] ^= s0[3].fold_16(coeff_128); - combined[4] ^= s0[4].fold_16(coeff_128); - combined[5] ^= s0[5].fold_16(coeff_128); - combined[6] ^= s0[6].fold_16(coeff_128); - combined[7] ^= s0[7].fold_16(coeff_128); - - // Handle any remaining block (odd tail) sequentially. - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128(&mut combined, &tail, coeff_128); - } - - fold_tail(combined, consts) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd_4way( - state: u64, - blocks: &[Block], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = fold_512b; - let coeff_128 = consts.fold_128b; - - let c384 = combine[0]; - let c256 = combine[1]; - let c128 = combine[2]; - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); - - let mut i = 4; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128(&mut s0, &b0, coeff_512); - fold_block_128(&mut s1, &b1, coeff_512); - fold_block_128(&mut s2, &b2, coeff_512); - fold_block_128(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut combined = s3; - combined[0] ^= s2[0].fold_16(c128); - combined[1] ^= s2[1].fold_16(c128); - combined[2] ^= s2[2].fold_16(c128); - combined[3] ^= s2[3].fold_16(c128); - combined[4] ^= s2[4].fold_16(c128); - combined[5] ^= s2[5].fold_16(c128); - combined[6] ^= s2[6].fold_16(c128); - combined[7] ^= s2[7].fold_16(c128); - - combined[0] ^= s1[0].fold_16(c256); - combined[1] ^= s1[1].fold_16(c256); - combined[2] ^= s1[2].fold_16(c256); - combined[3] ^= s1[3].fold_16(c256); - combined[4] ^= s1[4].fold_16(c256); - combined[5] ^= s1[5].fold_16(c256); - combined[6] ^= s1[6].fold_16(c256); - combined[7] ^= s1[7].fold_16(c256); - - combined[0] ^= s0[0].fold_16(c384); - combined[1] ^= s0[1].fold_16(c384); - combined[2] ^= s0[2].fold_16(c384); - combined[3] ^= s0[3].fold_16(c384); - combined[4] ^= s0[4].fold_16(c384); - combined[5] ^= s0[5].fold_16(c384); - combined[6] ^= s0[6].fold_16(c384); - combined[7] ^= s0[7].fold_16(c384); - - for block in &blocks[aligned..] { - let b = load_block(block); - fold_block_128(&mut combined, &b, coeff_128); - } - - fold_tail(combined, consts) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn update_simd_8way( - state: u64, - blocks: &[Block], - fold_1024b: (u64, u64), - combine: &[(u64, u64); 7], - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 8 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } - - let aligned = blocks.len().strict_div(8).strict_mul(8); - let coeff_1024 = fold_1024b; - let coeff_128 = consts.fold_128b; - - let mut streams = [ - load_block(&blocks[0]), - load_block(&blocks[1]), - load_block(&blocks[2]), - load_block(&blocks[3]), - load_block(&blocks[4]), - load_block(&blocks[5]), - load_block(&blocks[6]), - load_block(&blocks[7]), - ]; - - // Inject CRC into stream 0. - streams[0][0] ^= Simd::new(0, state); - - let mut i = 8usize; - while i < aligned { - let chunks = [ - load_block(&blocks[i]), - load_block(&blocks[i.strict_add(1)]), - load_block(&blocks[i.strict_add(2)]), - load_block(&blocks[i.strict_add(3)]), - load_block(&blocks[i.strict_add(4)]), - load_block(&blocks[i.strict_add(5)]), - load_block(&blocks[i.strict_add(6)]), - load_block(&blocks[i.strict_add(7)]), - ]; - - let mut stream = 0usize; - while stream < 8 { - fold_block_128(&mut streams[stream], &chunks[stream], coeff_1024); - stream = stream.strict_add(1); - } - - i = i.strict_add(8); - } - - // Merge: A^7·s0 ⊕ A^6·s1 ⊕ A^5·s2 ⊕ A^4·s3 ⊕ A^3·s4 ⊕ A^2·s5 ⊕ A·s6 ⊕ s7. - let mut combined = streams[7]; - let mut stream = 0usize; - while stream < 7 { - xor_fold_stream(&mut combined, &streams[stream], combine[stream]); - stream = stream.strict_add(1); - } - - for block in &blocks[aligned..] { - let chunk = load_block(block); - fold_block_128(&mut combined, &chunk, coeff_128); - } - - fold_tail(combined, consts) - } -} - -// ZVBC (vector carryless multiply) backend - -#[inline] -fn load_block_split(block: &Block) -> ([u64; 8], [u64; 8]) { - let mut hi = [0u64; 8]; - let mut lo = [0u64; 8]; - - let mut i = 0; - while i < 8 { - lo[i] = u64::from_le(block[i * 2]); - hi[i] = u64::from_le(block[i * 2 + 1]); - i = i.strict_add(1); - } - - (hi, lo) -} - -/// Carryless multiply of two `u64` values using ZVBC (returns 128-bit result as `{hi, lo}`). -/// -/// # Safety -/// -/// Requires RISC-V `v` + `zvbc`. -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn mul64_zvbc(a: u64, b: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let lo: u64; - let hi: u64; - asm!( - "vsetivli zero, 1, e64, m1, ta, ma", - "vmv.v.x v0, {a}", - "vclmul.vx v1, v0, {b}", - "vclmulh.vx v2, v0, {b}", - "vmv.x.s {lo}, v1", - "vmv.x.s {hi}, v2", - a = in(reg) a, - b = in(reg) b, - lo = lateout(reg) lo, - hi = lateout(reg) hi, - out("v0") _, - out("v1") _, - out("v2") _, - options(nostack) - ); - Simd::new(hi, lo) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_16_zvbc(x: Simd, coeff: (u64, u64)) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (coeff_high, coeff_low) = coeff; - mul64_zvbc(x.low_64(), coeff_low) ^ mul64_zvbc(x.high_64(), coeff_high) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_8_zvbc(x: Simd, coeff: u64) -> Simd { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let prod = mul64_zvbc(x.low_64(), coeff); - Simd::new(prod.high_64(), prod.low_64() ^ x.high_64()) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn barrett_zvbc(x: Simd, poly: u64, mu: u64) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let t1 = mul64_zvbc(x.low_64(), mu).low_64(); - let l = mul64_zvbc(t1, poly); - (x ^ l).high_64() ^ t1 - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_tail_zvbc(hi: [u64; 8], lo: [u64; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut acc = Simd::new(hi[7], lo[7]); - acc ^= fold_16_zvbc(Simd::new(hi[0], lo[0]), consts.tail_fold_16b[0]); - acc ^= fold_16_zvbc(Simd::new(hi[1], lo[1]), consts.tail_fold_16b[1]); - acc ^= fold_16_zvbc(Simd::new(hi[2], lo[2]), consts.tail_fold_16b[2]); - acc ^= fold_16_zvbc(Simd::new(hi[3], lo[3]), consts.tail_fold_16b[3]); - acc ^= fold_16_zvbc(Simd::new(hi[4], lo[4]), consts.tail_fold_16b[4]); - acc ^= fold_16_zvbc(Simd::new(hi[5], lo[5]), consts.tail_fold_16b[5]); - acc ^= fold_16_zvbc(Simd::new(hi[6], lo[6]), consts.tail_fold_16b[6]); - barrett_zvbc(fold_8_zvbc(acc, consts.fold_8b), consts.poly, consts.mu) - } -} - -#[inline] -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn fold_block_128_zvbc( - x_hi: &mut [u64; 8], - x_lo: &mut [u64; 8], - chunk_hi: &[u64; 8], - chunk_lo: &[u64; 8], - coeff_low: u64, - coeff_high: u64, -) { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let mut offset = 0usize; - while offset < 8 { - let remaining = 8 - offset; - let vl: usize; - asm!( - "vsetvli {vl}, {avl}, e64, m1, ta, ma", - "vle64.v v0, ({xlo})", - "vle64.v v1, ({xhi})", - "vclmul.vx v2, v0, {clo}", - "vclmulh.vx v3, v0, {clo}", - "vclmul.vx v4, v1, {chi}", - "vclmulh.vx v5, v1, {chi}", - "vxor.vv v2, v2, v4", - "vxor.vv v3, v3, v5", - "vle64.v v4, ({dlo})", - "vle64.v v5, ({dhi})", - "vxor.vv v2, v2, v4", - "vxor.vv v3, v3, v5", - "vse64.v v2, ({xlo})", - "vse64.v v3, ({xhi})", - vl = lateout(reg) vl, - avl = in(reg) remaining, - xlo = in(reg) x_lo.as_mut_ptr().add(offset), - xhi = in(reg) x_hi.as_mut_ptr().add(offset), - dlo = in(reg) chunk_lo.as_ptr().add(offset), - dhi = in(reg) chunk_hi.as_ptr().add(offset), - clo = in(reg) coeff_low, - chi = in(reg) coeff_high, - out("v0") _, - out("v1") _, - out("v2") _, - out("v3") _, - out("v4") _, - out("v5") _, - options(nostack) - ); - offset = offset.strict_add(vl); - } - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (mut x_hi, mut x_lo) = load_block_split(first); - - // XOR the initial CRC into the first lane. - x_lo[0] ^= state; - - let coeff_low = consts.fold_128b.1; - let coeff_high = consts.fold_128b.0; - - for block in rest { - let (chunk_hi, chunk_lo) = load_block_split(block); - fold_block_128_zvbc(&mut x_hi, &mut x_lo, &chunk_hi, &chunk_lo, coeff_low, coeff_high); - } - - fold_tail_zvbc(x_hi, x_lo, consts) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_2way( - state: u64, - blocks: &[Block], - fold_256b: (u64, u64), - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } - - let even = blocks.len() & !1usize; - - let coeff_256_low = fold_256b.1; - let coeff_256_high = fold_256b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state; - - let mut i = 2; - while i < even { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_256_low, coeff_256_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_256_low, coeff_256_high); - i = i.strict_add(2); - } - - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &s1_hi, &s1_lo, coeff_128_low, coeff_128_high); - - // Handle any remaining block (odd tail) sequentially. - if even != blocks.len() { - let (tail_hi, tail_lo) = load_block_split(&blocks[even]); - fold_block_128_zvbc( - &mut s0_hi, - &mut s0_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } - - fold_tail_zvbc(s0_hi, s0_lo, consts) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn update_simd_zvbc_4way( - state: u64, - blocks: &[Block], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - debug_assert!(!blocks.is_empty()); - - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd_zvbc(state, first, rest, consts); - } - - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512_low = fold_512b.1; - let coeff_512_high = fold_512b.0; - let coeff_128_low = consts.fold_128b.1; - let coeff_128_high = consts.fold_128b.0; - - let c384_low = combine[0].1; - let c384_high = combine[0].0; - let c256_low = combine[1].1; - let c256_high = combine[1].0; - let c128_low = combine[2].1; - let c128_high = combine[2].0; - - let (mut s0_hi, mut s0_lo) = load_block_split(&blocks[0]); - let (mut s1_hi, mut s1_lo) = load_block_split(&blocks[1]); - let (mut s2_hi, mut s2_lo) = load_block_split(&blocks[2]); - let (mut s3_hi, mut s3_lo) = load_block_split(&blocks[3]); - - // Inject CRC into stream 0. - s0_lo[0] ^= state; - - let mut i = 4; - while i < aligned { - let (b0_hi, b0_lo) = load_block_split(&blocks[i]); - let (b1_hi, b1_lo) = load_block_split(&blocks[i.strict_add(1)]); - let (b2_hi, b2_lo) = load_block_split(&blocks[i.strict_add(2)]); - let (b3_hi, b3_lo) = load_block_split(&blocks[i.strict_add(3)]); - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &b0_hi, &b0_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &b1_hi, &b1_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &b2_hi, &b2_lo, coeff_512_low, coeff_512_high); - fold_block_128_zvbc(&mut s3_hi, &mut s3_lo, &b3_hi, &b3_lo, coeff_512_low, coeff_512_high); - i = i.strict_add(4); - } - - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - // - // `fold_block_128_zvbc(x, chunk, c)` computes: `x = chunk ⊕ fold(x, c)`. - // We want: `combined = combined ⊕ fold(stream, c)`, so we fold each stream - // into the current combined value by using `chunk = combined` and storing the - // result back into that stream (which is no longer needed after this point). - let mut combined_hi = s3_hi; - let mut combined_lo = s3_lo; - - fold_block_128_zvbc(&mut s2_hi, &mut s2_lo, &combined_hi, &combined_lo, c128_low, c128_high); - combined_hi = s2_hi; - combined_lo = s2_lo; - - fold_block_128_zvbc(&mut s1_hi, &mut s1_lo, &combined_hi, &combined_lo, c256_low, c256_high); - combined_hi = s1_hi; - combined_lo = s1_lo; - - fold_block_128_zvbc(&mut s0_hi, &mut s0_lo, &combined_hi, &combined_lo, c384_low, c384_high); - combined_hi = s0_hi; - combined_lo = s0_lo; - - for block in &blocks[aligned..] { - let (tail_hi, tail_lo) = load_block_split(block); - fold_block_128_zvbc( - &mut combined_hi, - &mut combined_lo, - &tail_hi, - &tail_lo, - coeff_128_low, - coeff_128_high, - ); - } - - fold_tail_zvbc(combined_hi, combined_lo, consts) - } -} - -// Public kernels (XZ + NVME) - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - if let Some((first, rest)) = blocks.split_first() { - state = update_simd(state, first, rest, consts); - } - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc64_zvbc(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - if let Some((first, rest)) = blocks.split_first() { - state = update_simd_zvbc(state, first, rest, consts); - } - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc_2way( - mut state: u64, - bytes: &[u8], - fold_256b: (u64, u64), - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_2way(state, blocks, fold_256b, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc64_zvbc_2way( - mut state: u64, - bytes: &[u8], - fold_256b: (u64, u64), - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_zvbc_2way(state, blocks, fold_256b, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc_4way( - mut state: u64, - bytes: &[u8], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_4way(state, blocks, fold_512b, combine, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "zbc")] -unsafe fn crc64_zbc_8way( - mut state: u64, - bytes: &[u8], - fold_1024b: (u64, u64), - combine: &[(u64, u64); 7], - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_8way(state, blocks, fold_1024b, combine, consts); - } - - if !tail_u64.is_empty() { - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -#[target_feature(enable = "v", enable = "zvbc")] -unsafe fn crc64_zvbc_4way( - mut state: u64, - bytes: &[u8], - fold_512b: (u64, u64), - combine: &[(u64, u64); 3], - consts: &Crc64ClmulConstants, - tables: &[[u64; 256]; 16], -) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - let (left, middle, right) = bytes.align_to::(); - - state = super::portable::crc64_slice16(state, left, tables); - - let block_u64s = middle.len() & !15usize; - let (blocks_u64, tail_u64) = middle.split_at(block_u64s); - - if !blocks_u64.is_empty() { - // SAFETY: `blocks_u64` length is a multiple of 16, so casting to `[u64; 16]` is safe. - let blocks: &[Block] = core::slice::from_raw_parts(blocks_u64.as_ptr().cast(), blocks_u64.len().strict_div(16)); - state = update_simd_zvbc_4way(state, blocks, fold_512b, combine, consts); - } - - if !tail_u64.is_empty() { - // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); - state = super::portable::crc64_slice16(state, tail_bytes, tables); - } - - super::portable::crc64_slice16(state, right, tables) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding. -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using RVV Zvbc (vector carryless multiply) folding. -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_xz_zvbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_2way( - crc, - data, - XZ_FOLD_256B, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using RVV Zvbc (vector carryless multiply) folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_xz_zvbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_2way( - crc, - data, - XZ_FOLD_256B, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_4way( - crc, - data, - XZ_FOLD_512B, - &XZ_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using scalar Zbc carryless multiply folding (8-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_xz_zbc_8way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_8way( - crc, - data, - XZ_FOLD_1024B, - &XZ_COMBINE_8WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-XZ using RVV Zvbc (vector carryless multiply) folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_xz_zvbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_4way( - crc, - data, - XZ_FOLD_512B, - &XZ_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding. -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using RVV Zvbc (vector carryless multiply) folding. -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_nvme_zvbc(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc( - crc, - data, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_2way( - crc, - data, - NVME_FOLD_256B, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using RVV Zvbc (vector carryless multiply) folding (2-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_nvme_zvbc_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_2way( - crc, - data, - NVME_FOLD_256B, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_4way( - crc, - data, - NVME_FOLD_512B, - &NVME_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using scalar Zbc carryless multiply folding (8-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V Zbc extension. Caller must verify via -/// `crate::platform::caps().has(riscv::ZBC)`. -#[target_feature(enable = "zbc")] -pub(crate) unsafe fn crc64_nvme_zbc_8way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zbc_8way( - crc, - data, - NVME_FOLD_1024B, - &NVME_COMBINE_8WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -/// CRC-64-NVME using RVV Zvbc (vector carryless multiply) folding (4-way ILP variant). -/// -/// # Safety -/// -/// Requires the RISC-V vector extension and Zvbc. Caller must verify via -/// `crate::platform::caps().has(riscv::ZVBC)`. -#[target_feature(enable = "v", enable = "zvbc")] -pub(crate) unsafe fn crc64_nvme_zvbc_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { - crc64_zvbc_4way( - crc, - data, - NVME_FOLD_512B, - &NVME_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } -} - -// Safe wrappers - -#[inline] -pub fn crc64_xz_zbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc(crc, data) } -} - -#[inline] -pub fn crc64_xz_zvbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zvbc(crc, data) } -} - -#[inline] -pub fn crc64_xz_zbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zvbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zvbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc_4way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zbc_8way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zbc_8way(crc, data) } -} - -#[inline] -pub fn crc64_xz_zvbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_xz_zvbc_4way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zvbc_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zvbc(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zvbc_2way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zvbc_2way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc_4way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zbc_8way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zbc_8way(crc, data) } -} - -#[inline] -pub fn crc64_nvme_zvbc_4way_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Dispatch verifies the required RISC-V feature bits; local fixed-size - // buffers and register-only operands are established before this block. - unsafe { crc64_nvme_zvbc_4way(crc, data) } -} - -#[cfg(test)] -mod tests { - extern crate std; - - use alloc::vec::Vec; - - use super::*; - - const LENS: &[usize] = &[0, 1, 7, 15, 16, 31, 63, 64, 127, 128, 255, 256, 1023, 1024, 4096]; - const OFFSETS: &[usize] = &[0, 1, 7, 15]; - const STATES: &[u64] = &[0, 0x0123_4567_89ab_cdef, 0xa5a5_5a5a_dead_beef, u64::MAX]; - - fn assert_kernel(name: &str, kernel: fn(u64, &[u8]) -> u64, portable: fn(u64, &[u8]) -> u64) { - let input: Vec = (0..4111) - .map(|i| (i as u8).wrapping_mul(41).wrapping_add((i >> 8) as u8)) - .collect(); - for &state in STATES { - for &offset in OFFSETS { - for &len in LENS { - let slice = &input[offset..offset + len]; - assert_eq!( - kernel(state, slice), - portable(state, slice), - "{name} state={state:#018x} offset={offset} len={len}" - ); - } - } - } - } - - #[test] - fn zbc_kernels_match_portable() { - if !crate::platform::caps().has(crate::platform::caps::riscv::ZBC) { - return; - } - - for (name, kernel) in [ - ("xz/zbc", crc64_xz_zbc_safe as fn(u64, &[u8]) -> u64), - ("xz/zbc-2way", crc64_xz_zbc_2way_safe), - ("xz/zbc-4way", crc64_xz_zbc_4way_safe), - ("xz/zbc-8way", crc64_xz_zbc_8way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); - } - - for (name, kernel) in [ - ("nvme/zbc", crc64_nvme_zbc_safe as fn(u64, &[u8]) -> u64), - ("nvme/zbc-2way", crc64_nvme_zbc_2way_safe), - ("nvme/zbc-4way", crc64_nvme_zbc_4way_safe), - ("nvme/zbc-8way", crc64_nvme_zbc_8way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); - } - } - - #[test] - fn zvbc_kernels_match_portable() { - use crate::platform::caps::riscv; - - if !crate::platform::caps().has(riscv::V.union(riscv::ZVBC)) { - return; - } - - for (name, kernel) in [ - ("xz/zvbc", crc64_xz_zvbc_safe as fn(u64, &[u8]) -> u64), - ("xz/zvbc-2way", crc64_xz_zvbc_2way_safe), - ("xz/zvbc-4way", crc64_xz_zvbc_4way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_xz); - } - - for (name, kernel) in [ - ("nvme/zvbc", crc64_nvme_zvbc_safe as fn(u64, &[u8]) -> u64), - ("nvme/zvbc-2way", crc64_nvme_zvbc_2way_safe), - ("nvme/zvbc-4way", crc64_nvme_zvbc_4way_safe), - ] { - assert_kernel(name, kernel, super::super::portable::crc64_slice16_nvme); - } - } -} diff --git a/src/checksum/crc64/s390x.rs b/src/checksum/crc64/s390x.rs index c806e28d..29f517c4 100644 --- a/src/checksum/crc64/s390x.rs +++ b/src/checksum/crc64/s390x.rs @@ -8,15 +8,9 @@ //! Uses `unsafe` for s390x vector + inline assembly. Callers must ensure the //! required CPU features are available before executing the accelerated path //! (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher -// SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] -// This module is intrinsics-heavy; unsafe blocks are per-function with SAFETY justifications. use core::{ arch::asm, - mem::MaybeUninit, ops::{BitXor, BitXorAssign}, simd::i64x2, }; @@ -66,8 +60,11 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn vgfm(a: i64x2, b: i64x2) -> i64x2 { + fn vgfm(a: i64x2, b: i64x2) -> i64x2 { // SAFETY: Caller guarantees the s390x vector facility is available // (verified by dispatch). The VGFM instruction operates on pure register // values with no memory access. @@ -85,49 +82,47 @@ impl Simd { } #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn mul64(a: u64, b: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values. - unsafe { - let va = Self::new(0, a); - let vb = Self::new(0, b); - Self(Self::vgfm(va.0, vb.0)) - } + fn mul64(a: u64, b: u64) -> Self { + let va = Self::new(0, a); + let vb = Self::new(0, b); + Self(Self::vgfm(va.0, vb.0)) } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // vgfm operates on pure register values. - unsafe { Self(Self::vgfm(self.0, coeff.0)) } + fn fold_16(self, coeff: Self) -> Self { + Self(Self::vgfm(self.0, coeff.0)) } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn fold_8(self, coeff: u64) -> Self { - // SAFETY: Caller guarantees the s390x vector facility is available. - // mul64 and XOR are pure register computations. - unsafe { - let prod = Self::mul64(self.low_64(), coeff); - prod ^ Self::new(0, self.high_64()) - } + fn fold_8(self, coeff: u64) -> Self { + let prod = Self::mul64(self.low_64(), coeff); + prod ^ Self::new(0, self.high_64()) } /// Barrett reduction to finalize the CRC. #[inline] + /// # Safety + /// + /// Requires the s390x vector facility. #[target_feature(enable = "vector")] - unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available. - // mul64 and XOR are pure register computations. - unsafe { - let t1 = Self::mul64(self.low_64(), mu).low_64(); - let l = Self::mul64(t1, poly); - (self ^ l).high_64() ^ t1 - } + fn barrett(self, poly: u64, mu: u64) -> u64 { + let t1 = Self::mul64(self.low_64(), mu).low_64(); + let l = Self::mul64(t1, poly); + (self ^ l).high_64() ^ t1 } } @@ -155,153 +150,136 @@ const NVME_COMBINE_4WAY: [(u64, u64); 3] = [ #[inline(always)] fn load_block(block: &Block) -> [Simd; 8] { - let mut out = MaybeUninit::<[Simd; 8]>::uninit(); - let base = out.as_mut_ptr().cast::(); - - let mut i = 0; - while i < 8 { - let low = u64::from_le(block[i * 2]); - let high = u64::from_le(block[i * 2 + 1]); - // SAFETY: `base` points to a `[Simd; 8]` buffer and `i` is in-bounds. - unsafe { - base.add(i).write(Simd::new(high, low)); - } - i = i.strict_add(1); + let mut out = [Simd::new(0, 0); 8]; + for (lane, &[low, high]) in out.iter_mut().zip(block.as_chunks::<2>().0) { + *lane = Simd::new(u64::from_le(high), u64::from_le(low)); } - - // SAFETY: all 8 elements are initialized above. - unsafe { out.assume_init() } + out } // Folding helpers -#[inline(always)] -unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 / fold_8 / barrett calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - // Tail reduction (8×16B → 1×16B), unrolled for throughput. - let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); - let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); - let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); - let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); - let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); - let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); - let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); - - let mut acc = x[7]; - acc ^= x[0].fold_16(c0); - acc ^= x[1].fold_16(c1); - acc ^= x[2].fold_16(c2); - acc ^= x[3].fold_16(c3); - acc ^= x[4].fold_16(c4); - acc ^= x[5].fold_16(c5); - acc ^= x[6].fold_16(c6); - - acc.fold_8(consts.fold_8b).barrett(consts.poly, consts.mu) - } +#[inline] +/// # Safety +/// +/// Requires the s390x vector facility. +#[target_feature(enable = "vector")] +fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { + let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); + let c1 = Simd::new(consts.tail_fold_16b[1].0, consts.tail_fold_16b[1].1); + let c2 = Simd::new(consts.tail_fold_16b[2].0, consts.tail_fold_16b[2].1); + let c3 = Simd::new(consts.tail_fold_16b[3].0, consts.tail_fold_16b[3].1); + let c4 = Simd::new(consts.tail_fold_16b[4].0, consts.tail_fold_16b[4].1); + let c5 = Simd::new(consts.tail_fold_16b[5].0, consts.tail_fold_16b[5].1); + let c6 = Simd::new(consts.tail_fold_16b[6].0, consts.tail_fold_16b[6].1); + + let mut acc = x[7]; + acc ^= x[0].fold_16(c0); + acc ^= x[1].fold_16(c1); + acc ^= x[2].fold_16(c2); + acc ^= x[3].fold_16(c3); + acc ^= x[4].fold_16(c4); + acc ^= x[5].fold_16(c5); + acc ^= x[6].fold_16(c6); + + acc.fold_8(consts.fold_8b).barrett(consts.poly, consts.mu) } #[inline] +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold_16 calls are pure VGFM register computations. - // Array indexing is over fixed-size [Simd; 8] with constant indices 0..7. - unsafe { - x[0] = chunk[0] ^ x[0].fold_16(coeff); - x[1] = chunk[1] ^ x[1].fold_16(coeff); - x[2] = chunk[2] ^ x[2].fold_16(coeff); - x[3] = chunk[3] ^ x[3].fold_16(coeff); - x[4] = chunk[4] ^ x[4].fold_16(coeff); - x[5] = chunk[5] ^ x[5].fold_16(coeff); - x[6] = chunk[6] ^ x[6].fold_16(coeff); - x[7] = chunk[7] ^ x[7].fold_16(coeff); - } +fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { + x[0] = chunk[0] ^ x[0].fold_16(coeff); + x[1] = chunk[1] ^ x[1].fold_16(coeff); + x[2] = chunk[2] ^ x[2].fold_16(coeff); + x[3] = chunk[3] ^ x[3].fold_16(coeff); + x[4] = chunk[4] ^ x[4].fold_16(coeff); + x[5] = chunk[5] ^ x[5].fold_16(coeff); + x[6] = chunk[6] ^ x[6].fold_16(coeff); + x[7] = chunk[7] ^ x[7].fold_16(coeff); } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Block iteration is bounded by the slice length; no out-of-bounds access. - unsafe { - let mut x = load_block(first); +fn update_simd(state: u64, first: &Block, rest: &[Block], consts: &Crc64ClmulConstants) -> u64 { + let mut x = load_block(first); - // XOR the initial CRC into the first lane. - x[0] ^= Simd::new(0, state); + // XOR the initial CRC into the first lane. + x[0] ^= Simd::new(0, state); - // 128-byte folding. - let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - for block in rest { - let chunk = load_block(block); - fold_block_128(&mut x, &chunk, coeff); - } - - fold_tail(x, consts) + // 128-byte folding. + let coeff = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + for block in rest { + let chunk = load_block(block); + fold_block_128(&mut x, &chunk, coeff); } + + fold_tail(x, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_2way(state: u64, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc64ClmulConstants) -> u64 { +fn update_simd_2way(state: u64, blocks: &[Block], fold_256b: (u64, u64), consts: &Crc64ClmulConstants) -> u64 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 2 and is bounded by `even` (≤ blocks.len()), - // so `blocks[i]` and `blocks[i + 1]` are always in-bounds. - unsafe { - if blocks.len() < 2 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } + if blocks.len() < 2 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let even = blocks.len() & !1usize; + let even = blocks.len() & !1usize; - let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let coeff_256 = Simd::new(fold_256b.0, fold_256b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); + // Inject CRC into stream 0. + s0[0] ^= Simd::new(0, state); - let mut i = 2; - while i < even { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - fold_block_128(&mut s0, &b0, coeff_256); - fold_block_128(&mut s1, &b1, coeff_256); - i = i.strict_add(2); - } + let mut i = 2; + while i < even { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + fold_block_128(&mut s0, &b0, coeff_256); + fold_block_128(&mut s1, &b1, coeff_256); + i = i.strict_add(2); + } - // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). - let mut combined = s1; - combined[0] ^= s0[0].fold_16(coeff_128); - combined[1] ^= s0[1].fold_16(coeff_128); - combined[2] ^= s0[2].fold_16(coeff_128); - combined[3] ^= s0[3].fold_16(coeff_128); - combined[4] ^= s0[4].fold_16(coeff_128); - combined[5] ^= s0[5].fold_16(coeff_128); - combined[6] ^= s0[6].fold_16(coeff_128); - combined[7] ^= s0[7].fold_16(coeff_128); - - // Handle any remaining block (odd tail) sequentially. - if even != blocks.len() { - let tail = load_block(&blocks[even]); - fold_block_128(&mut combined, &tail, coeff_128); - } + // Merge streams: A·s0 ⊕ s1 (A = shift by 128B). + let mut combined = s1; + combined[0] ^= s0[0].fold_16(coeff_128); + combined[1] ^= s0[1].fold_16(coeff_128); + combined[2] ^= s0[2].fold_16(coeff_128); + combined[3] ^= s0[3].fold_16(coeff_128); + combined[4] ^= s0[4].fold_16(coeff_128); + combined[5] ^= s0[5].fold_16(coeff_128); + combined[6] ^= s0[6].fold_16(coeff_128); + combined[7] ^= s0[7].fold_16(coeff_128); + + // Handle any remaining block (odd tail) sequentially. + if even != blocks.len() { + let tail = load_block(&blocks[even]); + fold_block_128(&mut combined, &tail, coeff_128); + } - fold_tail(combined, consts) - } // unsafe + fold_tail(combined, consts) } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn update_simd_4way( +fn update_simd_4way( state: u64, blocks: &[Block], fold_512b: (u64, u64), @@ -310,92 +288,88 @@ unsafe fn update_simd_4way( ) -> u64 { debug_assert!(!blocks.is_empty()); - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // All fold operations are pure VGFM register computations. - // Loop index `i` advances by 4 and is bounded by `aligned` (≤ blocks.len()), - // so `blocks[i..i+3]` are always in-bounds. Tail processing uses iterator. - unsafe { - if blocks.len() < 4 { - let Some((first, rest)) = blocks.split_first() else { - return state; - }; - return update_simd(state, first, rest, consts); - } + if blocks.len() < 4 { + let Some((first, rest)) = blocks.split_first() else { + return state; + }; + return update_simd(state, first, rest, consts); + } - let aligned = blocks.len().strict_div(4).strict_mul(4); - - let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); - let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); - let c384 = Simd::new(combine[0].0, combine[0].1); - let c256 = Simd::new(combine[1].0, combine[1].1); - let c128 = Simd::new(combine[2].0, combine[2].1); - - let mut s0 = load_block(&blocks[0]); - let mut s1 = load_block(&blocks[1]); - let mut s2 = load_block(&blocks[2]); - let mut s3 = load_block(&blocks[3]); - - // Inject CRC into stream 0. - s0[0] ^= Simd::new(0, state); - - let mut i = 4; - while i < aligned { - let b0 = load_block(&blocks[i]); - let b1 = load_block(&blocks[i.strict_add(1)]); - let b2 = load_block(&blocks[i.strict_add(2)]); - let b3 = load_block(&blocks[i.strict_add(3)]); - fold_block_128(&mut s0, &b0, coeff_512); - fold_block_128(&mut s1, &b1, coeff_512); - fold_block_128(&mut s2, &b2, coeff_512); - fold_block_128(&mut s3, &b3, coeff_512); - i = i.strict_add(4); - } + let aligned = blocks.len().strict_div(4).strict_mul(4); + + let coeff_512 = Simd::new(fold_512b.0, fold_512b.1); + let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); + let c384 = Simd::new(combine[0].0, combine[0].1); + let c256 = Simd::new(combine[1].0, combine[1].1); + let c128 = Simd::new(combine[2].0, combine[2].1); + + let mut s0 = load_block(&blocks[0]); + let mut s1 = load_block(&blocks[1]); + let mut s2 = load_block(&blocks[2]); + let mut s3 = load_block(&blocks[3]); + + // Inject CRC into stream 0. + s0[0] ^= Simd::new(0, state); + + let mut i = 4; + while i < aligned { + let b0 = load_block(&blocks[i]); + let b1 = load_block(&blocks[i.strict_add(1)]); + let b2 = load_block(&blocks[i.strict_add(2)]); + let b3 = load_block(&blocks[i.strict_add(3)]); + fold_block_128(&mut s0, &b0, coeff_512); + fold_block_128(&mut s1, &b1, coeff_512); + fold_block_128(&mut s2, &b2, coeff_512); + fold_block_128(&mut s3, &b3, coeff_512); + i = i.strict_add(4); + } - // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. - let mut combined = s3; - combined[0] ^= s2[0].fold_16(c128); - combined[1] ^= s2[1].fold_16(c128); - combined[2] ^= s2[2].fold_16(c128); - combined[3] ^= s2[3].fold_16(c128); - combined[4] ^= s2[4].fold_16(c128); - combined[5] ^= s2[5].fold_16(c128); - combined[6] ^= s2[6].fold_16(c128); - combined[7] ^= s2[7].fold_16(c128); - - combined[0] ^= s1[0].fold_16(c256); - combined[1] ^= s1[1].fold_16(c256); - combined[2] ^= s1[2].fold_16(c256); - combined[3] ^= s1[3].fold_16(c256); - combined[4] ^= s1[4].fold_16(c256); - combined[5] ^= s1[5].fold_16(c256); - combined[6] ^= s1[6].fold_16(c256); - combined[7] ^= s1[7].fold_16(c256); - - combined[0] ^= s0[0].fold_16(c384); - combined[1] ^= s0[1].fold_16(c384); - combined[2] ^= s0[2].fold_16(c384); - combined[3] ^= s0[3].fold_16(c384); - combined[4] ^= s0[4].fold_16(c384); - combined[5] ^= s0[5].fold_16(c384); - combined[6] ^= s0[6].fold_16(c384); - combined[7] ^= s0[7].fold_16(c384); - - for block in &blocks[aligned..] { - let b = load_block(block); - fold_block_128(&mut combined, &b, coeff_128); - } + // Merge: A^3·s0 ⊕ A^2·s1 ⊕ A·s2 ⊕ s3. + let mut combined = s3; + combined[0] ^= s2[0].fold_16(c128); + combined[1] ^= s2[1].fold_16(c128); + combined[2] ^= s2[2].fold_16(c128); + combined[3] ^= s2[3].fold_16(c128); + combined[4] ^= s2[4].fold_16(c128); + combined[5] ^= s2[5].fold_16(c128); + combined[6] ^= s2[6].fold_16(c128); + combined[7] ^= s2[7].fold_16(c128); + + combined[0] ^= s1[0].fold_16(c256); + combined[1] ^= s1[1].fold_16(c256); + combined[2] ^= s1[2].fold_16(c256); + combined[3] ^= s1[3].fold_16(c256); + combined[4] ^= s1[4].fold_16(c256); + combined[5] ^= s1[5].fold_16(c256); + combined[6] ^= s1[6].fold_16(c256); + combined[7] ^= s1[7].fold_16(c256); + + combined[0] ^= s0[0].fold_16(c384); + combined[1] ^= s0[1].fold_16(c384); + combined[2] ^= s0[2].fold_16(c384); + combined[3] ^= s0[3].fold_16(c384); + combined[4] ^= s0[4].fold_16(c384); + combined[5] ^= s0[5].fold_16(c384); + combined[6] ^= s0[6].fold_16(c384); + combined[7] ^= s0[7].fold_16(c384); + + for block in &blocks[aligned..] { + let b = load_block(block); + fold_block_128(&mut combined, &b, coeff_128); + } - fold_tail(combined, consts) - } // unsafe + fold_tail(combined, consts) } // Public kernels (XZ + NVME) +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices. from_raw_parts length is exact because - // blocks_u64.len() is a multiple of 16 and tail_u64 is from the aligned middle. +fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16]) -> u64 { + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -414,7 +388,7 @@ unsafe fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, if !tail_u64.is_empty() { // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len() * 8); + let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); state = super::portable::crc64_slice16(state, tail_bytes, tables); } @@ -422,16 +396,19 @@ unsafe fn crc64_vgfm(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, } } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc64_vgfm_2way( +fn crc64_vgfm_2way( mut state: u64, bytes: &[u8], fold_256b: (u64, u64), consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16], ) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices. from_raw_parts length is exact. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -448,7 +425,7 @@ unsafe fn crc64_vgfm_2way( if !tail_u64.is_empty() { // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len() * 8); + let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); state = super::portable::crc64_slice16(state, tail_bytes, tables); } @@ -456,8 +433,11 @@ unsafe fn crc64_vgfm_2way( } } +/// # Safety +/// +/// Requires the s390x vector facility. #[target_feature(enable = "vector")] -unsafe fn crc64_vgfm_4way( +fn crc64_vgfm_4way( mut state: u64, bytes: &[u8], fold_512b: (u64, u64), @@ -465,8 +445,8 @@ unsafe fn crc64_vgfm_4way( consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 16], ) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - // align_to produces valid sub-slices. from_raw_parts length is exact. + // SAFETY: u64 and Block accept every bit pattern. The aligned middle is split + // at complete Block boundaries before constructing Block and byte slices. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -483,7 +463,7 @@ unsafe fn crc64_vgfm_4way( if !tail_u64.is_empty() { // SAFETY: `tail_u64` is a subslice of the aligned u64 middle region. - let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len() * 8); + let tail_bytes = core::slice::from_raw_parts(tail_u64.as_ptr().cast(), tail_u64.len().strict_mul(8)); state = super::portable::crc64_slice16(state, tail_bytes, tables); } @@ -499,15 +479,12 @@ unsafe fn crc64_vgfm_4way( /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_xz_vgfm(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm( - crc, - data, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } + crc64_vgfm( + crc, + data, + &crate::checksum::common::clmul::CRC64_XZ_CLMUL, + &super::kernel_tables::XZ_TABLES_16, + ) } /// CRC-64-XZ using VGFM folding (2-way ILP variant). @@ -518,16 +495,13 @@ pub(crate) unsafe fn crc64_xz_vgfm(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_xz_vgfm_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_2way( - crc, - data, - XZ_FOLD_256B, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } + crc64_vgfm_2way( + crc, + data, + XZ_FOLD_256B, + &crate::checksum::common::clmul::CRC64_XZ_CLMUL, + &super::kernel_tables::XZ_TABLES_16, + ) } /// CRC-64-XZ using VGFM folding (4-way ILP variant). @@ -538,17 +512,14 @@ pub(crate) unsafe fn crc64_xz_vgfm_2way(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_xz_vgfm_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_4way( - crc, - data, - XZ_FOLD_512B, - &XZ_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_XZ_CLMUL, - &super::kernel_tables::XZ_TABLES_16, - ) - } + crc64_vgfm_4way( + crc, + data, + XZ_FOLD_512B, + &XZ_COMBINE_4WAY, + &crate::checksum::common::clmul::CRC64_XZ_CLMUL, + &super::kernel_tables::XZ_TABLES_16, + ) } /// CRC-64-NVME using VGFM folding. @@ -559,15 +530,12 @@ pub(crate) unsafe fn crc64_xz_vgfm_4way(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_nvme_vgfm(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm( - crc, - data, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } + crc64_vgfm( + crc, + data, + &crate::checksum::common::clmul::CRC64_NVME_CLMUL, + &super::kernel_tables::NVME_TABLES_16, + ) } /// CRC-64-NVME using VGFM folding (2-way ILP variant). @@ -578,16 +546,13 @@ pub(crate) unsafe fn crc64_nvme_vgfm(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_nvme_vgfm_2way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_2way( - crc, - data, - NVME_FOLD_256B, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } + crc64_vgfm_2way( + crc, + data, + NVME_FOLD_256B, + &crate::checksum::common::clmul::CRC64_NVME_CLMUL, + &super::kernel_tables::NVME_TABLES_16, + ) } /// CRC-64-NVME using VGFM folding (4-way ILP variant). @@ -598,53 +563,50 @@ pub(crate) unsafe fn crc64_nvme_vgfm_2way(crc: u64, data: &[u8]) -> u64 { /// `crate::platform::caps().has(s390x::VECTOR)`. #[target_feature(enable = "vector")] pub(crate) unsafe fn crc64_nvme_vgfm_4way(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Caller guarantees the s390x vector facility is available (dispatch check). - unsafe { - crc64_vgfm_4way( - crc, - data, - NVME_FOLD_512B, - &NVME_COMBINE_4WAY, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_16, - ) - } + crc64_vgfm_4way( + crc, + data, + NVME_FOLD_512B, + &NVME_COMBINE_4WAY, + &crate::checksum::common::clmul::CRC64_NVME_CLMUL, + &super::kernel_tables::NVME_TABLES_16, + ) } // Safe wrappers #[inline] -pub fn crc64_xz_vgfm_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vgfm_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_xz_vgfm(crc, data) } } #[inline] -pub fn crc64_xz_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_xz_vgfm_2way(crc, data) } } #[inline] -pub fn crc64_xz_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_xz_vgfm_4way(crc, data) } } #[inline] -pub fn crc64_nvme_vgfm_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vgfm_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm(crc, data) } } #[inline] -pub fn crc64_nvme_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vgfm_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm_2way(crc, data) } } #[inline] -pub fn crc64_nvme_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vgfm_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VECTOR facility before selecting this kernel. unsafe { crc64_nvme_vgfm_4way(crc, data) } } diff --git a/src/checksum/crc64/x86_64.rs b/src/checksum/crc64/x86_64.rs index 68d625f9..4fb0e0fb 100644 --- a/src/checksum/crc64/x86_64.rs +++ b/src/checksum/crc64/x86_64.rs @@ -7,10 +7,7 @@ //! //! Uses `unsafe` for x86 SIMD intrinsics. Callers must ensure PCLMULQDQ is //! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(dead_code)] // Kernels wired up via dispatcher // SAFETY: All indexing is over fixed-size arrays with in-bounds constant indices. -#![allow(clippy::indexing_slicing)] use core::{ arch::x86_64::*, @@ -43,22 +40,33 @@ impl BitXorAssign for Simd { impl Simd { #[inline] #[target_feature(enable = "sse2")] + /// Constructs a SIMD value with `high` above `low`. + /// + /// # Safety + /// + /// The current CPU must support SSE2, which is guaranteed on x86-64. unsafe fn new(high: u64, low: u64) -> Self { - // SAFETY: SSE2 intrinsics are available via this function's #[target_feature] attribute. Self(_mm_set_epi64x(high.cast_signed(), low.cast_signed())) } /// Fold 16 bytes: `(coeff.low ⊗ self.low) ⊕ (coeff.high ⊗ self.high)`. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_16(self, coeff: Self) -> Self { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. let h = _mm_clmulepi64_si128::<0x11>(self.0, coeff.0); let l = _mm_clmulepi64_si128::<0x00>(self.0, coeff.0); Self(_mm_xor_si128(h, l)) } /// Fold 8 bytes: `self.high ⊕ (coeff ⊗ self.low)`. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn fold_8(self, coeff: u64) -> Self { @@ -70,6 +78,10 @@ impl Simd { } /// Barrett reduction to finalize the CRC. + /// + /// # Safety + /// + /// The current CPU must support SSE2 and PCLMULQDQ. #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn barrett(self, poly: u64, mu: u64) -> u64 { @@ -89,6 +101,11 @@ impl Simd { } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds an initial 128-byte block and its remaining block sequence. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. unsafe { @@ -125,8 +142,13 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: } #[inline(always)] +/// Reduces eight folded SIMD lanes to one CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. + // SAFETY: The caller's SSE2/PCLMULQDQ guarantee covers the constructor, folds, and reduction. unsafe { // Tail reduction (8×16B → 1×16B), unrolled for throughput. let c0 = Simd::new(consts.tail_fold_16b[0].0, consts.tail_fold_16b[0].1); @@ -154,6 +176,11 @@ unsafe fn fold_tail(x: [Simd; 8], consts: &Crc64ClmulConstants) -> u64 { #[inline] #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds one 128-byte block into eight SIMD lanes. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. unsafe { @@ -178,8 +205,13 @@ unsafe fn fold_block_128(x: &mut [Simd; 8], chunk: &[Simd; 8], coeff: Simd) { } #[inline(always)] +/// Folds and XORs one eight-lane stream into an accumulator. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn merge_lanes_xor(acc: &mut [Simd; 8], stream: &[Simd; 8], coeff: Simd) { - // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. + // SAFETY: The caller's SSE2/PCLMULQDQ guarantee covers all eight lane folds. unsafe { acc[0] ^= stream[0].fold_16(coeff); acc[1] ^= stream[1].fold_16(coeff); @@ -193,6 +225,14 @@ unsafe fn merge_lanes_xor(acc: &mut [Simd; 8], stream: &[Simd; 8], coeff: Simd) } /// 2-way PCLMUL (SSE) kernel with double-unrolling and software prefetch. +/// +/// # Panics +/// +/// Panics if `blocks` contains fewer than two blocks. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_2way( state: u64, @@ -229,16 +269,16 @@ unsafe fn update_simd_2way( prefetch_read_l1(prefetch_ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); - fold_block_128(&mut s0, &blocks[i + 2], coeff_256); - fold_block_128(&mut s1, &blocks[i + 3], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); + fold_block_128(&mut s0, &blocks[i.strict_add(2)], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(3)], coeff_256); i = i.strict_add(4); } // Handle remaining pair. while i < even { fold_block_128(&mut s0, &blocks[i], coeff_256); - fold_block_128(&mut s1, &blocks[i + 1], coeff_256); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_256); i = i.strict_add(2); } @@ -256,6 +296,14 @@ unsafe fn update_simd_2way( } /// 4-way PCLMUL (SSE) kernel with double-unrolling and software prefetch. +/// +/// # Panics +/// +/// Panics in debug builds if `blocks` is empty. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_4way( state: u64, @@ -279,7 +327,7 @@ unsafe fn update_simd_4way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = (blocks.len() / 4).strict_mul(4); // Account for starting at i=4: we need i+7 < blocks.len() for valid access let double_aligned = 4usize.strict_add(((blocks.len().strict_sub(4)) / 8).strict_mul(8)); @@ -305,15 +353,15 @@ unsafe fn update_simd_4way( // First group of 4 blocks fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); // Second group of 4 blocks - fold_block_128(&mut s0, &blocks[i + 4], coeff_512); - fold_block_128(&mut s1, &blocks[i + 5], coeff_512); - fold_block_128(&mut s2, &blocks[i + 6], coeff_512); - fold_block_128(&mut s3, &blocks[i + 7], coeff_512); + fold_block_128(&mut s0, &blocks[i.strict_add(4)], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(5)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(6)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(7)], coeff_512); i = i.strict_add(8); } @@ -321,9 +369,9 @@ unsafe fn update_simd_4way( // Handle remaining group. while i < aligned { fold_block_128(&mut s0, &blocks[i], coeff_512); - fold_block_128(&mut s1, &blocks[i + 1], coeff_512); - fold_block_128(&mut s2, &blocks[i + 2], coeff_512); - fold_block_128(&mut s3, &blocks[i + 3], coeff_512); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_512); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_512); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_512); i = i.strict_add(4); } @@ -365,6 +413,14 @@ unsafe fn update_simd_4way( } /// 7-way PCLMUL (SSE) kernel with software prefetch. +/// +/// # Panics +/// +/// Panics in debug builds if `blocks` is empty. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_7way( state: u64, @@ -388,7 +444,7 @@ unsafe fn update_simd_7way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 7) * 7; + let aligned = (blocks.len() / 7).strict_mul(7); let coeff_896 = Simd::new(fold_896b.0, fold_896b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -415,12 +471,12 @@ unsafe fn update_simd_7way( prefetch_read_l1(prefetch_ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); fold_block_128(&mut s0, &blocks[i], coeff_896); - fold_block_128(&mut s1, &blocks[i + 1], coeff_896); - fold_block_128(&mut s2, &blocks[i + 2], coeff_896); - fold_block_128(&mut s3, &blocks[i + 3], coeff_896); - fold_block_128(&mut s4, &blocks[i + 4], coeff_896); - fold_block_128(&mut s5, &blocks[i + 5], coeff_896); - fold_block_128(&mut s6, &blocks[i + 6], coeff_896); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_896); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_896); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_896); + fold_block_128(&mut s4, &blocks[i.strict_add(4)], coeff_896); + fold_block_128(&mut s5, &blocks[i.strict_add(5)], coeff_896); + fold_block_128(&mut s6, &blocks[i.strict_add(6)], coeff_896); i = i.strict_add(7); } @@ -447,6 +503,14 @@ unsafe fn update_simd_7way( } /// 8-way PCLMUL (SSE) kernel with software prefetch. +/// +/// # Panics +/// +/// Panics in debug builds if `blocks` is empty. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn update_simd_8way( state: u64, @@ -470,7 +534,7 @@ unsafe fn update_simd_8way( return update_simd(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = (blocks.len() / 8).strict_mul(8); let coeff_1024 = Simd::new(fold_1024b.0, fold_1024b.1); let coeff_128 = Simd::new(consts.fold_128b.0, consts.fold_128b.1); @@ -498,13 +562,13 @@ unsafe fn update_simd_8way( prefetch_read_l1(prefetch_ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); fold_block_128(&mut s0, &blocks[i], coeff_1024); - fold_block_128(&mut s1, &blocks[i + 1], coeff_1024); - fold_block_128(&mut s2, &blocks[i + 2], coeff_1024); - fold_block_128(&mut s3, &blocks[i + 3], coeff_1024); - fold_block_128(&mut s4, &blocks[i + 4], coeff_1024); - fold_block_128(&mut s5, &blocks[i + 5], coeff_1024); - fold_block_128(&mut s6, &blocks[i + 6], coeff_1024); - fold_block_128(&mut s7, &blocks[i + 7], coeff_1024); + fold_block_128(&mut s1, &blocks[i.strict_add(1)], coeff_1024); + fold_block_128(&mut s2, &blocks[i.strict_add(2)], coeff_1024); + fold_block_128(&mut s3, &blocks[i.strict_add(3)], coeff_1024); + fold_block_128(&mut s4, &blocks[i.strict_add(4)], coeff_1024); + fold_block_128(&mut s5, &blocks[i.strict_add(5)], coeff_1024); + fold_block_128(&mut s6, &blocks[i.strict_add(6)], coeff_1024); + fold_block_128(&mut s7, &blocks[i.strict_add(7)], coeff_1024); i = i.strict_add(8); } @@ -532,9 +596,15 @@ unsafe fn update_simd_8way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with PCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8]) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if let Some((first, rest)) = middle.split_first() { @@ -548,6 +618,11 @@ unsafe fn crc64_pclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstant } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with two PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_2way( mut state: u64, bytes: &[u8], @@ -556,7 +631,8 @@ unsafe fn crc64_pclmul_2way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -576,6 +652,11 @@ unsafe fn crc64_pclmul_2way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with four PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_4way( mut state: u64, bytes: &[u8], @@ -585,7 +666,8 @@ unsafe fn crc64_pclmul_4way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -599,6 +681,11 @@ unsafe fn crc64_pclmul_4way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with seven PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_7way( mut state: u64, bytes: &[u8], @@ -608,7 +695,8 @@ unsafe fn crc64_pclmul_7way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -622,6 +710,11 @@ unsafe fn crc64_pclmul_7way( } #[target_feature(enable = "sse2", enable = "pclmulqdq")] +/// Folds the 128-byte-aligned middle of `bytes` with eight PCLMUL streams. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. unsafe fn crc64_pclmul_8way( mut state: u64, bytes: &[u8], @@ -631,7 +724,8 @@ unsafe fn crc64_pclmul_8way( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if middle.is_empty() { @@ -648,6 +742,10 @@ unsafe fn crc64_pclmul_8way( /// /// This avoids constructing the full 128-byte fold state when only complete /// 16-byte lanes are available. +/// +/// # Safety +/// +/// The current CPU must support SSE2 and PCLMULQDQ. #[target_feature(enable = "sse2", enable = "pclmulqdq")] unsafe fn crc64_pclmul_small( mut state: u64, @@ -656,7 +754,8 @@ unsafe fn crc64_pclmul_small( tables: &[[u64; 256]; 8], ) -> u64 { // SAFETY: SSE2/PCLMULQDQ intrinsics are available via this function's #[target_feature] attribute. - // align_to is sound because Simd is repr(transparent) over __m128i. + // Every bit pattern is valid for `Simd`, which is transparent over `__m128i`, so `align_to` + // produces initialized, non-overlapping fragments. unsafe { let (left, middle, right) = bytes.align_to::(); @@ -686,28 +785,20 @@ unsafe fn crc64_pclmul_small( // VPCLMULQDQ (AVX-512) folding -#[inline] -#[target_feature(enable = "avx512f", enable = "vpclmulqdq")] -unsafe fn fold16_4x(x: __m512i, coeff: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. - let h = _mm512_clmulepi64_epi128::<0x11>(x, coeff); - let l = _mm512_clmulepi64_epi128::<0x00>(x, coeff); - _mm512_xor_si512(h, l) -} - /// Fold and XOR with data using VPTERNLOGD (3-way XOR in one instruction). /// /// Computes: `data ^ clmul_hi(x, coeff) ^ clmul_lo(x, coeff)` /// -/// This saves one XOR instruction per fold operation compared to the -/// two-step `data ^ fold16_4x(x, coeff)` pattern. The ternary logic +/// This saves one XOR instruction per fold operation compared with first +/// combining the two products and then XORing the data. The ternary logic /// immediate 0x96 encodes XOR(a, XOR(b, c)) = a ^ b ^ c. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and VPCLMULQDQ. #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn fold16_4x_ternlog(x: __m512i, data: __m512i, coeff: __m512i) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. let h = _mm512_clmulepi64_epi128::<0x11>(x, coeff); let l = _mm512_clmulepi64_epi128::<0x00>(x, coeff); // VPTERNLOGD: 3-way XOR (imm8 = 0x96 = a ^ b ^ c) @@ -716,9 +807,12 @@ unsafe fn fold16_4x_ternlog(x: __m512i, data: __m512i, coeff: __m512i) -> __m512 #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Broadcasts a coefficient pair across four 128-bit lanes. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F and VPCLMULQDQ. unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. _mm512_set_epi64( pair.0.cast_signed(), pair.1.cast_signed(), @@ -733,27 +827,53 @@ unsafe fn vpclmul_coeff(pair: (u64, u64)) -> __m512i { #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Loads one 64-byte AVX-512 vector. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. `ptr` must remain valid to read +/// 64 initialized bytes from one allocation. When `ALIGNED` is `true`, `ptr` +/// must also be aligned to 64 bytes; `false` permits any byte alignment. unsafe fn load_m512(ptr: *const u8) -> __m512i { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Caller ensures `ptr` is valid for 64-byte reads (aligned or unaligned as indicated - // by ALIGNED). + // SAFETY: The function attribute enables both intrinsics. A readable 64-byte source cannot be + // null. Casting `NonNull` changes only the pointee type and preserves provenance; the aligned + // specialization's caller additionally guarantees the alignment required by + // `_mm512_load_si512`, while `_mm512_loadu_si512` accepts any byte alignment. unsafe { if ALIGNED { debug_assert_eq!((ptr as usize) & 63, 0); - _mm512_load_si512(ptr.cast::<__m512i>()) + _mm512_load_si512( + core::ptr::NonNull::new_unchecked(ptr.cast_mut()) + .cast::<__m512i>() + .as_ptr(), + ) } else { - _mm512_loadu_si512(ptr.cast::<__m512i>()) + _mm512_loadu_si512( + core::ptr::NonNull::new_unchecked(ptr.cast_mut()) + .cast::<__m512i>() + .as_ptr(), + ) } } } #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Loads one 128-byte block as two adjacent AVX-512 vectors. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// `block` must also begin at a 64-byte-aligned address. unsafe fn load_128b_block(block: &[Simd; 8]) -> (__m512i, __m512i) { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. `block` is 128 bytes (8x16B); ptr and ptr+64 are both in-bounds. + // SAFETY: The function attribute enables the intrinsics, and the reference provides 128 + // initialized readable bytes. `from_ref` preserves its provenance; the first load reads bytes + // 0..64 and the second reads bytes 64..128. When `ALIGNED` is true, the caller's 64-byte + // alignment guarantee applies to both addresses. unsafe { - let ptr = block as *const [Simd; 8] as *const u8; + let ptr = core::ptr::from_ref(block).cast::(); // 8×16B lanes packed as 2×64B vectors (4 lanes each). let y0 = load_m512::(ptr); let y1 = load_m512::(ptr.add(64)); @@ -763,9 +883,15 @@ unsafe fn load_128b_block(block: &[Simd; 8]) -> (__m512i, _ #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Reduces two folded AVX-512 vectors to one CRC-64 state. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The function attribute enables the AVX-512 lane extractions. The caller's complete + // capability guarantee also covers the SSE2/PCLMULQDQ operations used by `fold_tail`. unsafe { // Reuse the well-tested 128-bit tail fold + Barrett reduction by extracting // the 8×16B lanes directly (avoids a store+reload round-trip). @@ -784,11 +910,19 @@ unsafe fn finalize_vpclmul_state(x0: __m512i, x1: __m512i, consts: &Crc64ClmulCo } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Folds one initial block and its remaining VPCLMUL block sequence. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. If `first` begins at a +/// 64-byte-aligned address, every block in `rest` must also be 64-byte aligned. unsafe fn update_simd_vpclmul(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller provides the complete capability set and the alignment relationship + // between `first` and `rest`. `addr` observes only the public address; both specializations + // receive the original references with their provenance unchanged. unsafe { - let aligned = ((first as *const [Simd; 8] as usize) & 63) == 0; + let aligned = (core::ptr::from_ref(first).addr() & 63) == 0; if aligned { update_simd_vpclmul_impl::(state, first, rest, consts) } else { @@ -799,14 +933,22 @@ unsafe fn update_simd_vpclmul(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Folds an initial 128-byte block and its remaining block sequence. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// `first` and every block in `rest` must begin at a 64-byte-aligned address. unsafe fn update_simd_vpclmul_impl( state: u64, first: &[Simd; 8], rest: &[[Simd; 8]], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The function attribute enables the AVX-512 intrinsics, and the caller supplies the + // complete capability set used by the final reduction. The references provide initialized + // 128-byte blocks; when ALIGNED is true, the caller's alignment guarantee covers every load. unsafe { let (mut x0, mut x1) = load_128b_block::(first); @@ -830,9 +972,18 @@ unsafe fn update_simd_vpclmul_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Folds the 128-byte middle of `bytes` with VPCLMULQDQ. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn crc64_vpclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8]) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. align_to is sound because Simd is repr(transparent) over __m128i. + // SAFETY: The function attribute enables the AVX-512 intrinsics, and the caller supplies the + // complete capability set used by the final reduction. Every bit pattern is valid for `Simd`, + // which is transparent over `__m128i`, so `align_to` produces initialized, non-overlapping + // fragments. Middle blocks are 128 bytes apart and therefore share their first block's mod-64 + // alignment, satisfying `update_simd_vpclmul`'s specialization contract. unsafe { let (left, middle, right) = bytes.align_to::<[Simd; 8]>(); if let Some((first, rest)) = middle.split_first() { @@ -848,16 +999,23 @@ unsafe fn crc64_vpclmul(mut state: u64, bytes: &[u8], consts: &Crc64ClmulConstan // VPCLMULQDQ multi-stream (2/4/7-way, 128B blocks) #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned two-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_2way( state: u64, blocks: &[[Simd; 8]], fold_256b: (u64, u64), consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_2way_impl::(state, blocks, fold_256b, consts) } else { @@ -869,6 +1027,12 @@ unsafe fn update_simd_vpclmul_2way( /// 2-way VPCLMUL kernel with double-unrolling and software prefetch. /// /// Processes 512 bytes per iteration (4 × 128B blocks in 2 streams). +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_2way_impl( state: u64, @@ -876,9 +1040,10 @@ unsafe fn update_simd_vpclmul_2way_impl( fold_256b: (u64, u64), consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `even`/`double_even` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 2. `double_even` is 2 + 4k and <= len, so each + // double-loop i + {0,1,2,3} is in-bounds; i < even makes each remaining-pair i + {0,1} + // in-bounds; and even < len guards the odd tail. The prefetch base uses the in-bounds i. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -918,16 +1083,16 @@ unsafe fn update_simd_vpclmul_2way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_256); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_256); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_256); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_256); // Second pair of blocks (256B) - let (z0, z1) = load_128b_block::(&blocks[i + 2]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_0 = fold16_4x_ternlog(x0_0, z0, coeff_256); x1_0 = fold16_4x_ternlog(x1_0, z1, coeff_256); - let (z0, z1) = load_128b_block::(&blocks[i + 3]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_1 = fold16_4x_ternlog(x0_1, z0, coeff_256); x1_1 = fold16_4x_ternlog(x1_1, z1, coeff_256); @@ -940,7 +1105,7 @@ unsafe fn update_simd_vpclmul_2way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_256); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_256); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_256); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_256); @@ -963,6 +1128,12 @@ unsafe fn update_simd_vpclmul_2way_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned four-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_4way( state: u64, blocks: &[[Simd; 8]], @@ -970,10 +1141,11 @@ unsafe fn update_simd_vpclmul_4way( combine: &[(u64, u64); 3], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_4way_impl::(state, blocks, fold_512b, combine, consts) } else { @@ -985,6 +1157,12 @@ unsafe fn update_simd_vpclmul_4way( /// 4-way VPCLMUL kernel with double-unrolling and software prefetch. /// /// Processes 1024 bytes per iteration (8 × 128B blocks in 4 streams). +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_4way_impl( state: u64, @@ -993,9 +1171,10 @@ unsafe fn update_simd_vpclmul_4way_impl( combine: &[(u64, u64); 3], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `aligned`/`double_aligned` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 4. `double_aligned` is 4 + 8k and <= len, so each + // double-loop i + {0..7} is in-bounds; i < aligned makes each remaining-group i + {0..3} + // in-bounds; and aligned <= len makes the final suffix valid. The prefetch base uses in-bounds i. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1008,7 +1187,7 @@ unsafe fn update_simd_vpclmul_4way_impl( return update_simd_vpclmul(state, first, rest, consts); } - let aligned = (blocks.len() / 4) * 4; + let aligned = (blocks.len() / 4).strict_mul(4); // Account for starting at i=4: we need i+7 < blocks.len() for valid access let double_aligned = 4usize.strict_add(((blocks.len().strict_sub(4)) / 8).strict_mul(8)); @@ -1038,32 +1217,32 @@ unsafe fn update_simd_vpclmul_4way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_512); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_512); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_512); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_512); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_512); // Second group of 4 blocks (512B) - let (z0, z1) = load_128b_block::(&blocks[i + 4]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(4)]); x0_0 = fold16_4x_ternlog(x0_0, z0, coeff_512); x1_0 = fold16_4x_ternlog(x1_0, z1, coeff_512); - let (z0, z1) = load_128b_block::(&blocks[i + 5]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(5)]); x0_1 = fold16_4x_ternlog(x0_1, z0, coeff_512); x1_1 = fold16_4x_ternlog(x1_1, z1, coeff_512); - let (z0, z1) = load_128b_block::(&blocks[i + 6]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(6)]); x0_2 = fold16_4x_ternlog(x0_2, z0, coeff_512); x1_2 = fold16_4x_ternlog(x1_2, z1, coeff_512); - let (z0, z1) = load_128b_block::(&blocks[i + 7]); + let (z0, z1) = load_128b_block::(&blocks[i.strict_add(7)]); x0_3 = fold16_4x_ternlog(x0_3, z0, coeff_512); x1_3 = fold16_4x_ternlog(x1_3, z1, coeff_512); @@ -1076,15 +1255,15 @@ unsafe fn update_simd_vpclmul_4way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_512); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_512); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_512); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_512); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_512); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_512); @@ -1111,6 +1290,12 @@ unsafe fn update_simd_vpclmul_4way_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned seven-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_7way( state: u64, blocks: &[[Simd; 8]], @@ -1118,10 +1303,11 @@ unsafe fn update_simd_vpclmul_7way( combine: &[(u64, u64); 6], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_7way_impl::(state, blocks, fold_896b, combine, consts) } else { @@ -1134,6 +1320,12 @@ unsafe fn update_simd_vpclmul_7way( /// /// Processes 896 bytes per iteration (7 × 128B blocks in 7 streams). /// Already has high ILP from 7-way parallelism; prefetch helps hide memory latency. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_7way_impl( state: u64, @@ -1142,9 +1334,10 @@ unsafe fn update_simd_vpclmul_7way_impl( combine: &[(u64, u64); 6], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `aligned` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 7. `aligned` is a multiple of 7 and <= len; i + // starts at 7 and advances by 7, so i < aligned makes every i + {0..6} and the prefetch base + // in-bounds. `aligned <= len` also makes the final suffix valid. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1157,7 +1350,7 @@ unsafe fn update_simd_vpclmul_7way_impl( return update_simd_vpclmul(state, first, rest, consts); } - let aligned = (blocks.len() / 7) * 7; + let aligned = (blocks.len() / 7).strict_mul(7); let (mut x0_0, mut x1_0) = load_128b_block::(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block::(&blocks[1]); @@ -1189,27 +1382,27 @@ unsafe fn update_simd_vpclmul_7way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_896); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_896); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_896); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_896); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 4]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(4)]); x0_4 = fold16_4x_ternlog(x0_4, y0, coeff_896); x1_4 = fold16_4x_ternlog(x1_4, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 5]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(5)]); x0_5 = fold16_4x_ternlog(x0_5, y0, coeff_896); x1_5 = fold16_4x_ternlog(x1_5, y1, coeff_896); - let (y0, y1) = load_128b_block::(&blocks[i + 6]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(6)]); x0_6 = fold16_4x_ternlog(x0_6, y0, coeff_896); x1_6 = fold16_4x_ternlog(x1_6, y1, coeff_896); @@ -1245,6 +1438,12 @@ unsafe fn update_simd_vpclmul_7way_impl( } #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] +/// Selects the aligned or unaligned eight-stream VPCLMUL fold. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. unsafe fn update_simd_vpclmul_8way( state: u64, blocks: &[[Simd; 8]], @@ -1252,10 +1451,11 @@ unsafe fn update_simd_vpclmul_8way( combine: &[(u64, u64); 7], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. + // SAFETY: The caller supplies the complete capability set. `addr` observes only the public + // address, and 128-byte array-element strides preserve its mod-64 alignment for every block; + // both specializations receive the original slice with its provenance unchanged. unsafe { - let aligned = ((blocks.as_ptr() as usize) & 63) == 0; + let aligned = (blocks.as_ptr().addr() & 63) == 0; if aligned { update_simd_vpclmul_8way_impl::(state, blocks, fold_1024b, combine, consts) } else { @@ -1268,6 +1468,12 @@ unsafe fn update_simd_vpclmul_8way( /// /// Processes 1024 bytes per iteration (8 × 128B blocks in 8 streams). /// Already has high ILP from 8-way parallelism; prefetch helps hide memory latency. +/// +/// # Safety +/// +/// The current CPU must support the features represented by +/// `crate::platform::caps::x86::VPCLMUL_READY`. When `ALIGNED` is `true`, +/// every block in `blocks` must begin at a 64-byte-aligned address. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn update_simd_vpclmul_8way_impl( state: u64, @@ -1276,9 +1482,10 @@ unsafe fn update_simd_vpclmul_8way_impl( combine: &[(u64, u64); 7], consts: &Crc64ClmulConstants, ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. Pointer arithmetic on `blocks.as_ptr()` is in-bounds: indices are bounded by - // `aligned` < `blocks.len()`. + // SAFETY: The caller supplies the complete capability set and the alignment required by the + // selected loads. After the fallback, len >= 8. `aligned` is a multiple of 8 and <= len; i + // starts at 8 and advances by 8, so i < aligned makes every i + {0..7} and the prefetch base + // in-bounds. `aligned <= len` also makes the final suffix valid. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1291,7 +1498,7 @@ unsafe fn update_simd_vpclmul_8way_impl( return update_simd_vpclmul(state, first, rest, consts); } - let aligned = (blocks.len() / 8) * 8; + let aligned = (blocks.len() / 8).strict_mul(8); let (mut x0_0, mut x1_0) = load_128b_block::(&blocks[0]); let (mut x0_1, mut x1_1) = load_128b_block::(&blocks[1]); @@ -1324,31 +1531,31 @@ unsafe fn update_simd_vpclmul_8way_impl( x0_0 = fold16_4x_ternlog(x0_0, y0, coeff_1024); x1_0 = fold16_4x_ternlog(x1_0, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 1]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(1)]); x0_1 = fold16_4x_ternlog(x0_1, y0, coeff_1024); x1_1 = fold16_4x_ternlog(x1_1, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 2]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(2)]); x0_2 = fold16_4x_ternlog(x0_2, y0, coeff_1024); x1_2 = fold16_4x_ternlog(x1_2, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 3]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(3)]); x0_3 = fold16_4x_ternlog(x0_3, y0, coeff_1024); x1_3 = fold16_4x_ternlog(x1_3, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 4]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(4)]); x0_4 = fold16_4x_ternlog(x0_4, y0, coeff_1024); x1_4 = fold16_4x_ternlog(x1_4, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 5]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(5)]); x0_5 = fold16_4x_ternlog(x0_5, y0, coeff_1024); x1_5 = fold16_4x_ternlog(x1_5, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 6]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(6)]); x0_6 = fold16_4x_ternlog(x0_6, y0, coeff_1024); x1_6 = fold16_4x_ternlog(x1_6, y1, coeff_1024); - let (y0, y1) = load_128b_block::(&blocks[i + 7]); + let (y0, y1) = load_128b_block::(&blocks[i.strict_add(7)]); x0_7 = fold16_4x_ternlog(x0_7, y0, coeff_1024); x1_7 = fold16_4x_ternlog(x1_7, y1, coeff_1024); @@ -1397,6 +1604,11 @@ unsafe fn update_simd_vpclmul_8way_impl( /// 1. Fold x0,x1 into x2,x3 (1024-bit shift = 128 bytes) /// 2. Fold result into single 512-bit register (512-bit shift = 64 bytes) /// 3. Extract 4×128-bit lanes and reduce to u64 +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[inline] #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn finalize_4x512_state( @@ -1448,6 +1660,11 @@ unsafe fn finalize_4x512_state( /// /// The double-unroll halves the number of loop-control steps. Software /// prefetch remains a target-dependent hint. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_4x512( mut state: u64, @@ -1456,10 +1673,11 @@ unsafe fn crc64_vpclmul_4x512( consts: &Crc64ClmulConstants, tables: &[[u64; 256]; 8], ) -> u64 { - // SAFETY: AVX-512/VPCLMULQDQ intrinsics are available via this function's #[target_feature] - // attribute. align_to is sound because Simd is repr(transparent) over __m128i. - // Pointer arithmetic: `ptr` stays within `aligned_bytes`; bounds checked via `ptr.add(...) <= end`. - // from_raw_parts: `ptr` is valid for `remaining` bytes (derived from `end.offset_from(ptr)`). + // SAFETY: The caller provides the full VPCLMUL_READY capability set. `ptr` and `end` retain the + // provenance and order of the same slice allocation. The initial unsigned-distance check makes + // `last_double_block` in-bounds, and every aligned 64-byte load and pointer advance is dominated + // by that bound or a matching tail-distance check. The final pointer and tail length therefore + // describe initialized bytes from that allocation. unsafe { use crate::checksum::common::prefetch::{LARGE_BLOCK_DISTANCE, prefetch_read_l1}; @@ -1493,10 +1711,10 @@ unsafe fn crc64_vpclmul_4x512( debug_assert_eq!((ptr as usize) & 63, 0); // Load first 256B block into 4 __m512i registers. - let mut x0 = _mm512_load_si512(ptr.cast::<__m512i>()); - let mut x1 = _mm512_load_si512(ptr.add(64).cast::<__m512i>()); - let mut x2 = _mm512_load_si512(ptr.add(128).cast::<__m512i>()); - let mut x3 = _mm512_load_si512(ptr.add(192).cast::<__m512i>()); + let mut x0 = load_m512::(ptr); + let mut x1 = load_m512::(ptr.add(64)); + let mut x2 = load_m512::(ptr.add(128)); + let mut x3 = load_m512::(ptr.add(192)); ptr = ptr.add(BLOCK_SIZE); // XOR the initial CRC into lane 0 (low 64 bits of first register). @@ -1508,41 +1726,44 @@ unsafe fn crc64_vpclmul_4x512( // Double-unrolled main loop: fold 512B (2 blocks) per iteration. // This reduces loop overhead by 50% and improves ILP. - while ptr.add(DOUBLE_BLOCK) <= end { - // Prefetch 2 iterations ahead (1KB) to hide memory latency. - prefetch_read_l1(ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); - - // First block (256B): load and fold - let y0 = _mm512_load_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_load_si512(ptr.add(64).cast::<__m512i>()); - let y2 = _mm512_load_si512(ptr.add(128).cast::<__m512i>()); - let y3 = _mm512_load_si512(ptr.add(192).cast::<__m512i>()); - - x0 = fold16_4x_ternlog(x0, y0, coeff_256); - x1 = fold16_4x_ternlog(x1, y1, coeff_256); - x2 = fold16_4x_ternlog(x2, y2, coeff_256); - x3 = fold16_4x_ternlog(x3, y3, coeff_256); - - // Second block (256B): load and fold - let z0 = _mm512_load_si512(ptr.add(256).cast::<__m512i>()); - let z1 = _mm512_load_si512(ptr.add(320).cast::<__m512i>()); - let z2 = _mm512_load_si512(ptr.add(384).cast::<__m512i>()); - let z3 = _mm512_load_si512(ptr.add(448).cast::<__m512i>()); - - x0 = fold16_4x_ternlog(x0, z0, coeff_256); - x1 = fold16_4x_ternlog(x1, z1, coeff_256); - x2 = fold16_4x_ternlog(x2, z2, coeff_256); - x3 = fold16_4x_ternlog(x3, z3, coeff_256); - - ptr = ptr.add(DOUBLE_BLOCK); + if end.offset_from_unsigned(ptr) >= DOUBLE_BLOCK { + let last_double_block = end.sub(DOUBLE_BLOCK); + while ptr <= last_double_block { + // Prefetch 2 iterations ahead (1KB) to hide memory latency. + prefetch_read_l1(ptr.wrapping_add(LARGE_BLOCK_DISTANCE)); + + // First block (256B): load and fold + let y0 = load_m512::(ptr); + let y1 = load_m512::(ptr.add(64)); + let y2 = load_m512::(ptr.add(128)); + let y3 = load_m512::(ptr.add(192)); + + x0 = fold16_4x_ternlog(x0, y0, coeff_256); + x1 = fold16_4x_ternlog(x1, y1, coeff_256); + x2 = fold16_4x_ternlog(x2, y2, coeff_256); + x3 = fold16_4x_ternlog(x3, y3, coeff_256); + + // Second block (256B): load and fold + let z0 = load_m512::(ptr.add(256)); + let z1 = load_m512::(ptr.add(320)); + let z2 = load_m512::(ptr.add(384)); + let z3 = load_m512::(ptr.add(448)); + + x0 = fold16_4x_ternlog(x0, z0, coeff_256); + x1 = fold16_4x_ternlog(x1, z1, coeff_256); + x2 = fold16_4x_ternlog(x2, z2, coeff_256); + x3 = fold16_4x_ternlog(x3, z3, coeff_256); + + ptr = ptr.add(DOUBLE_BLOCK); + } } // Handle remaining single block (if odd number of blocks). - if ptr.add(BLOCK_SIZE) <= end { - let y0 = _mm512_load_si512(ptr.cast::<__m512i>()); - let y1 = _mm512_load_si512(ptr.add(64).cast::<__m512i>()); - let y2 = _mm512_load_si512(ptr.add(128).cast::<__m512i>()); - let y3 = _mm512_load_si512(ptr.add(192).cast::<__m512i>()); + if end.offset_from_unsigned(ptr) >= BLOCK_SIZE { + let y0 = load_m512::(ptr); + let y1 = load_m512::(ptr.add(64)); + let y2 = load_m512::(ptr.add(128)); + let y3 = load_m512::(ptr.add(192)); x0 = fold16_4x_ternlog(x0, y0, coeff_256); x1 = fold16_4x_ternlog(x1, y1, coeff_256); @@ -1556,11 +1777,17 @@ unsafe fn crc64_vpclmul_4x512( state = finalize_4x512_state(x0, x1, x2, x3, consts.fold_128b, consts.tail_fold_16b[3], consts); // Process any remaining bytes. - let remaining = end.offset_from(ptr) as usize; + let remaining = end.offset_from_unsigned(ptr); super::portable::crc64_slice8(state, core::slice::from_raw_parts(ptr, remaining), tables) } } +/// Fold CRC-64 with two parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_2way( mut state: u64, @@ -1583,6 +1810,12 @@ unsafe fn crc64_vpclmul_2way( } } +/// Fold CRC-64 with four parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_4way( mut state: u64, @@ -1606,6 +1839,12 @@ unsafe fn crc64_vpclmul_4way( } } +/// Fold CRC-64 with seven parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_7way( mut state: u64, @@ -1629,6 +1868,12 @@ unsafe fn crc64_vpclmul_7way( } } +/// Fold CRC-64 with eight parallel VPCLMUL streams. +/// +/// # Safety +/// +/// Requires the x86 features represented by `crate::platform::caps::x86::VPCLMUL_READY`. +/// The caller must verify that capability set before calling this function. #[target_feature(enable = "avx512f", enable = "vpclmulqdq")] unsafe fn crc64_vpclmul_8way( mut state: u64, @@ -2131,199 +2376,169 @@ pub(crate) unsafe fn crc64_nvme_vpclmul_8way(crc: u64, data: &[u8]) -> u64 { } } -/// CRC-64-NVME using VPCLMULQDQ (4×512-bit variant). -/// -/// Processes 512 bytes per main-loop iteration across four fold streams. -/// -/// # Safety -/// -/// Requires VPCLMULQDQ + AVX-512. Caller must verify via -/// `crate::platform::caps().has(x86::VPCLMUL_READY)`. -#[target_feature(enable = "avx512f", enable = "vpclmulqdq")] -pub(crate) unsafe fn crc64_nvme_vpclmul_4x512(crc: u64, data: &[u8]) -> u64 { - // SAFETY: This wrapper relies on the function's safety contract (caller ensures VPCLMULQDQ/AVX-512 - // is available). - unsafe { - crc64_vpclmul_4x512( - crc, - data, - CRC64_NVME_STREAM.fold_256b, - &crate::checksum::common::clmul::CRC64_NVME_CLMUL, - &super::kernel_tables::NVME_TABLES_8, - ) - } -} - // Dispatcher Wrappers (safe interface) /// Safe wrapper for CRC-64-XZ PCLMUL kernel. #[inline] -pub fn crc64_xz_pclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL small-buffer kernel. #[inline] -pub fn crc64_xz_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_small(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 2-way kernel. #[inline] -pub fn crc64_xz_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 4-way kernel. #[inline] -pub fn crc64_xz_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 7-way kernel. #[inline] -pub fn crc64_xz_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-XZ PCLMUL 8-way kernel. #[inline] -pub fn crc64_xz_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_xz_pclmul_8way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL kernel. #[inline] -pub fn crc64_xz_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc64_xz_vpclmul(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 2-way kernel. #[inline] -pub fn crc64_xz_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 4-way kernel. #[inline] -pub fn crc64_xz_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 7-way kernel. #[inline] -pub fn crc64_xz_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 8-way kernel. #[inline] -pub fn crc64_xz_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_8way(crc, data) } } /// Safe wrapper for CRC-64-XZ VPCLMUL 4×512-bit kernel. #[inline] -pub fn crc64_xz_vpclmul_4x512_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_xz_vpclmul_4x512_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_xz_vpclmul_4x512(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL kernel. #[inline] -pub fn crc64_nvme_pclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL small-buffer kernel. #[inline] -pub fn crc64_nvme_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_small_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_small(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 2-way kernel. #[inline] -pub fn crc64_nvme_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 4-way kernel. #[inline] -pub fn crc64_nvme_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 7-way kernel. #[inline] -pub fn crc64_nvme_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-NVME PCLMUL 8-way kernel. #[inline] -pub fn crc64_nvme_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_pclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies PCLMULQDQ before selecting this kernel. unsafe { crc64_nvme_pclmul_8way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL kernel. #[inline] -pub fn crc64_nvme_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Dispatcher verifies VPCLMULQDQ + AVX-512 before selecting this kernel. unsafe { crc64_nvme_vpclmul(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 2-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_2way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_2way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 4-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_4way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_4way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 7-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_7way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_7way(crc, data) } } /// Safe wrapper for CRC-64-NVME VPCLMUL 8-way kernel. #[inline] -pub fn crc64_nvme_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { +pub(super) fn crc64_nvme_vpclmul_8way_safe(crc: u64, data: &[u8]) -> u64 { // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. unsafe { crc64_nvme_vpclmul_8way(crc, data) } } -/// Safe wrapper for CRC-64-NVME VPCLMUL 4×512-bit kernel. -#[inline] -pub fn crc64_nvme_vpclmul_4x512_safe(crc: u64, data: &[u8]) -> u64 { - // SAFETY: Callers must verify VPCLMUL_READY before selecting this kernel. - unsafe { crc64_nvme_vpclmul_4x512(crc, data) } -} - // Tests // Tests require SIMD intrinsics that Miri cannot interpret. @@ -2336,8 +2551,17 @@ mod tests { use super::*; fn make_data(len: usize) -> Vec { + let mut byte_index = 0u8; + let mut group_index = 0u8; (0..len) - .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 3) as u8)) + .map(|_| { + let byte = byte_index.wrapping_mul(17).wrapping_add(group_index); + byte_index = byte_index.wrapping_add(1); + if byte_index & 7 == 0 { + group_index = group_index.wrapping_add(1); + } + byte + }) .collect() } @@ -2485,37 +2709,108 @@ mod tests { #[test] fn test_crc64_xz_vpclmul_multiway_matches_portable_various_lengths() { - if !(std::arch::is_x86_feature_detected!("avx512f") && std::arch::is_x86_feature_detected!("vpclmulqdq")) { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { return; } - for len in [0usize, 1, 7, 16, 63, 64, 127, 128, 255, 256, 512, 1024, 4096, 16 * 1024] { + for len in [ + 0usize, + 1, + 7, + 16, + 63, + 64, + 127, + 128, + 255, + 256, + 512, + 1023, + 1024, + 1025, + 4096, + 16 * 1024, + ] { let data = make_data(len); let portable = super::super::portable::crc64_slice8_xz(!0, &data) ^ !0; let vp2 = crc64_xz_vpclmul_2way_safe(!0, &data) ^ !0; let vp4 = crc64_xz_vpclmul_4way_safe(!0, &data) ^ !0; let vp7 = crc64_xz_vpclmul_7way_safe(!0, &data) ^ !0; + let vp8 = crc64_xz_vpclmul_8way_safe(!0, &data) ^ !0; assert_eq!(vp2, portable, "2-way mismatch at len={len}"); assert_eq!(vp4, portable, "4-way mismatch at len={len}"); assert_eq!(vp7, portable, "7-way mismatch at len={len}"); + assert_eq!(vp8, portable, "8-way mismatch at len={len}"); + } + } + + #[test] + fn test_crc64_xz_vpclmul_4x512_matches_portable_at_alignment_boundaries() { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { + return; + } + + const BUFFER_LEN: usize = 8256; + const LENGTHS: &[usize] = &[ + 0, 255, 256, 257, 511, 512, 513, 767, 768, 769, 1023, 1024, 1025, 4096, 4097, 8193, + ]; + const STATES: &[u64] = &[0, u64::MAX, 0x0123_4567_89AB_CDEF, 0xDEAD_BEEF_CAFE_BABE]; + + #[repr(align(64))] + struct AlignedBytes([u8; BUFFER_LEN]); + + let mut data = AlignedBytes([0; BUFFER_LEN]); + data.0.copy_from_slice(&make_data(BUFFER_LEN)); + + for &state in STATES { + for offset in 0usize..64 { + for &len in LENGTHS { + let input = &data.0[offset..offset.strict_add(len)]; + let expected_xz = super::super::portable::crc64_slice8_xz(state, input); + assert_eq!( + crc64_xz_vpclmul_4x512_safe(state, input), + expected_xz, + "XZ mismatch at state={state:#018x}, offset={offset}, len={len}" + ); + } + } } } #[test] fn test_crc64_nvme_vpclmul_multiway_matches_portable_various_lengths() { - if !(std::arch::is_x86_feature_detected!("avx512f") && std::arch::is_x86_feature_detected!("vpclmulqdq")) { + if !crate::platform::caps().has(crate::platform::caps::x86::VPCLMUL_READY) { return; } - for len in [0usize, 1, 7, 16, 63, 64, 127, 128, 255, 256, 512, 1024, 4096, 16 * 1024] { + for len in [ + 0usize, + 1, + 7, + 16, + 63, + 64, + 127, + 128, + 255, + 256, + 512, + 1023, + 1024, + 1025, + 4096, + 16 * 1024, + ] { let data = make_data(len); let portable = super::super::portable::crc64_slice8_nvme(!0, &data) ^ !0; let vp2 = crc64_nvme_vpclmul_2way_safe(!0, &data) ^ !0; let vp4 = crc64_nvme_vpclmul_4way_safe(!0, &data) ^ !0; let vp7 = crc64_nvme_vpclmul_7way_safe(!0, &data) ^ !0; + let vp8 = crc64_nvme_vpclmul_8way_safe(!0, &data) ^ !0; assert_eq!(vp2, portable, "2-way mismatch at len={len}"); assert_eq!(vp4, portable, "4-way mismatch at len={len}"); assert_eq!(vp7, portable, "7-way mismatch at len={len}"); + assert_eq!(vp8, portable, "8-way mismatch at len={len}"); } } } diff --git a/src/checksum/diag.rs b/src/checksum/diag.rs index 44c07a81..ede99702 100644 --- a/src/checksum/diag.rs +++ b/src/checksum/diag.rs @@ -38,7 +38,9 @@ pub enum SelectionReason { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Crc32Polynomial { + /// CRC-32/ISO-HDLC, commonly called CRC-32/IEEE. Ieee, + /// CRC-32C using the Castagnoli polynomial. Castagnoli, } @@ -47,50 +49,103 @@ pub enum Crc32Polynomial { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum Crc64Polynomial { + /// CRC-64/XZ using the reflected ECMA-182 polynomial. Xz, + /// CRC-64/NVME. Nvme, } +/// Snapshot explaining the CRC-32 kernel selected for one input length. +/// +/// Capability booleans describe facts reported by the active selection policy; +/// use [`crate::platform`] when direct platform capability detection is needed. #[cfg(feature = "crc32")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Crc32SelectionDiag { + /// Polynomial variant evaluated by the selector. pub polynomial: Crc32Polynomial, + /// Input length supplied to the selector, in bytes. pub len: usize, + /// Detected architecture used to choose the dispatch table. pub arch: Arch, + /// High-level reason the reported kernel was selected. pub reason: SelectionReason, + /// Force request after clamping it to detected platform capabilities. pub effective_force: crate::checksum::config::Crc32Force, + /// Name of the policy family that produced this snapshot. pub policy_family: &'static str, + /// Stable name of the selected kernel. pub selected_kernel: &'static str, + /// Stream count reported by the policy. + /// + /// The table-backed policy reports `1`; wider stream selection is encoded in + /// [`Self::selected_kernel`]. pub selected_streams: u8, + /// First size boundary in the active CRC-32 dispatch table. + /// + /// The legacy field name does not guarantee that the next tier uses hardware + /// CRC instructions. pub portable_to_hwcrc: usize, + /// Hardware-CRC-to-fusion boundary reported by the policy. + /// + /// The table-backed policy currently reports its first size boundary here. pub hwcrc_to_fusion: usize, + /// Fusion-to-AVX-512 boundary, or [`usize::MAX`] when not modeled separately. pub fusion_to_avx512: usize, + /// Fusion-to-VPCLMUL boundary, or [`usize::MAX`] when not modeled separately. pub fusion_to_vpclmul: usize, + /// Minimum bytes per reported stream, or [`usize::MAX`] when unavailable. pub min_bytes_per_lane: usize, + /// Whether the policy classified this selection as memory-bound. pub memory_bound: bool, + /// Whether the policy reported a hardware CRC tier. pub has_hwcrc: bool, + /// Whether the policy reported a fused CRC/CLMUL tier. pub has_fusion: bool, + /// Whether the policy reported VPCLMUL support. pub has_vpclmul: bool, + /// Whether the policy reported AVX-512 support. pub has_avx512: bool, + /// Whether the policy reported an AArch64 EOR3 tier. pub has_eor3: bool, + /// Whether the policy reported an AArch64 SVE2 tier. pub has_sve2: bool, } +/// Snapshot explaining the CRC-64 kernel selected for one input length. #[cfg(feature = "crc64")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Crc64SelectionDiag { + /// Polynomial variant evaluated by the selector. pub polynomial: Crc64Polynomial, + /// Input length supplied to the selector, in bytes. pub len: usize, + /// Detected architecture used to choose the dispatch table. pub arch: Arch, + /// High-level reason the reported kernel was selected. pub reason: SelectionReason, + /// Force request after clamping it to detected platform capabilities. pub effective_force: crate::checksum::config::Crc64Force, + /// Name of the policy family that produced this snapshot. pub policy_family: &'static str, + /// Stable name of the selected kernel. pub selected_kernel: &'static str, + /// Stream count reported by the policy. + /// + /// The table-backed policy reports `1`; wider stream selection is encoded in + /// [`Self::selected_kernel`]. pub selected_streams: u8, + /// Boundary between portable and carryless-multiply tiers, in bytes. pub portable_to_clmul: usize, + /// Boundary between narrow and wide carryless-multiply tiers, in bytes. + /// + /// The legacy field name also represents equivalent non-x86 wide tiers. pub pclmul_to_vpclmul: usize, + /// Largest input size assigned to the small-buffer kernel, in bytes. pub small_kernel_max_bytes: usize, + /// Whether the policy selected the four-lane 512-bit VPCLMUL strategy. pub use_4x512: bool, + /// Minimum bytes per reported stream, or [`usize::MAX`] when unavailable. pub min_bytes_per_lane: usize, } diff --git a/src/checksum/dispatchers.rs b/src/checksum/dispatchers.rs index 53e9d9c1..561f77f4 100644 --- a/src/checksum/dispatchers.rs +++ b/src/checksum/dispatchers.rs @@ -17,7 +17,7 @@ /// /// Updated CRC state after processing the input data. #[cfg(feature = "crc16")] -pub type Crc16Fn = fn(u16, &[u8]) -> u16; +pub(in crate::checksum) type Crc16Fn = fn(u16, &[u8]) -> u16; // CRC-24 Kernel Function Type @@ -35,7 +35,7 @@ pub type Crc16Fn = fn(u16, &[u8]) -> u16; /// /// Updated CRC state with the result in the low 24 bits. #[cfg(feature = "crc24")] -pub type Crc24Fn = fn(u32, &[u8]) -> u32; +pub(in crate::checksum) type Crc24Fn = fn(u32, &[u8]) -> u32; // CRC-32 Kernel Function Type @@ -59,7 +59,7 @@ pub type Crc24Fn = fn(u32, &[u8]) -> u32; /// /// Updated CRC state after processing the input data. #[cfg(feature = "crc32")] -pub type Crc32Fn = fn(u32, &[u8]) -> u32; +pub(in crate::checksum) type Crc32Fn = fn(u32, &[u8]) -> u32; // CRC-64 Kernel Function Type @@ -84,4 +84,4 @@ pub type Crc32Fn = fn(u32, &[u8]) -> u32; /// /// Updated CRC state after processing the input data. #[cfg(feature = "crc64")] -pub type Crc64Fn = fn(u64, &[u8]) -> u64; +pub(in crate::checksum) type Crc64Fn = fn(u64, &[u8]) -> u64; diff --git a/src/checksum/kernel_table.rs b/src/checksum/kernel_table.rs index 974679e3..c8dce155 100644 --- a/src/checksum/kernel_table.rs +++ b/src/checksum/kernel_table.rs @@ -331,7 +331,7 @@ pub(crate) struct KernelSet { impl KernelSet { /// Extract the hot-path function pointer set. #[inline] - pub const fn fns(&self) -> KernelFnSet { + const fn fns(&self) -> KernelFnSet { KernelFnSet { #[cfg(feature = "crc16")] crc16_ccitt: self.crc16_ccitt, @@ -352,7 +352,7 @@ impl KernelSet { /// Extract the cold-path name set. #[inline] - pub const fn names(&self) -> KernelNameSet { + const fn names(&self) -> KernelNameSet { KernelNameSet { #[cfg(feature = "crc16")] crc16_ccitt_name: self.crc16_ccitt_name, @@ -410,7 +410,7 @@ const L: usize = 3; impl KernelTable { /// Construct a `KernelTable` from four [`KernelSet`] definitions, splitting /// function pointers from name strings at compile time. - pub const fn from_sets( + const fn from_sets( requires: Caps, boundaries: [usize; 3], xs: KernelSet, @@ -428,7 +428,7 @@ impl KernelTable { /// Select the hot-path function pointer set for the given buffer length. #[inline] - pub const fn select_fns(&self, len: usize) -> &KernelFnSet { + pub(in crate::checksum) const fn select_fns(&self, len: usize) -> &KernelFnSet { if len <= self.boundaries[0] { &self.fns[XS] } else if len <= self.boundaries[1] { @@ -444,7 +444,7 @@ impl KernelTable { /// /// Used only for introspection / diagnostics; never called on the hot path. #[inline] - pub const fn select_names(&self, len: usize) -> &KernelNameSet { + pub(in crate::checksum) const fn select_names(&self, len: usize) -> &KernelNameSet { if len <= self.boundaries[0] { &self.names[XS] } else if len <= self.boundaries[1] { @@ -460,10 +460,9 @@ impl KernelTable { /// /// A table is hardware-accelerated if it requires any CPU capabilities /// beyond the baseline (i.e., `requires != Caps::NONE`). - #[cfg(any(test, feature = "diag"))] - #[cfg_attr(test, allow(dead_code))] + #[cfg(feature = "diag")] #[inline] - pub const fn is_hardware_accelerated(&self) -> bool { + const fn is_hardware_accelerated(&self) -> bool { !self.requires.is_empty() } } @@ -474,6 +473,13 @@ impl KernelTable { /// Accepts the same field layout as the old monolithic `KernelTable { ... }` /// struct literal (with `requires`, `boundaries`, `xs`, `s`, `m`, `l` labels), /// converting it to `KernelTable::from_sets(...)` under the hood. +#[cfg(any( + target_arch = "aarch64", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" +))] macro_rules! kernel_table { ( requires: $req:expr, @@ -488,8 +494,7 @@ macro_rules! kernel_table { } /// Returns `true` if the active kernel table uses hardware-accelerated CRC kernels. -#[cfg(any(test, feature = "diag"))] -#[cfg_attr(test, allow(dead_code))] +#[cfg(feature = "diag")] #[inline] pub fn is_hardware_accelerated() -> bool { active_table().is_hardware_accelerated() @@ -669,18 +674,6 @@ fn capability_match(caps: Caps) -> Option<&'static KernelTable> { #[cfg(feature = "crc64")] #[inline] fn capability_match_crc64(caps: Caps) -> Option<&'static KernelTable> { - #[cfg(target_arch = "riscv64")] - { - use crate::platform::caps::riscv::{V, ZBC, ZVBC}; - let v_zvbc = V.union(ZVBC); - if caps.has(v_zvbc) { - return Some(&RISCV64_CRC64_ZVBC_TABLE); - } - if caps.has(ZBC) { - return Some(&RISCV64_CRC64_ZBC_TABLE); - } - } - capability_match(caps) } @@ -720,7 +713,7 @@ const PORTABLE_SET: KernelSet = KernelSet { crc64_nvme_name: "portable/slice16", }; -pub static PORTABLE_TABLE: KernelTable = KernelTable::from_sets( +static PORTABLE_TABLE: KernelTable = KernelTable::from_sets( Caps::NONE, [64, 256, 4096], PORTABLE_SET, @@ -743,14 +736,24 @@ mod aarch64_tables { #[cfg(feature = "crc64")] use crate::checksum::crc64::kernels::aarch64 as crc64_k; - #[cfg(all(feature = "crc16", not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all( + feature = "crc16", + feature = "std", + not(miri), + any(target_os = "linux", target_os = "android") + ))] const G3_CRC16_PMULL_EOR3_2WAY_MAX_LEN: usize = 262_144; /// Graviton3 CRC16/CCITT large-path PMULL+EOR3 hybrid. /// /// PMULL+EOR3 2-way improves lower "large" buffers, while PMULL+EOR3 1-way /// remains safer for very large buffers on G3. - #[cfg(all(feature = "crc16", not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all( + feature = "crc16", + feature = "std", + not(miri), + any(target_os = "linux", target_os = "android") + ))] #[inline] fn g3_crc16_ccitt_l_hybrid(crc: u16, data: &[u8]) -> u16 { if data.len() <= G3_CRC16_PMULL_EOR3_2WAY_MAX_LEN { @@ -764,7 +767,12 @@ mod aarch64_tables { /// /// Empirically, 2-way PMULL+EOR3 helps lower "large" buffers while 1-way /// PMULL+EOR3 remains safer for very large buffers on G3. - #[cfg(all(feature = "crc16", not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all( + feature = "crc16", + feature = "std", + not(miri), + any(target_os = "linux", target_os = "android") + ))] #[inline] fn g3_crc16_ibm_l_hybrid(crc: u16, data: &[u8]) -> u16 { if data.len() <= G3_CRC16_PMULL_EOR3_2WAY_MAX_LEN { @@ -777,7 +785,7 @@ mod aarch64_tables { // Apple M1-M3 table. // // Features: PMULL + SHA3 (EOR3) - pub static APPLE_M1M3_TABLE: KernelTable = kernel_table! { + pub(super) static APPLE_M1M3_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY .union(crate::platform::caps::aarch64::PMULL_EOR3_READY) .union(crate::platform::caps::aarch64::PMULL_READY), @@ -911,8 +919,8 @@ mod aarch64_tables { // Graviton2 table. // // Features: PMULL (no EOR3/SHA3) - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub static GRAVITON2_TABLE: KernelTable = kernel_table! { + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(super) static GRAVITON2_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), boundaries: [64, 256, 4096], @@ -1044,7 +1052,7 @@ mod aarch64_tables { // Graviton3 table. // // Features: PMULL + SHA3/EOR3 - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_XS: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: crc16_k::CCITT_PMULL_SMALL_KERNEL, @@ -1076,7 +1084,7 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull-small", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_S: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: crc16_k::CCITT_PMULL[0], @@ -1108,7 +1116,7 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_M: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: crc16_k::CCITT_PMULL_EOR3[0], // PMULL+EOR3 cuts XOR chain in large lanes. @@ -1140,7 +1148,7 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull-2way", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] const G3_L: KernelSet = KernelSet { #[cfg(feature = "crc16")] crc16_ccitt: g3_crc16_ccitt_l_hybrid, @@ -1172,8 +1180,8 @@ mod aarch64_tables { crc64_nvme_name: "aarch64/pmull-eor3", }; - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub static GRAVITON3_TABLE: KernelTable = KernelTable::from_sets( + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(super) static GRAVITON3_TABLE: KernelTable = KernelTable::from_sets( crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), [64, 256, 4096], G3_XS, @@ -1185,8 +1193,8 @@ mod aarch64_tables { // Graviton4 Table // // Starts from Graviton3, but keeps 2-way PMULL+EOR3 for CRC16 large classes. - #[cfg(all(not(miri), any(target_os = "linux", target_os = "android")))] - pub static GRAVITON4_TABLE: KernelTable = KernelTable::from_sets( + #[cfg(all(feature = "std", not(miri), any(target_os = "linux", target_os = "android")))] + pub(super) static GRAVITON4_TABLE: KernelTable = KernelTable::from_sets( crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_EOR3_READY), [64, 256, 4096], G3_XS, @@ -1200,7 +1208,26 @@ mod aarch64_tables { crc16_ibm: crc16_k::IBM_PMULL_EOR3[1], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-eor3-2way", - ..G3_M + #[cfg(feature = "crc24")] + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], + #[cfg(feature = "crc24")] + crc24_openpgp_name: "aarch64/pmull", + #[cfg(feature = "crc32")] + crc32_ieee: crc32_k::CRC32_PMULL_SMALL_KERNEL, + #[cfg(feature = "crc32")] + crc32_ieee_name: "aarch64/pmull-small", + #[cfg(feature = "crc32")] + crc32c: crc32_k::CRC32C_PMULL_SMALL_KERNEL, + #[cfg(feature = "crc32")] + crc32c_name: "aarch64/pmull-small", + #[cfg(feature = "crc64")] + crc64_xz: crc64_k::XZ_PMULL[0], + #[cfg(feature = "crc64")] + crc64_xz_name: "aarch64/pmull", + #[cfg(feature = "crc64")] + crc64_nvme: crc64_k::NVME_PMULL[1], + #[cfg(feature = "crc64")] + crc64_nvme_name: "aarch64/pmull-2way", }, KernelSet { #[cfg(feature = "crc16")] @@ -1211,7 +1238,26 @@ mod aarch64_tables { crc16_ibm: crc16_k::IBM_PMULL_EOR3[1], #[cfg(feature = "crc16")] crc16_ibm_name: "aarch64/pmull-eor3-2way", - ..G3_L + #[cfg(feature = "crc24")] + crc24_openpgp: crc24_k::OPENPGP_PMULL[0], + #[cfg(feature = "crc24")] + crc24_openpgp_name: "aarch64/pmull", + #[cfg(feature = "crc32")] + crc32_ieee: crc32_k::CRC32_PMULL_EOR3[0], + #[cfg(feature = "crc32")] + crc32_ieee_name: "aarch64/pmull-eor3-v9s3x2e-s3", + #[cfg(feature = "crc32")] + crc32c: crc32_k::CRC32C_PMULL_EOR3[0], + #[cfg(feature = "crc32")] + crc32c_name: "aarch64/pmull-eor3-v9s3x2e-s3", + #[cfg(feature = "crc64")] + crc64_xz: crc64_k::XZ_PMULL_EOR3[0], + #[cfg(feature = "crc64")] + crc64_xz_name: "aarch64/pmull-eor3", + #[cfg(feature = "crc64")] + crc64_nvme: crc64_k::NVME_PMULL_EOR3[0], + #[cfg(feature = "crc64")] + crc64_nvme_name: "aarch64/pmull-eor3", }, ); @@ -1219,12 +1265,12 @@ mod aarch64_tables { // // For unknown ARM platforms with PMULL + SHA3 features. // Uses Apple M1-M3 selections (good EOR3 support). - pub static GENERIC_ARM_PMULL_EOR3_TABLE: KernelTable = APPLE_M1M3_TABLE; + pub(super) static GENERIC_ARM_PMULL_EOR3_TABLE: KernelTable = APPLE_M1M3_TABLE; // Generic ARM PMULL Table (conservative) // // For unknown ARM platforms with CRC + PMULL but *without* SHA3/EOR3. - pub static GENERIC_ARM_PMULL_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_ARM_PMULL_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY.union(crate::platform::caps::aarch64::PMULL_READY), boundaries: [64, 256, 4096], @@ -1354,7 +1400,7 @@ mod aarch64_tables { }; /// PMULL-only table for platforms without the CRC extension. - pub static GENERIC_ARM_PMULL_NO_CRC_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_ARM_PMULL_NO_CRC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::PMULL_READY, boundaries: [64, 256, 4096], @@ -1484,7 +1530,7 @@ mod aarch64_tables { }; /// CRC-only table for platforms without PMULL. - pub static GENERIC_ARM_CRC_ONLY_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_ARM_CRC_ONLY_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::aarch64::CRC_READY, boundaries: [64, 256, 4096], @@ -1648,7 +1694,7 @@ mod x86_64_tables { // Zen4 table. // // Features: VPCLMULQDQ + AVX-512 - pub static ZEN4_TABLE: KernelTable = kernel_table! { + pub(super) static ZEN4_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::VPCLMUL_READY .union(crate::platform::caps::x86::PCLMUL_READY) .union(crate::platform::caps::x86::CRC32C_READY), @@ -1783,12 +1829,12 @@ mod x86_64_tables { // // For unknown x86-64 platforms with VPCLMULQDQ. // Uses Zen4 selections (good AVX-512/VPCLMUL support). - pub static GENERIC_X86_VPCLMUL_TABLE: KernelTable = ZEN4_TABLE; + pub(super) static GENERIC_X86_VPCLMUL_TABLE: KernelTable = ZEN4_TABLE; /// VPCLMUL table that never selects SSE4.2 CRC32C instructions/fusion. /// /// Use on systems with VPCLMUL but without SSE4.2 (`CRC32C_READY`). - pub static GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::VPCLMUL_READY.union(crate::platform::caps::x86::PCLMUL_READY), boundaries: [64, 256, 4096], @@ -1920,7 +1966,7 @@ mod x86_64_tables { // Generic x86-64 PCLMUL Table (conservative) // // Features: PCLMULQDQ only - pub static GENERIC_X86_PCLMUL_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_PCLMUL_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::PCLMUL_READY.union(crate::platform::caps::x86::CRC32C_READY), boundaries: [64, 256, 4096], @@ -2052,7 +2098,7 @@ mod x86_64_tables { /// PCLMUL table that never selects SSE4.2 CRC32C instructions/fusion. /// /// Use on systems with PCLMUL but without SSE4.2 (`CRC32C_READY`). - pub static GENERIC_X86_PCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_PCLMUL_NO_CRC32C_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::PCLMUL_READY, boundaries: [64, 256, 4096], @@ -2182,7 +2228,7 @@ mod x86_64_tables { }; /// SSE4.2-only table: accelerate CRC32C, keep other variants portable. - pub static GENERIC_X86_CRC32C_ONLY_TABLE: KernelTable = kernel_table! { + pub(super) static GENERIC_X86_CRC32C_ONLY_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::x86::CRC32C_READY, boundaries: [64, 256, 4096], @@ -2313,7 +2359,7 @@ mod x86_64_tables { } #[cfg(target_arch = "x86_64")] -pub use x86_64_tables::*; +use x86_64_tables::*; // s390x Platform Tables @@ -2329,7 +2375,7 @@ mod s390x_tables { #[cfg(feature = "crc64")] use crate::checksum::crc64::kernels::s390x as crc64_k; - pub static S390X_Z13_TABLE: KernelTable = kernel_table! { + pub(super) static S390X_Z13_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::s390x::Z13_READY, boundaries: [64, 256, 4096], xs: PORTABLE_SET, @@ -2396,7 +2442,7 @@ mod s390x_tables { }, }; - pub static S390X_Z14_TABLE: KernelTable = kernel_table! { + pub(super) static S390X_Z14_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::s390x::Z13_READY, boundaries: [64, 128, 4096], xs: PORTABLE_SET, @@ -2463,7 +2509,7 @@ mod s390x_tables { }, }; - pub static S390X_Z15_TABLE: KernelTable = kernel_table! { + pub(super) static S390X_Z15_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::s390x::Z13_READY, boundaries: [63, 63, 4096], xs: PORTABLE_SET, @@ -2532,7 +2578,7 @@ mod s390x_tables { } #[cfg(target_arch = "s390x")] -pub use s390x_tables::*; +use s390x_tables::*; // powerpc64 Platform Tables @@ -2548,7 +2594,7 @@ mod power_tables { #[cfg(feature = "crc64")] use crate::checksum::crc64::kernels::power as crc64_k; - pub static POWER8_TABLE: KernelTable = kernel_table! { + pub(super) static POWER8_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::power::VPMSUM_READY, boundaries: [64, 128, 4096], xs: PORTABLE_SET, @@ -2615,7 +2661,7 @@ mod power_tables { }, }; - pub static POWER9_TABLE: KernelTable = kernel_table! { + pub(super) static POWER9_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::power::VPMSUM_READY, boundaries: [64, 64, 4096], xs: PORTABLE_SET, @@ -2682,7 +2728,7 @@ mod power_tables { }, }; - pub static POWER10_TABLE: KernelTable = kernel_table! { + pub(super) static POWER10_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::power::VPMSUM_READY, boundaries: [63, 63, 4096], xs: PORTABLE_SET, @@ -2751,7 +2797,7 @@ mod power_tables { } #[cfg(target_arch = "powerpc64")] -pub use power_tables::*; +use power_tables::*; // riscv64 Platform Tables @@ -2764,10 +2810,8 @@ mod riscv64_tables { use crate::checksum::crc24::kernels::riscv64 as crc24_k; #[cfg(feature = "crc32")] use crate::checksum::crc32::kernels::riscv64 as crc32_k; - #[cfg(feature = "crc64")] - use crate::checksum::crc64::kernels::riscv64 as crc64_k; - pub static RISCV64_ZBC_TABLE: KernelTable = kernel_table! { + pub(super) static RISCV64_ZBC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::riscv::ZBC, boundaries: [63, 1024, 4096], xs: PORTABLE_SET, @@ -2866,7 +2910,7 @@ mod riscv64_tables { }, }; - pub static RISCV64_ZVBC_TABLE: KernelTable = kernel_table! { + pub(super) static RISCV64_ZVBC_TABLE: KernelTable = kernel_table! { requires: crate::platform::caps::riscv::V.union(crate::platform::caps::riscv::ZVBC), boundaries: [63, 1024, 4096], xs: PORTABLE_SET, @@ -2892,13 +2936,13 @@ mod riscv64_tables { #[cfg(feature = "crc32")] crc32c_name: "riscv64/zvbc", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_ZVBC[0], + crc64_xz: crate::checksum::crc64::portable::crc64_slice16_xz, #[cfg(feature = "crc64")] - crc64_xz_name: "riscv64/zvbc", + crc64_xz_name: "portable/slice16", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_ZVBC[0], + crc64_nvme: crate::checksum::crc64::portable::crc64_slice16_nvme, #[cfg(feature = "crc64")] - crc64_nvme_name: "riscv64/zvbc", + crc64_nvme_name: "portable/slice16", }, m: KernelSet { #[cfg(feature = "crc16")] @@ -2922,13 +2966,13 @@ mod riscv64_tables { #[cfg(feature = "crc32")] crc32c_name: "riscv64/zvbc-2way", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_ZVBC[1], + crc64_xz: crate::checksum::crc64::portable::crc64_slice16_xz, #[cfg(feature = "crc64")] - crc64_xz_name: "riscv64/zvbc-2way", + crc64_xz_name: "portable/slice16", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_ZVBC[1], + crc64_nvme: crate::checksum::crc64::portable::crc64_slice16_nvme, #[cfg(feature = "crc64")] - crc64_nvme_name: "riscv64/zvbc-2way", + crc64_nvme_name: "portable/slice16", }, l: KernelSet { #[cfg(feature = "crc16")] @@ -2952,44 +2996,19 @@ mod riscv64_tables { #[cfg(feature = "crc32")] crc32c_name: "riscv64/zvbc-4way", #[cfg(feature = "crc64")] - crc64_xz: crc64_k::XZ_ZVBC[2], + crc64_xz: crate::checksum::crc64::portable::crc64_slice16_xz, #[cfg(feature = "crc64")] - crc64_xz_name: "riscv64/zvbc-4way", + crc64_xz_name: "portable/slice16", #[cfg(feature = "crc64")] - crc64_nvme: crc64_k::NVME_ZVBC[2], + crc64_nvme: crate::checksum::crc64::portable::crc64_slice16_nvme, #[cfg(feature = "crc64")] - crc64_nvme_name: "riscv64/zvbc-4way", + crc64_nvme_name: "portable/slice16", }, }; - - #[cfg(feature = "crc64")] - // Keep CRC64 auto on portable slice-by-16 for now. - // - // No tracked target evidence currently justifies selecting the accelerated - // CRC64 kernels by default. - #[cfg(feature = "crc64")] - pub static RISCV64_CRC64_ZBC_TABLE: KernelTable = kernel_table! { - requires: crate::platform::caps::riscv::ZBC, - boundaries: [128, 4096, 16384], - xs: PORTABLE_SET, - s: PORTABLE_SET, - m: PORTABLE_SET, - l: PORTABLE_SET, - }; - - #[cfg(feature = "crc64")] - pub static RISCV64_CRC64_ZVBC_TABLE: KernelTable = kernel_table! { - requires: crate::platform::caps::riscv::V.union(crate::platform::caps::riscv::ZVBC), - boundaries: [63, 1024, 4096], - xs: PORTABLE_SET, - s: PORTABLE_SET, - m: PORTABLE_SET, - l: PORTABLE_SET, - }; } #[cfg(target_arch = "riscv64")] -pub use riscv64_tables::*; +use riscv64_tables::*; // Tests @@ -3004,14 +3023,55 @@ mod tests { // Verify portable table returns correct sets for each size class let table = &PORTABLE_TABLE; - assert!(core::ptr::eq(table.select_fns(0), &table.fns[XS])); - assert!(core::ptr::eq(table.select_fns(64), &table.fns[XS])); - assert!(core::ptr::eq(table.select_fns(65), &table.fns[S])); - assert!(core::ptr::eq(table.select_fns(256), &table.fns[S])); - assert!(core::ptr::eq(table.select_fns(257), &table.fns[M])); - assert!(core::ptr::eq(table.select_fns(4096), &table.fns[M])); - assert!(core::ptr::eq(table.select_fns(4097), &table.fns[L])); - assert!(core::ptr::eq(table.select_fns(1_000_000), &table.fns[L])); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(0)), + core::ptr::from_ref(&table.fns[XS]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(64)), + core::ptr::from_ref(&table.fns[XS]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(65)), + core::ptr::from_ref(&table.fns[S]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(256)), + core::ptr::from_ref(&table.fns[S]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(257)), + core::ptr::from_ref(&table.fns[M]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(4096)), + core::ptr::from_ref(&table.fns[M]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(4097)), + core::ptr::from_ref(&table.fns[L]) + )); + assert!(core::ptr::eq( + core::ptr::from_ref(table.select_fns(1_000_000)), + core::ptr::from_ref(&table.fns[L]) + )); + } + + #[test] + #[cfg(all(target_arch = "x86_64", feature = "crc64"))] + fn test_vpclmul_without_crc32c_selects_crc64_xz_4x512_above_4096() { + use crate::platform::caps::x86::{CRC32C_READY, VPCLMUL_READY}; + + assert!(!VPCLMUL_READY.has(CRC32C_READY)); + let table = select_crc64_table(VPCLMUL_READY); + assert!(core::ptr::eq(table, &raw const GENERIC_X86_VPCLMUL_NO_CRC32C_TABLE)); + assert_eq!(table.select_names(4096).crc64_xz_name, "x86_64/vpclmul-2way"); + assert_eq!(table.select_names(4097).crc64_xz_name, "x86_64/vpclmul-4x512"); + assert_eq!(table.select_names(4097).crc64_nvme_name, "x86_64/vpclmul-4way"); + assert!(core::ptr::fn_addr_eq( + table.select_fns(4097).crc64_xz, + crate::checksum::crc64::kernels::x86_64::XZ_VPCLMUL_4X512 + )); } #[test] @@ -3045,7 +3105,7 @@ mod tests { #[cfg(feature = "crc64")] { - let crc64_names = RISCV64_CRC64_ZVBC_TABLE.select_names(64); + let crc64_names = RISCV64_ZVBC_TABLE.select_names(64); assert_eq!(crc64_names.crc64_xz_name, "portable/slice16"); assert_eq!(crc64_names.crc64_nvme_name, "portable/slice16"); } @@ -3055,7 +3115,10 @@ mod tests { fn test_select_table_fallback() { // With no capabilities, should return portable table let table = select_table(Caps::NONE); - assert!(core::ptr::eq(table, &PORTABLE_TABLE)); + assert!(core::ptr::eq( + core::ptr::from_ref(table), + core::ptr::from_ref(&PORTABLE_TABLE) + )); } // Oneshot Function Tests @@ -3146,7 +3209,7 @@ mod tests { let sizes = [1, 64, 65, 256, 257, 4096, 4097, 65536]; for &size in &sizes { - let data: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + let data: Vec = (0u8..=u8::MAX).cycle().take(size).collect(); // Just verify no panics and consistent non-zero results for non-empty data #[cfg(feature = "crc64")] let _ = crc64_xz(&data); diff --git a/src/checksum/macros.rs b/src/checksum/macros.rs index 7737a4f7..509fcfcc 100644 --- a/src/checksum/macros.rs +++ b/src/checksum/macros.rs @@ -42,10 +42,7 @@ macro_rules! crc_vectored_dispatch { /// This covers the repeated "resolve once under std, otherwise use auto" /// pattern shared by CRC-16/32/64 variants while leaving the actual kernel /// wrappers and force-specific architecture helpers local to each module. -#[cfg_attr( - not(any(feature = "crc16", feature = "crc32", feature = "crc64")), - allow(unused_macros) -)] +#[cfg(any(feature = "crc16", feature = "crc32", feature = "crc64"))] macro_rules! define_crc_dispatch { ( word_ty: $word_ty:ty, @@ -190,7 +187,6 @@ macro_rules! define_buffered_crc { /// /// Data is buffered internally until enough accumulates for efficient /// SIMD processing. - #[allow(clippy::indexing_slicing)] // Safety: All slice indices are bounds-checked by the algorithm: // - self.len < buffer_size (invariant maintained by this function) // - fill = min(input.len(), space), so input[..fill] and buffer[len..len+fill] are valid @@ -250,7 +246,6 @@ macro_rules! define_buffered_crc { /// /// Flushes any remaining buffered data before computing the final CRC. #[must_use] - #[allow(clippy::indexing_slicing)] // Safety: self.len < buffer_size (invariant) pub fn finalize(&self) -> <$inner as $crate::Checksum>::Output { if self.len > 0 { diff --git a/src/checksum/mod.rs b/src/checksum/mod.rs index b0f1c2c3..a32de765 100644 --- a/src/checksum/mod.rs +++ b/src/checksum/mod.rs @@ -47,10 +47,10 @@ //! ```toml //! [dependencies] //! # Smallest CRC-32-only build -//! rscrypto = { version = "0.7", default-features = false, features = ["crc32"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["crc32"] } //! //! # All checksum families -//! rscrypto = { version = "0.7", default-features = false, features = ["checksums"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["checksums"] } //! ``` //! //! # API Conventions @@ -76,7 +76,7 @@ //! //! ```toml //! [dependencies] -//! rscrypto = { version = "0.7", default-features = false, features = ["crc32"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["crc32"] } //! ``` mod common; @@ -104,6 +104,7 @@ pub mod io; #[doc(hidden)] pub(crate) mod kernel_table; #[cfg(feature = "alloc")] +/// Buffered checksum adapters for workloads composed of many short updates. pub mod buffered { #[cfg(feature = "crc16")] pub use crate::checksum::crc16::{BufferedCrc16Ccitt, BufferedCrc16Ibm}; diff --git a/src/hashes/common/mod.rs b/src/hashes/common/mod.rs index 2875ebe6..3b8dd548 100644 --- a/src/hashes/common/mod.rs +++ b/src/hashes/common/mod.rs @@ -3,4 +3,4 @@ //! This module currently owns the AArch64 BLAKE3 prefetch helper. #[cfg(all(feature = "blake3", target_arch = "aarch64"))] -pub mod prefetch; +pub(crate) mod prefetch; diff --git a/src/hashes/common/prefetch.rs b/src/hashes/common/prefetch.rs index 83952c65..1887684f 100644 --- a/src/hashes/common/prefetch.rs +++ b/src/hashes/common/prefetch.rs @@ -8,7 +8,6 @@ // SAFETY: This module provides low-level prefetch intrinsics that require unsafe. // Prefetch instructions are hints to the CPU and cannot cause memory unsafety; // invalid addresses are silently ignored. -#![allow(unsafe_code)] /// Prefetch data for read into L1 cache (PLDL1KEEP). /// diff --git a/src/hashes/crypto/ascon.rs b/src/hashes/crypto/ascon.rs index a6032128..ab3e7810 100644 --- a/src/hashes/crypto/ascon.rs +++ b/src/hashes/crypto/ascon.rs @@ -2,8 +2,6 @@ //! //! Portable, `no_std`, pure Rust implementation. -#![allow(clippy::indexing_slicing)] // Fixed-size state + sponge buffering - use core::fmt; use crate::{ @@ -11,18 +9,21 @@ use crate::{ traits::{Digest, Xof}, }; -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", any(test, feature = "std")))] mod aarch64; #[doc(hidden)] +#[cfg(any(test, feature = "std"))] pub(crate) mod dispatch; #[doc(hidden)] +#[cfg(any(test, feature = "std"))] pub(crate) mod dispatch_tables; #[cfg(test)] mod kernel_test; +#[cfg(any(test, feature = "std"))] pub(crate) mod kernels; -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", any(test, feature = "std")))] mod x86_64_avx2; -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", any(test, feature = "std")))] mod x86_64_avx512; const RATE: usize = 8; @@ -48,7 +49,7 @@ impl Permuter for InlinePermuter { // Ascon permutation round constants (12 rounds). // Used by SIMD kernels; the shared portable permutation inlines the constants. -#[allow(dead_code)] +#[cfg(all(any(test, feature = "std"), any(target_arch = "aarch64", target_arch = "x86_64")))] const RC: [u64; 12] = [0xF0, 0xE1, 0xD2, 0xC3, 0xB4, 0xA5, 0x96, 0x87, 0x78, 0x69, 0x5A, 0x4B]; // Domain-specific IVs (from the Ascon hash/XOF specification). @@ -80,7 +81,7 @@ define_unit_error! { const fn pad(n: usize) -> u64 { // Produce the padding mask used by the reference construction: // XOR `pad(len)` into state[0], with state interpreted little-endian. - 0x01_u64 << (8 * n) + 0x01_u64 << n.strict_mul(8) } #[derive(Clone)] @@ -121,7 +122,7 @@ impl( ) { debug_assert_eq!(inputs.len(), N); let len = inputs[0].len(); - let full_bytes = len / RATE * RATE; + let full_bytes = len.strict_sub(len % RATE); for off in (0..full_bytes).step_by(RATE) { for (state, input) in states.iter_mut().zip(inputs.iter().copied()) { let mut block = [0u8; RATE]; - block.copy_from_slice(&input[off..off + RATE]); + block.copy_from_slice(&input[off..off.strict_add(RATE)]); state[0] ^= u64::from_le_bytes(block); } permute_many(states); } - let tail_len = len - full_bytes; + let tail_len = len.strict_sub(full_bytes); for (state, input) in states.iter_mut().zip(inputs.iter().copied()) { let mut block = [0u8; RATE]; block[..tail_len].copy_from_slice(&input[full_bytes..]); @@ -259,10 +260,10 @@ fn squeeze_hash256_group( let mut off = 0usize; while off < 24 { for (state, output) in states.iter().zip(outputs.iter_mut()) { - output[off..off + RATE].copy_from_slice(&state[0].to_le_bytes()); + output[off..off.strict_add(RATE)].copy_from_slice(&state[0].to_le_bytes()); } permute_many(states); - off += RATE; + off = off.strict_add(RATE); } for (state, output) in states.iter().zip(outputs.iter_mut()) { output[24..32].copy_from_slice(&state[0].to_le_bytes()); @@ -276,14 +277,14 @@ fn squeeze_xof_group( outputs: &mut [u8], permute_many: fn(&mut [[u64; 5]; N]), ) { - debug_assert_eq!(outputs.len(), N * out_len); + debug_assert_eq!(outputs.len(), N.strict_mul(out_len)); let mut produced = 0usize; while produced < out_len { - let take = core::cmp::min(RATE, out_len - produced); + let take = core::cmp::min(RATE, out_len.strict_sub(produced)); for (state, output) in states.iter().zip(outputs.chunks_exact_mut(out_len)) { - output[produced..produced + take].copy_from_slice(&state[0].to_le_bytes()[..take]); + output[produced..produced.strict_add(take)].copy_from_slice(&state[0].to_le_bytes()[..take]); } - produced += take; + produced = produced.strict_add(take); if produced < out_len { permute_many(states); } @@ -321,7 +322,7 @@ fn xof_many_equal_len_group( permute_many: fn(&mut [[u64; 5]; N]), ) { debug_assert_eq!(inputs.len(), N); - debug_assert_eq!(outputs.len(), N * out_len); + debug_assert_eq!(outputs.len(), N.strict_mul(out_len)); let mut states = init_states::(iv); absorb_equal_len_group(&mut states, inputs, permute_many); squeeze_xof_group(&mut states, out_len, outputs, permute_many); @@ -383,17 +384,19 @@ impl AsconHash256 { match kid { kernels::AsconPermute12KernelId::Portable => { - let mut input_groups = inputs.chunks_exact(1); - let mut output_groups = outputs.chunks_exact_mut(1); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<1>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<1>(); + debug_assert!(rem_inputs.is_empty()); + debug_assert!(rem_outputs.is_empty()); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<1>(group_inputs, group_outputs, HASH256_IV, permute_12_many_portable::<1>); } } #[cfg(target_arch = "aarch64")] kernels::AsconPermute12KernelId::Aarch64Neon => { - let mut input_groups = inputs.chunks_exact(2); - let mut output_groups = outputs.chunks_exact_mut(2); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<2>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<2>(); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<2>( group_inputs, group_outputs, @@ -401,8 +404,6 @@ impl AsconHash256 { aarch64::permute_12_aarch64_neon_x2, ); } - let rem_inputs = input_groups.remainder(); - let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::digest_many_with_kernel( dispatch::batch_fallback_kernel_id(kid, rem_inputs.len()), @@ -413,9 +414,9 @@ impl AsconHash256 { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx2 => { - let mut input_groups = inputs.chunks_exact(4); - let mut output_groups = outputs.chunks_exact_mut(4); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<4>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<4>(); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<4>( group_inputs, group_outputs, @@ -423,8 +424,6 @@ impl AsconHash256 { x86_64_avx2::permute_12_x86_avx2_x4, ); } - let rem_inputs = input_groups.remainder(); - let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::digest_many_with_kernel( dispatch::batch_fallback_kernel_id(kid, rem_inputs.len()), @@ -435,9 +434,9 @@ impl AsconHash256 { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx512 => { - let mut input_groups = inputs.chunks_exact(8); - let mut output_groups = outputs.chunks_exact_mut(8); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<8>(); + let (output_groups, rem_outputs) = outputs.as_chunks_mut::<8>(); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.iter_mut()) { digest_many_equal_len_group::<8>( group_inputs, group_outputs, @@ -445,8 +444,6 @@ impl AsconHash256 { x86_64_avx512::permute_12_x86_avx512_x8, ); } - let rem_inputs = input_groups.remainder(); - let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::digest_many_with_kernel( dispatch::batch_fallback_kernel_id(kid, rem_inputs.len()), @@ -472,12 +469,12 @@ impl AsconHash256 { let mut start = 0usize; while start < inputs.len() { let len = inputs[start].len(); - let mut end = start + 1; + let mut end = start.strict_add(1); while end < inputs.len() && inputs[end].len() == len { - end += 1; + end = end.strict_add(1); } - let kid = Self::batch_kernel_id_for_count(end - start); + let kid = Self::batch_kernel_id_for_count(end.strict_sub(start)); Self::digest_many_with_kernel(kid, &inputs[start..end], &mut outputs[start..end]); start = end; } @@ -531,12 +528,14 @@ impl fmt::Debug for AsconXof { } impl AsconXof { + /// Creates an empty Ascon-XOF128 state. #[inline] #[must_use] pub fn new() -> Self { Self::default() } + /// Absorbs `data` and returns an Ascon-XOF128 output reader. #[inline] #[must_use] pub fn xof(data: &[u8]) -> AsconXofReader { @@ -544,16 +543,19 @@ impl AsconXof { AsconXofReader::from_state(state) } + /// Absorbs more input into the XOF state. #[inline] pub fn update(&mut self, data: &[u8]) { self.sponge.update(data); } + /// Resets the XOF state to its initial value. #[inline] pub fn reset(&mut self) { *self = Self::default(); } + /// Finalizes the current state into an output reader without consuming it. #[inline] #[must_use] pub fn finalize_xof(&self) -> AsconXofReader { @@ -592,10 +594,10 @@ impl AsconXof { ) { assert_eq!( outputs.len(), - inputs.len() * out_len, + inputs.len().strict_mul(out_len), "input/output batch length mismatch" ); - if inputs.is_empty() { + if inputs.is_empty() || out_len == 0 { return; } @@ -604,17 +606,18 @@ impl AsconXof { if degree == 1 || !inputs_have_equal_len(inputs) { let scalar_kid = dispatch::scalar_kernel_id(); for (index, input) in inputs.iter().enumerate() { - let base = index * out_len; - Self::hash_into_with_kernel(scalar_kid, input, &mut outputs[base..base + out_len]); + let base = index.strict_mul(out_len); + Self::hash_into_with_kernel(scalar_kid, input, &mut outputs[base..base.strict_add(out_len)]); } return; } match kid { kernels::AsconPermute12KernelId::Portable => { - let mut input_groups = inputs.chunks_exact(1); + let (input_groups, rem_inputs) = inputs.as_chunks::<1>(); let mut output_groups = outputs.chunks_exact_mut(out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + debug_assert!(rem_inputs.is_empty()); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<1>( group_inputs, out_len, @@ -626,9 +629,9 @@ impl AsconXof { } #[cfg(target_arch = "aarch64")] kernels::AsconPermute12KernelId::Aarch64Neon => { - let mut input_groups = inputs.chunks_exact(2); - let mut output_groups = outputs.chunks_exact_mut(2 * out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<2>(); + let mut output_groups = outputs.chunks_exact_mut(2usize.strict_mul(out_len)); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<2>( group_inputs, out_len, @@ -637,7 +640,6 @@ impl AsconXof { aarch64::permute_12_aarch64_neon_x2, ); } - let rem_inputs = input_groups.remainder(); let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::hash_many_into_with_kernel( @@ -650,9 +652,9 @@ impl AsconXof { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx2 => { - let mut input_groups = inputs.chunks_exact(4); - let mut output_groups = outputs.chunks_exact_mut(4 * out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<4>(); + let mut output_groups = outputs.chunks_exact_mut(4usize.strict_mul(out_len)); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<4>( group_inputs, out_len, @@ -661,7 +663,6 @@ impl AsconXof { x86_64_avx2::permute_12_x86_avx2_x4, ); } - let rem_inputs = input_groups.remainder(); let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::hash_many_into_with_kernel( @@ -674,9 +675,9 @@ impl AsconXof { } #[cfg(target_arch = "x86_64")] kernels::AsconPermute12KernelId::X86Avx512 => { - let mut input_groups = inputs.chunks_exact(8); - let mut output_groups = outputs.chunks_exact_mut(8 * out_len); - for (group_inputs, group_outputs) in input_groups.by_ref().zip(output_groups.by_ref()) { + let (input_groups, rem_inputs) = inputs.as_chunks::<8>(); + let mut output_groups = outputs.chunks_exact_mut(8usize.strict_mul(out_len)); + for (group_inputs, group_outputs) in input_groups.iter().zip(output_groups.by_ref()) { xof_many_equal_len_group::<8>( group_inputs, out_len, @@ -685,7 +686,6 @@ impl AsconXof { x86_64_avx512::permute_12_x86_avx512_x8, ); } - let rem_inputs = input_groups.remainder(); let rem_outputs = output_groups.into_remainder(); if !rem_inputs.is_empty() { Self::hash_many_into_with_kernel( @@ -709,21 +709,24 @@ impl AsconXof { pub fn hash_many_into(inputs: &[&[u8]], out_len: usize, outputs: &mut [u8]) { assert_eq!( outputs.len(), - inputs.len() * out_len, + inputs.len().strict_mul(out_len), "input/output batch length mismatch" ); + if inputs.is_empty() || out_len == 0 { + return; + } let mut start = 0usize; while start < inputs.len() { let len = inputs[start].len(); - let mut end = start + 1; + let mut end = start.strict_add(1); while end < inputs.len() && inputs[end].len() == len { - end += 1; + end = end.strict_add(1); } - let kid = AsconHash256::batch_kernel_id_for_count(end - start); - let start_byte = start * out_len; - let end_byte = end * out_len; + let kid = AsconHash256::batch_kernel_id_for_count(end.strict_sub(start)); + let start_byte = start.strict_mul(out_len); + let end_byte = end.strict_mul(out_len); Self::hash_many_into_with_kernel(kid, &inputs[start..end], out_len, &mut outputs[start_byte..end_byte]); start = end; } @@ -766,7 +769,7 @@ impl Xof for AsconXofReader { #[inline(always)] fn squeeze(&mut self, mut out: &mut [u8]) { if self.pos < RATE && !out.is_empty() { - let take = core::cmp::min(RATE - self.pos, out.len()); + let take = core::cmp::min(RATE.strict_sub(self.pos), out.len()); out[..take].copy_from_slice(&self.buf[self.pos..self.pos.strict_add(take)]); self.pos = self.pos.strict_add(take); out = &mut out[take..]; diff --git a/src/hashes/crypto/ascon/aarch64.rs b/src/hashes/crypto/ascon/aarch64.rs index a6de44a7..8708590c 100644 --- a/src/hashes/crypto/ascon/aarch64.rs +++ b/src/hashes/crypto/ascon/aarch64.rs @@ -19,7 +19,11 @@ macro_rules! ror_u64x2 { } #[cfg(target_arch = "aarch64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] +/// Combine two scalar words into the low and high lanes of one NEON register. +/// +/// # Safety +/// +/// The caller must ensure the current CPU supports NEON. #[target_feature(enable = "neon")] unsafe fn combine_lanes(a: u64, b: u64) -> uint64x2_t { vcombine_u64(vcreate_u64(a), vcreate_u64(b)) @@ -99,7 +103,6 @@ pub(crate) fn permute_12_aarch64_neon(state: &mut [u64; 5]) { /// /// Caller must ensure the `neon` CPU feature is available. #[cfg(target_arch = "aarch64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[target_feature(enable = "neon")] #[inline] unsafe fn permute_12_aarch64_neon_x2_impl(states: &mut [[u64; 5]; 2]) { @@ -167,7 +170,6 @@ unsafe fn permute_12_aarch64_neon_x2_impl(states: &mut [[u64; 5]; 2]) { /// Apply the Ascon-p[12] permutation to two independent states in parallel. #[cfg(target_arch = "aarch64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[inline] pub(crate) fn permute_12_aarch64_neon_x2(states: &mut [[u64; 5]; 2]) { // SAFETY: Dispatch verifies aarch64::NEON before selecting this kernel. diff --git a/src/hashes/crypto/ascon/dispatch.rs b/src/hashes/crypto/ascon/dispatch.rs index c915b504..167f3c22 100644 --- a/src/hashes/crypto/ascon/dispatch.rs +++ b/src/hashes/crypto/ascon/dispatch.rs @@ -1,13 +1,17 @@ -#![cfg_attr(test, allow(dead_code))] - +#[cfg(feature = "diag")] +use super::kernels::permute_fn; use super::{ dispatch_tables::DispatchTable, - kernels::{AsconPermute12KernelId, permute_fn, required_caps}, + kernels::{AsconPermute12KernelId, required_caps}, }; -use crate::{backend::cache::OnceCache, platform::Caps}; +#[cfg(feature = "diag")] +use crate::backend::cache::OnceCache; +use crate::platform::Caps; +#[cfg(feature = "diag")] type PermuteFn = fn(&mut [u64; 5]); +#[cfg(feature = "diag")] #[derive(Clone, Copy)] struct ActiveDispatch { boundaries: [usize; 3], @@ -21,6 +25,7 @@ struct ActiveDispatch { l_name: &'static str, } +#[cfg(feature = "diag")] static ACTIVE: OnceCache = OnceCache::new(); #[inline] @@ -33,6 +38,7 @@ fn resolve(id: AsconPermute12KernelId, caps: Caps) -> AsconPermute12KernelId { } } +#[cfg(feature = "diag")] #[inline] #[must_use] fn active() -> ActiveDispatch { @@ -59,6 +65,7 @@ fn active() -> ActiveDispatch { }) } +#[cfg(feature = "diag")] #[inline] #[must_use] fn select(d: &ActiveDispatch, len: usize) -> (PermuteFn, &'static str) { @@ -74,29 +81,14 @@ fn select(d: &ActiveDispatch, len: usize) -> (PermuteFn, &'static str) { } } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { let d = active(); select(&d, len).1 } -/// Apply the configured Ascon permutation kernel for a specific workload size. -/// -/// The `len` parameter is a hint representing the total amount of work (bytes) -/// associated with the sponge operation. This allows tuned size-class tables to -/// take effect for one-shot and long-running streaming workloads. -/// -/// Production code uses `InlinePermuter` directly; this is retained for the -/// test/bench harness. -#[inline] -#[allow(dead_code)] -pub fn permute_12_for_len(state: &mut [u64; 5], len: usize) { - let d = active(); - (select(&d, len).0)(state); -} - #[cfg(any(test, feature = "std"))] #[inline] #[must_use] diff --git a/src/hashes/crypto/ascon/dispatch_tables.rs b/src/hashes/crypto/ascon/dispatch_tables.rs index 2e507958..7d102ba9 100644 --- a/src/hashes/crypto/ascon/dispatch_tables.rs +++ b/src/hashes/crypto/ascon/dispatch_tables.rs @@ -2,85 +2,88 @@ //! //! This table controls which `permute_12` kernel is used by Ascon hash and XOF. -pub use super::kernels::AsconPermute12KernelId as KernelId; +pub(crate) use super::kernels::AsconPermute12KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +#[cfg(feature = "diag")] +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { + #[cfg(feature = "diag")] pub boundaries: [usize; 3], pub xs: KernelId, + #[cfg(feature = "diag")] pub s: KernelId, + #[cfg(feature = "diag")] pub m: KernelId, + #[cfg(feature = "diag")] pub l: KernelId, } -impl DispatchTable { - #[allow(dead_code)] - #[inline] - #[must_use] - pub const fn kernel_for_len(&self, len: usize) -> KernelId { - let [xs_max, s_max, m_max] = self.boundaries; - if len <= xs_max { - self.xs - } else if len <= s_max { - self.s - } else if len <= m_max { - self.m - } else { - self.l - } - } -} - -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_NEON_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_NEON_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, // The single-state policy stays scalar: duplicating each of the five state // words across NEON lanes does not add independent work. The NEON x2 batch // path is wired separately. xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, // The single-state policy stays scalar: broadcasting each state word across // four AVX2 lanes does not add independent work. The AVX2 x4 batch path is // wired separately. xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512_TABLE: DispatchTable = DispatchTable { + #[cfg(feature = "diag")] boundaries: DEFAULT_BOUNDARIES, // The single-state policy stays scalar: broadcasting each state word across // eight AVX-512 lanes does not add independent work. The AVX-512 x8 batch // path is wired separately. xs: KernelId::Portable, + #[cfg(feature = "diag")] s: KernelId::Portable, + #[cfg(feature = "diag")] m: KernelId::Portable, + #[cfg(feature = "diag")] l: KernelId::Portable, }; #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "aarch64")] { use crate::platform::caps::aarch64; @@ -100,5 +103,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/ascon/kernel_test.rs b/src/hashes/crypto/ascon/kernel_test.rs index b194c5fd..16025888 100644 --- a/src/hashes/crypto/ascon/kernel_test.rs +++ b/src/hashes/crypto/ascon/kernel_test.rs @@ -3,9 +3,7 @@ use alloc::{vec, vec::Vec}; use super::kernels::{ALL, permute_fn, required_caps}; #[derive(Clone, Debug)] -#[allow(dead_code)] -pub struct KernelResult { - pub name: &'static str, +pub(super) struct KernelResult { pub state: [u64; 5], } @@ -25,7 +23,7 @@ fn state_from_bytes(data: &[u8]) -> [u64; 5] { } #[must_use] -pub fn run_all_ascon_p12_kernels(data: &[u8]) -> Vec { +pub(super) fn run_all_ascon_p12_kernels(data: &[u8]) -> Vec { let caps = crate::platform::caps(); let mut out = Vec::with_capacity(ALL.len()); let init = state_from_bytes(data); @@ -36,16 +34,13 @@ pub fn run_all_ascon_p12_kernels(data: &[u8]) -> Vec { } let mut st = init; (permute_fn(id))(&mut st); - out.push(KernelResult { - name: id.as_str(), - state: st, - }); + out.push(KernelResult { state: st }); } out } -pub fn verify_ascon_p12_kernels(data: &[u8]) -> Result<(), &'static str> { +pub(super) fn verify_ascon_p12_kernels(data: &[u8]) -> Result<(), &'static str> { let results = run_all_ascon_p12_kernels(data); let Some(first) = results.first() else { return Ok(()); @@ -106,7 +101,7 @@ mod tests { for (input, actual) in inputs.iter().zip(batch.iter()) { let expected = crate::hashes::crypto::AsconHash256::digest(input); - assert_eq!(*actual, expected, "digest_many mismatch for {}", id.as_str()); + assert_eq!(*actual, expected, "digest_many mismatch for {id:?}"); } } } @@ -138,8 +133,7 @@ mod tests { assert_eq!( &batch[base..base + out_len], expected.as_slice(), - "xof_many mismatch for {}", - id.as_str() + "xof_many mismatch for {id:?}" ); } } @@ -188,4 +182,16 @@ mod tests { assert_eq!(&actual[base..base + out_len], expected.as_slice()); } } + + #[test] + fn xof_many_accepts_zero_length_outputs() { + let input = b"same-length input".as_slice(); + let inputs = [input; 8]; + let mut output = []; + + crate::hashes::crypto::AsconXof::hash_many_into(&inputs, 0, &mut output); + for &id in ALL { + crate::hashes::crypto::AsconXof::hash_many_into_with_kernel(id, &inputs, 0, &mut output); + } + } } diff --git a/src/hashes/crypto/ascon/kernels.rs b/src/hashes/crypto/ascon/kernels.rs index 39e21c2c..b088b15e 100644 --- a/src/hashes/crypto/ascon/kernels.rs +++ b/src/hashes/crypto/ascon/kernels.rs @@ -4,8 +4,7 @@ use crate::platform::Caps; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] -pub enum AsconPermute12KernelId { +pub(crate) enum AsconPermute12KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Neon = 1, @@ -16,9 +15,10 @@ pub enum AsconPermute12KernelId { } impl AsconPermute12KernelId { + #[cfg(feature = "diag")] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -33,8 +33,7 @@ impl AsconPermute12KernelId { #[inline] #[must_use] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] -pub const fn simd_degree(id: AsconPermute12KernelId) -> usize { +pub(crate) const fn simd_degree(id: AsconPermute12KernelId) -> usize { match id { AsconPermute12KernelId::Portable => 1, #[cfg(target_arch = "aarch64")] @@ -47,7 +46,7 @@ pub const fn simd_degree(id: AsconPermute12KernelId) -> usize { } #[must_use] -pub fn permute_fn(id: AsconPermute12KernelId) -> fn(&mut [u64; 5]) { +pub(crate) fn permute_fn(id: AsconPermute12KernelId) -> fn(&mut [u64; 5]) { match id { AsconPermute12KernelId::Portable => permute_12_portable, #[cfg(target_arch = "aarch64")] @@ -61,7 +60,7 @@ pub fn permute_fn(id: AsconPermute12KernelId) -> fn(&mut [u64; 5]) { #[inline] #[must_use] -pub const fn required_caps(id: AsconPermute12KernelId) -> Caps { +pub(crate) const fn required_caps(id: AsconPermute12KernelId) -> Caps { match id { AsconPermute12KernelId::Portable => Caps::NONE, #[cfg(target_arch = "aarch64")] @@ -76,7 +75,7 @@ pub const fn required_caps(id: AsconPermute12KernelId) -> Caps { } #[cfg(test)] -pub const ALL: &[AsconPermute12KernelId] = &[ +pub(crate) const ALL: &[AsconPermute12KernelId] = &[ AsconPermute12KernelId::Portable, #[cfg(target_arch = "aarch64")] AsconPermute12KernelId::Aarch64Neon, diff --git a/src/hashes/crypto/ascon/x86_64_avx2.rs b/src/hashes/crypto/ascon/x86_64_avx2.rs index 6e790905..a2b4fd27 100644 --- a/src/hashes/crypto/ascon/x86_64_avx2.rs +++ b/src/hashes/crypto/ascon/x86_64_avx2.rs @@ -26,15 +26,15 @@ macro_rules! ror_epi64 { #[target_feature(enable = "avx2")] #[inline] unsafe fn permute_12_x86_avx2_impl(state: &mut [u64; 5]) { - let mut x0 = _mm256_set1_epi64x(state[0] as i64); - let mut x1 = _mm256_set1_epi64x(state[1] as i64); - let mut x2 = _mm256_set1_epi64x(state[2] as i64); - let mut x3 = _mm256_set1_epi64x(state[3] as i64); - let mut x4 = _mm256_set1_epi64x(state[4] as i64); + let mut x0 = _mm256_set1_epi64x(state[0].cast_signed()); + let mut x1 = _mm256_set1_epi64x(state[1].cast_signed()); + let mut x2 = _mm256_set1_epi64x(state[2].cast_signed()); + let mut x3 = _mm256_set1_epi64x(state[3].cast_signed()); + let mut x4 = _mm256_set1_epi64x(state[4].cast_signed()); let ones = _mm256_set1_epi64x(-1); for &c in &super::RC { - x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c as i64)); + x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c.cast_signed())); x0 = _mm256_xor_si256(x0, x4); x4 = _mm256_xor_si256(x4, x3); @@ -70,11 +70,11 @@ unsafe fn permute_12_x86_avx2_impl(state: &mut [u64; 5]) { x4 = _mm256_xor_si256(y4, _mm256_xor_si256(ror_epi64!(y4, 7, 57), ror_epi64!(y4, 41, 23))); } - state[0] = _mm256_extract_epi64::<0>(x0) as u64; - state[1] = _mm256_extract_epi64::<0>(x1) as u64; - state[2] = _mm256_extract_epi64::<0>(x2) as u64; - state[3] = _mm256_extract_epi64::<0>(x3) as u64; - state[4] = _mm256_extract_epi64::<0>(x4) as u64; + state[0] = _mm256_extract_epi64::<0>(x0).cast_unsigned(); + state[1] = _mm256_extract_epi64::<0>(x1).cast_unsigned(); + state[2] = _mm256_extract_epi64::<0>(x2).cast_unsigned(); + state[3] = _mm256_extract_epi64::<0>(x3).cast_unsigned(); + state[4] = _mm256_extract_epi64::<0>(x4).cast_unsigned(); } /// Apply the Ascon-p[12] permutation using x86_64 AVX2. @@ -91,44 +91,43 @@ pub(crate) fn permute_12_x86_avx2(state: &mut [u64; 5]) { /// /// Caller must ensure the `avx2` CPU feature is available. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[target_feature(enable = "avx2")] #[inline] unsafe fn permute_12_x86_avx2_x4_impl(states: &mut [[u64; 5]; 4]) { let mut x0 = _mm256_set_epi64x( - states[3][0] as i64, - states[2][0] as i64, - states[1][0] as i64, - states[0][0] as i64, + states[3][0].cast_signed(), + states[2][0].cast_signed(), + states[1][0].cast_signed(), + states[0][0].cast_signed(), ); let mut x1 = _mm256_set_epi64x( - states[3][1] as i64, - states[2][1] as i64, - states[1][1] as i64, - states[0][1] as i64, + states[3][1].cast_signed(), + states[2][1].cast_signed(), + states[1][1].cast_signed(), + states[0][1].cast_signed(), ); let mut x2 = _mm256_set_epi64x( - states[3][2] as i64, - states[2][2] as i64, - states[1][2] as i64, - states[0][2] as i64, + states[3][2].cast_signed(), + states[2][2].cast_signed(), + states[1][2].cast_signed(), + states[0][2].cast_signed(), ); let mut x3 = _mm256_set_epi64x( - states[3][3] as i64, - states[2][3] as i64, - states[1][3] as i64, - states[0][3] as i64, + states[3][3].cast_signed(), + states[2][3].cast_signed(), + states[1][3].cast_signed(), + states[0][3].cast_signed(), ); let mut x4 = _mm256_set_epi64x( - states[3][4] as i64, - states[2][4] as i64, - states[1][4] as i64, - states[0][4] as i64, + states[3][4].cast_signed(), + states[2][4].cast_signed(), + states[1][4].cast_signed(), + states[0][4].cast_signed(), ); let ones = _mm256_set1_epi64x(-1); for &c in &super::RC { - x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c as i64)); + x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c.cast_signed())); x0 = _mm256_xor_si256(x0, x4); x4 = _mm256_xor_si256(x4, x3); @@ -164,31 +163,30 @@ unsafe fn permute_12_x86_avx2_x4_impl(states: &mut [[u64; 5]; 4]) { x4 = _mm256_xor_si256(y4, _mm256_xor_si256(ror_epi64!(y4, 7, 57), ror_epi64!(y4, 41, 23))); } - states[0][0] = _mm256_extract_epi64::<0>(x0) as u64; - states[1][0] = _mm256_extract_epi64::<1>(x0) as u64; - states[2][0] = _mm256_extract_epi64::<2>(x0) as u64; - states[3][0] = _mm256_extract_epi64::<3>(x0) as u64; - states[0][1] = _mm256_extract_epi64::<0>(x1) as u64; - states[1][1] = _mm256_extract_epi64::<1>(x1) as u64; - states[2][1] = _mm256_extract_epi64::<2>(x1) as u64; - states[3][1] = _mm256_extract_epi64::<3>(x1) as u64; - states[0][2] = _mm256_extract_epi64::<0>(x2) as u64; - states[1][2] = _mm256_extract_epi64::<1>(x2) as u64; - states[2][2] = _mm256_extract_epi64::<2>(x2) as u64; - states[3][2] = _mm256_extract_epi64::<3>(x2) as u64; - states[0][3] = _mm256_extract_epi64::<0>(x3) as u64; - states[1][3] = _mm256_extract_epi64::<1>(x3) as u64; - states[2][3] = _mm256_extract_epi64::<2>(x3) as u64; - states[3][3] = _mm256_extract_epi64::<3>(x3) as u64; - states[0][4] = _mm256_extract_epi64::<0>(x4) as u64; - states[1][4] = _mm256_extract_epi64::<1>(x4) as u64; - states[2][4] = _mm256_extract_epi64::<2>(x4) as u64; - states[3][4] = _mm256_extract_epi64::<3>(x4) as u64; + states[0][0] = _mm256_extract_epi64::<0>(x0).cast_unsigned(); + states[1][0] = _mm256_extract_epi64::<1>(x0).cast_unsigned(); + states[2][0] = _mm256_extract_epi64::<2>(x0).cast_unsigned(); + states[3][0] = _mm256_extract_epi64::<3>(x0).cast_unsigned(); + states[0][1] = _mm256_extract_epi64::<0>(x1).cast_unsigned(); + states[1][1] = _mm256_extract_epi64::<1>(x1).cast_unsigned(); + states[2][1] = _mm256_extract_epi64::<2>(x1).cast_unsigned(); + states[3][1] = _mm256_extract_epi64::<3>(x1).cast_unsigned(); + states[0][2] = _mm256_extract_epi64::<0>(x2).cast_unsigned(); + states[1][2] = _mm256_extract_epi64::<1>(x2).cast_unsigned(); + states[2][2] = _mm256_extract_epi64::<2>(x2).cast_unsigned(); + states[3][2] = _mm256_extract_epi64::<3>(x2).cast_unsigned(); + states[0][3] = _mm256_extract_epi64::<0>(x3).cast_unsigned(); + states[1][3] = _mm256_extract_epi64::<1>(x3).cast_unsigned(); + states[2][3] = _mm256_extract_epi64::<2>(x3).cast_unsigned(); + states[3][3] = _mm256_extract_epi64::<3>(x3).cast_unsigned(); + states[0][4] = _mm256_extract_epi64::<0>(x4).cast_unsigned(); + states[1][4] = _mm256_extract_epi64::<1>(x4).cast_unsigned(); + states[2][4] = _mm256_extract_epi64::<2>(x4).cast_unsigned(); + states[3][4] = _mm256_extract_epi64::<3>(x4).cast_unsigned(); } /// Apply the Ascon-p[12] permutation to four independent states in parallel. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[inline] pub(crate) fn permute_12_x86_avx2_x4(states: &mut [[u64; 5]; 4]) { // SAFETY: Dispatch verifies x86::AVX2 before selecting this kernel. diff --git a/src/hashes/crypto/ascon/x86_64_avx512.rs b/src/hashes/crypto/ascon/x86_64_avx512.rs index 06c52167..a6e96731 100644 --- a/src/hashes/crypto/ascon/x86_64_avx512.rs +++ b/src/hashes/crypto/ascon/x86_64_avx512.rs @@ -31,15 +31,15 @@ macro_rules! ror_epi64x8 { #[target_feature(enable = "avx512f,avx512vl")] #[inline] unsafe fn permute_12_x86_avx512_impl(state: &mut [u64; 5]) { - let mut x0 = _mm256_set1_epi64x(state[0] as i64); - let mut x1 = _mm256_set1_epi64x(state[1] as i64); - let mut x2 = _mm256_set1_epi64x(state[2] as i64); - let mut x3 = _mm256_set1_epi64x(state[3] as i64); - let mut x4 = _mm256_set1_epi64x(state[4] as i64); + let mut x0 = _mm256_set1_epi64x(state[0].cast_signed()); + let mut x1 = _mm256_set1_epi64x(state[1].cast_signed()); + let mut x2 = _mm256_set1_epi64x(state[2].cast_signed()); + let mut x3 = _mm256_set1_epi64x(state[3].cast_signed()); + let mut x4 = _mm256_set1_epi64x(state[4].cast_signed()); let ones = _mm256_set1_epi64x(-1); for &c in &super::RC { - x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c as i64)); + x2 = _mm256_xor_si256(x2, _mm256_set1_epi64x(c.cast_signed())); x0 = _mm256_xor_si256(x0, x4); x4 = _mm256_xor_si256(x4, x3); @@ -75,11 +75,11 @@ unsafe fn permute_12_x86_avx512_impl(state: &mut [u64; 5]) { x4 = _mm256_xor_si256(y4, _mm256_xor_si256(ror_epi64!(y4, 57), ror_epi64!(y4, 23))); } - state[0] = _mm256_extract_epi64::<0>(x0) as u64; - state[1] = _mm256_extract_epi64::<0>(x1) as u64; - state[2] = _mm256_extract_epi64::<0>(x2) as u64; - state[3] = _mm256_extract_epi64::<0>(x3) as u64; - state[4] = _mm256_extract_epi64::<0>(x4) as u64; + state[0] = _mm256_extract_epi64::<0>(x0).cast_unsigned(); + state[1] = _mm256_extract_epi64::<0>(x1).cast_unsigned(); + state[2] = _mm256_extract_epi64::<0>(x2).cast_unsigned(); + state[3] = _mm256_extract_epi64::<0>(x3).cast_unsigned(); + state[4] = _mm256_extract_epi64::<0>(x4).cast_unsigned(); } /// Apply the Ascon-p[12] permutation using x86_64 AVX-512. @@ -96,64 +96,63 @@ pub(crate) fn permute_12_x86_avx512(state: &mut [u64; 5]) { /// /// Caller must ensure the `avx512f` and `avx512vl` CPU features are available. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[target_feature(enable = "avx512f,avx512vl")] #[inline] unsafe fn permute_12_x86_avx512_x8_impl(states: &mut [[u64; 5]; 8]) { let mut x0 = _mm512_set_epi64( - states[7][0] as i64, - states[6][0] as i64, - states[5][0] as i64, - states[4][0] as i64, - states[3][0] as i64, - states[2][0] as i64, - states[1][0] as i64, - states[0][0] as i64, + states[7][0].cast_signed(), + states[6][0].cast_signed(), + states[5][0].cast_signed(), + states[4][0].cast_signed(), + states[3][0].cast_signed(), + states[2][0].cast_signed(), + states[1][0].cast_signed(), + states[0][0].cast_signed(), ); let mut x1 = _mm512_set_epi64( - states[7][1] as i64, - states[6][1] as i64, - states[5][1] as i64, - states[4][1] as i64, - states[3][1] as i64, - states[2][1] as i64, - states[1][1] as i64, - states[0][1] as i64, + states[7][1].cast_signed(), + states[6][1].cast_signed(), + states[5][1].cast_signed(), + states[4][1].cast_signed(), + states[3][1].cast_signed(), + states[2][1].cast_signed(), + states[1][1].cast_signed(), + states[0][1].cast_signed(), ); let mut x2 = _mm512_set_epi64( - states[7][2] as i64, - states[6][2] as i64, - states[5][2] as i64, - states[4][2] as i64, - states[3][2] as i64, - states[2][2] as i64, - states[1][2] as i64, - states[0][2] as i64, + states[7][2].cast_signed(), + states[6][2].cast_signed(), + states[5][2].cast_signed(), + states[4][2].cast_signed(), + states[3][2].cast_signed(), + states[2][2].cast_signed(), + states[1][2].cast_signed(), + states[0][2].cast_signed(), ); let mut x3 = _mm512_set_epi64( - states[7][3] as i64, - states[6][3] as i64, - states[5][3] as i64, - states[4][3] as i64, - states[3][3] as i64, - states[2][3] as i64, - states[1][3] as i64, - states[0][3] as i64, + states[7][3].cast_signed(), + states[6][3].cast_signed(), + states[5][3].cast_signed(), + states[4][3].cast_signed(), + states[3][3].cast_signed(), + states[2][3].cast_signed(), + states[1][3].cast_signed(), + states[0][3].cast_signed(), ); let mut x4 = _mm512_set_epi64( - states[7][4] as i64, - states[6][4] as i64, - states[5][4] as i64, - states[4][4] as i64, - states[3][4] as i64, - states[2][4] as i64, - states[1][4] as i64, - states[0][4] as i64, + states[7][4].cast_signed(), + states[6][4].cast_signed(), + states[5][4].cast_signed(), + states[4][4].cast_signed(), + states[3][4].cast_signed(), + states[2][4].cast_signed(), + states[1][4].cast_signed(), + states[0][4].cast_signed(), ); let ones = _mm512_set1_epi64(-1); for &c in &super::RC { - x2 = _mm512_xor_si512(x2, _mm512_set1_epi64(c as i64)); + x2 = _mm512_xor_si512(x2, _mm512_set1_epi64(c.cast_signed())); x0 = _mm512_xor_si512(x0, x4); x4 = _mm512_xor_si512(x4, x3); @@ -213,7 +212,6 @@ unsafe fn permute_12_x86_avx512_x8_impl(states: &mut [[u64; 5]; 8]) { /// Apply the Ascon-p[12] permutation to eight independent states in parallel. #[cfg(target_arch = "x86_64")] -#[cfg_attr(not(any(test, feature = "std")), allow(dead_code))] #[inline] pub(crate) fn permute_12_x86_avx512_x8(states: &mut [[u64; 5]; 8]) { // SAFETY: Dispatch verifies x86::AVX512F + x86::AVX512VL before selecting this kernel. diff --git a/src/hashes/crypto/blake2b/kernels.rs b/src/hashes/crypto/blake2b/kernels.rs index 6996ebf4..73f342b2 100644 --- a/src/hashes/crypto/blake2b/kernels.rs +++ b/src/hashes/crypto/blake2b/kernels.rs @@ -1,5 +1,6 @@ //! Blake2b portable compression function and kernel dispatch (RFC 7693). +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] use crate::platform::Caps; #[cfg(target_arch = "riscv64")] use crate::platform::caps::riscv; @@ -7,6 +8,7 @@ use crate::platform::caps::riscv; use crate::platform::caps::wasm; #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; +use crate::traits::ct; /// Blake2b compress function pointer type. /// @@ -50,14 +52,19 @@ impl Blake2bCounter { pub(crate) const fn as_u128(self) -> u128 { ((self.hi as u128) << 64) | self.lo as u128 } + + #[inline(always)] + pub(crate) fn zeroize_no_fence(&mut self) { + ct::zeroize_words_no_fence(core::slice::from_mut(&mut self.lo)); + ct::zeroize_words_no_fence(core::slice::from_mut(&mut self.hi)); + } } /// Blake2b kernel identifier. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -#[cfg_attr(target_os = "macos", allow(dead_code))] -pub enum Blake2bKernelId { +pub(crate) enum Blake2bKernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Avx2 = 1, @@ -73,7 +80,7 @@ impl Blake2bKernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -115,14 +122,12 @@ fn compress_wasm_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u128, last: boo #[inline(always)] fn compress_blocks_with(h: &mut [u64; 8], blocks: &[u8], t: &mut Blake2bCounter, compress: CompressFn) { debug_assert_eq!(blocks.len() % 128, 0); - let mut chunks = blocks.chunks_exact(128); - for chunk in &mut chunks { + let (chunks, remainder) = blocks.as_chunks::<128>(); + for block in chunks { t.add_len(128); - // SAFETY: `chunks_exact(128)` yields slices of exactly 128 bytes. - let block = unsafe { &*chunk.as_ptr().cast::<[u8; 128]>() }; compress(h, block, t.as_u128(), false); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } fn compress_blocks_portable(h: &mut [u64; 8], blocks: &[u8], t: &mut Blake2bCounter) { @@ -181,10 +186,10 @@ pub(crate) fn compress_blocks_fn(id: Blake2bKernelId) -> CompressBlocksFn { } /// Capabilities required to run the given kernel. +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] #[inline] #[must_use] -#[allow(dead_code)] // Used by runtime dispatch on targets that don't bypass to a fixed kernel. -pub const fn required_caps(id: Blake2bKernelId) -> Caps { +pub(crate) const fn required_caps(id: Blake2bKernelId) -> Caps { match id { Blake2bKernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -200,7 +205,7 @@ pub const fn required_caps(id: Blake2bKernelId) -> Caps { /// All kernel IDs for agreement testing. #[cfg(test)] -pub const ALL: &[Blake2bKernelId] = &[ +pub(crate) const ALL: &[Blake2bKernelId] = &[ Blake2bKernelId::Portable, #[cfg(target_arch = "x86_64")] Blake2bKernelId::X86Avx2, @@ -237,7 +242,6 @@ pub(crate) fn compile_time_best() -> CompressFn { { return compress_x86_avx2; } - #[allow(unreachable_code)] compress } @@ -255,7 +259,6 @@ pub(crate) fn compile_time_best_blocks() -> CompressBlocksFn { { return compress_blocks_x86_avx2; } - #[allow(unreachable_code)] compress_blocks_portable } @@ -272,7 +275,6 @@ pub(crate) const IV: [u64; 8] = [ ]; /// Message-word permutation schedule (10 rows, reused cyclically for 12 rounds). -#[allow(dead_code)] // Used by target-specific SIMD backends that are not compiled on every host. pub(crate) const SIGMA: [[u8; 16]; 10] = [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], @@ -426,7 +428,6 @@ fn round(v: &mut [U64x4; 4], m: &[u64; 16], s: &[u8; 16]) { /// Load 16 little-endian u64 message words from a 128-byte block. #[inline(always)] -#[allow(clippy::indexing_slicing)] // i is always in 0..16 (loop bound). pub(crate) fn load_msg(block: &[u8; 128]) -> [u64; 16] { let mut m = [0u64; 16]; let src = block.as_ptr(); @@ -439,18 +440,28 @@ pub(crate) fn load_msg(block: &[u8; 128]) -> [u64; 16] { m } +#[inline(always)] +fn split_counter(counter: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = counter.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) +} + /// Initialize the 16-word working vector from state, IV, counter, and finalization flag. #[cfg(any(target_arch = "x86_64", target_arch = "wasm32", target_arch = "riscv64"))] #[inline(always)] pub(crate) fn init_v(h: &[u64; 8], t: u128, last: bool) -> [u64; 16] { + let (t0, t1) = split_counter(t); let mut v = [0u64; 16]; v[..8].copy_from_slice(h); v[8] = IV[0]; v[9] = IV[1]; v[10] = IV[2]; v[11] = IV[3]; - v[12] = IV[4] ^ (t as u64); - v[13] = IV[5] ^ ((t >> 64) as u64); + v[12] = IV[4] ^ t0; + v[13] = IV[5] ^ t1; v[14] = if last { IV[6] ^ u64::MAX } else { IV[6] }; v[15] = IV[7]; v @@ -460,11 +471,9 @@ pub(crate) fn init_v(h: &[u64; 8], t: u128, last: bool) -> [u64; 16] { /// /// `t` is the total number of input bytes after this block (inclusive). /// `last` is `true` for the final block (sets the finalization flag). -#[allow(clippy::indexing_slicing)] pub(crate) fn compress(h: &mut [u64; 8], block: &[u8; 128], t: u128, last: bool) { let m = load_msg(block); - let t0 = t as u64; - let t1 = (t >> 64) as u64; + let (t0, t1) = split_counter(t); let f0 = if last { u64::MAX } else { 0 }; let mut v = [ diff --git a/src/hashes/crypto/blake2b/mod.rs b/src/hashes/crypto/blake2b/mod.rs index b56fad26..a54d46f8 100644 --- a/src/hashes/crypto/blake2b/mod.rs +++ b/src/hashes/crypto/blake2b/mod.rs @@ -65,9 +65,12 @@ use kernels::{Blake2bCounter, IV}; use super::Blake2Error; use crate::traits::{Digest, ct}; -const BLOCK_SIZE: usize = 128; -const MAX_KEY_LEN: usize = 64; -const MAX_OUTPUT_LEN: usize = 64; +const BLOCK_SIZE_U8: u8 = 128; +const BLOCK_SIZE: usize = BLOCK_SIZE_U8 as usize; +const MAX_KEY_LEN_U8: u8 = 64; +const MAX_KEY_LEN: usize = MAX_KEY_LEN_U8 as usize; +const MAX_OUTPUT_LEN_U8: u8 = 64; +const MAX_OUTPUT_LEN: usize = MAX_OUTPUT_LEN_U8 as usize; /// Validated 1–64 byte Blake2b key. /// @@ -124,14 +127,14 @@ fn validate_key(key: &[u8]) -> Result<(), Blake2Error> { #[inline] fn validate_output_len(output_len: usize) -> Result { - if !(1..=MAX_OUTPUT_LEN).contains(&output_len) { + let output_len = u8::try_from(output_len).map_err(|_| Blake2Error::InvalidOutputLength)?; + if !(1..=MAX_OUTPUT_LEN_U8).contains(&output_len) { return Err(Blake2Error::InvalidOutputLength); } - Ok(output_len as u8) + Ok(output_len) } -#[cfg(any(test, feature = "diag"))] -#[allow(dead_code)] +#[cfg(feature = "diag")] #[inline] #[must_use] pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { @@ -182,7 +185,6 @@ impl Core { /// Create a new Blake2b state with output length `nn`, optional `key`, and /// spec-defined `salt` + `personal` parameter-block values (RFC 7693 §2.5). - #[allow(clippy::indexing_slicing)] fn new_with_params(nn: u8, key: &[u8], salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> Self { assert!( nn >= 1 && nn as usize <= MAX_OUTPUT_LEN, @@ -190,7 +192,7 @@ impl Core { ); assert!(key.len() <= MAX_KEY_LEN, "Blake2b key must be at most 64 bytes"); - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let h = init_state_with_params(nn, kk, salt, personal); let stored_key = if kk > 0 { @@ -204,7 +206,7 @@ impl Core { let mut buf = [0u8; BLOCK_SIZE]; let buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -228,30 +230,23 @@ impl Core { fn zeroize_key_if_any(&mut self) { if self.kk > 0 { // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. - unsafe { ct::zeroize_no_fence(&mut *self.key.as_mut_ptr()) }; + unsafe { ct::zeroize_no_fence(self.key.assume_init_mut()) }; } } #[inline(always)] fn wipe(&mut self) { - for word in self.h.iter_mut() { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut self.h); ct::zeroize_no_fence(&mut self.buf); self.zeroize_key_if_any(); - // SAFETY: fields are valid, aligned, dereferenceable pointers. - unsafe { - core::ptr::write_volatile(&mut self.buf_len, 0); - core::ptr::write_volatile(&mut self.t, Blake2bCounter::zero()); - core::ptr::write_volatile(&mut self.nn, 0); - core::ptr::write_volatile(&mut self.kk, 0); - } + ct::zeroize_no_fence(core::slice::from_mut(&mut self.buf_len)); + self.t.zeroize_no_fence(); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.nn)); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.kk)); core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } /// Feed data into the hash state. - #[allow(clippy::indexing_slicing)] fn update(&mut self, data: &[u8]) { if data.is_empty() { return; @@ -265,7 +260,7 @@ impl Core { if self.buf_len > 0 && (self.buf_len as usize).strict_add(data_len) > BLOCK_SIZE { let fill = BLOCK_SIZE.strict_sub(self.buf_len as usize); self.buf[self.buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); - self.t.add_len(BLOCK_SIZE as u64); + self.t.add_len(u64::from(BLOCK_SIZE_U8)); (self.compress)(&mut self.h, &self.buf, self.t.as_u128(), false); self.buf_len = 0; offset = fill; @@ -275,7 +270,7 @@ impl Core { // in the buffer so finalize can set the final flag. let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); (self.compress_blocks)(&mut self.h, &data[offset..end], &mut self.t); offset = end; @@ -286,12 +281,12 @@ impl Core { if remaining > 0 { let start = self.buf_len as usize; self.buf[start..start.strict_add(remaining)].copy_from_slice(&data[offset..]); - self.buf_len = self.buf_len.strict_add(remaining as u8); + let remaining = u8::try_from(remaining).expect("Blake2b buffer remainder fits in u8"); + self.buf_len = self.buf_len.strict_add(remaining); } } /// Finalize and write the hash into `out` (must be exactly `nn` bytes). - #[allow(clippy::indexing_slicing)] fn finalize_into(&self, out: &mut [u8]) { debug_assert!(out.len() == self.nn as usize); @@ -308,16 +303,10 @@ impl Core { write_output(&h, self.nn, out); - for word in h.iter_mut() { - // SAFETY: Volatile zeroization of a stack word is sound because: - // 1. `word` comes from `h.iter_mut()`, so it is valid and uniquely borrowed. - // 2. `u64` has no invalid bit patterns; writing zero preserves validity. - // 3. The pointer does not escape this loop iteration. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } - #[cfg(any(test, feature = "diag"))] + #[cfg(any(test, all(feature = "diag", feature = "argon2")))] fn new_with_compress_for_test( nn: u8, key: &[u8], @@ -330,7 +319,7 @@ impl Core { ); assert!(key.len() <= MAX_KEY_LEN, "Blake2b key must be at most 64 bytes"); - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let h = init_state(nn, kk); let stored_key = if kk > 0 { let mut bytes = [0u8; MAX_KEY_LEN]; @@ -346,7 +335,7 @@ impl Core { if let Some(dst) = buf.get_mut(..key.len()) { dst.copy_from_slice(key); } - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -368,19 +357,16 @@ impl Core { /// Reset to the initial state (including re-buffering the key if keyed and /// re-applying any salt/personalization parameter-block bytes). - #[allow(clippy::indexing_slicing)] fn reset(&mut self) { self.h = init_state_with_params(self.nn, self.kk, &self.salt, &self.personal); if self.kk > 0 { let key_len = self.kk as usize; self.buf = [0u8; BLOCK_SIZE]; - // SAFETY: when `kk > 0`, `self.key` was initialized in `new`, and `self.buf` - // has at least `key_len` bytes available. - unsafe { - core::ptr::copy_nonoverlapping(self.key.as_ptr().cast::(), self.buf.as_mut_ptr(), key_len); - } - self.buf_len = BLOCK_SIZE as u8; + // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. + let key = unsafe { self.key.assume_init_ref() }; + self.buf[..key_len].copy_from_slice(&key[..key_len]); + self.buf_len = BLOCK_SIZE_U8; } else { self.buf = [0u8; BLOCK_SIZE]; self.buf_len = 0; @@ -421,7 +407,6 @@ const BLAKE2B512_H0: [u64; 8] = init_unkeyed_fixed_state(64); /// digest length / key length / fanout / depth bits, and `h[4..8]` carry the /// salt and personalization words. #[inline] -#[allow(clippy::indexing_slicing)] fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> [u64; 8] { let p0 = nn as u64 | ((kk as u64) << 8) | (1u64 << 16) | (1u64 << 24); let mut h = IV; @@ -439,7 +424,6 @@ fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_output(h: &[u64; 8], nn: u8, out: &mut [u8]) { let nn = nn as usize; let full_words = nn / 8; @@ -468,7 +452,6 @@ fn write_output(h: &[u64; 8], nn: u8, out: &mut [u8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_small_into_with_params( nn: u8, key: &[u8], @@ -477,7 +460,7 @@ fn oneshot_small_into_with_params( out: &mut [u8], compress: kernels::CompressFn, ) { - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -495,30 +478,26 @@ fn oneshot_small_into_with_params( block[..key.len()].copy_from_slice(key); if data.is_empty() { - compress(&mut h, &block, BLOCK_SIZE as u128, true); + compress(&mut h, &block, u128::from(BLOCK_SIZE_U8), true); ct::zeroize(&mut block); } else { - compress(&mut h, &block, BLOCK_SIZE as u128, false); + compress(&mut h, &block, u128::from(BLOCK_SIZE_U8), false); ct::zeroize(&mut block); block[..data.len()].copy_from_slice(data); compress( &mut h, &block, - (BLOCK_SIZE as u128).strict_add(data.len() as u128), + u128::from(BLOCK_SIZE_U8).strict_add(data.len() as u128), true, ); } write_output(&h, nn, out); - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_hash_into_inner( nn: u8, key: &[u8], @@ -539,7 +518,7 @@ fn oneshot_hash_into_inner( return; } - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -547,7 +526,7 @@ fn oneshot_hash_into_inner( let mut buf = [0u8; BLOCK_SIZE]; let mut buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -560,7 +539,7 @@ fn oneshot_hash_into_inner( if fill > 0 { buf[buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); } - t.add_len(BLOCK_SIZE as u64); + t.add_len(u64::from(BLOCK_SIZE_U8)); compress(&mut h, &buf, t.as_u128(), false); ct::zeroize(&mut buf); buf_len = 0; @@ -569,7 +548,7 @@ fn oneshot_hash_into_inner( let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); let compress_blocks = dispatch::compress_blocks_dispatch(); compress_blocks(&mut h, &data[offset..end], &mut t); @@ -579,7 +558,7 @@ fn oneshot_hash_into_inner( let remaining = data_len.strict_sub(offset); if remaining > 0 { buf[..remaining].copy_from_slice(&data[offset..]); - buf_len = remaining as u8; + buf_len = u8::try_from(remaining).expect("Blake2b buffer remainder fits in u8"); } t.add_len(buf_len as u64); @@ -587,10 +566,7 @@ fn oneshot_hash_into_inner( write_output(&h, nn, out); if kk > 0 { - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); ct::zeroize(&mut buf); } } @@ -707,12 +683,11 @@ impl Blake2bParams { /// Set a validated MAC key. Omit this method for unkeyed hashing. #[must_use] - #[allow(clippy::indexing_slicing)] pub fn key(mut self, key: Blake2bKey<'_>) -> Self { let key = key.as_bytes(); self.key_buf = [0u8; MAX_KEY_LEN]; self.key_buf[..key.len()].copy_from_slice(key); - self.key_len = key.len() as u8; + self.key_len = u8::try_from(key.len()).expect("validated Blake2b key length fits in u8"); self } @@ -730,7 +705,6 @@ impl Blake2bParams { self } - #[allow(clippy::indexing_slicing)] fn key_slice(&self) -> &[u8] { &self.key_buf[..usize::from(self.key_len)] } @@ -877,7 +851,6 @@ impl Blake2b256 { #[must_use] pub fn new_keyed(key: Blake2bKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); Self(Core::new(32, key)) } @@ -885,14 +858,12 @@ impl Blake2b256 { #[must_use] pub fn keyed_digest(key: Blake2bKey<'_>, data: &[u8]) -> [u8; 32] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); oneshot_hash_array::<32>(32, key, data) } } impl Blake2b256 { - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn new_with_compress_for_test( compress: kernels::CompressFn, compress_blocks: kernels::CompressBlocksFn, @@ -900,8 +871,7 @@ impl Blake2b256 { Self(Core::new_with_compress_for_test(32, &[], compress, compress_blocks)) } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn keyed_with_compress_for_test( key: &[u8], compress: kernels::CompressFn, @@ -914,6 +884,7 @@ impl Blake2b256 { #[cfg(feature = "diag")] #[must_use] +/// Derive a keyed Blake2b-256 diagnostic digest with the portable backend. pub fn diag_blake2b256_keyed_digest_portable(key: &[u8; 32]) -> [u8; 32] { let mut out = [0u8; 32]; oneshot_small_into_with_params( @@ -927,7 +898,7 @@ pub fn diag_blake2b256_keyed_digest_portable(key: &[u8; 32]) -> [u8; 32] { out } -#[cfg(any(test, all(feature = "diag", feature = "argon2")))] +#[cfg(all(feature = "diag", feature = "argon2"))] pub(crate) fn diag_hash_parts_portable(output_len: u8, parts: &[&[u8]], out: &mut [u8]) { assert_eq!( out.len(), @@ -1022,7 +993,6 @@ impl Blake2b512 { #[must_use] pub fn new_keyed(key: Blake2bKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); Self(Core::new(64, key)) } @@ -1030,14 +1000,12 @@ impl Blake2b512 { #[must_use] pub fn keyed_digest(key: Blake2bKey<'_>, data: &[u8]) -> [u8; 64] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); oneshot_hash_array::<64>(64, key, data) } } impl Blake2b512 { - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn new_with_compress_for_test( compress: kernels::CompressFn, compress_blocks: kernels::CompressBlocksFn, @@ -1156,7 +1124,6 @@ impl Blake2b { pub fn new_keyed(output_len: usize, key: Blake2bKey<'_>) -> Result { let output_len = validate_output_len(output_len)?; let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); Ok(Self { core: Core::new(output_len, key), output_len, @@ -1222,7 +1189,6 @@ impl Blake2b { pub fn keyed_digest_into(key: Blake2bKey<'_>, data: &[u8], out: &mut [u8]) -> Result<(), Blake2Error> { let output_len = validate_output_len(out.len())?; let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); oneshot_hash_into(output_len, key, data, out); Ok(()) } @@ -1237,7 +1203,8 @@ impl Blake2b { const { assert!(N >= 1 && N <= MAX_OUTPUT_LEN, "Blake2b output length N must be 1..=64"); } - oneshot_hash_array::(N as u8, &[], data) + let output_len = u8::try_from(N).expect("const assertion ensures the Blake2b output length fits in u8"); + oneshot_hash_array::(output_len, &[], data) } /// Compute a keyed Blake2b hash in one shot, returning a fixed-size array. @@ -1249,15 +1216,15 @@ impl Blake2b { assert!(N >= 1 && N <= MAX_OUTPUT_LEN, "Blake2b output length N must be 1..=64"); } let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2b key must not be empty"); - oneshot_hash_array::(N as u8, key, data) + let output_len = u8::try_from(N).expect("const assertion ensures the Blake2b output length fits in u8"); + oneshot_hash_array::(output_len, key, data) } #[cfg(feature = "argon2")] #[inline] pub(crate) fn new_validated(output_len: usize) -> Self { debug_assert!((1..=MAX_OUTPUT_LEN).contains(&output_len)); - let output_len = output_len as u8; + let output_len = u8::try_from(output_len).expect("validated Blake2b output length fits in u8"); Self { core: Core::new(output_len, &[]), output_len, @@ -1286,10 +1253,12 @@ impl Drop for Blake2b { mod tests { use alloc::vec; - use blake2::{Blake2b as OracleBlake2b, Blake2bMac, Digest as _}; - use digest::{ - KeyInit, - consts::{U32, U64}, + use blake2::{ + Blake2b as OracleBlake2b, Blake2bMac, + digest::{ + Digest as _, Mac as _, + consts::{U32, U64}, + }, }; use super::*; @@ -1299,7 +1268,7 @@ mod tests { type OracleBlake2bMac256 = Blake2bMac; fn validated_key(key: &[u8]) -> Blake2bKey<'_> { - Blake2bKey::new(key).unwrap() + Blake2bKey::new(key).expect("test key length must satisfy the Blake2b key contract") } fn oracle_hash_256(data: &[u8]) -> [u8; 32] { @@ -1363,7 +1332,7 @@ mod tests { } #[test] - #[cfg(feature = "diag")] + #[cfg(all(feature = "diag", feature = "argon2"))] fn diag_hash_parts_portable_handles_multiblock_parts() { let first = [0x11u8; 4]; let second = [0x22u8; 1024]; @@ -1438,9 +1407,10 @@ mod tests { let key = b"secret-key"; let data = b"hello world"; - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, data); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, data); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let actual = Blake2b256::keyed_digest(validated_key(key), data); assert_eq!(actual, expected); @@ -1449,9 +1419,10 @@ mod tests { #[test] fn blake2b256_keyed_empty_data() { let key = b"key"; - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, b""); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, b""); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let actual = Blake2b256::keyed_digest(validated_key(key), b""); assert_eq!(actual, expected); @@ -1462,9 +1433,10 @@ mod tests { let key = &[0xAA; 64]; // max key length let data = &[0xBB; 512]; - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, data); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, data); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let actual = Blake2b256::keyed_digest(validated_key(key), data); assert_eq!(actual, expected); @@ -1498,7 +1470,7 @@ mod tests { #[test] fn variable_output_1_byte() { let mut out = [0u8; 1]; - Blake2b::digest_into(b"test", &mut out).unwrap(); + Blake2b::digest_into(b"test", &mut out).expect("one-byte Blake2b output is valid"); assert_ne!(out, [0u8; 1]); } @@ -1506,13 +1478,13 @@ mod tests { fn variable_output_matches_fixed() { // 32-byte variable output should match Blake2b256 let mut var_out = [0u8; 32]; - Blake2b::digest_into(b"hello", &mut var_out).unwrap(); + Blake2b::digest_into(b"hello", &mut var_out).expect("32-byte Blake2b output is valid"); let fixed_out = Blake2b256::digest(b"hello"); assert_eq!(var_out, fixed_out); // 64-byte variable output should match Blake2b512 let mut var_out = [0u8; 64]; - Blake2b::digest_into(b"hello", &mut var_out).unwrap(); + Blake2b::digest_into(b"hello", &mut var_out).expect("64-byte Blake2b output is valid"); let fixed_out = Blake2b512::digest(b"hello"); assert_eq!(var_out, fixed_out); } @@ -1526,7 +1498,7 @@ mod tests { let expected_arr: [u8; $nn] = oracle.finalize().into(); let mut actual = [0u8; $nn]; - Blake2b::digest_into(data, &mut actual).unwrap(); + Blake2b::digest_into(data, &mut actual).expect("oracle output length is valid for Blake2b"); assert_eq!(actual, expected_arr, "Blake2b nn={} oracle mismatch", $nn); }}; } @@ -1553,7 +1525,7 @@ mod tests { fn variable_output_oracle_spread_multiblock() { // Longer input (> one block) to stress the multi-block path against the oracle. use digest::consts::{U16, U32, U48, U64}; - let data: [u8; 300] = core::array::from_fn(|i| (i & 0xff) as u8); + let data: [u8; 300] = core::array::from_fn(|i| i.to_le_bytes()[0]); assert_blake2b_var_matches!(16, U16, &data); assert_blake2b_var_matches!(32, U32, &data); assert_blake2b_var_matches!(48, U48, &data); @@ -1563,23 +1535,25 @@ mod tests { #[test] fn variable_output_all_lengths_self_consistent() { // 1..=64 oneshot == streaming(single update) == streaming(chunked) - let data: [u8; 200] = core::array::from_fn(|i| ((i * 31 + 17) & 0xff) as u8); + let data: [u8; 200] = core::array::from_fn(|i| i.strict_mul(31).strict_add(17).to_le_bytes()[0]); for nn in 1usize..=64 { let mut oneshot = vec![0u8; nn]; - Blake2b::digest_into(&data, &mut oneshot).unwrap(); + Blake2b::digest_into(&data, &mut oneshot).expect("loop output length is valid for Blake2b"); - let mut h = Blake2b::new(nn).unwrap(); + let mut h = Blake2b::new(nn).expect("loop output length is valid for Blake2b"); h.update(&data); let mut single_update = vec![0u8; nn]; - h.finalize_into(&mut single_update).unwrap(); + h.finalize_into(&mut single_update) + .expect("final output length matches the configured Blake2b length"); assert_eq!(oneshot, single_update, "single-update nn={nn}"); - let mut h = Blake2b::new(nn).unwrap(); + let mut h = Blake2b::new(nn).expect("loop output length is valid for Blake2b"); for chunk in data.chunks(37) { h.update(chunk); } let mut chunked = vec![0u8; nn]; - h.finalize_into(&mut chunked).unwrap(); + h.finalize_into(&mut chunked) + .expect("final output length matches the configured Blake2b length"); assert_eq!(oneshot, chunked, "chunked nn={nn}"); } } @@ -1589,14 +1563,15 @@ mod tests { let data = [0x5Au8; 300]; for nn in [1usize, 16, 32, 48, 63, 64] { let mut expected = vec![0u8; nn]; - Blake2b::digest_into(&data, &mut expected).unwrap(); + Blake2b::digest_into(&data, &mut expected).expect("case output length is valid for Blake2b"); - let mut h = Blake2b::new(nn).unwrap(); + let mut h = Blake2b::new(nn).expect("case output length is valid for Blake2b"); for chunk in data.chunks(37) { h.update(chunk); } let mut actual = vec![0u8; nn]; - h.finalize_into(&mut actual).unwrap(); + h.finalize_into(&mut actual) + .expect("final output length matches the configured Blake2b length"); assert_eq!(actual, expected, "streaming mismatch nn={nn}"); } } @@ -1609,11 +1584,11 @@ mod tests { let mut actual_32 = [0u8; 32]; let key = validated_key(key); - Blake2b::keyed_digest_into(key, data, &mut actual_32).unwrap(); + Blake2b::keyed_digest_into(key, data, &mut actual_32).expect("32-byte keyed output is valid"); assert_eq!(actual_32, Blake2b256::keyed_digest(key, data)); let mut actual_64 = [0u8; 64]; - Blake2b::keyed_digest_into(key, data, &mut actual_64).unwrap(); + Blake2b::keyed_digest_into(key, data, &mut actual_64).expect("64-byte keyed output is valid"); assert_eq!(actual_64, Blake2b512::keyed_digest(key, data)); } @@ -1623,10 +1598,11 @@ mod tests { let data = b"message"; for nn in [1usize, 16, 40, 48, 63] { let mut keyed = vec![0u8; nn]; - Blake2b::keyed_digest_into(validated_key(key), data, &mut keyed).unwrap(); + Blake2b::keyed_digest_into(validated_key(key), data, &mut keyed) + .expect("case output length is valid for keyed Blake2b"); let mut unkeyed = vec![0u8; nn]; - Blake2b::digest_into(data, &mut unkeyed).unwrap(); + Blake2b::digest_into(data, &mut unkeyed).expect("case output length is valid for Blake2b"); assert_ne!(keyed, unkeyed, "keyed vs unkeyed nn={nn}"); } @@ -1640,26 +1616,28 @@ mod tests { let mut a = vec![0u8; nn]; let mut b = vec![0u8; nn]; let key = validated_key(key); - Blake2b::keyed_digest_into(key, data, &mut a).unwrap(); - Blake2b::keyed_digest_into(key, data, &mut b).unwrap(); + Blake2b::keyed_digest_into(key, data, &mut a).expect("case output length is valid for keyed Blake2b"); + Blake2b::keyed_digest_into(key, data, &mut b).expect("case output length is valid for keyed Blake2b"); assert_eq!(a, b, "determinism nn={nn}"); } } #[test] fn variable_output_reset_preserves_length() { - let mut h = Blake2b::new(40).unwrap(); + let mut h = Blake2b::new(40).expect("40-byte Blake2b output is valid"); h.update(b"first"); let mut first = [0u8; 40]; - h.finalize_into(&mut first).unwrap(); + h.finalize_into(&mut first) + .expect("final output length matches the configured Blake2b length"); h.reset(); h.update(b"second"); let mut second = [0u8; 40]; - h.finalize_into(&mut second).unwrap(); + h.finalize_into(&mut second) + .expect("final output length matches the configured Blake2b length"); let mut expected = [0u8; 40]; - Blake2b::digest_into(b"second", &mut expected).unwrap(); + Blake2b::digest_into(b"second", &mut expected).expect("40-byte Blake2b output is valid"); assert_eq!(second, expected); assert_ne!(first, second); } @@ -1668,7 +1646,7 @@ mod tests { fn variable_output_digest_array_matches_into() { let arr = Blake2b::digest_array::<20>(b"hello world"); let mut expected = [0u8; 20]; - Blake2b::digest_into(b"hello world", &mut expected).unwrap(); + Blake2b::digest_into(b"hello world", &mut expected).expect("20-byte Blake2b output is valid"); assert_eq!(arr, expected); } @@ -1678,13 +1656,16 @@ mod tests { .salt(*b"domain-salt\0\0\0\0\0") .personal(*b"app-v1\0\0\0\0\0\0\0\0\0\0"); let mut oneshot = [0u8; 24]; - params.hash_into(b"hello world", &mut oneshot).unwrap(); + params + .hash_into(b"hello world", &mut oneshot) + .expect("24-byte parameterized Blake2b output is valid"); - let mut h = params.build(24).unwrap(); + let mut h = params.build(24).expect("24-byte Blake2b output is valid"); h.update(b"hello "); h.update(b"world"); let mut streamed = [0u8; 24]; - h.finalize_into(&mut streamed).unwrap(); + h.finalize_into(&mut streamed) + .expect("final output length matches the configured Blake2b length"); assert_eq!(oneshot, streamed); } @@ -1693,10 +1674,12 @@ mod tests { // Empty key/salt/personal + variable length should match direct Blake2b. for nn in [1usize, 17, 32, 48, 64] { let mut via_params = vec![0u8; nn]; - Blake2bParams::new().hash_into(b"msg", &mut via_params).unwrap(); + Blake2bParams::new() + .hash_into(b"msg", &mut via_params) + .expect("case output length is valid for parameterized Blake2b"); let mut direct = vec![0u8; nn]; - Blake2b::digest_into(b"msg", &mut direct).unwrap(); + Blake2b::digest_into(b"msg", &mut direct).expect("case output length is valid for Blake2b"); assert_eq!(via_params, direct); } } @@ -1712,12 +1695,18 @@ mod tests { #[test] fn variable_output_zero_is_rejected() { - assert_eq!(Blake2b::new(0).unwrap_err(), Blake2Error::InvalidOutputLength); + assert_eq!( + Blake2b::new(0).expect_err("zero-length Blake2b output must be rejected"), + Blake2Error::InvalidOutputLength + ); } #[test] fn variable_output_over_64_is_rejected() { - assert_eq!(Blake2b::new(65).unwrap_err(), Blake2Error::InvalidOutputLength); + assert_eq!( + Blake2b::new(65).expect_err("oversized Blake2b output must be rejected"), + Blake2Error::InvalidOutputLength + ); } #[test] @@ -1730,7 +1719,7 @@ mod tests { #[test] fn variable_output_finalize_wrong_len_is_rejected() { - let h = Blake2b::new(32).unwrap(); + let h = Blake2b::new(32).expect("32-byte Blake2b output is valid"); let mut out = [0u8; 16]; assert_eq!(h.finalize_into(&mut out), Err(Blake2Error::OutputLengthMismatch)); } @@ -1766,12 +1755,18 @@ mod tests { #[test] fn keyed_empty_key_is_rejected() { - assert_eq!(Blake2bKey::new(b"").unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2bKey::new(b"").expect_err("empty Blake2b keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } #[test] fn keyed_overlength_key_is_rejected() { - assert_eq!(Blake2bKey::new(&[0u8; 65]).unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2bKey::new(&[0u8; 65]).expect_err("oversized Blake2b keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } // ── Finalize is non-destructive ─────────────────────────────────────── @@ -1893,7 +1888,7 @@ mod tests { .personal(*b"personal\0\0\0\0\0\0\0\0"); let mut h = params.build_256(); h.update(b"first"); - let _ = h.finalize(); + let _first = h.finalize(); h.reset(); h.update(b"hello world"); @@ -1955,9 +1950,10 @@ mod tests { // Keyed Blake2b-256 for &key in &[&b"key"[..], &[0xAA; 64]] { - let mut oracle = OracleBlake2bMac256::new_from_slice(key).unwrap(); - hmac::Mac::update(&mut oracle, b"message"); - let expected: [u8; 32] = hmac::Mac::finalize(oracle).into_bytes().into(); + let mut oracle = + OracleBlake2bMac256::new_from_slice(key).expect("test key length must satisfy the oracle MAC contract"); + blake2::digest::Mac::update(&mut oracle, b"message"); + let expected: [u8; 32] = blake2::digest::Mac::finalize(oracle).into_bytes().into(); let mut h = Blake2b256::keyed_with_compress_for_test(key, compress, compress_blocks); h.update(b"message"); diff --git a/src/hashes/crypto/blake2b/riscv64.rs b/src/hashes/crypto/blake2b/riscv64.rs index 4964e191..74bec92a 100644 --- a/src/hashes/crypto/blake2b/riscv64.rs +++ b/src/hashes/crypto/blake2b/riscv64.rs @@ -15,9 +15,6 @@ //! //! Requires the V extension. Caller must verify `riscv::V`. -#![allow(unsafe_code)] -#![allow(clippy::cast_possible_truncation, clippy::indexing_slicing)] - use super::kernels::{SIGMA, init_v, load_msg}; // ─── Inline asm helpers ─────────────────────────────────────────────────── @@ -70,47 +67,26 @@ fn vxor(a: [u64; 2], b: [u64; 2]) -> [u64; 2] { // ─── G function on register pairs ───────────────────────────────────────── +struct PairState { + a: [[u64; 2]; 2], + b: [[u64; 2]; 2], + c: [[u64; 2]; 2], + d: [[u64; 2]; 2], +} + /// Blake2b G mixing on 2-wide pairs. #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn g2( - a0: &mut [u64; 2], - a1: &mut [u64; 2], - b0: &mut [u64; 2], - b1: &mut [u64; 2], - c0: &mut [u64; 2], - c1: &mut [u64; 2], - d0: &mut [u64; 2], - d1: &mut [u64; 2], - mx0: [u64; 2], - mx1: [u64; 2], - my0: [u64; 2], - my1: [u64; 2], -) { - // a += b + mx - *a0 = vadd(vadd(*a0, *b0), mx0); - *a1 = vadd(vadd(*a1, *b1), mx1); - // d = (d ^ a) >>> 32 - *d0 = ror32(vxor(*d0, *a0)); - *d1 = ror32(vxor(*d1, *a1)); - // c += d - *c0 = vadd(*c0, *d0); - *c1 = vadd(*c1, *d1); - // b = (b ^ c) >>> 24 - *b0 = ror24(vxor(*b0, *c0)); - *b1 = ror24(vxor(*b1, *c1)); - // a += b + my - *a0 = vadd(vadd(*a0, *b0), my0); - *a1 = vadd(vadd(*a1, *b1), my1); - // d = (d ^ a) >>> 16 - *d0 = ror16(vxor(*d0, *a0)); - *d1 = ror16(vxor(*d1, *a1)); - // c += d - *c0 = vadd(*c0, *d0); - *c1 = vadd(*c1, *d1); - // b = (b ^ c) >>> 63 - *b0 = ror63(vxor(*b0, *c0)); - *b1 = ror63(vxor(*b1, *c1)); +fn g2(state: &mut PairState, mx: [[u64; 2]; 2], my: [[u64; 2]; 2]) { + for pair in 0..2 { + state.a[pair] = vadd(vadd(state.a[pair], state.b[pair]), mx[pair]); + state.d[pair] = ror32(vxor(state.d[pair], state.a[pair])); + state.c[pair] = vadd(state.c[pair], state.d[pair]); + state.b[pair] = ror24(vxor(state.b[pair], state.c[pair])); + state.a[pair] = vadd(vadd(state.a[pair], state.b[pair]), my[pair]); + state.d[pair] = ror16(vxor(state.d[pair], state.a[pair])); + state.c[pair] = vadd(state.c[pair], state.d[pair]); + state.b[pair] = ror63(vxor(state.b[pair], state.c[pair])); + } } // ─── Diagonalize / Un-diagonalize ───────────────────────────────────────── @@ -121,54 +97,32 @@ fn g2( /// new_lo = [old_lo[1], old_hi[0]] /// new_hi = [old_hi[1], old_lo[0]] #[inline(always)] -fn diagonalize( - b0: &mut [u64; 2], - b1: &mut [u64; 2], - c0: &mut [u64; 2], - c1: &mut [u64; 2], - d0: &mut [u64; 2], - d1: &mut [u64; 2], -) { +fn diagonalize(state: &mut PairState) { // B: rotate left 1 - let tb0 = *b0; - let tb1 = *b1; - *b0 = [tb0[1], tb1[0]]; - *b1 = [tb1[1], tb0[0]]; + let [b0, b1] = state.b; + state.b = [[b0[1], b1[0]], [b1[1], b0[0]]]; // C: rotate left 2 = swap lo/hi - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 3 = rotate right 1 - let td0 = *d0; - let td1 = *d1; - *d0 = [td1[1], td0[0]]; - *d1 = [td0[1], td1[0]]; + let [d0, d1] = state.d; + state.d = [[d1[1], d0[0]], [d0[1], d1[0]]]; } /// Un-diagonalize: reverse the rotations. #[inline(always)] -fn undiagonalize( - b0: &mut [u64; 2], - b1: &mut [u64; 2], - c0: &mut [u64; 2], - c1: &mut [u64; 2], - d0: &mut [u64; 2], - d1: &mut [u64; 2], -) { +fn undiagonalize(state: &mut PairState) { // B: rotate right 1 (undo left 1) - let tb0 = *b0; - let tb1 = *b1; - *b0 = [tb1[1], tb0[0]]; - *b1 = [tb0[1], tb1[0]]; + let [b0, b1] = state.b; + state.b = [[b1[1], b0[0]], [b0[1], b1[0]]]; // C: swap back - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 1 (undo right 1) - let td0 = *d0; - let td1 = *d1; - *d0 = [td0[1], td1[0]]; - *d1 = [td1[1], td0[0]]; + let [d0, d1] = state.d; + state.d = [[d0[1], d1[0]], [d1[1], d0[0]]]; } // ─── Compress entry point ───────────────────────────────────────────────── @@ -188,14 +142,12 @@ pub(super) unsafe fn compress_rvv(h: &mut [u64; 8], block: &[u8; 128], t: u128, let v = init_v(h, t, last); // Pack into 2-wide pairs: (lo, hi) for each row - let mut a0 = [v[0], v[1]]; - let mut a1 = [v[2], v[3]]; - let mut b0 = [v[4], v[5]]; - let mut b1 = [v[6], v[7]]; - let mut c0 = [v[8], v[9]]; - let mut c1 = [v[10], v[11]]; - let mut d0 = [v[12], v[13]]; - let mut d1 = [v[14], v[15]]; + let mut state = PairState { + a: [[v[0], v[1]], [v[2], v[3]]], + b: [[v[4], v[5]], [v[6], v[7]]], + c: [[v[8], v[9]], [v[10], v[11]]], + d: [[v[12], v[13]], [v[14], v[15]]], + }; // 12 rounds for round in 0..12u8 { @@ -207,11 +159,9 @@ pub(super) unsafe fn compress_rvv(h: &mut [u64; 8], block: &[u8; 128], t: u128, let my0 = [m[s[1] as usize], m[s[3] as usize]]; let my1 = [m[s[5] as usize], m[s[7] as usize]]; - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - diagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + diagonalize(&mut state); // Diagonal step let mx0 = [m[s[8] as usize], m[s[10] as usize]]; @@ -219,20 +169,18 @@ pub(super) unsafe fn compress_rvv(h: &mut [u64; 8], block: &[u8; 128], t: u128, let my0 = [m[s[9] as usize], m[s[11] as usize]]; let my1 = [m[s[13] as usize], m[s[15] as usize]]; - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - undiagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + undiagonalize(&mut state); } // Finalize: h[i] ^= v[i] ^ v[i+8] - h[0] ^= a0[0] ^ c0[0]; - h[1] ^= a0[1] ^ c0[1]; - h[2] ^= a1[0] ^ c1[0]; - h[3] ^= a1[1] ^ c1[1]; - h[4] ^= b0[0] ^ d0[0]; - h[5] ^= b0[1] ^ d0[1]; - h[6] ^= b1[0] ^ d1[0]; - h[7] ^= b1[1] ^ d1[1]; + h[0] ^= state.a[0][0] ^ state.c[0][0]; + h[1] ^= state.a[0][1] ^ state.c[0][1]; + h[2] ^= state.a[1][0] ^ state.c[1][0]; + h[3] ^= state.a[1][1] ^ state.c[1][1]; + h[4] ^= state.b[0][0] ^ state.d[0][0]; + h[5] ^= state.b[0][1] ^ state.d[0][1]; + h[6] ^= state.b[1][0] ^ state.d[1][0]; + h[7] ^= state.b[1][1] ^ state.d[1][1]; } diff --git a/src/hashes/crypto/blake2b/wasm.rs b/src/hashes/crypto/blake2b/wasm.rs index d1c1cd1e..b0632e6b 100644 --- a/src/hashes/crypto/blake2b/wasm.rs +++ b/src/hashes/crypto/blake2b/wasm.rs @@ -11,8 +11,6 @@ //! //! Requires WASM SIMD128. Caller must verify `wasm::SIMD128`. -#![allow(clippy::cast_possible_truncation, clippy::indexing_slicing)] - #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; @@ -55,45 +53,39 @@ fn ror63(x: v128) -> v128 { /// Blake2b G mixing on SIMD rows (2-wide). #[cfg(target_arch = "wasm32")] #[inline(always)] -#[allow(clippy::too_many_arguments)] -fn g2( - a0: &mut v128, - a1: &mut v128, - b0: &mut v128, - b1: &mut v128, - c0: &mut v128, - c1: &mut v128, - d0: &mut v128, - d1: &mut v128, - mx0: v128, - mx1: v128, - my0: v128, - my1: v128, -) { +fn g2(state: &mut WorkingState, mx: [v128; 2], my: [v128; 2]) { // a += b + mx - *a0 = i64x2_add(i64x2_add(*a0, *b0), mx0); - *a1 = i64x2_add(i64x2_add(*a1, *b1), mx1); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), mx[0]); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), mx[1]); // d = (d ^ a) >>> 32 - *d0 = ror32(v128_xor(*d0, *a0)); - *d1 = ror32(v128_xor(*d1, *a1)); + state.d[0] = ror32(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror32(v128_xor(state.d[1], state.a[1])); // c += d - *c0 = i64x2_add(*c0, *d0); - *c1 = i64x2_add(*c1, *d1); + state.c[0] = i64x2_add(state.c[0], state.d[0]); + state.c[1] = i64x2_add(state.c[1], state.d[1]); // b = (b ^ c) >>> 24 - *b0 = ror24(v128_xor(*b0, *c0)); - *b1 = ror24(v128_xor(*b1, *c1)); + state.b[0] = ror24(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror24(v128_xor(state.b[1], state.c[1])); // a += b + my - *a0 = i64x2_add(i64x2_add(*a0, *b0), my0); - *a1 = i64x2_add(i64x2_add(*a1, *b1), my1); + state.a[0] = i64x2_add(i64x2_add(state.a[0], state.b[0]), my[0]); + state.a[1] = i64x2_add(i64x2_add(state.a[1], state.b[1]), my[1]); // d = (d ^ a) >>> 16 - *d0 = ror16(v128_xor(*d0, *a0)); - *d1 = ror16(v128_xor(*d1, *a1)); + state.d[0] = ror16(v128_xor(state.d[0], state.a[0])); + state.d[1] = ror16(v128_xor(state.d[1], state.a[1])); // c += d - *c0 = i64x2_add(*c0, *d0); - *c1 = i64x2_add(*c1, *d1); + state.c[0] = i64x2_add(state.c[0], state.d[0]); + state.c[1] = i64x2_add(state.c[1], state.d[1]); // b = (b ^ c) >>> 63 - *b0 = ror63(v128_xor(*b0, *c0)); - *b1 = ror63(v128_xor(*b1, *c1)); + state.b[0] = ror63(v128_xor(state.b[0], state.c[0])); + state.b[1] = ror63(v128_xor(state.b[1], state.c[1])); +} + +#[cfg(target_arch = "wasm32")] +struct WorkingState { + a: [v128; 2], + b: [v128; 2], + c: [v128; 2], + d: [v128; 2], } // ─── Diagonalize / Un-diagonalize ───────────────────────────────────────── @@ -103,41 +95,41 @@ fn g2( /// `i64x2_shuffle` indices: 0,1 = lanes from first operand, 2,3 = from second. #[cfg(target_arch = "wasm32")] #[inline(always)] -fn diagonalize(b0: &mut v128, b1: &mut v128, c0: &mut v128, c1: &mut v128, d0: &mut v128, d1: &mut v128) { +fn diagonalize(state: &mut WorkingState) { // B: rotate left 1: (v4,v5,v6,v7) -> (v5,v6,v7,v4) - let tb0 = *b0; - let tb1 = *b1; - *b0 = i64x2_shuffle::<1, 2>(tb0, tb1); // [b0[1], b1[0]] = [v5, v6] - *b1 = i64x2_shuffle::<1, 2>(tb1, tb0); // [b1[1], b0[0]] = [v7, v4] + let tb0 = state.b[0]; + let tb1 = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb0, tb1); // [b0[1], b1[0]] = [v5, v6] + state.b[1] = i64x2_shuffle::<1, 2>(tb1, tb0); // [b1[1], b0[0]] = [v7, v4] // C: rotate left 2 = swap lo/hi - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 3 = rotate right 1: (v12,v13,v14,v15) -> (v15,v12,v13,v14) - let td0 = *d0; - let td1 = *d1; - *d0 = i64x2_shuffle::<1, 2>(td1, td0); // [d1[1], d0[0]] = [v15, v12] - *d1 = i64x2_shuffle::<1, 2>(td0, td1); // [d0[1], d1[0]] = [v13, v14] + let td0 = state.d[0]; + let td1 = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td1, td0); // [d1[1], d0[0]] = [v15, v12] + state.d[1] = i64x2_shuffle::<1, 2>(td0, td1); // [d0[1], d1[0]] = [v13, v14] } /// Un-diagonalize: reverse the rotations. #[cfg(target_arch = "wasm32")] #[inline(always)] -fn undiagonalize(b0: &mut v128, b1: &mut v128, c0: &mut v128, c1: &mut v128, d0: &mut v128, d1: &mut v128) { +fn undiagonalize(state: &mut WorkingState) { // B: rotate right 1 (undo left 1) - let tb0 = *b0; - let tb1 = *b1; - *b0 = i64x2_shuffle::<1, 2>(tb1, tb0); - *b1 = i64x2_shuffle::<1, 2>(tb0, tb1); + let tb0 = state.b[0]; + let tb1 = state.b[1]; + state.b[0] = i64x2_shuffle::<1, 2>(tb1, tb0); + state.b[1] = i64x2_shuffle::<1, 2>(tb0, tb1); // C: swap back - core::mem::swap(c0, c1); + state.c.swap(0, 1); // D: rotate left 1 (undo right 1) - let td0 = *d0; - let td1 = *d1; - *d0 = i64x2_shuffle::<1, 2>(td0, td1); - *d1 = i64x2_shuffle::<1, 2>(td1, td0); + let td0 = state.d[0]; + let td1 = state.d[1]; + state.d[0] = i64x2_shuffle::<1, 2>(td0, td1); + state.d[1] = i64x2_shuffle::<1, 2>(td1, td0); } // ─── Load helpers ───────────────────────────────────────────────────────── @@ -149,12 +141,18 @@ fn load_msg_pair(m: &[u64; 16], i0: u8, i1: u8) -> v128 { u64x2(m[i0 as usize], m[i1 as usize]) } -/// Load 2 consecutive u64 values from a pointer as v128. +/// Load two consecutive words into a SIMD vector. #[cfg(target_arch = "wasm32")] #[inline(always)] -unsafe fn vload_u64_pair(p: *const u64) -> v128 { - // SAFETY: caller ensures p is valid for 2 x u64 (16 bytes). - unsafe { v128_load(p as *const v128) } +fn load_u64_pair(words: &[u64; N], offset: usize) -> v128 { + u64x2(words[offset], words[offset.strict_add(1)]) +} + +#[cfg(target_arch = "wasm32")] +#[inline(always)] +fn store_u64_pair(words: &mut [u64; 8], offset: usize, value: v128) { + words[offset] = u64x2_extract_lane::<0>(value); + words[offset.strict_add(1)] = u64x2_extract_lane::<1>(value); } // ─── Compress entry point ───────────────────────────────────────────────── @@ -170,16 +168,13 @@ pub(super) unsafe fn compress_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u1 let m = load_msg(block); let v = init_v(h, t, last); - // Pack into 2-wide SIMD rows: (lo, hi) for each row - // SAFETY: v is a [u64; 16] — pointer arithmetic is within bounds. - let mut a0 = unsafe { vload_u64_pair(v.as_ptr()) }; // v[0], v[1] - let mut a1 = unsafe { vload_u64_pair(v.as_ptr().add(2)) }; // v[2], v[3] - let mut b0 = unsafe { vload_u64_pair(v.as_ptr().add(4)) }; // v[4], v[5] - let mut b1 = unsafe { vload_u64_pair(v.as_ptr().add(6)) }; // v[6], v[7] - let mut c0 = unsafe { vload_u64_pair(v.as_ptr().add(8)) }; // v[8], v[9] - let mut c1 = unsafe { vload_u64_pair(v.as_ptr().add(10)) }; // v[10], v[11] - let mut d0 = unsafe { vload_u64_pair(v.as_ptr().add(12)) }; // v[12], v[13] - let mut d1 = unsafe { vload_u64_pair(v.as_ptr().add(14)) }; // v[14], v[15] + // Pack into 2-wide SIMD rows: (lo, hi) for each row. + let mut state = WorkingState { + a: [load_u64_pair(&v, 0), load_u64_pair(&v, 2)], + b: [load_u64_pair(&v, 4), load_u64_pair(&v, 6)], + c: [load_u64_pair(&v, 8), load_u64_pair(&v, 10)], + d: [load_u64_pair(&v, 12), load_u64_pair(&v, 14)], + }; // 12 rounds for round in 0..12u8 { @@ -191,11 +186,9 @@ pub(super) unsafe fn compress_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u1 let my0 = load_msg_pair(&m, s[1], s[3]); let my1 = load_msg_pair(&m, s[5], s[7]); - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - diagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + diagonalize(&mut state); // Diagonal step let mx0 = load_msg_pair(&m, s[8], s[10]); @@ -203,30 +196,24 @@ pub(super) unsafe fn compress_simd128(h: &mut [u64; 8], block: &[u8; 128], t: u1 let my0 = load_msg_pair(&m, s[9], s[11]); let my1 = load_msg_pair(&m, s[13], s[15]); - g2( - &mut a0, &mut a1, &mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1, mx0, mx1, my0, my1, - ); + g2(&mut state, [mx0, mx1], [my0, my1]); - undiagonalize(&mut b0, &mut b1, &mut c0, &mut c1, &mut d0, &mut d1); + undiagonalize(&mut state); } // Finalize: h[i] ^= v[i] ^ v[i+8] - // SAFETY: h is a [u64; 8] — pointer arithmetic is within bounds. - let h0 = unsafe { vload_u64_pair(h.as_ptr()) }; - let h1 = unsafe { vload_u64_pair(h.as_ptr().add(2)) }; - let h2 = unsafe { vload_u64_pair(h.as_ptr().add(4)) }; - let h3 = unsafe { vload_u64_pair(h.as_ptr().add(6)) }; - - let r0 = v128_xor(h0, v128_xor(a0, c0)); - let r1 = v128_xor(h1, v128_xor(a1, c1)); - let r2 = v128_xor(h2, v128_xor(b0, d0)); - let r3 = v128_xor(h3, v128_xor(b1, d1)); - - // SAFETY: h is a [u64; 8] — pointer arithmetic is within bounds. - unsafe { - v128_store(h.as_mut_ptr() as *mut v128, r0); - v128_store(h.as_mut_ptr().add(2) as *mut v128, r1); - v128_store(h.as_mut_ptr().add(4) as *mut v128, r2); - v128_store(h.as_mut_ptr().add(6) as *mut v128, r3); - } + let h0 = load_u64_pair(h, 0); + let h1 = load_u64_pair(h, 2); + let h2 = load_u64_pair(h, 4); + let h3 = load_u64_pair(h, 6); + + let r0 = v128_xor(h0, v128_xor(state.a[0], state.c[0])); + let r1 = v128_xor(h1, v128_xor(state.a[1], state.c[1])); + let r2 = v128_xor(h2, v128_xor(state.b[0], state.d[0])); + let r3 = v128_xor(h3, v128_xor(state.b[1], state.d[1])); + + store_u64_pair(h, 0, r0); + store_u64_pair(h, 2, r1); + store_u64_pair(h, 4, r2); + store_u64_pair(h, 6, r3); } diff --git a/src/hashes/crypto/blake2b/x86_64.rs b/src/hashes/crypto/blake2b/x86_64.rs index 30f94eb7..812854ff 100644 --- a/src/hashes/crypto/blake2b/x86_64.rs +++ b/src/hashes/crypto/blake2b/x86_64.rs @@ -12,8 +12,6 @@ //! //! `compress_avx2` requires `avx2`. `compress_avx512vl` requires `avx512f` + `avx512vl`. -#![allow(clippy::cast_possible_truncation, clippy::indexing_slicing)] - use core::arch::x86_64::*; use super::kernels::{SIGMA, init_v, load_msg}; @@ -23,6 +21,10 @@ use super::kernels::{SIGMA, init_v, load_msg}; // ─── AVX2 rotation helpers ─────────────────────────────────────────────────── /// ROR 32: swap 32-bit halves within each 64-bit lane. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror32_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -31,17 +33,19 @@ unsafe fn ror32_avx2(x: __m256i) -> __m256i { } /// ROR 24: byte shuffle within each 128-bit lane. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror24_avx2(x: __m256i) -> __m256i { - // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. // Each 64-bit lane rotates its bytes right by 3 (= rotate_right(24)). #[repr(align(32))] struct Align32([u8; 32]); static ROT24: Align32 = Align32([ 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, 3, 4, 5, 6, 7, 0, 1, 2, 11, 12, 13, 14, 15, 8, 9, 10, ]); - // SAFETY: `ROT24` is 32-byte aligned by construction, so loading it with an - // aligned AVX2 load is valid and the subsequent shuffle stays in registers. + // SAFETY: the caller provides AVX2, and `ROT24` is 32-byte aligned for the aligned load. unsafe { let mask = _mm256_load_si256(ROT24.0.as_ptr().cast()); _mm256_shuffle_epi8(x, mask) @@ -49,17 +53,19 @@ unsafe fn ror24_avx2(x: __m256i) -> __m256i { } /// ROR 16: byte shuffle within each 128-bit lane. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror16_avx2(x: __m256i) -> __m256i { - // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. // Each 64-bit lane rotates its bytes right by 2 (= rotate_right(16)). #[repr(align(32))] struct Align32([u8; 32]); static ROT16: Align32 = Align32([ 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, 2, 3, 4, 5, 6, 7, 0, 1, 10, 11, 12, 13, 14, 15, 8, 9, ]); - // SAFETY: `ROT16` is 32-byte aligned by construction, so loading it with an - // aligned AVX2 load is valid and the subsequent shuffle stays in registers. + // SAFETY: the caller provides AVX2, and `ROT16` is 32-byte aligned for the aligned load. unsafe { let mask = _mm256_load_si256(ROT16.0.as_ptr().cast()); _mm256_shuffle_epi8(x, mask) @@ -67,6 +73,10 @@ unsafe fn ror16_avx2(x: __m256i) -> __m256i { } /// ROR 63: (x >> 63) | (x << 1) = (x + x) ^ (x >> 63). +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn ror63_avx2(x: __m256i) -> __m256i { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -76,6 +86,10 @@ unsafe fn ror63_avx2(x: __m256i) -> __m256i { // ─── AVX2 G function ───────────────────────────────────────────────────────── /// Blake2b quarter-round G on 4-wide AVX2 rows. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn g_avx2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m256i, mx: __m256i, my: __m256i) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -102,6 +116,10 @@ unsafe fn g_avx2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m2 // ─── Diagonalize / un-diagonalize ──────────────────────────────────────────── /// Diagonalize: rotate B left by 1, C left by 2, D left by 3. +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn diagonalize(b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -116,6 +134,10 @@ unsafe fn diagonalize(b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { } /// Un-diagonalize: reverse the rotations (B right 1, C swap, D left 1). +/// +/// # Safety +/// +/// Caller must ensure AVX2 is available. #[inline(always)] unsafe fn undiagonalize(b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. @@ -156,16 +178,16 @@ pub(super) unsafe fn compress_avx2(h: &mut [u64; 8], block: &[u8; 128], t: u128, // Column step: G on (0,4,8,12), (1,5,9,13), (2,6,10,14), (3,7,11,15) // _mm256_set_epi64x takes args in HIGH→LOW order: lane3, lane2, lane1, lane0 let mx = _mm256_set_epi64x( - m[s[6] as usize] as i64, - m[s[4] as usize] as i64, - m[s[2] as usize] as i64, - m[s[0] as usize] as i64, + m[s[6] as usize].cast_signed(), + m[s[4] as usize].cast_signed(), + m[s[2] as usize].cast_signed(), + m[s[0] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[7] as usize] as i64, - m[s[5] as usize] as i64, - m[s[3] as usize] as i64, - m[s[1] as usize] as i64, + m[s[7] as usize].cast_signed(), + m[s[5] as usize].cast_signed(), + m[s[3] as usize].cast_signed(), + m[s[1] as usize].cast_signed(), ); g_avx2(&mut a, &mut b, &mut c, &mut d, mx, my); @@ -174,16 +196,16 @@ pub(super) unsafe fn compress_avx2(h: &mut [u64; 8], block: &[u8; 128], t: u128, // Diagonal step: G on (0,5,10,15), (1,6,11,12), (2,7,8,13), (3,4,9,14) let mx = _mm256_set_epi64x( - m[s[14] as usize] as i64, - m[s[12] as usize] as i64, - m[s[10] as usize] as i64, - m[s[8] as usize] as i64, + m[s[14] as usize].cast_signed(), + m[s[12] as usize].cast_signed(), + m[s[10] as usize].cast_signed(), + m[s[8] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[15] as usize] as i64, - m[s[13] as usize] as i64, - m[s[11] as usize] as i64, - m[s[9] as usize] as i64, + m[s[15] as usize].cast_signed(), + m[s[13] as usize].cast_signed(), + m[s[11] as usize].cast_signed(), + m[s[9] as usize].cast_signed(), ); g_avx2(&mut a, &mut b, &mut c, &mut d, mx, my); @@ -212,6 +234,10 @@ pub(super) unsafe fn compress_avx2(h: &mut [u64; 8], block: &[u8; 128], t: u128, /// Blake2b quarter-round G on 4-wide AVX-512VL rows. /// /// All rotations use `VPRORQ` (`_mm256_ror_epi64`) — no shuffle tables needed. +/// +/// # Safety +/// +/// Caller must ensure AVX-512F and AVX-512VL are available. #[inline(always)] unsafe fn g_avx512vl(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m256i, mx: __m256i, my: __m256i) { // SAFETY: AVX-512VL intrinsics are available via the caller's #[target_feature] attribute. @@ -265,16 +291,16 @@ pub(super) unsafe fn compress_avx512vl(h: &mut [u64; 8], block: &[u8; 128], t: u // Column step let mx = _mm256_set_epi64x( - m[s[6] as usize] as i64, - m[s[4] as usize] as i64, - m[s[2] as usize] as i64, - m[s[0] as usize] as i64, + m[s[6] as usize].cast_signed(), + m[s[4] as usize].cast_signed(), + m[s[2] as usize].cast_signed(), + m[s[0] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[7] as usize] as i64, - m[s[5] as usize] as i64, - m[s[3] as usize] as i64, - m[s[1] as usize] as i64, + m[s[7] as usize].cast_signed(), + m[s[5] as usize].cast_signed(), + m[s[3] as usize].cast_signed(), + m[s[1] as usize].cast_signed(), ); g_avx512vl(&mut a, &mut b, &mut c, &mut d, mx, my); @@ -283,16 +309,16 @@ pub(super) unsafe fn compress_avx512vl(h: &mut [u64; 8], block: &[u8; 128], t: u // Diagonal step let mx = _mm256_set_epi64x( - m[s[14] as usize] as i64, - m[s[12] as usize] as i64, - m[s[10] as usize] as i64, - m[s[8] as usize] as i64, + m[s[14] as usize].cast_signed(), + m[s[12] as usize].cast_signed(), + m[s[10] as usize].cast_signed(), + m[s[8] as usize].cast_signed(), ); let my = _mm256_set_epi64x( - m[s[15] as usize] as i64, - m[s[13] as usize] as i64, - m[s[11] as usize] as i64, - m[s[9] as usize] as i64, + m[s[15] as usize].cast_signed(), + m[s[13] as usize].cast_signed(), + m[s[11] as usize].cast_signed(), + m[s[9] as usize].cast_signed(), ); g_avx512vl(&mut a, &mut b, &mut c, &mut d, mx, my); diff --git a/src/hashes/crypto/blake2s/kernels.rs b/src/hashes/crypto/blake2s/kernels.rs index 73f6c76e..5ea6add5 100644 --- a/src/hashes/crypto/blake2s/kernels.rs +++ b/src/hashes/crypto/blake2s/kernels.rs @@ -1,5 +1,6 @@ //! Blake2s portable compression function and kernel dispatch (RFC 7693). +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] use crate::platform::Caps; #[cfg(target_arch = "riscv64")] use crate::platform::caps::riscv; @@ -18,8 +19,7 @@ pub(crate) type CompressBlocksFn = fn(&mut [u32; 8], &[u8], &mut u64); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -#[cfg_attr(target_os = "macos", allow(dead_code))] -pub enum Blake2sKernelId { +pub(crate) enum Blake2sKernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Avx2 = 1, @@ -35,7 +35,7 @@ impl Blake2sKernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -77,14 +77,12 @@ fn compress_x86_avx512vl(h: &mut [u32; 8], block: &[u8; 64], t: u64, last: bool) #[inline(always)] fn compress_blocks_with(h: &mut [u32; 8], blocks: &[u8], t: &mut u64, compress: CompressFn) { debug_assert_eq!(blocks.len() % 64, 0); - let mut chunks = blocks.chunks_exact(64); - for chunk in &mut chunks { + let (chunks, remainder) = blocks.as_chunks::<64>(); + for block in chunks { *t = t.strict_add(64); - // SAFETY: `chunks_exact(64)` yields slices of exactly 64 bytes. - let block = unsafe { &*chunk.as_ptr().cast::<[u8; 64]>() }; compress(h, block, *t, false); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } fn compress_blocks_portable(h: &mut [u32; 8], blocks: &[u8], t: &mut u64) { @@ -143,10 +141,10 @@ pub(crate) fn compress_blocks_fn(id: Blake2sKernelId) -> CompressBlocksFn { } /// Capabilities required to run the given kernel. +#[cfg(any(test, not(all(target_arch = "aarch64", target_os = "macos"))))] #[inline] #[must_use] -#[allow(dead_code)] // Used by runtime dispatch on targets that don't bypass to a fixed kernel. -pub const fn required_caps(id: Blake2sKernelId) -> Caps { +pub(crate) const fn required_caps(id: Blake2sKernelId) -> Caps { match id { Blake2sKernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -162,7 +160,7 @@ pub const fn required_caps(id: Blake2sKernelId) -> Caps { /// All kernel IDs for agreement testing. #[cfg(test)] -pub const ALL: &[Blake2sKernelId] = &[ +pub(crate) const ALL: &[Blake2sKernelId] = &[ Blake2sKernelId::Portable, #[cfg(target_arch = "x86_64")] Blake2sKernelId::X86Avx2, @@ -197,7 +195,6 @@ pub(crate) fn compile_time_best() -> CompressFn { { return compress_x86_avx2; } - #[allow(unreachable_code)] compress } @@ -215,7 +212,6 @@ pub(crate) fn compile_time_best_blocks() -> CompressBlocksFn { { return compress_blocks_x86_avx2; } - #[allow(unreachable_code)] compress_blocks_portable } @@ -232,7 +228,6 @@ pub(crate) const IV: [u32; 8] = [ ]; /// Message-word permutation schedule (10 rows, reused cyclically for 10 rounds). -#[allow(dead_code)] // Used by target-specific SIMD backends that are not compiled on every host. pub(crate) const SIGMA: [[u8; 16]; 10] = [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], @@ -386,7 +381,6 @@ fn round(v: &mut [U32x4; 4], m: &[u32; 16], s: &[u8; 16]) { /// Load 16 little-endian u32 message words from a 64-byte block. #[inline(always)] -#[allow(clippy::indexing_slicing)] pub(crate) fn load_msg(block: &[u8; 64]) -> [u32; 16] { let mut m = [0u32; 16]; let src = block.as_ptr(); @@ -398,18 +392,28 @@ pub(crate) fn load_msg(block: &[u8; 64]) -> [u32; 16] { m } +#[inline(always)] +fn split_counter(counter: u64) -> (u32, u32) { + let [b0, b1, b2, b3, b4, b5, b6, b7] = counter.to_le_bytes(); + ( + u32::from_le_bytes([b0, b1, b2, b3]), + u32::from_le_bytes([b4, b5, b6, b7]), + ) +} + /// Initialize the 16-word working vector. #[cfg(any(target_arch = "x86_64", target_arch = "wasm32", target_arch = "riscv64"))] #[inline(always)] pub(crate) fn init_v(h: &[u32; 8], t: u64, last: bool) -> [u32; 16] { + let (t0, t1) = split_counter(t); let mut v = [0u32; 16]; v[..8].copy_from_slice(h); v[8] = IV[0]; v[9] = IV[1]; v[10] = IV[2]; v[11] = IV[3]; - v[12] = IV[4] ^ (t as u32); - v[13] = IV[5] ^ ((t >> 32) as u32); + v[12] = IV[4] ^ t0; + v[13] = IV[5] ^ t1; v[14] = if last { IV[6] ^ u32::MAX } else { IV[6] }; v[15] = IV[7]; v @@ -419,11 +423,9 @@ pub(crate) fn init_v(h: &[u32; 8], t: u64, last: bool) -> [u32; 16] { /// /// `t` is the total number of input bytes after this block (inclusive). /// `last` is `true` for the final block (sets the finalization flag). -#[allow(clippy::indexing_slicing)] pub(crate) fn compress(h: &mut [u32; 8], block: &[u8; 64], t: u64, last: bool) { let m = load_msg(block); - let t0 = t as u32; - let t1 = (t >> 32) as u32; + let (t0, t1) = split_counter(t); let f0 = if last { u32::MAX } else { 0 }; let mut v = [ diff --git a/src/hashes/crypto/blake2s/mod.rs b/src/hashes/crypto/blake2s/mod.rs index 010f16b4..3b7dbfeb 100644 --- a/src/hashes/crypto/blake2s/mod.rs +++ b/src/hashes/crypto/blake2s/mod.rs @@ -47,12 +47,13 @@ use kernels::IV; use super::Blake2Error; use crate::traits::{Digest, ct}; -const BLOCK_SIZE: usize = 64; -const MAX_KEY_LEN: usize = 32; -const MAX_OUTPUT_LEN: usize = 32; +const BLOCK_SIZE_U8: u8 = 64; +const BLOCK_SIZE: usize = BLOCK_SIZE_U8 as usize; +const MAX_KEY_LEN_U8: u8 = 32; +const MAX_KEY_LEN: usize = MAX_KEY_LEN_U8 as usize; +const MAX_OUTPUT_LEN_U8: u8 = 32; -#[cfg(any(test, feature = "diag"))] -#[allow(dead_code)] +#[cfg(feature = "diag")] #[inline] #[must_use] pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { @@ -155,15 +156,14 @@ impl Core { Self::new_with_params(nn, key, &[0u8; SALT_LEN], &[0u8; PERSONAL_LEN]) } - #[allow(clippy::indexing_slicing)] fn new_with_params(nn: u8, key: &[u8], salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> Self { assert!( - nn >= 1 && nn as usize <= MAX_OUTPUT_LEN, + (1..=MAX_OUTPUT_LEN_U8).contains(&nn), "Blake2s output length must be 1-32" ); assert!(key.len() <= MAX_KEY_LEN, "Blake2s key must be at most 32 bytes"); - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); let h = init_state_with_params(nn, kk, salt, personal); let stored_key = if kk > 0 { @@ -177,7 +177,7 @@ impl Core { let mut buf = [0u8; BLOCK_SIZE]; let buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -201,29 +201,23 @@ impl Core { fn zeroize_key_if_any(&mut self) { if self.kk > 0 { // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. - unsafe { ct::zeroize_no_fence(&mut *self.key.as_mut_ptr()) }; + unsafe { ct::zeroize_no_fence(self.key.assume_init_mut()) }; } } #[inline(always)] fn wipe(&mut self) { - for word in self.h.iter_mut() { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut self.h); ct::zeroize_no_fence(&mut self.buf); self.zeroize_key_if_any(); - // SAFETY: fields are valid, aligned, dereferenceable pointers. - unsafe { - core::ptr::write_volatile(&mut self.buf_len, 0); - core::ptr::write_volatile(&mut self.t, 0); - core::ptr::write_volatile(&mut self.nn, 0); - core::ptr::write_volatile(&mut self.kk, 0); - } + ct::zeroize_no_fence(core::slice::from_mut(&mut self.buf_len)); + ct::zeroize_words_no_fence(core::slice::from_mut(&mut self.t)); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.nn)); + ct::zeroize_no_fence(core::slice::from_mut(&mut self.kk)); core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } - #[cfg(any(test, feature = "diag"))] + #[cfg(test)] fn new_with_compress_for_test( nn: u8, key: &[u8], @@ -236,7 +230,6 @@ impl Core { core } - #[allow(clippy::indexing_slicing)] fn update(&mut self, data: &[u8]) { if data.is_empty() { return; @@ -248,7 +241,7 @@ impl Core { if self.buf_len > 0 && (self.buf_len as usize).strict_add(data_len) > BLOCK_SIZE { let fill = BLOCK_SIZE.strict_sub(self.buf_len as usize); self.buf[self.buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); - self.t = self.t.strict_add(BLOCK_SIZE as u64); + self.t = self.t.strict_add(u64::from(BLOCK_SIZE_U8)); (self.compress)(&mut self.h, &self.buf, self.t, false); self.buf_len = 0; offset = fill; @@ -256,7 +249,7 @@ impl Core { let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); (self.compress_blocks)(&mut self.h, &data[offset..end], &mut self.t); offset = end; @@ -266,11 +259,11 @@ impl Core { if remaining > 0 { let start = self.buf_len as usize; self.buf[start..start.strict_add(remaining)].copy_from_slice(&data[offset..]); - self.buf_len = self.buf_len.strict_add(remaining as u8); + let remaining = u8::try_from(remaining).expect("Blake2s buffer remainder fits in u8"); + self.buf_len = self.buf_len.strict_add(remaining); } } - #[allow(clippy::indexing_slicing)] fn finalize_into(&self, out: &mut [u8]) { debug_assert!(out.len() == self.nn as usize); @@ -287,30 +280,21 @@ impl Core { write_output(&h, self.nn, out); - for word in h.iter_mut() { - // SAFETY: Volatile zeroization of a stack word is sound because: - // 1. `word` comes from `h.iter_mut()`, so it is valid and uniquely borrowed. - // 2. `u32` has no invalid bit patterns; writing zero preserves validity. - // 3. The pointer does not escape this loop iteration. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } /// Reset to the initial state (including re-buffering the key if keyed and /// re-applying any salt/personalization parameter-block bytes). - #[allow(clippy::indexing_slicing)] fn reset(&mut self) { self.h = init_state_with_params(self.nn, self.kk, &self.salt, &self.personal); if self.kk > 0 { let key_len = self.kk as usize; self.buf = [0u8; BLOCK_SIZE]; - // SAFETY: when `kk > 0`, `self.key` was initialized in `new`, and `self.buf` - // has at least `key_len` bytes available. - unsafe { - core::ptr::copy_nonoverlapping(self.key.as_ptr().cast::(), self.buf.as_mut_ptr(), key_len); - } - self.buf_len = BLOCK_SIZE as u8; + // SAFETY: when `kk > 0`, `self.key` was initialized in `new`. + let key = unsafe { self.key.assume_init_ref() }; + self.buf[..key_len].copy_from_slice(&key[..key_len]); + self.buf_len = BLOCK_SIZE_U8; } else { self.buf = [0u8; BLOCK_SIZE]; self.buf_len = 0; @@ -346,7 +330,6 @@ const BLAKE2S256_H0: [u32; 8] = init_unkeyed_fixed_state(32); /// digest length / key length / fanout / depth bits, and `h[4..8]` carry the /// salt and personalization words. #[inline] -#[allow(clippy::indexing_slicing)] fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; PERSONAL_LEN]) -> [u32; 8] { let p0 = nn as u32 | ((kk as u32) << 8) | (1u32 << 16) | (1u32 << 24); let mut h = IV; @@ -364,7 +347,6 @@ fn init_state_with_params(nn: u8, kk: u8, salt: &[u8; SALT_LEN], personal: &[u8; } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn write_output(h: &[u32; 8], nn: u8, out: &mut [u8]) { let nn = nn as usize; let full_words = nn / 4; @@ -393,7 +375,6 @@ fn write_output(h: &[u32; 8], nn: u8, out: &mut [u8]) { } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_small_into_with_params( nn: u8, key: &[u8], @@ -402,7 +383,7 @@ fn oneshot_small_into_with_params( out: &mut [u8], compress: kernels::CompressFn, ) { - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -420,10 +401,10 @@ fn oneshot_small_into_with_params( key_block[..key.len()].copy_from_slice(key); if data.is_empty() { - compress(&mut h, &key_block, BLOCK_SIZE as u64, true); + compress(&mut h, &key_block, u64::from(BLOCK_SIZE_U8), true); ct::zeroize(&mut key_block); } else { - compress(&mut h, &key_block, BLOCK_SIZE as u64, false); + compress(&mut h, &key_block, u64::from(BLOCK_SIZE_U8), false); ct::zeroize(&mut key_block); let mut data_block = [0u8; BLOCK_SIZE]; @@ -431,20 +412,16 @@ fn oneshot_small_into_with_params( compress( &mut h, &data_block, - (BLOCK_SIZE as u64).strict_add(data.len() as u64), + u64::from(BLOCK_SIZE_U8).strict_add(data.len() as u64), true, ); } write_output(&h, nn, out); - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); } #[inline(always)] -#[allow(clippy::indexing_slicing)] fn oneshot_hash_into_inner( nn: u8, key: &[u8], @@ -454,7 +431,7 @@ fn oneshot_hash_into_inner( ) { debug_assert!(out.len() == nn as usize); assert!( - nn >= 1 && nn as usize <= MAX_OUTPUT_LEN, + (1..=MAX_OUTPUT_LEN_U8).contains(&nn), "Blake2s output length must be 1-32" ); assert!(key.len() <= MAX_KEY_LEN, "Blake2s key must be at most 32 bytes"); @@ -465,7 +442,7 @@ fn oneshot_hash_into_inner( return; } - let kk = key.len() as u8; + let kk = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); let mut h = match params { Some((salt, personal)) => init_state_with_params(nn, kk, salt, personal), None => init_state(nn, kk), @@ -473,7 +450,7 @@ fn oneshot_hash_into_inner( let mut buf = [0u8; BLOCK_SIZE]; let mut buf_len = if kk > 0 { buf[..key.len()].copy_from_slice(key); - BLOCK_SIZE as u8 + BLOCK_SIZE_U8 } else { 0 }; @@ -486,7 +463,7 @@ fn oneshot_hash_into_inner( if fill > 0 { buf[buf_len as usize..BLOCK_SIZE].copy_from_slice(&data[..fill]); } - t = t.strict_add(BLOCK_SIZE as u64); + t = t.strict_add(u64::from(BLOCK_SIZE_U8)); compress(&mut h, &buf, t, false); ct::zeroize(&mut buf); buf_len = 0; @@ -495,7 +472,7 @@ fn oneshot_hash_into_inner( let available = data_len.strict_sub(offset); if available > BLOCK_SIZE { - let blocks_len = available.strict_sub(1) / BLOCK_SIZE * BLOCK_SIZE; + let blocks_len = (available.strict_sub(1) / BLOCK_SIZE).strict_mul(BLOCK_SIZE); let end = offset.strict_add(blocks_len); let compress_blocks = dispatch::compress_blocks_dispatch(); compress_blocks(&mut h, &data[offset..end], &mut t); @@ -505,7 +482,7 @@ fn oneshot_hash_into_inner( let remaining = data_len.strict_sub(offset); if remaining > 0 { buf[..remaining].copy_from_slice(&data[offset..]); - buf_len = remaining as u8; + buf_len = u8::try_from(remaining).expect("Blake2s buffer remainder fits in u8"); } t = t.strict_add(buf_len as u64); @@ -513,10 +490,7 @@ fn oneshot_hash_into_inner( write_output(&h, nn, out); if kk > 0 { - for word in &mut h { - // SAFETY: word is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(word, 0) }; - } + ct::zeroize_words_no_fence(&mut h); ct::zeroize(&mut buf); } } @@ -633,12 +607,11 @@ impl Blake2sParams { /// Set a validated MAC key. Omit this method for unkeyed hashing. #[must_use] - #[allow(clippy::indexing_slicing)] pub fn key(mut self, key: Blake2sKey<'_>) -> Self { let key = key.as_bytes(); self.key_buf = [0u8; MAX_KEY_LEN]; self.key_buf[..key.len()].copy_from_slice(key); - self.key_len = key.len() as u8; + self.key_len = u8::try_from(key.len()).expect("validated Blake2s key length fits in u8"); self } @@ -656,7 +629,6 @@ impl Blake2sParams { self } - #[allow(clippy::indexing_slicing)] fn key_slice(&self) -> &[u8] { &self.key_buf[..usize::from(self.key_len)] } @@ -748,7 +720,6 @@ impl Blake2s256 { #[must_use] pub fn new_keyed(key: Blake2sKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); Self(Core::new(32, key)) } @@ -756,12 +727,10 @@ impl Blake2s256 { #[must_use] pub fn keyed_digest(key: Blake2sKey<'_>, data: &[u8]) -> [u8; 32] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); oneshot_hash_array::<32>(32, key, data) } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn new_with_compress_for_test( compress: kernels::CompressFn, compress_blocks: kernels::CompressBlocksFn, @@ -769,8 +738,7 @@ impl Blake2s256 { Self(Core::new_with_compress_for_test(32, &[], compress, compress_blocks)) } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(test)] pub(crate) fn keyed_with_compress_for_test( key: &[u8], compress: kernels::CompressFn, @@ -782,6 +750,7 @@ impl Blake2s256 { } #[cfg(feature = "diag")] +/// Derive a keyed Blake2s-256 diagnostic digest with the portable backend. #[must_use] pub fn diag_blake2s256_keyed_digest_portable(key: &[u8; 32]) -> [u8; 32] { let mut out = [0u8; 32]; @@ -867,7 +836,6 @@ impl Blake2s128 { #[must_use] pub fn new_keyed(key: Blake2sKey<'_>) -> Self { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); Self(Core::new(16, key)) } @@ -875,7 +843,6 @@ impl Blake2s128 { #[must_use] pub fn keyed_digest(key: Blake2sKey<'_>, data: &[u8]) -> [u8; 16] { let key = key.as_bytes(); - assert!(!key.is_empty(), "validated Blake2s key must not be empty"); oneshot_hash_array::<16>(16, key, data) } @@ -939,9 +906,13 @@ impl_std_io_write_for_digest!(Blake2s128); #[cfg(test)] mod tests { - use blake2::{Blake2s128 as OracleBlake2s128, Blake2s256 as OracleBlake2s256, Blake2sMac, Digest as _}; - use digest::typenum::{U16, U32}; - use hmac::{Mac as _, digest::KeyInit}; + use blake2::{ + Blake2s as OracleBlake2s, Blake2s256 as OracleBlake2s256, Blake2sMac, + digest::{ + Digest as _, Mac as _, + consts::{U16, U32}, + }, + }; use super::{ kernels::{ @@ -953,9 +924,10 @@ mod tests { type OracleBlake2sMac128 = Blake2sMac; type OracleBlake2sMac256 = Blake2sMac; + type OracleBlake2s128 = OracleBlake2s; fn validated_key(key: &[u8]) -> Blake2sKey<'_> { - Blake2sKey::new(key).unwrap() + Blake2sKey::new(key).expect("test Blake2s key length is valid") } const ORACLE_CASES: &[&[u8]] = &[ @@ -1064,7 +1036,7 @@ mod tests { let key = b"secret-key"; let data = b"hello world"; - let mut oracle = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle = OracleBlake2sMac256::new_from_slice(key).expect("oracle accepts a valid Blake2s-256 key"); oracle.update(data); let expected = oracle.finalize().into_bytes(); @@ -1077,7 +1049,7 @@ mod tests { let key = b"tiny-key"; let data = b"hello world"; - let mut oracle = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle = OracleBlake2sMac128::new_from_slice(key).expect("oracle accepts a valid Blake2s-128 key"); oracle.update(data); let expected = oracle.finalize().into_bytes(); @@ -1090,7 +1062,7 @@ mod tests { let key = &[0xAA; 32]; let data = &[0xBB; 512]; - let mut oracle = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle = OracleBlake2sMac256::new_from_slice(key).expect("oracle accepts a valid Blake2s-256 key"); oracle.update(data); let expected = oracle.finalize().into_bytes(); @@ -1134,12 +1106,18 @@ mod tests { #[test] fn keyed_empty_key_is_rejected() { - assert_eq!(Blake2sKey::new(b"").unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2sKey::new(b"").expect_err("empty Blake2s keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } #[test] fn keyed_overlength_key_is_rejected() { - assert_eq!(Blake2sKey::new(&[0u8; 33]).unwrap_err(), Blake2Error::InvalidKeyLength); + assert_eq!( + Blake2sKey::new(&[0u8; 33]).expect_err("overlength Blake2s keys must be rejected"), + Blake2Error::InvalidKeyLength + ); } // ── Params (salt + personalization) ─────────────────────────────────── @@ -1244,7 +1222,7 @@ mod tests { let params = Blake2sParams::new().salt(*b"salty\0\0\0").personal(*b"tagging\0"); let mut h = params.build_256(); h.update(b"first"); - let _ = h.finalize(); + let _first = h.finalize(); h.reset(); h.update(b"hello world"); @@ -1291,7 +1269,8 @@ mod tests { (&[0xAA; 16][..], &[0x55; 257][..]), (&[0xCC; 32][..], &[0x11; 512][..]), ] { - let mut oracle_128 = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle_128 = + OracleBlake2sMac128::new_from_slice(key).expect("oracle accepts the forced-kernel Blake2s-128 key"); oracle_128.update(msg); let expected_128 = oracle_128.finalize().into_bytes(); @@ -1306,7 +1285,8 @@ mod tests { key.len(), ); - let mut oracle_256 = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle_256 = + OracleBlake2sMac256::new_from_slice(key).expect("oracle accepts the forced-kernel Blake2s-256 key"); oracle_256.update(msg); let expected_256 = oracle_256.finalize().into_bytes(); diff --git a/src/hashes/crypto/blake2s/riscv64.rs b/src/hashes/crypto/blake2s/riscv64.rs index dc66ef2a..b289dc26 100644 --- a/src/hashes/crypto/blake2s/riscv64.rs +++ b/src/hashes/crypto/blake2s/riscv64.rs @@ -5,8 +5,6 @@ //! the compiler lower the hot pair/quad operations appropriately for current //! RISC-V vector hardware. -#![allow(clippy::indexing_slicing)] - use super::kernels::{SIGMA, init_v, load_msg}; #[inline(always)] diff --git a/src/hashes/crypto/blake2s/wasm.rs b/src/hashes/crypto/blake2s/wasm.rs index 1a425f05..35e69e23 100644 --- a/src/hashes/crypto/blake2s/wasm.rs +++ b/src/hashes/crypto/blake2s/wasm.rs @@ -4,8 +4,6 @@ //! `v128`, diagonalization is lane shuffling, and the 32-bit rotates map to //! byte shuffles plus shift/or. -#![allow(clippy::indexing_slicing)] - use core::arch::wasm32::*; use super::kernels::{SIGMA, init_v, load_msg}; @@ -62,9 +60,21 @@ fn load_msg_quad(m: &[u32; 16], i0: u8, i1: u8, i2: u8, i3: u8) -> v128 { } #[inline(always)] -unsafe fn vload_u32_quad(p: *const u32) -> v128 { - // SAFETY: caller ensures `p` is valid for 16 bytes / 4 u32 lanes. - unsafe { v128_load(p.cast()) } +fn load_u32_quad(words: &[u32; N], offset: usize) -> v128 { + u32x4( + words[offset], + words[offset.strict_add(1)], + words[offset.strict_add(2)], + words[offset.strict_add(3)], + ) +} + +#[inline(always)] +fn store_u32_quad(words: &mut [u32; 8], offset: usize, value: v128) { + words[offset] = u32x4_extract_lane::<0>(value); + words[offset.strict_add(1)] = u32x4_extract_lane::<1>(value); + words[offset.strict_add(2)] = u32x4_extract_lane::<2>(value); + words[offset.strict_add(3)] = u32x4_extract_lane::<3>(value); } /// Blake2s WASM SIMD128 compress. @@ -77,10 +87,10 @@ pub(super) unsafe fn compress_simd128(h: &mut [u32; 8], block: &[u8; 64], t: u64 let m = load_msg(block); let v = init_v(h, t, last); - let mut a = unsafe { vload_u32_quad(v.as_ptr()) }; - let mut b = unsafe { vload_u32_quad(v.as_ptr().add(4)) }; - let mut c = unsafe { vload_u32_quad(v.as_ptr().add(8)) }; - let mut d = unsafe { vload_u32_quad(v.as_ptr().add(12)) }; + let mut a = load_u32_quad(&v, 0); + let mut b = load_u32_quad(&v, 4); + let mut c = load_u32_quad(&v, 8); + let mut d = load_u32_quad(&v, 12); for round in 0..10u8 { let s = &SIGMA[round as usize]; @@ -98,11 +108,9 @@ pub(super) unsafe fn compress_simd128(h: &mut [u32; 8], block: &[u8; 64], t: u64 undiagonalize(&mut b, &mut c, &mut d); } - let h0 = unsafe { vload_u32_quad(h.as_ptr()) }; - let h1 = unsafe { vload_u32_quad(h.as_ptr().add(4)) }; + let h0 = load_u32_quad(h, 0); + let h1 = load_u32_quad(h, 4); - unsafe { - v128_store(h.as_mut_ptr().cast(), v128_xor(h0, v128_xor(a, c))); - v128_store(h.as_mut_ptr().add(4).cast(), v128_xor(h1, v128_xor(b, d))); - } + store_u32_quad(h, 0, v128_xor(h0, v128_xor(a, c))); + store_u32_quad(h, 4, v128_xor(h1, v128_xor(b, d))); } diff --git a/src/hashes/crypto/blake2s/x86_64.rs b/src/hashes/crypto/blake2s/x86_64.rs index 202ee7db..5445d47e 100644 --- a/src/hashes/crypto/blake2s/x86_64.rs +++ b/src/hashes/crypto/blake2s/x86_64.rs @@ -6,38 +6,36 @@ //! //! This is a real SIMD backend, not a dispatch stub. -#![allow(clippy::cast_possible_wrap, clippy::indexing_slicing)] - use core::arch::x86_64::*; use super::kernels::{SIGMA, init_v, load_msg}; #[inline(always)] -unsafe fn ror16_avx2(x: __m128i) -> __m128i { +fn ror16_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 16), _mm_slli_epi32(x, 16)) } } #[inline(always)] -unsafe fn ror12_avx2(x: __m128i) -> __m128i { +fn ror12_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 20)) } } #[inline(always)] -unsafe fn ror8_avx2(x: __m128i) -> __m128i { +fn ror8_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 24)) } } #[inline(always)] -unsafe fn ror7_avx2(x: __m128i) -> __m128i { +fn ror7_avx2(x: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics operate only on the provided SIMD register. unsafe { _mm_or_si128(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 25)) } } #[inline(always)] -unsafe fn g_avx2(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m128i, mx: __m128i, my: __m128i) { +fn g_avx2(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m128i, mx: __m128i, my: __m128i) { // SAFETY: all operations stay within SIMD registers and mutate disjoint rows. unsafe { *a = _mm_add_epi32(_mm_add_epi32(*a, *b), mx); @@ -52,7 +50,7 @@ unsafe fn g_avx2(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m1 } #[inline(always)] -unsafe fn diagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { +fn diagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { // SAFETY: shuffle permutes only the provided SIMD registers. unsafe { *b = _mm_shuffle_epi32(*b, 0x39); @@ -62,7 +60,7 @@ unsafe fn diagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { } #[inline(always)] -unsafe fn undiagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { +fn undiagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { // SAFETY: shuffle permutes only the provided SIMD registers. unsafe { *b = _mm_shuffle_epi32(*b, 0x93); @@ -72,14 +70,14 @@ unsafe fn undiagonalize(b: &mut __m128i, c: &mut __m128i, d: &mut __m128i) { } #[inline(always)] -unsafe fn load_msg_quad(m: &[u32; 16], i0: u8, i1: u8, i2: u8, i3: u8) -> __m128i { +fn load_msg_quad(m: &[u32; 16], i0: u8, i1: u8, i2: u8, i3: u8) -> __m128i { // SAFETY: `_mm_set_epi32` constructs a register from in-bounds message words. unsafe { _mm_set_epi32( - m[i3 as usize] as i32, - m[i2 as usize] as i32, - m[i1 as usize] as i32, - m[i0 as usize] as i32, + m[i3 as usize].cast_signed(), + m[i2 as usize].cast_signed(), + m[i1 as usize].cast_signed(), + m[i0 as usize].cast_signed(), ) } } @@ -126,6 +124,12 @@ pub(super) unsafe fn compress_avx2(h: &mut [u32; 8], block: &[u8; 64], t: u64, l } #[inline(always)] +/// Applies one BLAKE2s mixing step with AVX-512VL rotates. +/// +/// # Safety +/// +/// The caller must execute this function only when AVX-512F and AVX-512VL are +/// enabled for the current execution scope. unsafe fn g_avx512vl(a: &mut __m128i, b: &mut __m128i, c: &mut __m128i, d: &mut __m128i, mx: __m128i, my: __m128i) { // SAFETY: all operations stay within SIMD registers and the required target // features are enabled by the caller. diff --git a/src/hashes/crypto/blake3/aarch64.rs b/src/hashes/crypto/blake3/aarch64.rs index 3364b302..028b4fec 100644 --- a/src/hashes/crypto/blake3/aarch64.rs +++ b/src/hashes/crypto/blake3/aarch64.rs @@ -12,13 +12,8 @@ //! //! # Safety //! -//! All functions in this module are marked `unsafe` and require NEON -//! to be present. Callers must verify CPU capabilities before calling. - -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] +//! Unsafe entry points require NEON to be present. Callers must verify CPU +//! capabilities before calling them. #[cfg(target_arch = "aarch64")] use core::arch::aarch64::*; @@ -32,11 +27,32 @@ mod asm; // Constants use super::{BLOCK_LEN, CHUNK_LEN, CHUNK_START, IV, MSG_SCHEDULE, OUT_LEN, PARENT, words16_from_le_bytes_64}; +const BLOCK_LEN_U32: u32 = 64; + +#[inline(always)] +fn counter_words(counter: u64) -> (u32, u32) { + let [b0, b1, b2, b3, b4, b5, b6, b7] = counter.to_le_bytes(); + ( + u32::from_le_bytes([b0, b1, b2, b3]), + u32::from_le_bytes([b4, b5, b6, b7]), + ) +} + +#[inline(always)] +fn counter_lanes(counter: u64, deltas: [u64; 4]) -> ([u32; 4], [u32; 4]) { + let [d0, d1, d2, d3] = deltas; + let w0 = counter_words(counter.wrapping_add(d0)); + let w1 = counter_words(counter.wrapping_add(d1)); + let w2 = counter_words(counter.wrapping_add(d2)); + let w3 = counter_words(counter.wrapping_add(d3)); + ([w0.0, w1.0, w2.0, w3.0], [w0.1, w1.1, w2.1, w3.1]) +} + #[cfg(any(target_os = "linux", target_os = "macos"))] #[inline(always)] fn ptr_is_aligned(ptr: *const u8, align: usize) -> bool { debug_assert!(align.is_power_of_two()); - (ptr as usize) & (align - 1) == 0 + (ptr as usize) & align.strict_sub(1) == 0 } // Strategy: @@ -69,6 +85,14 @@ struct ChunkStateAsmScratch { #[cfg(any(target_os = "linux", target_os = "macos"))] #[inline(always)] +/// Dispatches full blocks to the platform assembly compressor. +/// +/// # Safety +/// +/// The current CPU must support NEON. `blocks` must be aligned as required by +/// the assembly backend and readable for `num_blocks * BLOCK_LEN` bytes. +/// `chaining_value` must reference eight writable `u32`s, and +/// `blocks_compressed` must reference one writable byte. unsafe fn chunk_compress_blocks_asm( blocks: *const u8, chaining_value: *mut u32, @@ -112,6 +136,10 @@ static ROT8_TABLE: [u8; 16] = [1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, /// Rotate right by 16 bits (each u32 lane). /// Uses vrev32q_u16 which reverses 16-bit elements within 32-bit containers. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr16(v: uint32x4_t) -> uint32x4_t { @@ -125,6 +153,10 @@ unsafe fn rotr16(v: uint32x4_t) -> uint32x4_t { /// Rotate right by 12 bits (each u32 lane). /// Uses shift+OR which allows parallel execution on superscalar cores. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr12(v: uint32x4_t) -> uint32x4_t { @@ -138,6 +170,10 @@ unsafe fn rotr12(v: uint32x4_t) -> uint32x4_t { /// Rotate right by 8 bits (each u32 lane). /// Uses vqtbl1q_u8 (table lookup) for byte-level rotation - single cycle. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr8_tbl(v: uint32x4_t, tbl: uint8x16_t) -> uint32x4_t { @@ -153,6 +189,10 @@ unsafe fn rotr8_tbl(v: uint32x4_t, tbl: uint8x16_t) -> uint32x4_t { /// /// Some cores have relatively high-latency `tbl`/`vtbl` paths; for the per-block /// compressor (latency-sensitive), prefer a shift/or implementation. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr8(v: uint32x4_t) -> uint32x4_t { @@ -163,6 +203,10 @@ unsafe fn rotr8(v: uint32x4_t) -> uint32x4_t { /// Rotate right by 7 bits (each u32 lane). /// Uses shift+OR which allows parallel execution on superscalar cores. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rotr7(v: uint32x4_t) -> uint32x4_t { @@ -176,6 +220,10 @@ unsafe fn rotr7(v: uint32x4_t) -> uint32x4_t { // Lane rotation helpers (for diagonalization) /// Rotate lanes left by 1: [a, b, c, d] -> [b, c, d, a] +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rot_lanes_left_1(v: uint32x4_t) -> uint32x4_t { @@ -184,6 +232,10 @@ unsafe fn rot_lanes_left_1(v: uint32x4_t) -> uint32x4_t { } /// Rotate lanes left by 2: [a, b, c, d] -> [c, d, a, b] +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rot_lanes_left_2(v: uint32x4_t) -> uint32x4_t { @@ -192,6 +244,10 @@ unsafe fn rot_lanes_left_2(v: uint32x4_t) -> uint32x4_t { } /// Rotate lanes left by 3: [a, b, c, d] -> [d, a, b, c] +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn rot_lanes_left_3(v: uint32x4_t) -> uint32x4_t { @@ -218,6 +274,10 @@ unsafe fn rot_lanes_left_3(v: uint32x4_t) -> uint32x4_t { /// vecs[2] = [a2, b2, c2, d2] /// vecs[3] = [a3, b3, c3, d3] /// ``` +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn transpose_vecs(vecs: &mut [uint32x4_t; 4]) { @@ -237,6 +297,11 @@ unsafe fn transpose_vecs(vecs: &mut [uint32x4_t; 4]) { /// Load and transpose message words from 4 input blocks. /// After transpose, m[i] contains word i from all 4 inputs. +/// +/// # Safety +/// +/// The current CPU must support NEON. Each input pointer must be readable for +/// `block_offset + block_len` bytes, and `block_len` must not exceed `BLOCK_LEN`. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, block_len: usize) -> [uint32x4_t; 16] { @@ -248,6 +313,9 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, } #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `src` must be readable for 16 bytes. unsafe fn loadu_128(src: *const u8) -> uint32x4_t { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -265,8 +333,8 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, let mut out = [vdupq_n_u32(0); 16]; // Load four 16-byte chunks per input and transpose each 4x4. - for lane_block in 0..4 { - let off = block_offset + lane_block * 16; + for lane_block in 0usize..4 { + let off = block_offset.strict_add(lane_block.strict_mul(16)); let mut vecs = [ loadu_128(inputs[0].add(off)), loadu_128(inputs[1].add(off)), @@ -274,10 +342,11 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, loadu_128(inputs[3].add(off)), ]; transpose_vecs(&mut vecs); - out[lane_block * 4] = vecs[0]; - out[lane_block * 4 + 1] = vecs[1]; - out[lane_block * 4 + 2] = vecs[2]; - out[lane_block * 4 + 3] = vecs[3]; + let dst = lane_block.strict_mul(4); + out[dst] = vecs[0]; + out[dst.strict_add(1)] = vecs[1]; + out[dst.strict_add(2)] = vecs[2]; + out[dst.strict_add(3)] = vecs[3]; } return out; @@ -313,6 +382,11 @@ unsafe fn load_msg_vecs_transposed(inputs: [*const u8; 4], block_offset: usize, /// - lane 3: `base + 3 * CHUNK_LEN` /// /// This path is only valid for full 64-byte blocks. +/// +/// # Safety +/// +/// The current CPU must support NEON. `base` must be readable for four full +/// chunks, and `block_offset` must identify a complete block within each chunk. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usize) -> [uint32x4_t; 16] { @@ -324,6 +398,9 @@ unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usi } #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `src` must be readable for 16 bytes. unsafe fn loadu_128(src: *const u8) -> uint32x4_t { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -339,18 +416,19 @@ unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usi let mut out = [vdupq_n_u32(0); 16]; macro_rules! load_and_transpose { ($dst:expr, $off:expr) => {{ - let off = block_offset + $off; + let off = block_offset.strict_add($off); + let dst: usize = $dst; let mut vecs = [ loadu_128(base.add(off)), - loadu_128(base.add(CHUNK_LEN + off)), - loadu_128(base.add(2 * CHUNK_LEN + off)), - loadu_128(base.add(3 * CHUNK_LEN + off)), + loadu_128(base.add(CHUNK_LEN.strict_add(off))), + loadu_128(base.add(CHUNK_LEN.strict_mul(2).strict_add(off))), + loadu_128(base.add(CHUNK_LEN.strict_mul(3).strict_add(off))), ]; transpose_vecs(&mut vecs); - out[$dst] = vecs[0]; - out[$dst + 1] = vecs[1]; - out[$dst + 2] = vecs[2]; - out[$dst + 3] = vecs[3]; + out[dst] = vecs[0]; + out[dst.strict_add(1)] = vecs[1]; + out[dst.strict_add(2)] = vecs[2]; + out[dst.strict_add(3)] = vecs[3]; }}; } @@ -365,6 +443,10 @@ unsafe fn load_msg_vecs_transposed_contiguous(base: *const u8, block_offset: usi // Single-block compression (NEON accelerated) /// Helper: take the low 64-bit lane (2x u32) from each input and concatenate. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn concat_low64_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { @@ -384,6 +466,10 @@ unsafe fn concat_low64_u32(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { /// Each round uses the same access pattern for message words, and the message /// vectors are permuted between rounds. This avoids the expensive per-round /// gather/extract machinery that dominated the previous per-block compressor. +/// +/// # Safety +/// +/// The current CPU must support NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn permute_msg(m0: uint32x4_t, m1: uint32x4_t, m2: uint32x4_t, m3: uint32x4_t) -> [uint32x4_t; 4] { @@ -418,6 +504,12 @@ unsafe fn permute_msg(m0: uint32x4_t, m1: uint32x4_t, m2: uint32x4_t, m3: uint32 #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] #[inline] +/// Runs the NEON compression core without its final output transform. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `block` must be readable for +/// `BLOCK_LEN` bytes. unsafe fn compress_neon_core( chaining_value: &[u32; 8], block: *const u8, @@ -433,6 +525,9 @@ unsafe fn compress_neon_core( } #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `src` must be readable for 16 bytes. unsafe fn loadu_u32x4(src: *const u8) -> uint32x4_t { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -452,8 +547,7 @@ unsafe fn compress_neon_core( let mut row2 = vld1q_u32(IV.as_ptr()); // Build row3 from counter, block_len, and flags. - let counter_lo = counter as u32; - let counter_hi = (counter >> 32) as u32; + let (counter_lo, counter_hi) = counter_words(counter); let row3_arr: [u32; 4] = [counter_lo, counter_hi, block_len, flags]; let mut row3 = vld1q_u32(row3_arr.as_ptr()); @@ -547,6 +641,12 @@ pub(crate) unsafe fn compress_neon( #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] +/// Compresses one byte-oriented block and returns the full compression output. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `block` must be readable for +/// `BLOCK_LEN` bytes. unsafe fn compress_neon_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -577,23 +677,27 @@ unsafe fn compress_neon_bytes( // 4-way parallel hashing (hash4_neon) +#[derive(Clone, Copy)] +struct Hash4Flags { + base: u32, + start: u32, + end: u32, +} + /// Parallel G function for 4 independent chunks. /// /// In this model: /// - v[0..4] are the four rows of state from 4 different chunks /// - Each vector lane i corresponds to chunk i +/// +/// # Safety +/// +/// The current CPU must support NEON. Every index in `indices` must be less +/// than 16 and the four indices must be distinct. #[cfg(target_arch = "aarch64")] #[inline(always)] -unsafe fn g4( - v: &mut [uint32x4_t; 16], - a: usize, - b: usize, - c: usize, - d: usize, - mx: uint32x4_t, - my: uint32x4_t, - rot8_tbl: uint8x16_t, -) { +unsafe fn g4(v: &mut [uint32x4_t; 16], indices: [usize; 4], mx: uint32x4_t, my: uint32x4_t, rot8_tbl: uint8x16_t) { + let [a, b, c, d] = indices; // SAFETY: NEON intrinsics are available via this function's #[target_feature] attribute. unsafe { // a = a + b + mx @@ -622,6 +726,11 @@ unsafe fn g4( } /// One round of the parallel compression function for 4 chunks. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `r` must be less than +/// `MSG_SCHEDULE.len()`. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_tbl: uint8x16_t) { @@ -635,24 +744,30 @@ unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_ // Column step: G(0,4,8,12), G(1,5,9,13), G(2,6,10,14), G(3,7,11,15) // SAFETY: `s` contains fixed schedule indices in 0..16, and `m` is `[T; 16]`. - g4(v, 0, 4, 8, 12, *m.get_unchecked(s[0]), *m.get_unchecked(s[1]), rot8_tbl); - g4(v, 1, 5, 9, 13, *m.get_unchecked(s[2]), *m.get_unchecked(s[3]), rot8_tbl); g4( v, - 2, - 6, - 10, - 14, + [0, 4, 8, 12], + *m.get_unchecked(s[0]), + *m.get_unchecked(s[1]), + rot8_tbl, + ); + g4( + v, + [1, 5, 9, 13], + *m.get_unchecked(s[2]), + *m.get_unchecked(s[3]), + rot8_tbl, + ); + g4( + v, + [2, 6, 10, 14], *m.get_unchecked(s[4]), *m.get_unchecked(s[5]), rot8_tbl, ); g4( v, - 3, - 7, - 11, - 15, + [3, 7, 11, 15], *m.get_unchecked(s[6]), *m.get_unchecked(s[7]), rot8_tbl, @@ -661,40 +776,28 @@ unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_ // Diagonal step: G(0,5,10,15), G(1,6,11,12), G(2,7,8,13), G(3,4,9,14) g4( v, - 0, - 5, - 10, - 15, + [0, 5, 10, 15], *m.get_unchecked(s[8]), *m.get_unchecked(s[9]), rot8_tbl, ); g4( v, - 1, - 6, - 11, - 12, + [1, 6, 11, 12], *m.get_unchecked(s[10]), *m.get_unchecked(s[11]), rot8_tbl, ); g4( v, - 2, - 7, - 8, - 13, + [2, 7, 8, 13], *m.get_unchecked(s[12]), *m.get_unchecked(s[13]), rot8_tbl, ); g4( v, - 3, - 4, - 9, - 14, + [3, 4, 9, 14], *m.get_unchecked(s[14]), *m.get_unchecked(s[15]), rot8_tbl, @@ -708,18 +811,17 @@ unsafe fn round4(v: &mut [uint32x4_t; 16], m: &[uint32x4_t; 16], r: usize, rot8_ /// /// # Safety /// -/// Caller must ensure NEON is available. +/// The current CPU must support NEON. Each input pointer must be readable for +/// `input_len` bytes, and `input_len` must be in `1..=CHUNK_LEN`. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] -pub(crate) unsafe fn hash4_neon( +unsafe fn hash4_neon( inputs: [*const u8; 4], input_len: usize, key: &[u32; 8], counter: u64, increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, + flags: Hash4Flags, out: &mut [[u8; OUT_LEN]; 4], ) { debug_assert!(input_len > 0); @@ -730,10 +832,13 @@ pub(crate) unsafe fn hash4_neon( // can skip the generic load/pad path and use a tighter contiguous loader. let contiguous_full_chunks = input_len == CHUNK_LEN && (inputs[1] as usize).wrapping_sub(inputs[0] as usize) == CHUNK_LEN - && (inputs[2] as usize).wrapping_sub(inputs[0] as usize) == 2 * CHUNK_LEN - && (inputs[3] as usize).wrapping_sub(inputs[0] as usize) == 3 * CHUNK_LEN; + && (inputs[2] as usize).wrapping_sub(inputs[0] as usize) == CHUNK_LEN.strict_mul(2) + && (inputs[3] as usize).wrapping_sub(inputs[0] as usize) == CHUNK_LEN.strict_mul(3); #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `dest` must be writable for 16 bytes. unsafe fn storeu_128(src: uint32x4_t, dest: *mut u8) { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -763,41 +868,26 @@ pub(crate) unsafe fn hash4_neon( } else { (0u64, 0, 0, 0) }; - let counter_low_vec = vld1q_u32( - [ - (counter.wrapping_add(d0) as u32), - (counter.wrapping_add(d1) as u32), - (counter.wrapping_add(d2) as u32), - (counter.wrapping_add(d3) as u32), - ] - .as_ptr(), - ); - let counter_high_vec = vld1q_u32( - [ - ((counter.wrapping_add(d0) >> 32) as u32), - ((counter.wrapping_add(d1) >> 32) as u32), - ((counter.wrapping_add(d2) >> 32) as u32), - ((counter.wrapping_add(d3) >> 32) as u32), - ] - .as_ptr(), - ); + let (counter_low, counter_high) = counter_lanes(counter, [d0, d1, d2, d3]); + let counter_low_vec = vld1q_u32(counter_low.as_ptr()); + let counter_high_vec = vld1q_u32(counter_high.as_ptr()); // Process each block - let mut block_flags = flags | flags_start; + let mut block_flags = flags.base | flags.start; let rot8_tbl = vld1q_u8(ROT8_TABLE.as_ptr()); for block_idx in 0..num_blocks { - let block_offset = block_idx * BLOCK_LEN; - let is_last = block_idx == num_blocks - 1; + let block_offset = block_idx.strict_mul(BLOCK_LEN); + let is_last = block_idx == num_blocks.strict_sub(1); // Calculate block length for last block let block_len = if is_last && !input_len.is_multiple_of(BLOCK_LEN) { - (input_len % BLOCK_LEN) as u32 + u32::try_from(input_len % BLOCK_LEN).expect("a partial BLAKE3 block length fits in u32") } else { - BLOCK_LEN as u32 + BLOCK_LEN_U32 }; if is_last { - block_flags |= flags_end; + block_flags |= flags.end; } // Load and transpose message blocks (pads the last block if needed). @@ -846,7 +936,7 @@ pub(crate) unsafe fn hash4_neon( h_vecs[6] = veorq_u32(v[6], v[14]); h_vecs[7] = veorq_u32(v[7], v[15]); - block_flags = flags; + block_flags = flags.base; } // Transpose the CV vectors so we can store each output contiguously. @@ -855,7 +945,7 @@ pub(crate) unsafe fn hash4_neon( transpose_vecs(&mut lo); transpose_vecs(&mut hi); - for lane in 0..4 { + for lane in 0usize..4 { let dst = out[lane].as_mut_ptr(); storeu_128(lo[lane], dst.add(0)); storeu_128(hi[lane], dst.add(16)); @@ -867,6 +957,11 @@ pub(crate) unsafe fn hash4_neon( /// /// This is the hot-path worker used by `hash_many_contiguous_neon` to avoid /// generic variable-size handling and repeated contiguous-shape checks. +/// +/// # Safety +/// +/// The current CPU must support NEON. `input0` must be readable for four full +/// chunks, and `out` must be writable for four outputs. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] unsafe fn hash4_contiguous_full_chunks_neon_to_out( @@ -877,6 +972,9 @@ unsafe fn hash4_contiguous_full_chunks_neon_to_out( out: *mut u8, ) { #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `dest` must be writable for 16 bytes. unsafe fn storeu_128(src: uint32x4_t, dest: *mut u8) { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -899,34 +997,19 @@ unsafe fn hash4_contiguous_full_chunks_neon_to_out( vdupq_n_u32(key[7]), ]; - let counter_low_vec = vld1q_u32( - [ - counter as u32, - counter.wrapping_add(1) as u32, - counter.wrapping_add(2) as u32, - counter.wrapping_add(3) as u32, - ] - .as_ptr(), - ); - let counter_high_vec = vld1q_u32( - [ - (counter >> 32) as u32, - (counter.wrapping_add(1) >> 32) as u32, - (counter.wrapping_add(2) >> 32) as u32, - (counter.wrapping_add(3) >> 32) as u32, - ] - .as_ptr(), - ); + let (counter_low, counter_high) = counter_lanes(counter, [0, 1, 2, 3]); + let counter_low_vec = vld1q_u32(counter_low.as_ptr()); + let counter_high_vec = vld1q_u32(counter_high.as_ptr()); - let block_len_vec = vdupq_n_u32(BLOCK_LEN as u32); + let block_len_vec = vdupq_n_u32(BLOCK_LEN_U32); let mut block_flags = flags | CHUNK_START; let rot8_tbl = vld1q_u8(ROT8_TABLE.as_ptr()); for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { - if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + if block_idx.strict_add(1) == (CHUNK_LEN / BLOCK_LEN) { block_flags |= super::CHUNK_END; } - let msg = load_msg_vecs_transposed_contiguous(input0, block_idx * BLOCK_LEN); + let msg = load_msg_vecs_transposed_contiguous(input0, block_idx.strict_mul(BLOCK_LEN)); let block_flags_vec = vdupq_n_u32(block_flags); let mut v = [ @@ -973,8 +1056,8 @@ unsafe fn hash4_contiguous_full_chunks_neon_to_out( transpose_vecs(&mut lo); transpose_vecs(&mut hi); - for lane in 0..4 { - let dst = out.add(lane * OUT_LEN); + for lane in 0usize..4 { + let dst = out.add(lane.strict_mul(OUT_LEN)); storeu_128(lo[lane], dst.add(0)); storeu_128(hi[lane], dst.add(16)); } @@ -1009,7 +1092,9 @@ pub(crate) unsafe fn root_hash_one_chunk_root_aarch64(input: *const u8, key: &[u let mut out = [0u8; OUT_LEN]; for (j, word) in out_words.iter().copied().enumerate() { - out[j * 4..j * 4 + 4].copy_from_slice(&word.to_le_bytes()); + let start = j.strict_mul(4); + let end = start.strict_add(4); + out[start..end].copy_from_slice(&word.to_le_bytes()); } return out; } @@ -1056,29 +1141,20 @@ pub(crate) unsafe fn chunk_cv_one_chunk_aarch64_out( // Fallback: per-block NEON compressor. let mut cv = *key; for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { - let block_bytes: &[u8; BLOCK_LEN] = { - let src = input.add(block_idx * BLOCK_LEN); - &*(src as *const [u8; BLOCK_LEN]) - }; + let block = input.add(block_idx.strict_mul(BLOCK_LEN)); let start = if block_idx == 0 { CHUNK_START } else { 0 }; - let end = if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + let end = if block_idx.strict_add(1) == (CHUNK_LEN / BLOCK_LEN) { super::CHUNK_END } else { 0 }; - cv = compress_cv_neon_bytes( - &cv, - block_bytes.as_ptr(), - counter, - BLOCK_LEN as u32, - flags | start | end, - ); + cv = compress_cv_neon_bytes(&cv, block, counter, BLOCK_LEN_U32, flags | start | end); } for (j, &word) in cv.iter().enumerate() { let bytes = word.to_le_bytes(); - core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j * 4), 4); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j.strict_mul(4)), 4); } } } @@ -1169,21 +1245,18 @@ pub(crate) unsafe fn chunk_state_one_chunk_aarch64_out( // Fallback: per-block NEON compressor for blocks 0..14, then copy the final block bytes. let mut cv = *key; - for block_idx in 0..15 { - let block_bytes: &[u8; BLOCK_LEN] = { - let src = input.add(block_idx * BLOCK_LEN); - &*(src as *const [u8; BLOCK_LEN]) - }; + for block_idx in 0usize..15 { + let block = input.add(block_idx.strict_mul(BLOCK_LEN)); let start = if block_idx == 0 { CHUNK_START } else { 0 }; - cv = compress_cv_neon_bytes(&cv, block_bytes.as_ptr(), counter, BLOCK_LEN as u32, flags | start); + cv = compress_cv_neon_bytes(&cv, block, counter, BLOCK_LEN_U32, flags | start); } // Store cv. core::ptr::copy_nonoverlapping(cv.as_ptr(), out_cv, 8); // Copy final block bytes. - core::ptr::copy_nonoverlapping(input.add(15 * BLOCK_LEN), out_last_block, BLOCK_LEN); + core::ptr::copy_nonoverlapping(input.add(15usize.strict_mul(BLOCK_LEN)), out_last_block, BLOCK_LEN); } } @@ -1212,9 +1285,11 @@ pub(crate) unsafe fn root_hash_one_chunk_neon(input: *const u8, key: &[u32; 8], key, 0, false, - flags, - CHUNK_START, - super::CHUNK_END | super::ROOT, + Hash4Flags { + base: flags, + start: CHUNK_START, + end: super::CHUNK_END | super::ROOT, + }, &mut out, ); out[0] @@ -1247,16 +1322,17 @@ pub(crate) unsafe fn hash_many_contiguous_neon( // Process 8 chunks per loop iteration when possible: two specialized 4-chunk // kernels with fixed contiguous/full-chunk shape. - while idx + 8 <= num_chunks { + while num_chunks.strict_sub(idx) >= 8 { // SAFETY: caller guarantees `input` is valid for `num_chunks * CHUNK_LEN`. - let src0 = input.add(idx * CHUNK_LEN); + let src0 = input.add(idx.strict_mul(CHUNK_LEN)); // SAFETY: caller guarantees `out` is valid for `num_chunks * OUT_LEN`. - let dst0 = out.add(idx * OUT_LEN); + let dst0 = out.add(idx.strict_mul(OUT_LEN)); // Prefetch next batch while processing current. - if idx + 16 <= num_chunks { + if num_chunks.strict_sub(idx) >= 16 { // SAFETY: prefetch is a CPU hint — invalid addresses are silently ignored by hardware. - crate::hashes::common::prefetch::prefetch_read_l1(input.add((idx + 8) * CHUNK_LEN)); + let next = idx.strict_add(8).strict_mul(CHUNK_LEN); + crate::hashes::common::prefetch::prefetch_read_l1(input.add(next)); } hash4_contiguous_full_chunks_neon_to_out(src0, key, counter, flags, dst0); @@ -1267,24 +1343,25 @@ pub(crate) unsafe fn hash_many_contiguous_neon( flags, dst0.add(4 * OUT_LEN), ); - idx += 8; + idx = idx.strict_add(8); counter = counter.wrapping_add(8); } - while idx + 4 <= num_chunks { + while num_chunks.strict_sub(idx) >= 4 { // SAFETY: caller guarantees `input` is valid for `num_chunks * CHUNK_LEN`. - let src = input.add(idx * CHUNK_LEN); + let src = input.add(idx.strict_mul(CHUNK_LEN)); // SAFETY: caller guarantees `out` is valid for `num_chunks * OUT_LEN`. - let dst = out.add(idx * OUT_LEN); + let dst = out.add(idx.strict_mul(OUT_LEN)); // Prefetch next batch while processing current. - if idx + 8 <= num_chunks { + if num_chunks.strict_sub(idx) >= 8 { // SAFETY: prefetch is a CPU hint — invalid addresses are silently ignored by hardware. - crate::hashes::common::prefetch::prefetch_read_l1(input.add((idx + 4) * CHUNK_LEN)); + let next = idx.strict_add(4).strict_mul(CHUNK_LEN); + crate::hashes::common::prefetch::prefetch_read_l1(input.add(next)); } hash4_contiguous_full_chunks_neon_to_out(src, key, counter, flags, dst); - idx += 4; + idx = idx.strict_add(4); counter = counter.wrapping_add(4); } @@ -1300,9 +1377,10 @@ pub(crate) unsafe fn hash_many_contiguous_neon( for lane in 0..remaining { // SAFETY: `lane < remaining <= num_chunks - idx`, so per-lane source and // destination pointers are within caller-provided buffers. - let src = input.add((idx + lane) * CHUNK_LEN); - let dst = out.add((idx + lane) * OUT_LEN); - let chunk_counter = counter.wrapping_add(lane as u64); + let item = idx.strict_add(lane); + let src = input.add(item.strict_mul(CHUNK_LEN)); + let dst = out.add(item.strict_mul(OUT_LEN)); + let chunk_counter = counter.wrapping_add(u64::try_from(lane).expect("BLAKE3 tail lane fits in u64")); chunk_cv_one_chunk_aarch64_out(src, key, chunk_counter, flags, dst); } } @@ -1313,6 +1391,11 @@ pub(crate) unsafe fn hash_many_contiguous_neon( /// `children` is interpreted as `[left0, right0, left1, right1, ...]`. /// This path is alignment-agnostic and is the canonical aarch64 parent combine /// fast path (asm is only used for chunk-state loops). +/// +/// # Safety +/// +/// The current CPU must support NEON. `children.len()` must equal +/// `2 * out.len()`. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] pub(crate) unsafe fn parent_cvs_many_neon( @@ -1321,40 +1404,62 @@ pub(crate) unsafe fn parent_cvs_many_neon( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } let parent_flags = PARENT | flags; - debug_assert!(parent_flags <= u8::MAX as u32); // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. unsafe { let mut idx = 0usize; while idx < out.len() { - let rem = core::cmp::min(4usize, out.len() - idx); - let last_ptr = children[2 * (idx + rem - 1)].as_ptr(); + let rem = core::cmp::min(4usize, out.len().strict_sub(idx)); + let last_child = idx.strict_add(rem).strict_sub(1).strict_mul(2); + let last_ptr = children[last_child].as_ptr(); let mut ptrs = [last_ptr; 4]; - for lane in 0..rem { - ptrs[lane] = children[2 * (idx + lane)].as_ptr(); + for (lane, ptr) in ptrs.iter_mut().enumerate().take(rem) { + let child = idx.strict_add(lane).strict_mul(2); + *ptr = children[child].as_ptr(); } if rem == 4 { - // SAFETY: - // - `idx + 4 <= out.len()` by `rem == 4`. - // - `out` is `[[u8; OUT_LEN]]`, so taking a 4-element window as `[[u8; OUT_LEN]; 4]` is - // layout-compatible. - // - We write exactly those 4 outputs. - let out4 = &mut *(out.as_mut_ptr().add(idx) as *mut [[u8; OUT_LEN]; 4]); - hash4_neon(ptrs, BLOCK_LEN, &key_words, 0, false, parent_flags, 0, 0, out4); + let out4 = out[idx..] + .first_chunk_mut::<4>() + .expect("four remaining BLAKE3 parents provide four output slots"); + hash4_neon( + ptrs, + BLOCK_LEN, + &key_words, + 0, + false, + Hash4Flags { + base: parent_flags, + start: 0, + end: 0, + }, + out4, + ); } else { let mut tmp = [[0u8; OUT_LEN]; 4]; - hash4_neon(ptrs, BLOCK_LEN, &key_words, 0, false, parent_flags, 0, 0, &mut tmp); - out[idx..idx + rem].copy_from_slice(&tmp[..rem]); + hash4_neon( + ptrs, + BLOCK_LEN, + &key_words, + 0, + false, + Hash4Flags { + base: parent_flags, + start: 0, + end: 0, + }, + &mut tmp, + ); + out[idx..idx.strict_add(rem)].copy_from_slice(&tmp[..rem]); } - idx += rem; + idx = idx.strict_add(rem); } } } @@ -1365,7 +1470,8 @@ pub(crate) unsafe fn parent_cvs_many_neon( /// /// # Safety /// -/// Caller must ensure NEON is available. +/// The current CPU must support NEON. `blocks` must contain only complete +/// BLAKE3 blocks, and `blocks_compressed` must remain within one chunk. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] pub(crate) unsafe fn chunk_compress_blocks_neon( @@ -1395,7 +1501,7 @@ pub(crate) unsafe fn chunk_compress_blocks_neon( chaining_value.as_mut_ptr(), chunk_counter, flags, - blocks_compressed as *mut u8, + core::ptr::from_mut(blocks_compressed), num_blocks, ); return; @@ -1410,10 +1516,10 @@ pub(crate) unsafe fn chunk_compress_blocks_neon( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -1422,6 +1528,11 @@ pub(crate) unsafe fn chunk_compress_blocks_neon( /// /// Returns the 8-word chaining value result (row0^row2, row1^row3), avoiding /// materializing the full 16-word compression output. +/// +/// # Safety +/// +/// The current CPU must support NEON, and `block` must be readable for +/// `BLOCK_LEN` bytes. #[cfg(target_arch = "aarch64")] #[target_feature(enable = "neon")] pub(crate) unsafe fn compress_cv_neon_bytes( @@ -1469,7 +1580,7 @@ pub(crate) unsafe fn parent_cv_neon( &key_words, block_words.as_ptr().cast(), 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, ) } @@ -1494,6 +1605,9 @@ pub(crate) unsafe fn root_output_blocks4_neon( out: *mut u8, ) { #[inline(always)] + /// # Safety + /// + /// The current CPU must support NEON, and `dest` must be writable for 16 bytes. unsafe fn storeu_128(src: uint32x4_t, dest: *mut u8) { // SAFETY: NEON intrinsics and pointer ops are sound: intrinsics require NEON via #[target_feature], // pointers are within bounds of caller-provided slices. @@ -1533,24 +1647,9 @@ pub(crate) unsafe fn root_output_blocks4_neon( vdupq_n_u32(block_words[15]), ]; - let counter_low_vec = vld1q_u32( - [ - counter as u32, - counter.wrapping_add(1) as u32, - counter.wrapping_add(2) as u32, - counter.wrapping_add(3) as u32, - ] - .as_ptr(), - ); - let counter_high_vec = vld1q_u32( - [ - (counter >> 32) as u32, - (counter.wrapping_add(1) >> 32) as u32, - (counter.wrapping_add(2) >> 32) as u32, - (counter.wrapping_add(3) >> 32) as u32, - ] - .as_ptr(), - ); + let (counter_low, counter_high) = counter_lanes(counter, [0, 1, 2, 3]); + let counter_low_vec = vld1q_u32(counter_low.as_ptr()); + let counter_high_vec = vld1q_u32(counter_high.as_ptr()); let block_len_vec = vdupq_n_u32(block_len); let flags_vec = vdupq_n_u32(flags); @@ -1616,8 +1715,8 @@ pub(crate) unsafe fn root_output_blocks4_neon( transpose_vecs(&mut g2); transpose_vecs(&mut g3); - for lane in 0..4 { - let base = out.add(lane * 64); + for lane in 0usize..4 { + let base = out.add(lane.strict_mul(BLOCK_LEN)); storeu_128(g0[lane], base); storeu_128(g1[lane], base.add(16)); storeu_128(g2[lane], base.add(32)); diff --git a/src/hashes/crypto/blake3/aarch64/asm.rs b/src/hashes/crypto/blake3/aarch64/asm.rs index da10523e..432c235a 100644 --- a/src/hashes/crypto/blake3/aarch64/asm.rs +++ b/src/hashes/crypto/blake3/aarch64/asm.rs @@ -9,8 +9,6 @@ //! - last-block byte outputs require 8-byte alignment //! - unaligned callers must use the NEON paths in `aarch64.rs`. -#![allow(unsafe_code)] - use core::arch::global_asm; pub(crate) const ASM_ALIGN_INPUT: usize = 8; @@ -25,13 +23,13 @@ global_asm!(include_str!("asm/rscrypto_blake3_hash1_chunk_aarch64_apple_darwin.s #[cfg(target_os = "linux")] unsafe extern "C" { - pub fn rscrypto_blake3_hash1_chunk_root_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_hash1_chunk_root_aarch64_unix_linux( input: *const u8, key: *const u32, flags: u32, out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_cv_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_hash1_chunk_cv_aarch64_unix_linux( input: *const u8, key: *const u32, counter: u64, @@ -39,7 +37,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_state_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_hash1_chunk_state_aarch64_unix_linux( input: *const u8, key: *const u32, counter: u64, @@ -48,7 +46,7 @@ unsafe extern "C" { out_last_block: *mut u8, ); - pub fn rscrypto_blake3_chunk_compress_blocks_aarch64_unix_linux( + pub(super) fn rscrypto_blake3_chunk_compress_blocks_aarch64_unix_linux( blocks: *const u8, chaining_value: *mut u32, chunk_counter: u64, @@ -60,13 +58,13 @@ unsafe extern "C" { #[cfg(target_os = "macos")] unsafe extern "C" { - pub fn rscrypto_blake3_hash1_chunk_root_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_hash1_chunk_root_aarch64_apple_darwin( input: *const u8, key: *const u32, flags: u32, out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_cv_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_hash1_chunk_cv_aarch64_apple_darwin( input: *const u8, key: *const u32, counter: u64, @@ -74,7 +72,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_hash1_chunk_state_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_hash1_chunk_state_aarch64_apple_darwin( input: *const u8, key: *const u32, counter: u64, @@ -83,7 +81,7 @@ unsafe extern "C" { out_last_block: *mut u8, ); - pub fn rscrypto_blake3_chunk_compress_blocks_aarch64_apple_darwin( + pub(super) fn rscrypto_blake3_chunk_compress_blocks_aarch64_apple_darwin( blocks: *const u8, chaining_value: *mut u32, chunk_counter: u64, diff --git a/src/hashes/crypto/blake3/control.rs b/src/hashes/crypto/blake3/control.rs index ea45b042..95401f12 100644 --- a/src/hashes/crypto/blake3/control.rs +++ b/src/hashes/crypto/blake3/control.rs @@ -25,7 +25,7 @@ fn compute_derive_context_key_words(context: &str) -> [u32; 8] { } #[inline] -#[cfg_attr(feature = "std", allow(dead_code))] +#[cfg(not(feature = "std"))] pub(super) fn derive_context_key_words(context: &str) -> [u32; 8] { let context_bytes = context.as_bytes(); let kernel_ctx = dispatch::hasher_dispatch().size_class_kernel(context_bytes.len()); @@ -171,23 +171,23 @@ pub(super) fn streaming_parallel_threads_for_flags( #[cfg(feature = "parallel")] #[inline] -#[allow(clippy::manual_saturating_arithmetic)] // Explicit clamp semantics are preferred here. fn clamp_add_usize(lhs: usize, rhs: usize) -> usize { - lhs.checked_add(rhs).unwrap_or(usize::MAX) + lhs.saturating_add(rhs) } #[cfg(feature = "parallel")] #[inline] -#[allow(clippy::manual_saturating_arithmetic)] // Explicit clamp semantics are preferred here. fn clamp_mul_usize(lhs: usize, rhs: usize) -> usize { - lhs.checked_mul(rhs).unwrap_or(usize::MAX) + lhs.saturating_mul(rhs) } #[cfg(feature = "parallel")] #[inline] fn ceil_div_usize(value: usize, divisor: usize) -> usize { let adjusted = clamp_add_usize(value, divisor.strict_sub(1)); - adjusted / divisor + adjusted + .checked_div(divisor) + .expect("parallel policy divisor is nonzero") } #[cfg(feature = "parallel")] @@ -252,7 +252,7 @@ pub(super) fn parallel_admission_decision( threads: candidate, }; } - candidate -= 1; + candidate = candidate.strict_sub(1); } ParallelAdmissionDecision { would_parallelize: false, @@ -270,7 +270,10 @@ fn parallel_merge_divisor(mode: ParallelPolicyKind, commit_full_chunks: usize, t ParallelPolicyKind::Update | ParallelPolicyKind::KeyedUpdate | ParallelPolicyKind::DeriveUpdate => 1, ParallelPolicyKind::Xof | ParallelPolicyKind::KeyedXof | ParallelPolicyKind::DeriveXof => 3, }; - 1 + chunk_depth + thread_depth + mode_bias + 1usize + .strict_add(chunk_depth) + .strict_add(thread_depth) + .strict_add(mode_bias) } #[cfg(feature = "parallel")] diff --git a/src/hashes/crypto/blake3/dispatch.rs b/src/hashes/crypto/blake3/dispatch.rs index d64cfd0e..6d6fe5b1 100644 --- a/src/hashes/crypto/blake3/dispatch.rs +++ b/src/hashes/crypto/blake3/dispatch.rs @@ -1,5 +1,3 @@ -#![cfg_attr(test, allow(dead_code))] - #[cfg(feature = "parallel")] use super::dispatch_tables::ParallelTable; #[cfg(any( @@ -64,7 +62,10 @@ struct ResolvedDispatch { avx2_hash_many_one_chunk_fast_path: bool, #[cfg(target_arch = "x86_64")] hash_many_wide_pipeline: bool, - #[cfg(target_arch = "x86_64")] + #[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") + ))] avx2_available: bool, } @@ -269,7 +270,10 @@ fn resolved() -> ResolvedDispatch { avx2_hash_many_one_chunk_fast_path: allow_avx2_hash_many_one_chunk_fast_path(caps), #[cfg(target_arch = "x86_64")] hash_many_wide_pipeline: is_wide_pipeline_for_hash_many(caps), - #[cfg(target_arch = "x86_64")] + #[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") + ))] avx2_available: caps.has(required_caps(Blake3KernelId::X86Avx2)), } }) @@ -303,17 +307,17 @@ fn select(d: &ActiveDispatch, len: usize) -> Entry { } } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { let d = active(); select(&d, len).kernel.name } #[inline] #[must_use] -pub fn xof(data: &[u8]) -> super::Blake3XofReader { +pub(crate) fn xof(data: &[u8]) -> super::Blake3XofReader { let d = active(); let kernel = select(&d, data.len()).kernel; @@ -360,7 +364,10 @@ pub(crate) fn hash_many_wide_pipeline() -> bool { resolved().hash_many_wide_pipeline } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline] #[must_use] pub(crate) fn avx2_available() -> bool { diff --git a/src/hashes/crypto/blake3/dispatch_tables.rs b/src/hashes/crypto/blake3/dispatch_tables.rs index 75919a20..55e61722 100644 --- a/src/hashes/crypto/blake3/dispatch_tables.rs +++ b/src/hashes/crypto/blake3/dispatch_tables.rs @@ -2,7 +2,7 @@ //! //! This module stores compact per-family profiles used by runtime dispatch. -pub use super::kernels::Blake3KernelId as KernelId; +pub(crate) use super::kernels::Blake3KernelId as KernelId; use crate::platform::Caps; #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; @@ -15,7 +15,7 @@ use crate::platform::caps::s390x; #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; // Architecture-specific thresholds for when to switch from table bulk kernel to size-class // selection. These are tuned based on SIMD width and latency characteristics. @@ -28,12 +28,19 @@ const THRESHOLD_NEON: usize = 16 * 1024; // NEON - slightly higher due to differ #[cfg(not(target_arch = "x86_64"))] const THRESHOLD_PORTABLE: usize = 32 * 1024; // Conservative for scalar +#[cfg(feature = "parallel")] const DEFAULT_PAR_SPAWN_COST_BYTES: usize = 24 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_MERGE_COST_BYTES: usize = 16 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_BYTES_PER_CORE_SMALL: usize = 256 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_BYTES_PER_CORE_MEDIUM: usize = 128 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_BYTES_PER_CORE_LARGE: usize = 64 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_SMALL_LIMIT_BYTES: usize = 256 * 1024; +#[cfg(feature = "parallel")] const DEFAULT_PAR_MEDIUM_LIMIT_BYTES: usize = 2 * 1024 * 1024; /// Parallel hashing policy for large inputs (std-only). @@ -41,9 +48,9 @@ const DEFAULT_PAR_MEDIUM_LIMIT_BYTES: usize = 2 * 1024 * 1024; /// The scheduler treats this as an explicit cost model: /// - fixed terms: `spawn_cost_bytes`, `merge_cost_bytes` /// - work terms: `bytes_per_core_*` for small/medium/large payload classes -#[cfg_attr(not(feature = "parallel"), allow(dead_code))] +#[cfg(feature = "parallel")] #[derive(Clone, Copy, Debug)] -pub struct ParallelTable { +pub(crate) struct ParallelTable { /// Minimum total input bytes before parallel hashing is considered. pub min_bytes: usize, /// Minimum number of full chunks to commit in one batch. @@ -76,14 +83,14 @@ pub struct ParallelTable { /// `bulk_sizeclass_threshold` is the minimum input length to use size-class-based bulk kernel /// selection instead of the table's default bulk kernel. This is architecture-specific. #[derive(Clone, Copy, Debug)] -pub struct StreamingTable { +pub(crate) struct StreamingTable { pub stream: KernelId, pub bulk: KernelId, pub bulk_sizeclass_threshold: usize, } #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -92,12 +99,13 @@ pub struct DispatchTable { } /// Compact profile for one microarchitecture family. -#[cfg_attr(not(feature = "parallel"), allow(dead_code))] #[derive(Clone, Copy, Debug)] -pub struct FamilyProfile { +pub(crate) struct FamilyProfile { pub dispatch: DispatchTable, pub streaming: StreamingTable, + #[cfg(feature = "parallel")] pub parallel: ParallelTable, + #[cfg(feature = "parallel")] pub streaming_parallel: ParallelTable, } @@ -216,6 +224,7 @@ const DEFAULT_STREAM_KERNEL: KernelId = KernelId::Portable; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] const DEFAULT_BULK_KERNEL: KernelId = SIMD_KERNEL; +#[cfg(feature = "parallel")] #[derive(Clone, Copy, Debug)] struct ParallelCostModel { spawn_cost_bytes: usize, @@ -227,6 +236,7 @@ struct ParallelCostModel { medium_limit_bytes: usize, } +#[cfg(feature = "parallel")] #[inline] #[must_use] const fn parallel_cost_model( @@ -249,6 +259,7 @@ const fn parallel_cost_model( } } +#[cfg(feature = "parallel")] #[inline] #[must_use] const fn parallel_table( @@ -271,7 +282,7 @@ const fn parallel_table( } } -#[cfg(any(target_arch = "s390x", target_arch = "powerpc64"))] +#[cfg(all(feature = "parallel", any(target_arch = "s390x", target_arch = "powerpc64")))] macro_rules! parallel_costs { ( $min_bytes:expr, @@ -302,6 +313,7 @@ macro_rules! parallel_costs { }; } +#[cfg(feature = "parallel")] #[inline] #[must_use] const fn default_parallel_costs(min_bytes: usize, min_chunks: usize, max_threads: u8) -> ParallelTable { @@ -323,7 +335,7 @@ const fn default_parallel_costs(min_bytes: usize, min_chunks: usize, max_threads #[inline] #[must_use] -#[cfg(any(target_arch = "s390x", target_arch = "powerpc64"))] +#[cfg(all(feature = "parallel", any(target_arch = "s390x", target_arch = "powerpc64")))] const fn scalar_profile_parallel( min_bytes: usize, min_chunks: usize, @@ -431,14 +443,14 @@ const fn default_kind_streaming_table() -> StreamingTable { } } -#[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] +#[cfg(all(feature = "parallel", any(target_arch = "x86_64", target_arch = "riscv64")))] #[inline] #[must_use] const fn default_kind_parallel_table() -> ParallelTable { default_parallel_costs(128 * 1024, 64, 0) } -#[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] +#[cfg(all(feature = "parallel", any(target_arch = "x86_64", target_arch = "riscv64")))] #[inline] #[must_use] const fn default_kind_streaming_parallel_table() -> ParallelTable { @@ -452,7 +464,9 @@ const fn default_kind_profile() -> FamilyProfile { FamilyProfile { dispatch: default_kind_table(), streaming: default_kind_streaming_table(), + #[cfg(feature = "parallel")] parallel: default_kind_parallel_table(), + #[cfg(feature = "parallel")] streaming_parallel: default_kind_streaming_parallel_table(), } } @@ -474,21 +488,20 @@ const fn portable_profile() -> FamilyProfile { bulk: KernelId::Portable, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: default_parallel_costs(128 * 1024, 64, 0), + #[cfg(feature = "parallel")] streaming_parallel: default_parallel_costs(128 * 1024, 64, 0), } } #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] -// Family Profile: DEFAULT_KIND -pub static PROFILE_DEFAULT_KIND: FamilyProfile = default_kind_profile(); -// Family Profile: PORTABLE +pub(crate) static PROFILE_DEFAULT_KIND: FamilyProfile = default_kind_profile(); #[cfg(not(target_arch = "x86_64"))] -pub static PROFILE_PORTABLE: FamilyProfile = portable_profile(); +pub(crate) static PROFILE_PORTABLE: FamilyProfile = portable_profile(); -// Family Profile: INTEL_SAPPHIRE_RAPIDS #[cfg(target_arch = "x86_64")] -pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 1024, 4096], // This profile uses AVX-512 for every size class; its selector has no @@ -503,6 +516,7 @@ pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { bulk: KernelId::X86Avx512, bulk_sizeclass_threshold: THRESHOLD_AVX512, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -515,6 +529,7 @@ pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { small_limit_bytes: 1048576, medium_limit_bytes: 4194304, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -528,9 +543,8 @@ pub static PROFILE_INTEL_SAPPHIRE_RAPIDS: FamilyProfile = FamilyProfile { medium_limit_bytes: 2097152, }, }; -// Family Profile: X86_AVX512 #[cfg(target_arch = "x86_64")] -pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 1024, 4096], // This profile uses AVX-512 for every size class; its selector has no @@ -545,6 +559,7 @@ pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { bulk: KernelId::X86Avx512, bulk_sizeclass_threshold: THRESHOLD_AVX512, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -557,6 +572,7 @@ pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 2097152, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -572,7 +588,7 @@ pub static PROFILE_X86_AVX512: FamilyProfile = FamilyProfile { }; // Family Profile: AARCH64_NEON #[cfg(target_arch = "aarch64")] -pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 4095, 4096], // Graviton-class attribution shows that for short inputs (<4KiB), the @@ -589,6 +605,7 @@ pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { bulk: KernelId::Aarch64Neon, bulk_sizeclass_threshold: THRESHOLD_NEON, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -601,6 +618,7 @@ pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 2097152, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -616,33 +634,37 @@ pub static PROFILE_AARCH64_NEON: FamilyProfile = FamilyProfile { }; // Family Profile: Z13 #[cfg(target_arch = "s390x")] -pub static PROFILE_Z13: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_Z13: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: S390X_VECTOR_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), }; // Family Profile: Z14 #[cfg(target_arch = "s390x")] -pub static PROFILE_Z14: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_Z14: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: S390X_VECTOR_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), }; // Family Profile: Z15 #[cfg(target_arch = "s390x")] -pub static PROFILE_Z15: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_Z15: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 256, 4096], xs: KernelId::Portable, @@ -655,6 +677,7 @@ pub static PROFILE_Z15: FamilyProfile = FamilyProfile { bulk: S390X_VECTOR_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -667,6 +690,7 @@ pub static PROFILE_Z15: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 8388608, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -682,46 +706,52 @@ pub static PROFILE_Z15: FamilyProfile = FamilyProfile { }; // Family Profile: POWER7 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER7: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER7: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(256 * 1024, 128, 8, 0), }; // Family Profile: POWER8 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER8: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER8: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(192 * 1024, 96, 8, 1), }; // Family Profile: POWER9 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER9: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER9: FamilyProfile = FamilyProfile { dispatch: default_kind_table(), streaming: StreamingTable { stream: KernelId::Portable, bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: scalar_profile_parallel(128 * 1024, 64, 16, 3), + #[cfg(feature = "parallel")] streaming_parallel: scalar_profile_parallel(128 * 1024, 64, 16, 3), }; // Family Profile: POWER10 #[cfg(target_arch = "powerpc64")] -pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { +pub(crate) static PROFILE_POWER10: FamilyProfile = FamilyProfile { dispatch: DispatchTable { boundaries: [64, 256, 4096], xs: KernelId::Portable, @@ -734,6 +764,7 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { bulk: POWER_VSX_KERNEL, bulk_sizeclass_threshold: THRESHOLD_PORTABLE, }, + #[cfg(feature = "parallel")] parallel: ParallelTable { min_bytes: 65536, min_chunks: 64, @@ -746,6 +777,7 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { small_limit_bytes: 262144, medium_limit_bytes: 8388608, }, + #[cfg(feature = "parallel")] streaming_parallel: ParallelTable { min_bytes: 0, min_chunks: 0, @@ -762,7 +794,7 @@ pub static PROFILE_POWER10: FamilyProfile = FamilyProfile { #[inline] #[must_use] -pub fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { +pub(crate) fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { #[cfg(target_arch = "x86_64")] { if caps.has(x86::AVX512_READY) { @@ -837,27 +869,27 @@ pub fn select_profile_for_caps(caps: Caps) -> &'static FamilyProfile { #[inline] #[must_use] -pub fn select_table_for_caps(caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_table_for_caps(caps: Caps) -> &'static DispatchTable { &select_profile_for_caps(caps).dispatch } #[inline] #[must_use] -pub fn select_streaming_table_for_caps(caps: Caps) -> &'static StreamingTable { +pub(crate) fn select_streaming_table_for_caps(caps: Caps) -> &'static StreamingTable { &select_profile_for_caps(caps).streaming } #[cfg(feature = "parallel")] #[inline] #[must_use] -pub fn select_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { +pub(crate) fn select_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { &select_profile_for_caps(caps).parallel } #[cfg(feature = "parallel")] #[inline] #[must_use] -pub fn select_streaming_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { +pub(crate) fn select_streaming_parallel_table_for_caps(caps: Caps) -> &'static ParallelTable { &select_profile_for_caps(caps).streaming_parallel } @@ -876,19 +908,19 @@ mod tests { let sapphire_rapids = x86::AVX512_READY | x86::INTEL_SAPPHIRE_RAPIDS; assert!(core::ptr::eq( select_profile_for_caps(sapphire_rapids), - &PROFILE_INTEL_SAPPHIRE_RAPIDS + &raw const PROFILE_INTEL_SAPPHIRE_RAPIDS )); assert!(core::ptr::eq( select_profile_for_caps(sapphire_rapids | ALL_AMX), - &PROFILE_INTEL_SAPPHIRE_RAPIDS + &raw const PROFILE_INTEL_SAPPHIRE_RAPIDS )); assert!(core::ptr::eq( select_profile_for_caps(x86::AVX512_READY), - &PROFILE_X86_AVX512 + &raw const PROFILE_X86_AVX512 )); assert!(core::ptr::eq( select_profile_for_caps(x86::AVX512_READY | ALL_AMX), - &PROFILE_X86_AVX512 + &raw const PROFILE_X86_AVX512 )); } } diff --git a/src/hashes/crypto/blake3/kernel_test.rs b/src/hashes/crypto/blake3/kernel_test.rs index 2dab9a5e..01c4cff6 100644 --- a/src/hashes/crypto/blake3/kernel_test.rs +++ b/src/hashes/crypto/blake3/kernel_test.rs @@ -25,7 +25,7 @@ const ALL: &[Blake3KernelId] = &[ ]; #[derive(Clone, Debug)] -pub struct KernelResult { +pub(super) struct KernelResult { pub digest: [u8; 32], } @@ -48,7 +48,7 @@ fn digest_with_kernel(id: Blake3KernelId, data: &[u8]) -> [u8; 32] { } #[must_use] -pub fn run_all_blake3_kernels(data: &[u8]) -> Vec { +pub(super) fn run_all_blake3_kernels(data: &[u8]) -> Vec { let caps = crate::platform::caps(); let mut out = Vec::with_capacity(ALL.len()); for &id in ALL { @@ -61,7 +61,7 @@ pub fn run_all_blake3_kernels(data: &[u8]) -> Vec { out } -pub fn verify_blake3_kernels(data: &[u8]) -> Result<(), &'static str> { +pub(super) fn verify_blake3_kernels(data: &[u8]) -> Result<(), &'static str> { let results = run_all_blake3_kernels(data); let Some(first) = results.first() else { return Ok(()); @@ -93,7 +93,9 @@ mod tests { } fn pattern(len: usize) -> Vec { - (0..len).map(|i| (i % 251) as u8).collect() + (0..len) + .map(|i| u8::try_from(i % 251).expect("test pattern byte fits in u8")) + .collect() } #[test] @@ -335,7 +337,9 @@ mod tests { for chunk_idx in 0..num_chunks { let base = chunk_idx * CHUNK_LEN; for i in 0..CHUNK_LEN { - input[base + i] = ((i % 251) as u8).wrapping_add(chunk_idx as u8); + let byte = u8::try_from(i % 251).expect("test pattern byte fits in u8"); + let chunk = u8::try_from(chunk_idx).expect("test chunk index fits in u8"); + input[base + i] = byte.wrapping_add(chunk); } } @@ -383,7 +387,9 @@ mod tests { for chunk_idx in 0..num_chunks { let base = chunk_idx * CHUNK_LEN; for i in 0..CHUNK_LEN { - input[base + i] = ((i % 251) as u8).wrapping_add(chunk_idx as u8); + let byte = u8::try_from(i % 251).expect("test pattern byte fits in u8"); + let chunk = u8::try_from(chunk_idx).expect("test chunk index fits in u8"); + input[base + i] = byte.wrapping_add(chunk); } } diff --git a/src/hashes/crypto/blake3/kernels.rs b/src/hashes/crypto/blake3/kernels.rs index eb68b377..70ad4f61 100644 --- a/src/hashes/crypto/blake3/kernels.rs +++ b/src/hashes/crypto/blake3/kernels.rs @@ -1,6 +1,15 @@ use super::{ BLOCK_LEN, CHUNK_LEN, CHUNK_START, OUT_LEN, PARENT, first_8_words, words8_from_le_bytes_32, words16_from_le_bytes_64, }; +#[cfg(any( + test, + feature = "diag", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "powerpc64", + target_arch = "riscv64" +))] use crate::platform::Caps; #[cfg(target_arch = "aarch64")] use crate::platform::caps::aarch64; @@ -13,6 +22,33 @@ use crate::platform::caps::s390x; #[cfg(target_arch = "x86_64")] use crate::platform::caps::x86; +const BLOCK_LEN_U32: u32 = 64; + +#[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") +))] +#[inline(always)] +fn assembly_flags(flags: u32) -> u8 { + u8::try_from(flags).expect("BLAKE3 flags must fit the assembly ABI") +} + +#[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] +#[inline(always)] +fn counter_words(counter: u64) -> (u32, u32) { + let [c0, c1, c2, c3, c4, c5, c6, c7] = counter.to_le_bytes(); + ( + u32::from_le_bytes([c0, c1, c2, c3]), + u32::from_le_bytes([c4, c5, c6, c7]), + ) +} + +#[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] +#[inline(always)] +fn wrapping_add4(lhs: core::simd::u32x4, rhs: core::simd::u32x4) -> core::simd::u32x4 { + core::ops::Add::add(lhs, rhs) +} + // Kernel function types /// Core compression function (single block). @@ -61,8 +97,7 @@ pub(crate) struct Kernel { #[cfg(all(feature = "diag", target_arch = "x86_64"))] pub(crate) force_x86_avx512_exact_block_asm: bool, /// Kernel name for debugging/tuning. - #[cfg(any(test, feature = "diag"))] - #[cfg_attr(test, allow(dead_code))] + #[cfg(feature = "diag")] pub(crate) name: &'static str, } @@ -71,7 +106,7 @@ pub(crate) struct Kernel { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Blake3KernelId { +pub(crate) enum Blake3KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Sse41 = 2, @@ -93,7 +128,7 @@ impl Blake3KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -116,7 +151,7 @@ impl Blake3KernelId { /// Returns the SIMD degree for this kernel. #[inline] #[must_use] - pub const fn simd_degree(self) -> usize { + pub(crate) const fn simd_degree(self) -> usize { match self { Self::Portable => 1, #[cfg(target_arch = "x86_64")] @@ -153,7 +188,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "x86_64")] @@ -172,7 +207,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "x86_64")] @@ -191,7 +226,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "x86_64")] @@ -210,7 +245,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { owned_x86_hash_many: false, #[cfg(all(feature = "diag", target_arch = "x86_64"))] force_x86_avx512_exact_block_asm: false, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "aarch64")] @@ -223,7 +258,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_neon_wrapper, chunk_compress_blocks: chunk_compress_blocks_neon_wrapper, hash_many_contiguous: hash_many_contiguous_neon_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "s390x")] @@ -232,7 +267,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_s390x_vector_wrapper, chunk_compress_blocks: chunk_compress_blocks_s390x_vector_wrapper, hash_many_contiguous: hash_many_contiguous_s390x_vector_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "powerpc64")] @@ -241,7 +276,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_power_vsx_wrapper, chunk_compress_blocks: chunk_compress_blocks_power_vsx_wrapper, hash_many_contiguous: hash_many_contiguous_power_vsx_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, #[cfg(target_arch = "riscv64")] @@ -250,7 +285,7 @@ pub(crate) fn kernel(id: Blake3KernelId) -> Kernel { compress: compress_riscv_v_wrapper, chunk_compress_blocks: chunk_compress_blocks_riscv_v_wrapper, hash_many_contiguous: hash_many_contiguous_riscv_v_wrapper, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }, } @@ -366,7 +401,7 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, block.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, ); } @@ -379,7 +414,7 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, block.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, ); } @@ -392,7 +427,7 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, block.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, ); } @@ -408,12 +443,19 @@ pub(crate) unsafe fn compress_block_asm_inline( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags, )); } #[inline(always)] +/// Dispatches contiguous full chunks through a selected kernel. +/// +/// # Safety +/// +/// `input` must reference `num_chunks * CHUNK_LEN` readable bytes, `out` must +/// reference `num_chunks * OUT_LEN` writable bytes, and the CPU must support +/// every target feature required by `id`. pub(crate) unsafe fn hash_many_contiguous_inline( id: Blake3KernelId, input: *const u8, @@ -545,8 +587,8 @@ fn write_root_output_words(out: &mut [u8; 2 * OUT_LEN], words: &[u32; 16]) { } for (idx, word) in words.iter().copied().enumerate() { - let offset = idx * 4; - out[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + let offset = idx.strict_mul(4); + out[offset..offset.strict_add(4)].copy_from_slice(&word.to_le_bytes()); } } @@ -1039,12 +1081,12 @@ where { debug_assert!(rem != 0); debug_assert!(rem <= DEGREE); - let last_ptr = ptr_at(total - 1); + let last_ptr = ptr_at(total.strict_sub(1)); let mut ptrs = [last_ptr; DEGREE]; let mut lane = 0usize; while lane < rem { - ptrs[lane] = ptr_at(start + lane); - lane += 1; + ptrs[lane] = ptr_at(start.strict_add(lane)); + lane = lane.strict_add(1); } ptrs } @@ -1064,15 +1106,15 @@ fn reduce_parent_blocks_lanes( let mut tmp = [[0u8; OUT_LEN]; DEGREE]; let mut i = 0usize; while i < count { - let rem = core::cmp::min(DEGREE, count - i); + let rem = core::cmp::min(DEGREE, count.strict_sub(i)); let ptrs = parent_block_ptrs::(i, rem, count, &mut ptr_at); hash_many(&ptrs, rem, &mut tmp); let mut lane = 0usize; while lane < rem { - sink(i + lane, &tmp[lane]); - lane += 1; + sink(i.strict_add(lane), &tmp[lane]); + lane = lane.strict_add(1); } - i += rem; + i = i.strict_add(rem); } } @@ -1087,24 +1129,24 @@ pub(crate) fn diag_chunk_cvs_many_avx2_pair_from_bytes( debug_assert!(!input.is_empty()); debug_assert_eq!(input.len() % CHUNK_LEN, 0); let chunks = input.len() / CHUNK_LEN; - debug_assert_eq!(out.len(), chunks * OUT_LEN); + debug_assert_eq!(out.len(), chunks.strict_mul(OUT_LEN)); let mut idx = 0usize; - while idx + 1 < chunks { + while idx.strict_add(1) < chunks { // SAFETY: Diagnostic owned AVX2 two-chunk batch because: // 1. Diagnostic availability checks AVX2 support before this helper is called. // 2. `idx + 1 < chunks`, so `input + idx * CHUNK_LEN` is readable for two full chunks. // 3. `out + idx * OUT_LEN` is writable for two OUT_LEN-byte CV outputs. unsafe { super::x86_64::avx2::hash2_chunks_owned( - input.as_ptr().add(idx * CHUNK_LEN), + input.as_ptr().add(idx.strict_mul(CHUNK_LEN)), &key_words, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("chunk index must fit u64")), flags, - out.as_mut_ptr().add(idx * OUT_LEN), + out.as_mut_ptr().add(idx.strict_mul(OUT_LEN)), ); } - idx += 2; + idx = idx.strict_add(2); } if idx < chunks { @@ -1114,11 +1156,11 @@ pub(crate) fn diag_chunk_cvs_many_avx2_pair_from_bytes( // 3. Diagnostic availability checks AVX2 support before this helper is called. unsafe { hash_one_chunk_avx2_owned_serial( - input.as_ptr().add(idx * CHUNK_LEN), + input.as_ptr().add(idx.strict_mul(CHUNK_LEN)), &key_words, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("chunk index must fit u64")), flags, - out.as_mut_ptr().add(idx * OUT_LEN), + out.as_mut_ptr().add(idx.strict_mul(OUT_LEN)), ); } } @@ -1131,7 +1173,7 @@ pub(crate) fn diag_parent_cvs_many_avx2_owned_from_bytes( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -1139,24 +1181,24 @@ pub(crate) fn diag_parent_cvs_many_avx2_owned_from_bytes( let parent_flags = PARENT | flags; reduce_parent_blocks_lanes::<{ super::x86_64::avx2::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { // SAFETY: Diagnostic owned AVX2 parent batch because: // 1. Diagnostic availability checks AVX2 support before this helper is called. // 2. `parent_block_ptrs` fills every lane with a valid 64-byte parent block pointer. // 3. `tmp` has space for all 8 output CV lanes; callers copy only real lanes. unsafe { - super::x86_64::avx2::hash8_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1170,10 +1212,13 @@ pub(crate) fn diag_parent_cvs_many_avx2_pair_from_bytes( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); let mut idx = 0usize; - while idx + 1 < out.len() { - let parents = [children[2 * idx].as_ptr(), children[2 * (idx + 1)].as_ptr()]; + while idx.strict_add(1) < out.len() { + let parents = [ + children[2usize.strict_mul(idx)].as_ptr(), + children[2usize.strict_mul(idx.strict_add(1))].as_ptr(), + ]; // SAFETY: Diagnostic owned AVX2 two-parent batch because: // 1. Diagnostic availability checks AVX2 support before this helper is called. // 2. `parents` points to two adjacent 64-byte parent blocks. @@ -1181,7 +1226,7 @@ pub(crate) fn diag_parent_cvs_many_avx2_pair_from_bytes( unsafe { super::x86_64::avx2::parent_cv2_owned(&parents, &key_words, flags, out[idx].as_mut_ptr()); } - idx += 2; + idx = idx.strict_add(2); } if idx < out.len() { @@ -1190,12 +1235,26 @@ pub(crate) fn diag_parent_cvs_many_avx2_pair_from_bytes( // 2. `idx < out.len()`, so `out[idx]` is writable for one OUT_LEN-byte CV. // 3. Diagnostic availability checks AVX2 support before this helper is called. unsafe { - parent_one_avx2_owned_serial_from_block(children[2 * idx].as_ptr(), key_words, flags, &mut out[idx]); + parent_one_avx2_owned_serial_from_block( + children[2usize.strict_mul(idx)].as_ptr(), + key_words, + flags, + &mut out[idx], + ); } } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] +/// Hash one packed parent block through the serial AVX2 path. +/// +/// # Safety +/// +/// AVX2 must be available, `parent_block` must be readable for 64 bytes, and +/// `out` must be writable for one chaining value. unsafe fn parent_one_avx2_owned_serial_from_block( parent_block: *const u8, key_words: [u32; 8], @@ -1240,7 +1299,7 @@ pub(crate) fn diag_parent_cvs_many_avx512_owned_from_bytes( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -1248,24 +1307,24 @@ pub(crate) fn diag_parent_cvs_many_avx512_owned_from_bytes( let parent_flags = PARENT | flags; reduce_parent_blocks_lanes::<{ super::x86_64::avx512::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { // SAFETY: Diagnostic owned AVX-512 parent batch because: // 1. Diagnostic availability checks AVX-512F/VL/DQ plus AVX2 support before this helper is called. // 2. `parent_block_ptrs` fills every lane with a valid 64-byte parent block pointer. // 3. `tmp` has space for all 16 output CV lanes; callers copy only real lanes. unsafe { - super::x86_64::avx512::hash16_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx512::hash16_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1284,13 +1343,13 @@ pub(crate) fn parent_cvs_many_from_cvs_inline( flags: u32, out: &mut [[u32; 8]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } if id == Blake3KernelId::Portable { - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { *out_cv = parent_cv_inline(id, pair[0], pair[1], key_words, flags); } return; @@ -1313,7 +1372,7 @@ pub(crate) fn parent_cvs_many_from_cvs_inline( #[cfg(not(target_endian = "little"))] { // Big-endian fallback keeps explicit LE conversion. - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { *out_cv = parent_cv_inline(id, pair[0], pair[1], key_words, flags); } } @@ -1334,7 +1393,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( flags: u32, out: &mut [[u8; OUT_LEN]], ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -1368,21 +1427,21 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( let parent_flags = PARENT | flags; reduce_parent_blocks_lanes::<{ super::x86_64::sse41::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { // SAFETY: SSE4.1 is available per dispatch; pointers and outputs are valid. unsafe { - super::x86_64::sse41::hash4( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::sse41::hash4(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1395,10 +1454,9 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( { let parent_flags = PARENT | flags; const DEGREE: usize = 16; - debug_assert!(parent_flags <= u8::MAX as u32); reduce_parent_blocks_lanes::( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, rem, tmp| { if rem == 15 && avx512_owned_hash_many_available() { // SAFETY: Use the owned AVX-512 parent tail because: @@ -1408,35 +1466,35 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // real parent block. // 4. `tmp` has space for all 16 OUT_LEN-byte outputs. unsafe { - super::x86_64::avx512::hash16_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx512::hash16_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } return; } // SAFETY: AVX-512 is available per dispatch; pointers are valid for // one parent block each, `rem <= DEGREE`, and output is large enough. unsafe { - super::x86_64::asm::hash_many_avx512( - ptrs.as_ptr(), - rem, - 1, - key_words.as_ptr(), - 0, - false, - parent_flags as u8, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::asm::hash_many_avx512(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: rem, + blocks: 1, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: assembly_flags(parent_flags), + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } }, |idx, bytes| out[idx] = *bytes, @@ -1449,7 +1507,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // Keep AVX-512 semantics on non-asm targets as well: run the parent // fold through the AVX-512 per-parent entrypoint instead of delegating // into AVX2. - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { let left = words8_from_le_bytes_32(&pair[0]); let right = words8_from_le_bytes_32(&pair[1]); *out_cv = super::words8_to_le_bytes(&parent_cv_inline( @@ -1516,25 +1574,24 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( parent_block_ptr_from_children(children, 2), parent_block_ptr_from_children(children, 3), ]; - debug_assert!(parent_flags <= u8::MAX as u32); // SAFETY: Keep the four-parent AVX2 assembly tail but write it directly because: // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. // 2. `ptrs` points to four readable packed 64-byte parent blocks. // 3. `out` is writable for four contiguous OUT_LEN-byte parent CV outputs. // 4. Direct output avoids the generic reducer's temporary copy. unsafe { - super::x86_64::asm::hash_many_avx2( - ptrs.as_ptr(), - 4, - 1, - key_words.as_ptr(), - 0, - false, - parent_flags as u8, - 0, - 0, - out[0].as_mut_ptr(), - ); + super::x86_64::asm::hash_many_avx2(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: 4, + blocks: 1, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: assembly_flags(parent_flags), + flags_start: 0, + flags_end: 0, + out: out[0].as_mut_ptr(), + }); } return; } @@ -1551,7 +1608,17 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 3. `out` is writable for 8 contiguous OUT_LEN-byte parent CV outputs. // 4. Direct output avoids the generic reducer's temporary copy. unsafe { - super::x86_64::avx2::hash8_owned(&ptrs, 1, &key_words, 0, false, parent_flags, 0, 0, out[0].as_mut_ptr()); + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: out[0].as_mut_ptr(), + }); } return; } @@ -1559,12 +1626,11 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( reduce_parent_blocks_lanes::<{ super::x86_64::avx2::DEGREE }, _, _, _>( out.len(), - |idx| children[2 * idx].as_ptr(), + |idx| children[2usize.strict_mul(idx)].as_ptr(), |ptrs, _rem, tmp| { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { let rem = _rem; - debug_assert!(parent_flags <= u8::MAX as u32); if rem == 1 { // SAFETY: Use the owned AVX2 one-parent tail because: // 1. Dispatch selected the AVX2 kernel, so AVX2 is available. @@ -1585,17 +1651,17 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // 4. The `rem` guard keeps 2/3/4 generic-reducer tails on assembly and promotes only 5/6/7, which // won on Sapphire Rapids. Exact 2/3/8 parent reductions use direct fast paths above. unsafe { - super::x86_64::avx2::hash8_owned( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } return; } @@ -1603,18 +1669,18 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( // dispatch. Each pointer is valid for one 64-byte parent block. // `rem <= DEGREE`, and `tmp` is large enough. unsafe { - super::x86_64::asm::hash_many_avx2( - ptrs.as_ptr(), - rem, - 1, - key_words.as_ptr(), - 0, - false, - parent_flags as u8, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ); + super::x86_64::asm::hash_many_avx2(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: rem, + blocks: 1, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: assembly_flags(parent_flags), + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }); } } @@ -1622,17 +1688,17 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( { // SAFETY: AVX2 is available for this wrapper; pointers and outputs are valid. unsafe { - super::x86_64::avx2::hash8( - ptrs, - 1, - &key_words, - 0, - false, - parent_flags, - 0, - 0, - tmp.as_mut_ptr().cast::(), - ) + super::x86_64::avx2::hash8(super::x86_64::HashManyRequest { + inputs: ptrs, + blocks: 1, + key: &key_words, + counter: 0, + increment_counter: false, + flags: parent_flags, + flags_start: 0, + flags_end: 0, + out: tmp.as_mut_ptr().cast::(), + }) }; } }, @@ -1643,7 +1709,7 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( } // Scalar fallback. - for (pair, out_cv) in children.chunks_exact(2).zip(out.iter_mut()) { + for (pair, out_cv) in children.as_chunks::<2>().0.iter().zip(out.iter_mut()) { let left = words8_from_le_bytes_32(&pair[0]); let right = words8_from_le_bytes_32(&pair[1]); *out_cv = super::words8_to_le_bytes(&parent_cv_inline(id, left, right, key_words, flags)); @@ -1656,7 +1722,16 @@ pub(crate) fn parent_cvs_many_from_bytes_inline( /// `[left_cv (8 words), right_cv (8 words)]`. #[inline] #[must_use] -pub const fn required_caps(id: Blake3KernelId) -> Caps { +#[cfg(any( + test, + feature = "diag", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "powerpc64", + target_arch = "riscv64" +))] +pub(crate) const fn required_caps(id: Blake3KernelId) -> Caps { match id { Blake3KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -1702,7 +1777,7 @@ pub const fn required_caps(id: Blake3KernelId) -> Caps { #[cfg(all(feature = "diag", target_arch = "x86_64"))] #[inline] #[must_use] -pub const fn required_caps_owned_hash_many(id: Blake3KernelId) -> Caps { +pub(crate) const fn required_caps_owned_hash_many(id: Blake3KernelId) -> Caps { match id { Blake3KernelId::X86Avx2 => required_caps(Blake3KernelId::X86Avx2), Blake3KernelId::X86Avx512 => x86::AVX512F @@ -1718,7 +1793,7 @@ pub const fn required_caps_owned_hash_many(id: Blake3KernelId) -> Caps { #[cfg(all(feature = "diag", target_arch = "x86_64"))] #[inline] #[must_use] -pub const fn required_caps_owned_compress(id: Blake3KernelId) -> Caps { +pub(crate) const fn required_caps_owned_compress(id: Blake3KernelId) -> Caps { match id { Blake3KernelId::X86Avx512 => required_caps(Blake3KernelId::X86Avx512), _ => required_caps(id), @@ -1735,27 +1810,24 @@ fn chunk_compress_blocks_portable( blocks_compressed: &mut u8, blocks: &[u8], ) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); + let (block_slices, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); // Hot path for streaming callers that feed one full block at a time. - if blocks.len() == BLOCK_LEN { - // SAFETY: `blocks` is exactly one block, and `[u8; BLOCK_LEN]` has 1-byte alignment. - let block_bytes: &[u8; BLOCK_LEN] = unsafe { &*(blocks.as_ptr().cast()) }; + if let [block_bytes] = block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); *chaining_value = first_8_words((super::compress)( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, )); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); return; } - let (block_slices, remainder) = blocks.as_chunks::(); - debug_assert!(remainder.is_empty()); for block_bytes in block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); @@ -1763,10 +1835,10 @@ fn chunk_compress_blocks_portable( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, )); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); } } @@ -1779,11 +1851,17 @@ fn parent_cv_portable(left_child_cv: [u32; 8], right_child_cv: [u32; 8], key_wor &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } +/// Hashes contiguous chunks with the portable compression function. +/// +/// # Safety +/// +/// `input` must reference `num_chunks * CHUNK_LEN` readable bytes and `out` +/// must reference `num_chunks * OUT_LEN` writable bytes. unsafe fn hash_many_contiguous_portable( input: *const u8, num_chunks: usize, @@ -1795,13 +1873,17 @@ unsafe fn hash_many_contiguous_portable( debug_assert!(num_chunks != 0); for chunk_idx in 0..num_chunks { - let chunk_counter = counter.wrapping_add(chunk_idx as u64); + let chunk_counter = counter.wrapping_add(u64::try_from(chunk_idx).expect("BLAKE3 chunk index fits in u64")); let mut cv = *key; - for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { + for block_idx in 0..CHUNK_LEN.strict_div(BLOCK_LEN) { // SAFETY: caller guarantees `input` is valid for `num_chunks * CHUNK_LEN`. let block_words = unsafe { - let src = input.add(chunk_idx * CHUNK_LEN + block_idx * BLOCK_LEN); + let src = input.add( + chunk_idx + .strict_mul(CHUNK_LEN) + .strict_add(block_idx.strict_mul(BLOCK_LEN)), + ); // SAFETY: // - Caller guarantees `src` is valid for `BLOCK_LEN` bytes. // - `[u8; BLOCK_LEN]` has alignment 1, so this reference doesn't assume any alignment of the input @@ -1810,7 +1892,7 @@ unsafe fn hash_many_contiguous_portable( }; let start = if block_idx == 0 { CHUNK_START } else { 0 }; - let end = if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + let end = if block_idx.strict_add(1) == CHUNK_LEN.strict_div(BLOCK_LEN) { super::CHUNK_END } else { 0 @@ -1820,26 +1902,32 @@ unsafe fn hash_many_contiguous_portable( &cv, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, block_flags, )); } for (j, &word) in cv.iter().enumerate() { let bytes = word.to_le_bytes(); + let offset = chunk_idx.strict_mul(OUT_LEN).strict_add(j.strict_mul(4)); // SAFETY: caller guarantees out is valid for `num_chunks * OUT_LEN`. - unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(chunk_idx * OUT_LEN + j * 4), 4) }; + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(offset), 4) }; } } } #[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] #[inline(always)] +/// Writes one chaining value in little-endian byte order. +/// +/// # Safety +/// +/// `out` must reference at least `OUT_LEN` writable bytes. unsafe fn write_cv_words(out: *mut u8, cv: &[u32; 8]) { for (j, &word) in cv.iter().enumerate() { let bytes = word.to_le_bytes(); // SAFETY: caller guarantees one full CV output is writable at `out`. - unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j * 4), 4) }; + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), out.add(j.strict_mul(4)), 4) }; } } @@ -1866,8 +1954,8 @@ fn rot_lanes_left_3(v: core::simd::u32x4) -> core::simd::u32x4 { fn rotr32(v: core::simd::u32x4) -> core::simd::u32x4 { debug_assert!(N > 0 && N < 32); let s0 = core::simd::u32x4::splat(N); - let s1 = core::simd::u32x4::splat(32 - N); - (v >> s0) | (v << s1) + let s1 = core::simd::u32x4::splat(32u32.strict_sub(N)); + core::ops::BitOr::bitor(core::ops::Shr::shr(v, s0), core::ops::Shl::shl(v, s1)) } #[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] @@ -1919,22 +2007,23 @@ fn compress_simd_leaf( chaining_value[7], ]); let mut row2 = Vec4::from_array([super::IV[0], super::IV[1], super::IV[2], super::IV[3]]); - let mut row3 = Vec4::from_array([counter as u32, (counter >> 32) as u32, block_len, flags]); + let (counter_low, counter_high) = counter_words(counter); + let mut row3 = Vec4::from_array([counter_low, counter_high, block_len, flags]); macro_rules! g { ($mx:expr, $my:expr) => {{ - row0 += row1; - row0 += $mx; + row0 = wrapping_add4(row0, row1); + row0 = wrapping_add4(row0, $mx); row3 ^= row0; row3 = rotr32::<16>(row3); - row2 += row3; + row2 = wrapping_add4(row2, row3); row1 ^= row2; row1 = rotr32::<12>(row1); - row0 += row1; - row0 += $my; + row0 = wrapping_add4(row0, row1); + row0 = wrapping_add4(row0, $my); row3 ^= row0; row3 = rotr32::<8>(row3); - row2 += row3; + row2 = wrapping_add4(row2, row3); row1 ^= row2; row1 = rotr32::<7>(row1); }}; @@ -1999,16 +2088,29 @@ fn load_msg_lanes4_contiguous(base: *const u8, block_offset: usize) -> [core::si // SAFETY: caller provides 4 contiguous full chunks; each block load is in-bounds. let b0 = unsafe { words16_from_le_bytes_64(&*base.add(block_offset).cast::<[u8; BLOCK_LEN]>()) }; // SAFETY: see above. - let b1 = unsafe { words16_from_le_bytes_64(&*base.add(CHUNK_LEN + block_offset).cast::<[u8; BLOCK_LEN]>()) }; + let b1 = + unsafe { words16_from_le_bytes_64(&*base.add(CHUNK_LEN.strict_add(block_offset)).cast::<[u8; BLOCK_LEN]>()) }; // SAFETY: see above. - let b2 = unsafe { words16_from_le_bytes_64(&*base.add(2 * CHUNK_LEN + block_offset).cast::<[u8; BLOCK_LEN]>()) }; + let b2 = unsafe { + words16_from_le_bytes_64( + &*base + .add(2usize.strict_mul(CHUNK_LEN).strict_add(block_offset)) + .cast::<[u8; BLOCK_LEN]>(), + ) + }; // SAFETY: see above. - let b3 = unsafe { words16_from_le_bytes_64(&*base.add(3 * CHUNK_LEN + block_offset).cast::<[u8; BLOCK_LEN]>()) }; + let b3 = unsafe { + words16_from_le_bytes_64( + &*base + .add(3usize.strict_mul(CHUNK_LEN).strict_add(block_offset)) + .cast::<[u8; BLOCK_LEN]>(), + ) + }; let mut out = [core::simd::u32x4::splat(0); 16]; let mut i = 0usize; while i < 16 { out[i] = core::simd::u32x4::from_array([b0[i], b1[i], b2[i], b3[i]]); - i += 1; + i = i.strict_add(1); } out } @@ -2024,18 +2126,18 @@ fn g4_simd( mx: core::simd::u32x4, my: core::simd::u32x4, ) { - v[a] += v[b]; - v[a] += mx; + v[a] = wrapping_add4(v[a], v[b]); + v[a] = wrapping_add4(v[a], mx); v[d] ^= v[a]; v[d] = rotr32::<16>(v[d]); - v[c] += v[d]; + v[c] = wrapping_add4(v[c], v[d]); v[b] ^= v[c]; v[b] = rotr32::<12>(v[b]); - v[a] += v[b]; - v[a] += my; + v[a] = wrapping_add4(v[a], v[b]); + v[a] = wrapping_add4(v[a], my); v[d] ^= v[a]; v[d] = rotr32::<8>(v[d]); - v[c] += v[d]; + v[c] = wrapping_add4(v[c], v[d]); v[b] ^= v[c]; v[b] = rotr32::<7>(v[b]); } @@ -2059,27 +2161,27 @@ fn round4_simd(v: &mut [core::simd::u32x4; 16], m: &[core::simd::u32x4; 16], r: fn load_parent_msg_lanes4(children: &[[u8; OUT_LEN]], base: usize, rem: usize) -> [core::simd::u32x4; 16] { type Vec4 = core::simd::u32x4; debug_assert!(rem > 0 && rem <= 4); - let last = base + rem - 1; + let last = base.strict_add(rem).strict_sub(1); let idx0 = base; - let idx1 = if rem > 1 { base + 1 } else { last }; - let idx2 = if rem > 2 { base + 2 } else { last }; - let idx3 = if rem > 3 { base + 3 } else { last }; - - let l0 = words8_from_le_bytes_32(&children[2 * idx0]); - let r0 = words8_from_le_bytes_32(&children[2 * idx0 + 1]); - let l1 = words8_from_le_bytes_32(&children[2 * idx1]); - let r1 = words8_from_le_bytes_32(&children[2 * idx1 + 1]); - let l2 = words8_from_le_bytes_32(&children[2 * idx2]); - let r2 = words8_from_le_bytes_32(&children[2 * idx2 + 1]); - let l3 = words8_from_le_bytes_32(&children[2 * idx3]); - let r3 = words8_from_le_bytes_32(&children[2 * idx3 + 1]); + let idx1 = if rem > 1 { base.strict_add(1) } else { last }; + let idx2 = if rem > 2 { base.strict_add(2) } else { last }; + let idx3 = if rem > 3 { base.strict_add(3) } else { last }; + + let l0 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx0)]); + let r0 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx0).strict_add(1)]); + let l1 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx1)]); + let r1 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx1).strict_add(1)]); + let l2 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx2)]); + let r2 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx2).strict_add(1)]); + let l3 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx3)]); + let r3 = words8_from_le_bytes_32(&children[2usize.strict_mul(idx3).strict_add(1)]); let mut msg = [Vec4::splat(0); 16]; let mut w = 0usize; while w < 8 { msg[w] = Vec4::from_array([l0[w], l1[w], l2[w], l3[w]]); - msg[w + 8] = Vec4::from_array([r0[w], r1[w], r2[w], r3[w]]); - w += 1; + msg[w.strict_add(8)] = Vec4::from_array([r0[w], r1[w], r2[w], r3[w]]); + w = w.strict_add(1); } msg } @@ -2088,7 +2190,7 @@ fn load_parent_msg_lanes4(children: &[[u8; OUT_LEN]], base: usize, rem: usize) - #[inline(always)] fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: u32, out: &mut [[u8; OUT_LEN]]) { type Vec4 = core::simd::u32x4; - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if out.is_empty() { return; } @@ -2096,7 +2198,7 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: let mut i = 0usize; while i < out.len() { - let rem = core::cmp::min(4, out.len() - i); + let rem = core::cmp::min(4, out.len().strict_sub(i)); let msg = load_parent_msg_lanes4(children, i, rem); let mut v = [ @@ -2114,7 +2216,7 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: Vec4::splat(super::IV[3]), Vec4::splat(0), Vec4::splat(0), - Vec4::splat(BLOCK_LEN as u32), + Vec4::splat(BLOCK_LEN_U32), Vec4::splat(parent_flags), ]; @@ -2129,8 +2231,8 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: let mut x = [Vec4::splat(0); 8]; let mut j = 0usize; while j < 8 { - x[j] = v[j] ^ v[j + 8]; - j += 1; + x[j] = v[j] ^ v[j.strict_add(8)]; + j = j.strict_add(1); } let xa0 = x[0].to_array(); let xa1 = x[1].to_array(); @@ -2143,17 +2245,23 @@ fn parent_cvs_many4_simd(children: &[[u8; OUT_LEN]], key_words: [u32; 8], flags: let mut lane = 0usize; while lane < rem { - out[i + lane] = super::words8_to_le_bytes(&[ + out[i.strict_add(lane)] = super::words8_to_le_bytes(&[ xa0[lane], xa1[lane], xa2[lane], xa3[lane], xa4[lane], xa5[lane], xa6[lane], xa7[lane], ]); - lane += 1; + lane = lane.strict_add(1); } - i += rem; + i = i.strict_add(rem); } } #[cfg(any(target_arch = "s390x", target_arch = "powerpc64", target_arch = "riscv64"))] #[inline(always)] +/// Hashes four contiguous full chunks with the portable SIMD kernel. +/// +/// # Safety +/// +/// `input` must reference four readable chunks and `out` must reference four +/// writable chaining values. unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { type Vec4 = core::simd::u32x4; let mut h = [ @@ -2166,25 +2274,22 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co Vec4::splat(key[6]), Vec4::splat(key[7]), ]; - let counter_low = Vec4::from_array([ - counter as u32, - counter.wrapping_add(1) as u32, - counter.wrapping_add(2) as u32, - counter.wrapping_add(3) as u32, - ]); - let counter_high = Vec4::from_array([ - (counter >> 32) as u32, - (counter.wrapping_add(1) >> 32) as u32, - (counter.wrapping_add(2) >> 32) as u32, - (counter.wrapping_add(3) >> 32) as u32, - ]); + let [(low0, high0), (low1, high1), (low2, high2), (low3, high3)] = [ + counter, + counter.wrapping_add(1), + counter.wrapping_add(2), + counter.wrapping_add(3), + ] + .map(counter_words); + let counter_low = Vec4::from_array([low0, low1, low2, low3]); + let counter_high = Vec4::from_array([high0, high1, high2, high3]); let mut block_idx = 0usize; - while block_idx < (CHUNK_LEN / BLOCK_LEN) { - let msg = load_msg_lanes4_contiguous(input, block_idx * BLOCK_LEN); + while block_idx < CHUNK_LEN.strict_div(BLOCK_LEN) { + let msg = load_msg_lanes4_contiguous(input, block_idx.strict_mul(BLOCK_LEN)); let block_flags = flags | if block_idx == 0 { CHUNK_START } else { 0 } - | if block_idx + 1 == (CHUNK_LEN / BLOCK_LEN) { + | if block_idx.strict_add(1) == CHUNK_LEN.strict_div(BLOCK_LEN) { super::CHUNK_END } else { 0 @@ -2204,7 +2309,7 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co Vec4::splat(super::IV[3]), counter_low, counter_high, - Vec4::splat(BLOCK_LEN as u32), + Vec4::splat(BLOCK_LEN_U32), Vec4::splat(block_flags), ]; round4_simd(&mut v, &msg, 0); @@ -2216,10 +2321,10 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co round4_simd(&mut v, &msg, 6); let mut i = 0usize; while i < 8 { - h[i] = v[i] ^ v[i + 8]; - i += 1; + h[i] = v[i] ^ v[i.strict_add(8)]; + i = i.strict_add(1); } - block_idx += 1; + block_idx = block_idx.strict_add(1); } let h0 = h[0].to_array(); @@ -2234,7 +2339,7 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co let mut lane = 0usize; while lane < 4 { // SAFETY: caller guarantees `out` is valid for 4 contiguous `OUT_LEN` outputs. - let dst = unsafe { out.add(lane * OUT_LEN) }; + let dst = unsafe { out.add(lane.strict_mul(OUT_LEN)) }; let words = [ h0[lane], h1[lane], h2[lane], h3[lane], h4[lane], h5[lane], h6[lane], h7[lane], ]; @@ -2242,16 +2347,21 @@ unsafe fn hash4_contiguous_full_chunks_simd(input: *const u8, key: &[u32; 8], co while word < 8 { let bytes = words[word].to_le_bytes(); // SAFETY: caller guarantees out is valid for 4 CV outputs. - unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), dst.add(word * 4), 4) }; - word += 1; + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), dst.add(word.strict_mul(4)), 4) }; + word = word.strict_add(1); } - lane += 1; + lane = lane.strict_add(1); } } // s390x vector wrappers #[cfg(target_arch = "s390x")] +/// Compresses one block with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn compress_s390x_vector( chaining_value: &[u32; 8], @@ -2276,6 +2386,12 @@ fn compress_s390x_vector_wrapper( } #[cfg(target_arch = "s390x")] +/// Writes one root-output block with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility, and `out` must reference +/// one writable BLAKE3 output block. #[target_feature(enable = "vector")] unsafe fn root_output_blocks1_s390x_vector( chaining_value: &[u32; 8], @@ -2293,6 +2409,11 @@ unsafe fn root_output_blocks1_s390x_vector( } #[cfg(target_arch = "s390x")] +/// Compresses full chunk blocks with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn chunk_compress_blocks_s390x_vector( chaining_value: &mut [u32; 8], @@ -2323,11 +2444,11 @@ unsafe fn chunk_compress_blocks_s390x_vector( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -2342,11 +2463,11 @@ unsafe fn chunk_compress_blocks_s390x_vector( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } @@ -2363,6 +2484,11 @@ fn chunk_compress_blocks_s390x_vector_wrapper( } #[cfg(target_arch = "s390x")] +/// Compresses a parent node with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility. #[target_feature(enable = "vector")] unsafe fn parent_cv_s390x_vector( left_child_cv: [u32; 8], @@ -2377,7 +2503,7 @@ unsafe fn parent_cv_s390x_vector( &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } @@ -2394,6 +2520,12 @@ fn parent_cv_s390x_vector_wrapper( } #[cfg(target_arch = "s390x")] +/// Hashes one full chunk with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility, `input` must reference +/// one readable chunk, and `out` must reference one writable chaining value. #[target_feature(enable = "vector")] unsafe fn hash_one_chunk_s390x_vector(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; @@ -2412,12 +2544,19 @@ unsafe fn hash_one_chunk_s390x_vector(input: *const u8, key: &[u32; 8], counter: let start = if blocks_compressed == 0 { CHUNK_START } else { 0 }; let tail_flags = flags | start | super::CHUNK_END; // SAFETY: `compress_s390x_vector` is gated by the current function's vector target feature. - cv = first_8_words(unsafe { compress_s390x_vector(&cv, &tail_words, counter, BLOCK_LEN as u32, tail_flags) }); + cv = first_8_words(unsafe { compress_s390x_vector(&cv, &tail_words, counter, BLOCK_LEN_U32, tail_flags) }); // SAFETY: caller guarantees one full CV output is writable at `out`. unsafe { write_cv_words(out, &cv) }; } #[cfg(target_arch = "s390x")] +/// Hashes contiguous chunks with the s390x vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the vector facility, `input` must reference +/// `num_chunks` readable chunks, and `out` must reference `num_chunks` writable +/// chaining values. #[target_feature(enable = "vector")] unsafe fn hash_many_contiguous_s390x_vector( input: *const u8, @@ -2430,36 +2569,43 @@ unsafe fn hash_many_contiguous_s390x_vector( debug_assert!(num_chunks != 0); let mut idx = 0usize; - while idx + 4 <= num_chunks { + while idx.strict_add(4) <= num_chunks { // SAFETY: `idx + 4 <= num_chunks` guarantees four full contiguous chunks and outputs remain. unsafe { hash4_contiguous_full_chunks_simd( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 4; + idx = idx.strict_add(4); } while idx < num_chunks { // SAFETY: `idx < num_chunks` guarantees one full chunk/output remains. unsafe { hash_one_chunk_s390x_vector( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 1; + idx = idx.strict_add(1); } } #[cfg(target_arch = "s390x")] +/// Dispatches contiguous chunks through the s390x vector implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the executing CPU must support +/// the vector facility. unsafe fn hash_many_contiguous_s390x_vector_wrapper( input: *const u8, num_chunks: usize, @@ -2476,6 +2622,11 @@ unsafe fn hash_many_contiguous_s390x_vector_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Compresses one block with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX. unsafe fn compress_power_vsx( chaining_value: &[u32; 8], block_words: &[u32; 16], @@ -2500,6 +2651,12 @@ fn compress_power_vsx_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Writes one root-output block with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX and `out` must reference one writable +/// BLAKE3 output block. unsafe fn root_output_blocks1_power_vsx( chaining_value: &[u32; 8], block_words: &[u32; 16], @@ -2517,6 +2674,11 @@ unsafe fn root_output_blocks1_power_vsx( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Compresses full chunk blocks with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX. unsafe fn chunk_compress_blocks_power_vsx( chaining_value: &mut [u32; 8], chunk_counter: u64, @@ -2524,11 +2686,10 @@ unsafe fn chunk_compress_blocks_power_vsx( blocks_compressed: &mut u8, blocks: &[u8], ) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); + let (block_slices, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); - if blocks.len() == BLOCK_LEN { - // SAFETY: `blocks` is exactly one block, and `[u8; BLOCK_LEN]` has 1-byte alignment. - let block_bytes: &[u8; BLOCK_LEN] = unsafe { &*(blocks.as_ptr().cast()) }; + if let [block_bytes] = block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); // SAFETY: `compress_power_vsx` is gated by the current function's VSX target feature. @@ -2537,16 +2698,14 @@ unsafe fn chunk_compress_blocks_power_vsx( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); return; } - let (block_slices, remainder) = blocks.as_chunks::(); - debug_assert!(remainder.is_empty()); for block_bytes in block_slices { let start = if *blocks_compressed == 0 { CHUNK_START } else { 0 }; let block_words = words16_from_le_bytes_64(block_bytes); @@ -2556,11 +2715,11 @@ unsafe fn chunk_compress_blocks_power_vsx( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = (*blocks_compressed).strict_add(1); } } @@ -2578,6 +2737,11 @@ fn chunk_compress_blocks_power_vsx_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Compresses a parent node with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX. unsafe fn parent_cv_power_vsx( left_child_cv: [u32; 8], right_child_cv: [u32; 8], @@ -2591,7 +2755,7 @@ unsafe fn parent_cv_power_vsx( &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } @@ -2609,10 +2773,16 @@ fn parent_cv_power_vsx_wrapper( #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Hashes one full chunk with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX, `input` must reference one readable +/// chunk, and `out` must reference one writable chaining value. unsafe fn hash_one_chunk_power_vsx(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; let mut blocks_compressed = 0u8; - let body_len = CHUNK_LEN - BLOCK_LEN; + let body_len = CHUNK_LEN.strict_sub(BLOCK_LEN); // SAFETY: caller guarantees one full chunk is readable from `input`. let body = unsafe { core::slice::from_raw_parts(input, body_len) }; // SAFETY: caller upholds the chunk pointer/length precondition. @@ -2626,13 +2796,20 @@ unsafe fn hash_one_chunk_power_vsx(input: *const u8, key: &[u32; 8], counter: u6 let start = if blocks_compressed == 0 { CHUNK_START } else { 0 }; let tail_flags = flags | start | super::CHUNK_END; // SAFETY: `compress_power_vsx` is gated by the current function's VSX target feature. - cv = first_8_words(unsafe { compress_power_vsx(&cv, &tail_words, counter, BLOCK_LEN as u32, tail_flags) }); + cv = first_8_words(unsafe { compress_power_vsx(&cv, &tail_words, counter, BLOCK_LEN_U32, tail_flags) }); // SAFETY: caller guarantees one full CV output is writable at `out`. unsafe { write_cv_words(out, &cv) }; } #[cfg(target_arch = "powerpc64")] #[target_feature(enable = "vsx")] +/// Hashes contiguous chunks with the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// The current CPU must support VSX, `input` must reference `num_chunks` +/// readable chunks, and `out` must reference `num_chunks` writable chaining +/// values. unsafe fn hash_many_contiguous_power_vsx( input: *const u8, num_chunks: usize, @@ -2644,36 +2821,42 @@ unsafe fn hash_many_contiguous_power_vsx( debug_assert!(num_chunks != 0); let mut idx = 0usize; - while idx + 4 <= num_chunks { + while idx.strict_add(4) <= num_chunks { // SAFETY: `idx + 4 <= num_chunks` guarantees four full contiguous chunks and outputs remain. unsafe { hash4_contiguous_full_chunks_simd( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 4; + idx = idx.strict_add(4); } while idx < num_chunks { // SAFETY: `idx < num_chunks` guarantees one full chunk/output remains. unsafe { hash_one_chunk_power_vsx( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 1; + idx = idx.strict_add(1); } } #[cfg(target_arch = "powerpc64")] +/// Dispatches contiguous chunks through the PowerPC64 VSX implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the current CPU must support VSX. unsafe fn hash_many_contiguous_power_vsx_wrapper( input: *const u8, num_chunks: usize, @@ -2703,7 +2886,7 @@ fn compress_simd_leaf_riscv( let mut w = 0usize; while w < 16 { msg[w] = Vec4::splat(block_words[w]); - w += 1; + w = w.strict_add(1); } let mut v = [ @@ -2719,8 +2902,8 @@ fn compress_simd_leaf_riscv( Vec4::splat(super::IV[1]), Vec4::splat(super::IV[2]), Vec4::splat(super::IV[3]), - Vec4::splat(counter as u32), - Vec4::splat((counter >> 32) as u32), + Vec4::splat(counter_words(counter).0), + Vec4::splat(counter_words(counter).1), Vec4::splat(block_len), Vec4::splat(flags), ]; @@ -2756,6 +2939,11 @@ fn compress_simd_leaf_riscv( } #[cfg(target_arch = "riscv64")] +/// Compresses one block with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn compress_riscv_v( chaining_value: &[u32; 8], @@ -2780,6 +2968,12 @@ fn compress_riscv_v_wrapper( } #[cfg(target_arch = "riscv64")] +/// Writes one root-output block with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, and `out` must +/// reference one writable BLAKE3 output block. #[target_feature(enable = "v")] unsafe fn root_output_blocks1_riscv_v( chaining_value: &[u32; 8], @@ -2797,6 +2991,11 @@ unsafe fn root_output_blocks1_riscv_v( } #[cfg(target_arch = "riscv64")] +/// Compresses full chunk blocks with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn chunk_compress_blocks_riscv_v( chaining_value: &mut [u32; 8], @@ -2818,11 +3017,11 @@ unsafe fn chunk_compress_blocks_riscv_v( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -2837,11 +3036,11 @@ unsafe fn chunk_compress_blocks_riscv_v( chaining_value, &block_words, chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ) }); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } @@ -2858,6 +3057,11 @@ fn chunk_compress_blocks_riscv_v_wrapper( } #[cfg(target_arch = "riscv64")] +/// Compresses a parent node with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension. #[target_feature(enable = "v")] unsafe fn parent_cv_riscv_v( left_child_cv: [u32; 8], @@ -2872,7 +3076,7 @@ unsafe fn parent_cv_riscv_v( &key_words, &block_words, 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, PARENT | flags, )) } @@ -2889,11 +3093,17 @@ fn parent_cv_riscv_v_wrapper( } #[cfg(target_arch = "riscv64")] +/// Hashes one full chunk with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, `input` must reference +/// one readable chunk, and `out` must reference one writable chaining value. #[target_feature(enable = "v")] unsafe fn hash_one_chunk_riscv_v(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; let mut blocks_compressed = 0u8; - let body_len = CHUNK_LEN - BLOCK_LEN; + let body_len = CHUNK_LEN.strict_sub(BLOCK_LEN); // SAFETY: caller guarantees one full chunk is readable from `input`. let body = unsafe { core::slice::from_raw_parts(input, body_len) }; // SAFETY: caller upholds the chunk pointer/length precondition. @@ -2907,12 +3117,19 @@ unsafe fn hash_one_chunk_riscv_v(input: *const u8, key: &[u32; 8], counter: u64, let start = if blocks_compressed == 0 { CHUNK_START } else { 0 }; let tail_flags = flags | start | super::CHUNK_END; // SAFETY: this kernel is only dispatched when `riscv::V` is available. - cv = first_8_words(unsafe { compress_riscv_v(&cv, &tail_words, counter, BLOCK_LEN as u32, tail_flags) }); + cv = first_8_words(unsafe { compress_riscv_v(&cv, &tail_words, counter, BLOCK_LEN_U32, tail_flags) }); // SAFETY: caller guarantees one full CV output is writable at `out`. unsafe { write_cv_words(out, &cv) }; } #[cfg(target_arch = "riscv64")] +/// Hashes contiguous chunks with the RISC-V vector implementation. +/// +/// # Safety +/// +/// The executing CPU must support the RISC-V V extension, `input` must reference +/// `num_chunks` readable chunks, and `out` must reference `num_chunks` writable +/// chaining values. #[target_feature(enable = "v")] unsafe fn hash_many_contiguous_riscv_v( input: *const u8, @@ -2925,36 +3142,43 @@ unsafe fn hash_many_contiguous_riscv_v( debug_assert!(num_chunks != 0); let mut idx = 0usize; - while idx + 4 <= num_chunks { + while idx.strict_add(4) <= num_chunks { // SAFETY: `idx + 4 <= num_chunks` guarantees four full contiguous chunks and outputs remain. unsafe { hash4_contiguous_full_chunks_simd( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 4; + idx = idx.strict_add(4); } while idx < num_chunks { // SAFETY: `idx < num_chunks` guarantees one full chunk/output remains. unsafe { hash_one_chunk_riscv_v( - input.add(idx * CHUNK_LEN), + input.add(idx.strict_mul(CHUNK_LEN)), key, - counter.wrapping_add(idx as u64), + counter.wrapping_add(u64::try_from(idx).expect("BLAKE3 chunk index fits in u64")), flags, - out.add(idx * OUT_LEN), + out.add(idx.strict_mul(OUT_LEN)), ); } - idx += 1; + idx = idx.strict_add(1); } } #[cfg(target_arch = "riscv64")] +/// Dispatches contiguous chunks through the RISC-V vector implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the executing CPU must support +/// the RISC-V V extension. unsafe fn hash_many_contiguous_riscv_v_wrapper( input: *const u8, num_chunks: usize, @@ -2983,6 +3207,11 @@ fn x86_compress_cv_portable_wrapper( } #[cfg(target_arch = "x86_64")] +/// Compress one block with the SSE4.1 kernel. +/// +/// # Safety +/// +/// SSE4.1 must be available and `block` must be readable for 64 bytes. unsafe fn x86_compress_cv_sse41_wrapper( cv: &[u32; 8], block: *const u8, @@ -2996,6 +3225,11 @@ unsafe fn x86_compress_cv_sse41_wrapper( } #[cfg(target_arch = "x86_64")] +/// Compress one block with the AVX2 kernel. +/// +/// # Safety +/// +/// AVX2 must be available and `block` must be readable for 64 bytes. unsafe fn x86_compress_cv_avx2_wrapper( cv: &[u32; 8], block: *const u8, @@ -3009,6 +3243,12 @@ unsafe fn x86_compress_cv_avx2_wrapper( } #[cfg(target_arch = "x86_64")] +/// Compress one block with the AVX-512 kernel. +/// +/// # Safety +/// +/// The AVX-512 dispatch requirements must hold and `block` must be readable +/// for 64 bytes. unsafe fn x86_compress_cv_avx512_wrapper( cv: &[u32; 8], block: *const u8, @@ -3031,6 +3271,12 @@ unsafe fn x86_compress_cv_avx512_wrapper( } #[cfg(all(feature = "diag", target_arch = "x86_64"))] +/// Compress one block with the owned AVX-512 kernel. +/// +/// # Safety +/// +/// The owned AVX-512 dispatch requirements must hold and `block` must be +/// readable for 64 bytes. unsafe fn x86_compress_cv_avx512_owned_wrapper( cv: &[u32; 8], block: *const u8, @@ -3129,11 +3375,11 @@ fn chunk_compress_blocks_avx512_owned_wrapper( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } @@ -3186,6 +3432,12 @@ fn parent_cv_sse41_wrapper( } #[cfg(target_arch = "x86_64")] +/// Hash contiguous chunks with the SSE4.1 kernel. +/// +/// # Safety +/// +/// SSE4.1 must be available. `input` and `out` must cover `num_chunks` full +/// chunks and chaining values, respectively, without overlapping. unsafe fn hash_many_contiguous_sse41_wrapper( input: *const u8, mut num_chunks: usize, @@ -3210,22 +3462,22 @@ unsafe fn hash_many_contiguous_sse41_wrapper( // SAFETY: dispatch selects this kernel only when SSE4.1 is available; the // caller guarantees `input`/`out` cover the full `num_chunks` buffer. unsafe { - super::x86_64::sse41::hash4( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::sse41::hash4(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::sse41::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::sse41::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::sse41::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::sse41::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::sse41::DEGREE as u64); - num_chunks -= super::x86_64::sse41::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::sse41::DEGREE); } if num_chunks != 0 { @@ -3233,7 +3485,7 @@ unsafe fn hash_many_contiguous_sse41_wrapper( // the final chunk pointer into unused lanes and only copy the needed // outputs. // SAFETY: `num_chunks != 0`, and `input` is valid for `num_chunks * CHUNK_LEN` bytes. - let last = unsafe { input.add((num_chunks - 1) * CHUNK_LEN) }; + let last = unsafe { input.add(num_chunks.strict_sub(1).strict_mul(CHUNK_LEN)) }; // SAFETY: all pointers are within the caller-provided `input` buffer. let ptrs = unsafe { [ @@ -3248,23 +3500,26 @@ unsafe fn hash_many_contiguous_sse41_wrapper( // SAFETY: SSE4.1 is available for this wrapper, `ptrs` are in-bounds for // full chunks, and `tmp`/`out` are large enough for the copied outputs. unsafe { - super::x86_64::sse41::hash4( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::sse41::hash4(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, - tmp.as_mut_ptr(), - ); - core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks * OUT_LEN); + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, + out: tmp.as_mut_ptr(), + }); + core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] /// Hash a sub-degree contiguous AVX2 chunk tail by duplicating the final lane. /// /// # Safety @@ -3288,7 +3543,7 @@ unsafe fn hash_many_avx2_owned_duplicate_tail( // SAFETY: Computing the final real chunk pointer because: // 1. The caller guarantees `num_chunks` is non-zero. // 2. The caller guarantees `input` is readable for `num_chunks * CHUNK_LEN` bytes. - let last = unsafe { input.add((num_chunks - 1) * CHUNK_LEN) }; + let last = unsafe { input.add(num_chunks.strict_sub(1).strict_mul(CHUNK_LEN)) }; // SAFETY: Constructing a full 8-lane batch because: // 1. Lanes `< num_chunks` point at distinct real chunks in the caller-provided input. // 2. Extra lanes duplicate `last`, which is valid for one full chunk. @@ -3312,23 +3567,32 @@ unsafe fn hash_many_avx2_owned_duplicate_tail( // 2. `ptrs` is a valid full 8-lane batch and `tmp` holds all lane outputs. // 3. `out` is writable for the first `num_chunks * OUT_LEN` bytes. unsafe { - super::x86_64::avx2::hash8_owned( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, - tmp.as_mut_ptr(), - ); - core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks * OUT_LEN); + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, + out: tmp.as_mut_ptr(), + }); + core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline(always)] +/// Hash one full chunk through the serial AVX2 compressor. +/// +/// # Safety +/// +/// AVX2 must be available. `input` must be readable for one full chunk, and +/// `out` must be writable for one chaining value without overlapping `input`. unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { let mut cv = *key; for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { @@ -3336,7 +3600,7 @@ unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], cou if block_idx == 0 { block_flags |= CHUNK_START; } - if block_idx + 1 == CHUNK_LEN / BLOCK_LEN { + if block_idx.strict_add(1) == CHUNK_LEN / BLOCK_LEN { block_flags |= super::CHUNK_END; } @@ -3348,9 +3612,9 @@ unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], cou cv = unsafe { super::x86_64::compress_cv_avx2_bytes( &cv, - input.add(block_idx * BLOCK_LEN), + input.add(block_idx.strict_mul(BLOCK_LEN)), counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, block_flags, ) }; @@ -3364,6 +3628,12 @@ unsafe fn hash_one_chunk_avx2_owned_serial(input: *const u8, key: &[u32; 8], cou } #[cfg(target_arch = "x86_64")] +/// Hash contiguous chunks with the production AVX2 routing policy. +/// +/// # Safety +/// +/// AVX2 must be available. `input` and `out` must cover `num_chunks` full +/// chunks and chaining values, respectively, without overlapping. unsafe fn hash_many_contiguous_avx2_inner( input: *const u8, mut num_chunks: usize, @@ -3396,19 +3666,19 @@ unsafe fn hash_many_contiguous_avx2_inner( // 3. `out` is valid for `DEGREE * OUT_LEN` bytes. // 4. Sub-degree tails are handled below by the explicit per-degree policy. unsafe { - super::x86_64::avx2::hash8_owned( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::avx2::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx2::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx2::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx2::DEGREE.strict_mul(OUT_LEN)); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] @@ -3417,29 +3687,28 @@ unsafe fn hash_many_contiguous_avx2_inner( // caller guarantees `input`/`out` cover the full `num_chunks` buffer. // SAFETY: AVX2 is available for this wrapper, `ptrs` are in-bounds for // full chunks, and `out` is large enough for `DEGREE * OUT_LEN` bytes. - super::x86_64::avx2::hash8( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::avx2::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx2::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx2::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx2::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx2::DEGREE as u64); - num_chunks -= super::x86_64::avx2::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx2::DEGREE); } if num_chunks != 0 { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { debug_assert!(num_chunks < super::x86_64::avx2::DEGREE); - debug_assert!(flags <= u8::MAX as u32); if num_chunks == 1 { // SAFETY: Use the owned AVX2 one-chunk tail because: // 1. This wrapper is only selected after AVX2 dispatch. @@ -3479,32 +3748,28 @@ unsafe fn hash_many_contiguous_avx2_inner( // Use the upstream-grade AVX2 asm `hash_many` backend for the remaining // sub-degree tails. Passing `num_inputs = num_chunks` avoids wasting lanes // for the four-chunk shape assigned to assembly by the tail policy. - // SAFETY: This wrapper is only selected when AVX2 is available (checked - // by dispatch). `input` is valid for `num_chunks * CHUNK_LEN` bytes, so - // each `input.add(i * CHUNK_LEN)` stays in-bounds. `out` is valid for - // `num_chunks * OUT_LEN` bytes. let mut ptrs = [input; super::x86_64::avx2::DEGREE]; for (i, ptr) in ptrs.iter_mut().enumerate().take(num_chunks) { // SAFETY: `i < num_chunks` and the caller guarantees `input` is valid // for `num_chunks * CHUNK_LEN` bytes. - *ptr = unsafe { input.add(i * CHUNK_LEN) }; + *ptr = unsafe { input.add(i.strict_mul(CHUNK_LEN)) }; } // SAFETY: AVX2 is available for this kernel per dispatch. `ptrs` points // to `num_chunks` valid chunk inputs, and `out` is valid for // `num_chunks * OUT_LEN` bytes. unsafe { - super::x86_64::asm::hash_many_avx2( - ptrs.as_ptr(), - num_chunks, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), + super::x86_64::asm::hash_many_avx2(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: num_chunks, + blocks: CHUNK_LEN / BLOCK_LEN, + key: key.as_ptr(), counter, - true, - flags as u8, - CHUNK_START as u8, - super::CHUNK_END as u8, + increment_counter: true, + flags: assembly_flags(flags), + flags_start: assembly_flags(CHUNK_START), + flags_end: assembly_flags(super::CHUNK_END), out, - ); + }); } } @@ -3513,7 +3778,7 @@ unsafe fn hash_many_contiguous_avx2_inner( // Non-Linux fallback: hash an 8-lane batch with duplicated final pointers // and copy only the needed outputs. // SAFETY: `num_chunks != 0`, and `input` is valid for `num_chunks * CHUNK_LEN` bytes. - let last = unsafe { input.add((num_chunks - 1) * CHUNK_LEN) }; + let last = unsafe { input.add(num_chunks.strict_sub(1).strict_mul(CHUNK_LEN)) }; // SAFETY: all pointers are within the caller-provided `input` buffer. let ptrs = unsafe { [ @@ -3532,24 +3797,30 @@ unsafe fn hash_many_contiguous_avx2_inner( // SAFETY: AVX2 is available for this wrapper, `ptrs` are in-bounds for // full chunks, and `tmp`/`out` are large enough for the copied outputs. unsafe { - super::x86_64::avx2::hash8( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, - tmp.as_mut_ptr(), - ); - core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks * OUT_LEN); + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, + out: tmp.as_mut_ptr(), + }); + core::ptr::copy_nonoverlapping(tmp.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } } } #[cfg(target_arch = "x86_64")] +/// Forward contiguous chunks into the production AVX2 implementation. +/// +/// # Safety +/// +/// AVX2 must be available. `input` and `out` must satisfy +/// [`HashManyContiguousFn`]'s buffer and non-overlap contract. unsafe fn hash_many_contiguous_avx2_wrapper( input: *const u8, num_chunks: usize, @@ -3564,7 +3835,10 @@ unsafe fn hash_many_contiguous_avx2_wrapper( unsafe { hash_many_contiguous_avx2_inner(input, num_chunks, key, counter, flags, out) }; } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline] fn avx512_owned_hash_many_available() -> bool { crate::platform::caps().has( @@ -3577,7 +3851,17 @@ fn avx512_owned_hash_many_available() -> bool { ) } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows") +))] +/// Hash a sub-degree contiguous AVX-512 chunk tail with duplicate lanes. +/// +/// # Safety +/// +/// The owned AVX-512 dispatch requirements must hold. `num_chunks` must be in +/// `1..avx512::DEGREE`, and `input`/`out` must cover that many chunks/CVs +/// without overlapping. unsafe fn hash_many_avx512_owned_duplicate_tail( input: *const u8, num_chunks: usize, @@ -3598,14 +3882,14 @@ unsafe fn hash_many_avx512_owned_duplicate_tail( // 3. Source and destination ranges are disjoint allocations. unsafe { core::ptr::copy_nonoverlapping( - input.add(i * CHUNK_LEN), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + input.add(i.strict_mul(CHUNK_LEN)), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } } - let last_src_offset = (num_chunks - 1) * CHUNK_LEN; + let last_src_offset = num_chunks.strict_sub(1).strict_mul(CHUNK_LEN); for i in num_chunks..super::x86_64::avx512::DEGREE { // SAFETY: Duplicating the final real tail chunk because: // 1. `num_chunks != 0`, so `last_src_offset` names an initialized lane. @@ -3614,7 +3898,7 @@ unsafe fn hash_many_avx512_owned_duplicate_tail( unsafe { core::ptr::copy_nonoverlapping( tmp_input.as_ptr().add(last_src_offset), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } @@ -3626,11 +3910,18 @@ unsafe fn hash_many_avx512_owned_duplicate_tail( // 3. `out` is writable for the first `num_chunks * OUT_LEN` bytes. unsafe { super::x86_64::avx512::hash16_contiguous_owned(tmp_input.as_ptr(), key, counter, flags, tmp_out.as_mut_ptr()); - core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks * OUT_LEN); + core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } #[cfg(target_arch = "x86_64")] +/// Hash contiguous chunks with the production AVX-512 routing policy. +/// +/// # Safety +/// +/// The AVX-512 dispatch requirements must hold. `input` and `out` must cover +/// `num_chunks` full chunks and chaining values, respectively, without +/// overlapping. unsafe fn hash_many_contiguous_avx512_wrapper( input: *const u8, mut num_chunks: usize, @@ -3646,7 +3937,6 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // by dispatch), the constructed `ptrs` all stay in-bounds for full chunks, // and `out` is valid for `DEGREE * OUT_LEN` bytes. unsafe { - debug_assert!(flags <= u8::MAX as u32); let ptrs = [ input, input.add(CHUNK_LEN), @@ -3665,20 +3955,20 @@ unsafe fn hash_many_contiguous_avx512_wrapper( input.add(14 * CHUNK_LEN), input.add(15 * CHUNK_LEN), ]; - super::x86_64::asm::hash_many_avx512( - ptrs.as_ptr(), - super::x86_64::avx512::DEGREE, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), + super::x86_64::asm::hash_many_avx512(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: super::x86_64::avx512::DEGREE, + blocks: CHUNK_LEN / BLOCK_LEN, + key: key.as_ptr(), counter, - true, - flags as u8, - CHUNK_START as u8, - super::CHUNK_END as u8, + increment_counter: true, + flags: assembly_flags(flags), + flags_start: assembly_flags(CHUNK_START), + flags_end: assembly_flags(super::CHUNK_END), out, - ); - input = input.add(super::x86_64::avx512::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx512::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx512::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx512::DEGREE.strict_mul(OUT_LEN)); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] @@ -3686,18 +3976,17 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // SAFETY: dispatch selects this kernel only when AVX-512 is available; // the caller guarantees `input`/`out` cover the full `num_chunks` buffer. super::x86_64::avx512::hash16_contiguous(input, key, counter, flags, out); - input = input.add(super::x86_64::avx512::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx512::DEGREE * OUT_LEN); + input = input.add(super::x86_64::avx512::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx512::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx512::DEGREE as u64); - num_chunks -= super::x86_64::avx512::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx512::DEGREE); } if num_chunks != 0 { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { debug_assert!(num_chunks < super::x86_64::avx512::DEGREE); - debug_assert!(flags <= u8::MAX as u32); // Use the AVX-512 asm `hash_many` backend for the sub-degree tail // (1–15 chunks). The assembly performs its internal cascade with lane // masking. @@ -3716,30 +4005,28 @@ unsafe fn hash_many_contiguous_avx512_wrapper( return; } - // SAFETY: This wrapper is only selected when the AVX-512 kernel is - // available per dispatch. `input`/`out` cover `num_chunks` full chunks. let mut ptrs = [input; super::x86_64::avx512::DEGREE]; for (i, ptr) in ptrs.iter_mut().enumerate().take(num_chunks) { // SAFETY: `i < num_chunks` and the caller guarantees `input` is valid // for `num_chunks * CHUNK_LEN` bytes. - *ptr = unsafe { input.add(i * CHUNK_LEN) }; + *ptr = unsafe { input.add(i.strict_mul(CHUNK_LEN)) }; } // SAFETY: AVX-512 is available for this kernel per dispatch. `ptrs` // points to `num_chunks` valid chunk inputs, and `out` is valid for // `num_chunks * OUT_LEN` bytes. unsafe { - super::x86_64::asm::hash_many_avx512( - ptrs.as_ptr(), - num_chunks, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), + super::x86_64::asm::hash_many_avx512(super::x86_64::asm::HashManyRequest { + inputs: ptrs.as_ptr(), + num_inputs: num_chunks, + blocks: CHUNK_LEN / BLOCK_LEN, + key: key.as_ptr(), counter, - true, - flags as u8, - CHUNK_START as u8, - super::CHUNK_END as u8, + increment_counter: true, + flags: assembly_flags(flags), + flags_start: assembly_flags(CHUNK_START), + flags_end: assembly_flags(super::CHUNK_END), out, - ); + }); } } @@ -3756,21 +4043,21 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // `DEGREE * CHUNK_LEN`, so each destination lane is in-bounds. unsafe { core::ptr::copy_nonoverlapping( - input.add(i * CHUNK_LEN), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + input.add(i.strict_mul(CHUNK_LEN)), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } } - let last_src_offset = (num_chunks - 1) * CHUNK_LEN; + let last_src_offset = num_chunks.strict_sub(1).strict_mul(CHUNK_LEN); for i in num_chunks..super::x86_64::avx512::DEGREE { // SAFETY: `last_src_offset` points to a previously materialized lane in // `tmp_input`, and destination lane `i` is within the fixed-size buffer. unsafe { core::ptr::copy_nonoverlapping( tmp_input.as_ptr().add(last_src_offset), - tmp_input.as_mut_ptr().add(i * CHUNK_LEN), + tmp_input.as_mut_ptr().add(i.strict_mul(CHUNK_LEN)), CHUNK_LEN, ); } @@ -3780,13 +4067,19 @@ unsafe fn hash_many_contiguous_avx512_wrapper( // valid for the full 16-lane contiguous contract. unsafe { super::x86_64::avx512::hash16_contiguous(tmp_input.as_ptr(), key, counter, flags, tmp_out.as_mut_ptr()); - core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks * OUT_LEN); + core::ptr::copy_nonoverlapping(tmp_out.as_ptr(), out, num_chunks.strict_mul(OUT_LEN)); } } } } #[cfg(all(feature = "diag", target_arch = "x86_64"))] +/// Hash contiguous chunks with the owned AVX2 diagnostic kernel. +/// +/// # Safety +/// +/// The owned AVX2 dispatch requirements must hold. `input` and `out` must +/// cover `num_chunks` full chunks and chaining values without overlapping. unsafe fn hash_many_contiguous_avx2_owned_wrapper( input: *const u8, mut num_chunks: usize, @@ -3819,22 +4112,22 @@ unsafe fn hash_many_contiguous_avx2_owned_wrapper( // 2. `ptrs` cover 8 full chunks and `out` covers 8 output CVs. // 3. Pointer advancement stays within the caller-provided input/output ranges. unsafe { - super::x86_64::avx2::hash8_owned( - &ptrs, - CHUNK_LEN / BLOCK_LEN, + super::x86_64::avx2::hash8_owned(super::x86_64::HashManyRequest { + inputs: &ptrs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - CHUNK_START, - super::CHUNK_END, + flags_start: CHUNK_START, + flags_end: super::CHUNK_END, out, - ); - input = input.add(super::x86_64::avx2::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx2::DEGREE * OUT_LEN); + }); + input = input.add(super::x86_64::avx2::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx2::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx2::DEGREE as u64); - num_chunks -= super::x86_64::avx2::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx2::DEGREE); } if num_chunks != 0 { @@ -3848,6 +4141,12 @@ unsafe fn hash_many_contiguous_avx2_owned_wrapper( } #[cfg(all(feature = "diag", target_arch = "x86_64"))] +/// Hash contiguous chunks with the owned AVX-512 diagnostic kernel. +/// +/// # Safety +/// +/// The owned AVX-512 dispatch requirements must hold. `input` and `out` must +/// cover `num_chunks` full chunks and chaining values without overlapping. unsafe fn hash_many_contiguous_avx512_owned_wrapper( input: *const u8, mut num_chunks: usize, @@ -3864,11 +4163,11 @@ unsafe fn hash_many_contiguous_avx512_owned_wrapper( // 3. `out` is writable for `DEGREE * OUT_LEN` bytes. unsafe { super::x86_64::avx512::hash16_contiguous_owned(input, key, counter, flags, out); - input = input.add(super::x86_64::avx512::DEGREE * CHUNK_LEN); - out = out.add(super::x86_64::avx512::DEGREE * OUT_LEN); + input = input.add(super::x86_64::avx512::DEGREE.strict_mul(CHUNK_LEN)); + out = out.add(super::x86_64::avx512::DEGREE.strict_mul(OUT_LEN)); } counter = counter.wrapping_add(super::x86_64::avx512::DEGREE as u64); - num_chunks -= super::x86_64::avx512::DEGREE; + num_chunks = num_chunks.strict_sub(super::x86_64::avx512::DEGREE); } if num_chunks != 0 { @@ -3920,6 +4219,13 @@ fn parent_cv_neon_wrapper( } #[cfg(target_arch = "aarch64")] +/// Dispatches contiguous chunks through the AArch64 NEON implementation. +/// +/// # Safety +/// +/// `input` must reference `num_chunks` readable chunks, `out` must reference +/// `num_chunks` writable chaining values, and the current CPU must support +/// NEON. unsafe fn hash_many_contiguous_neon_wrapper( input: *const u8, num_chunks: usize, diff --git a/src/hashes/crypto/blake3/mod.rs b/src/hashes/crypto/blake3/mod.rs index e2844434..7224d34c 100644 --- a/src/hashes/crypto/blake3/mod.rs +++ b/src/hashes/crypto/blake3/mod.rs @@ -2,16 +2,10 @@ //! //! This is a portable, dependency-free implementation suitable for `no_std`. -#![cfg_attr(not(test), deny(clippy::unwrap_used))] -#![cfg_attr(not(test), deny(clippy::expect_used))] -#![allow(clippy::indexing_slicing)] // Audited fixed-size parsing + perf-critical inner loops. - #[cfg(feature = "std")] use core::cell::RefCell; #[cfg(any(feature = "parallel", not(target_endian = "little")))] use core::slice; -#[cfg(all(feature = "parallel", test))] -use core::sync::atomic::{AtomicBool, Ordering}; use core::{cmp::min, mem::MaybeUninit, ptr}; #[cfg(feature = "std")] use std::thread_local; @@ -37,6 +31,10 @@ use self::{control::ParallelPolicyKind, kernels::Kernel}; const OUT_LEN: usize = 32; const KEY_LEN: usize = 32; const BLOCK_LEN: usize = 64; +#[cfg(target_arch = "x86_64")] +const BLOCK_LEN_U8: u8 = 64; +#[cfg(target_arch = "x86_64")] +const BLOCK_LEN_U32: u32 = 64; const CHUNK_LEN: usize = 1024; const OUTPUT_BLOCK_LEN: usize = 2 * OUT_LEN; // Max CV stack depth for incremental hashing. @@ -215,9 +213,8 @@ fn thread_range(thread_index: usize, threads_total: usize, total: usize) -> (usi #[cfg(feature = "parallel")] #[inline] -fn run_parallel_task(task: impl FnOnce()) -> bool { +fn run_parallel_task(task: impl FnOnce()) { task(); - true } #[cfg(feature = "parallel")] @@ -282,12 +279,12 @@ fn reduce_power_of_two_cvs_in_place( let mut cur_is_0 = true; while cur_len > 1 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs != 0); if cur_is_0 { kernels::parent_cvs_many_from_cvs_inline( kernel.id, - &scratch0[..2 * pairs], + &scratch0[..2usize.strict_mul(pairs)], key_words, flags, &mut scratch1[..pairs], @@ -296,7 +293,7 @@ fn reduce_power_of_two_cvs_in_place( } else { kernels::parent_cvs_many_from_cvs_inline( kernel.id, - &scratch1[..2 * pairs], + &scratch1[..2usize.strict_mul(pairs)], key_words, flags, &mut scratch0[..pairs], @@ -319,7 +316,7 @@ fn hash_full_chunks_cvs_serial( input: &[u8], out: &mut [[u32; 8]], ) { - debug_assert_eq!(input.len(), out.len() * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(CHUNK_LEN)); const MAX_SIMD_DEGREE: usize = 16; @@ -330,7 +327,7 @@ fn hash_full_chunks_cvs_serial( #[cfg(target_endian = "little")] { while written < out.len() { - let remaining = out.len() - written; + let remaining = out.len().strict_sub(written); let batch = remaining.min(MAX_SIMD_DEGREE); // SAFETY: @@ -342,11 +339,11 @@ fn hash_full_chunks_cvs_serial( (kernel.hash_many_contiguous)(input_ptr, batch, &key_words, chunk_counter, flags, out_ptr); } - written += batch; + written = written.strict_add(batch); chunk_counter = chunk_counter.wrapping_add(batch as u64); // SAFETY: advancing within `input` by whole chunks. unsafe { - input_ptr = input_ptr.add(batch * CHUNK_LEN); + input_ptr = input_ptr.add(batch.strict_mul(CHUNK_LEN)); } } } @@ -356,7 +353,7 @@ fn hash_full_chunks_cvs_serial( #[cfg(not(target_endian = "little"))] while written < out.len() { - let remaining = out.len() - written; + let remaining = out.len().strict_sub(written); let batch = remaining.min(MAX_SIMD_DEGREE); // SAFETY: @@ -366,18 +363,17 @@ fn hash_full_chunks_cvs_serial( (kernel.hash_many_contiguous)(input_ptr, batch, &key_words, chunk_counter, flags, out_buf.as_mut_ptr()); } - for i in 0..batch { - let offset = i * OUT_LEN; - // SAFETY: `out_buf` is `OUT_LEN * MAX_SIMD_DEGREE` bytes, and `i < batch <= MAX_SIMD_DEGREE`. - let cv = unsafe { words8_from_le_bytes_32(&*(out_buf.as_ptr().add(offset) as *const [u8; OUT_LEN])) }; - out[written + i] = cv; + let (out_cvs, remainder) = out_buf.as_chunks::(); + debug_assert!(remainder.is_empty()); + for (i, cv_bytes) in out_cvs[..batch].iter().enumerate() { + out[written.strict_add(i)] = words8_from_le_bytes_32(cv_bytes); } - written += batch; + written = written.strict_add(batch); chunk_counter = chunk_counter.wrapping_add(batch as u64); // SAFETY: advancing within `input` by whole chunks. unsafe { - input_ptr = input_ptr.add(batch * CHUNK_LEN); + input_ptr = input_ptr.add(batch.strict_mul(CHUNK_LEN)); } } } @@ -392,7 +388,7 @@ fn hash_full_chunks_cvs_parallel_rayon( out: &mut [[u32; 8]], threads_total: usize, ) { - debug_assert_eq!(input.len(), out.len() * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(CHUNK_LEN)); if threads_total <= 1 || out.len() < 2 { hash_full_chunks_cvs_serial(kernel, key_words, flags, base_counter, input, out); @@ -400,82 +396,40 @@ fn hash_full_chunks_cvs_parallel_rayon( } let threads_total = threads_total.min(out.len()).max(1); - #[cfg(test)] - let failed = AtomicBool::new(false); let out_ptr = SendPtr(out.as_mut_ptr()); let out_len = out.len(); rayon::scope(|s| { - #[cfg(test)] - let failed = &failed; for t in 1..threads_total { let (start, end) = thread_range(t, threads_total, out_len); if start == end { continue; } - let input = &input[start * CHUNK_LEN..end * CHUNK_LEN]; + let input = &input[start.strict_mul(CHUNK_LEN)..end.strict_mul(CHUNK_LEN)]; let counter = base_counter.wrapping_add(start as u64); s.spawn(move |_| { - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { // SAFETY: `out_ptr` is valid for `out_len` elements and this task's // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; hash_full_chunks_cvs_serial(kernel, key_words, flags, counter, input, out); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - // SAFETY: `out_ptr` is valid for `out_len` elements and this task's - // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - hash_full_chunks_cvs_serial(kernel, key_words, flags, counter, input, out); - }); - } }); } let (start, end) = thread_range(0, threads_total, out_len); - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { hash_full_chunks_cvs_serial( kernel, key_words, flags, base_counter.wrapping_add(start as u64), - &input[start * CHUNK_LEN..end * CHUNK_LEN], + &input[start.strict_mul(CHUNK_LEN)..end.strict_mul(CHUNK_LEN)], // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, + unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }, ); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - hash_full_chunks_cvs_serial( - kernel, - key_words, - flags, - base_counter.wrapping_add(start as u64), - &input[start * CHUNK_LEN..end * CHUNK_LEN], - // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, - ); - }); - } }); - - #[cfg(test)] - if failed.load(Ordering::Relaxed) { - hash_full_chunks_cvs_serial(kernel, key_words, flags, base_counter, input, out); - } } #[cfg(feature = "parallel")] @@ -487,7 +441,7 @@ fn parent_cvs_many_from_cvs_parallel_rayon( out: &mut [[u32; 8]], threads_total: usize, ) { - debug_assert_eq!(children.len(), out.len() * 2); + debug_assert_eq!(children.len(), out.len().strict_mul(2)); if threads_total <= 1 || out.len() < 2 { kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); @@ -496,78 +450,37 @@ fn parent_cvs_many_from_cvs_parallel_rayon( let pairs = out.len(); let threads_total = threads_total.min(pairs).max(1); - #[cfg(test)] - let failed = AtomicBool::new(false); let out_ptr = SendPtr(out.as_mut_ptr()); rayon::scope(|s| { - #[cfg(test)] - let failed = &failed; for t in 1..threads_total { let (start, end) = thread_range(t, threads_total, pairs); if start == end { continue; } - let children = &children[2 * start..2 * end]; + let children = &children[start.strict_mul(2)..end.strict_mul(2)]; s.spawn(move |_| { - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { // SAFETY: `out_ptr` is valid for `pairs` outputs and this task's // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - // SAFETY: `out_ptr` is valid for `pairs` outputs and this task's - // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); - }); - } }); } let (start, end) = thread_range(0, threads_total, pairs); - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { kernels::parent_cvs_many_from_cvs_inline( kernel.id, - &children[2 * start..2 * end], + &children[start.strict_mul(2)..end.strict_mul(2)], key_words, flags, // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, + unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }, ); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - kernels::parent_cvs_many_from_cvs_inline( - kernel.id, - &children[2 * start..2 * end], - key_words, - flags, - // SAFETY: disjoint partition for thread 0. - unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }, - ); - }); - } }); - - #[cfg(test)] - if failed.load(Ordering::Relaxed) { - kernels::parent_cvs_many_from_cvs_inline(kernel.id, children, key_words, flags, out); - } } #[cfg(feature = "parallel")] @@ -583,12 +496,14 @@ fn hash_power_of_two_subtree_roots_serial( ) { debug_assert!(subtree_chunks.is_power_of_two()); debug_assert_ne!(subtree_chunks, 0); - debug_assert_eq!(input.len(), out.len() * subtree_chunks * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(subtree_chunks).strict_mul(CHUNK_LEN)); with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = base_counter.wrapping_add((i * subtree_chunks) as u64); - let bytes = &input[i * subtree_chunks * CHUNK_LEN..(i + 1) * subtree_chunks * CHUNK_LEN]; + let chunk_offset = i.strict_mul(subtree_chunks); + let chunk_end = i.strict_add(1).strict_mul(subtree_chunks); + let chunk_base = base_counter.wrapping_add(chunk_offset as u64); + let bytes = &input[chunk_offset.strict_mul(CHUNK_LEN)..chunk_end.strict_mul(CHUNK_LEN)]; hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); } @@ -621,7 +536,7 @@ fn hash_power_of_two_subtree_roots_parallel_rayon(req: SubtreeRootsRequest<'_>) } = req; debug_assert!(subtree_chunks.is_power_of_two()); debug_assert_ne!(subtree_chunks, 0); - debug_assert_eq!(input.len(), out.len() * subtree_chunks * CHUNK_LEN); + debug_assert_eq!(input.len(), out.len().strict_mul(subtree_chunks).strict_mul(CHUNK_LEN)); if threads_total <= 1 || out.len() < 2 { hash_power_of_two_subtree_roots_serial(kernel, key_words, flags, base_counter, input, subtree_chunks, out); @@ -629,101 +544,58 @@ fn hash_power_of_two_subtree_roots_parallel_rayon(req: SubtreeRootsRequest<'_>) } let threads_total = threads_total.min(out.len()).max(1); - #[cfg(test)] - let failed = AtomicBool::new(false); let out_ptr = SendPtr(out.as_mut_ptr()); let out_len = out.len(); rayon::scope(|s| { - #[cfg(test)] - let failed = &failed; for t in 1..threads_total { let (start, end) = thread_range(t, threads_total, out_len); if start == end { continue; } - let input = &input[start * subtree_chunks * CHUNK_LEN..end * subtree_chunks * CHUNK_LEN]; - let counter = base_counter.wrapping_add((start * subtree_chunks) as u64); + let start_chunk = start.strict_mul(subtree_chunks); + let end_chunk = end.strict_mul(subtree_chunks); + let input = &input[start_chunk.strict_mul(CHUNK_LEN)..end_chunk.strict_mul(CHUNK_LEN)]; + let counter = base_counter.wrapping_add(start_chunk as u64); s.spawn(move |_| { - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { // SAFETY: `out_ptr` is valid for `out_len` outputs and this task's // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[i * subtree_chunks * CHUNK_LEN..(i + 1) * subtree_chunks * CHUNK_LEN]; + let chunk_offset = i.strict_mul(subtree_chunks); + let chunk_end = i.strict_add(1).strict_mul(subtree_chunks); + let chunk_base = counter.wrapping_add(chunk_offset as u64); + let bytes = &input[chunk_offset.strict_mul(CHUNK_LEN)..chunk_end.strict_mul(CHUNK_LEN)]; hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); } }); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { - // SAFETY: `out_ptr` is valid for `out_len` outputs and this task's - // range is disjoint from every other task. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[i * subtree_chunks * CHUNK_LEN..(i + 1) * subtree_chunks * CHUNK_LEN]; - hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); - *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); - } - }); - }); - } }); } let (start, end) = thread_range(0, threads_total, out_len); - #[cfg(test)] - let ok = run_parallel_task(|| { + run_parallel_task(|| { with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { - let counter = base_counter.wrapping_add((start * subtree_chunks) as u64); + let start_chunk = start.strict_mul(subtree_chunks); + let counter = base_counter.wrapping_add(start_chunk as u64); // SAFETY: disjoint partition for thread 0. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end.strict_sub(start)) }; for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[(start + i) * subtree_chunks * CHUNK_LEN..(start + i + 1) * subtree_chunks * CHUNK_LEN]; + let chunk_offset = i.strict_mul(subtree_chunks); + let global_chunk = start.strict_add(i).strict_mul(subtree_chunks); + let global_chunk_end = start.strict_add(i).strict_add(1).strict_mul(subtree_chunks); + let chunk_base = counter.wrapping_add(chunk_offset as u64); + let bytes = &input[global_chunk.strict_mul(CHUNK_LEN)..global_chunk_end.strict_mul(CHUNK_LEN)]; hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); } }); }); - #[cfg(test)] - if !ok { - failed.store(true, Ordering::Relaxed); - } - #[cfg(not(test))] - { - run_parallel_task(|| { - with_subtree_scratch(subtree_chunks, flags, |scratch0, scratch1| { - let counter = base_counter.wrapping_add((start * subtree_chunks) as u64); - // SAFETY: disjoint partition for thread 0. - let out = unsafe { slice::from_raw_parts_mut(out_ptr.get().add(start), end - start) }; - for (i, slot) in out.iter_mut().enumerate() { - let chunk_base = counter + (i * subtree_chunks) as u64; - let bytes = &input[(start + i) * subtree_chunks * CHUNK_LEN..(start + i + 1) * subtree_chunks * CHUNK_LEN]; - hash_full_chunks_cvs_serial(kernel, key_words, flags, chunk_base, bytes, scratch0); - *slot = reduce_power_of_two_cvs_in_place(kernel, key_words, flags, scratch0, scratch1); - } - }); - }); - } }); - - #[cfg(test)] - if failed.load(Ordering::Relaxed) { - hash_power_of_two_subtree_roots_serial(kernel, key_words, flags, base_counter, input, subtree_chunks, out); - } } /// BLAKE3 message schedule. @@ -754,21 +626,9 @@ fn words8_from_le_bytes_32(bytes: &[u8; 32]) -> [u32; 8] { // 1-byte alignment of `[u8; 32]`. unsafe { ptr::read_unaligned(bytes.as_ptr().cast::<[u32; 8]>()) } } else { - let src = bytes.as_ptr() as *const u32; - // SAFETY: `bytes` is exactly 32 bytes; `read_unaligned` supports the - // 1-byte alignment of `[u8; 32]`. - unsafe { - [ - u32::from_le(ptr::read_unaligned(src.add(0))), - u32::from_le(ptr::read_unaligned(src.add(1))), - u32::from_le(ptr::read_unaligned(src.add(2))), - u32::from_le(ptr::read_unaligned(src.add(3))), - u32::from_le(ptr::read_unaligned(src.add(4))), - u32::from_le(ptr::read_unaligned(src.add(5))), - u32::from_le(ptr::read_unaligned(src.add(6))), - u32::from_le(ptr::read_unaligned(src.add(7))), - ] - } + let (words, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + core::array::from_fn(|i| u32::from_le_bytes(words[i])) } } @@ -779,29 +639,9 @@ fn words16_from_le_bytes_64(bytes: &[u8; 64]) -> [u32; 16] { // 1-byte alignment of `[u8; 64]`. unsafe { ptr::read_unaligned(bytes.as_ptr().cast::<[u32; 16]>()) } } else { - let src = bytes.as_ptr() as *const u32; - // SAFETY: `bytes` is exactly 64 bytes; `read_unaligned` supports the - // 1-byte alignment of `[u8; 64]`. - unsafe { - [ - u32::from_le(ptr::read_unaligned(src.add(0))), - u32::from_le(ptr::read_unaligned(src.add(1))), - u32::from_le(ptr::read_unaligned(src.add(2))), - u32::from_le(ptr::read_unaligned(src.add(3))), - u32::from_le(ptr::read_unaligned(src.add(4))), - u32::from_le(ptr::read_unaligned(src.add(5))), - u32::from_le(ptr::read_unaligned(src.add(6))), - u32::from_le(ptr::read_unaligned(src.add(7))), - u32::from_le(ptr::read_unaligned(src.add(8))), - u32::from_le(ptr::read_unaligned(src.add(9))), - u32::from_le(ptr::read_unaligned(src.add(10))), - u32::from_le(ptr::read_unaligned(src.add(11))), - u32::from_le(ptr::read_unaligned(src.add(12))), - u32::from_le(ptr::read_unaligned(src.add(13))), - u32::from_le(ptr::read_unaligned(src.add(14))), - u32::from_le(ptr::read_unaligned(src.add(15))), - ] - } + let (words, remainder) = bytes.as_chunks::<4>(); + debug_assert!(remainder.is_empty()); + core::array::from_fn(|i| u32::from_le_bytes(words[i])) } } @@ -813,8 +653,8 @@ fn words16_to_le_bytes_64(words: &[u32; 16]) -> [u8; 64] { unsafe { ptr::copy_nonoverlapping(words.as_ptr().cast::(), out.as_mut_ptr(), 64) }; } else { for (idx, word) in words.iter().copied().enumerate() { - let offset = idx * 4; - out[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + let offset = idx.strict_mul(4); + out[offset..offset.strict_add(4)].copy_from_slice(&word.to_le_bytes()); } } out @@ -823,7 +663,7 @@ fn words16_to_le_bytes_64(words: &[u32; 16]) -> [u8; 64] { #[inline(always)] fn pow2_floor(n: usize) -> usize { debug_assert!(n != 0); - 1usize << (usize::BITS - 1 - n.leading_zeros()) + 1usize.strict_shl(usize::BITS.strict_sub(1).strict_sub(n.leading_zeros())) } #[cfg(any(target_endian = "little", feature = "parallel"))] @@ -842,9 +682,15 @@ fn reduce_power_of_two_chunk_cvs(kernel: Kernel, key_words: [u32; 8], flags: u32 let mut cur_len = cvs.len(); while cur_len > 1 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs <= 8); - kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cur[..2 * pairs], key_words, flags, &mut next[..pairs]); + kernels::parent_cvs_many_from_cvs_inline( + kernel.id, + &cur[..2usize.strict_mul(pairs)], + key_words, + flags, + &mut next[..pairs], + ); cur[..pairs].copy_from_slice(&next[..pairs]); cur_len = pairs; } @@ -874,9 +720,15 @@ fn reduce_power_of_two_chunk_cvs_bytes(kernel: Kernel, key_words: [u32; 8], flag let mut cur_len = cvs.len(); while cur_len > 1 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs <= 8); - kernels::parent_cvs_many_from_bytes_inline(kernel.id, &cur[..2 * pairs], key_words, flags, &mut next[..pairs]); + kernels::parent_cvs_many_from_bytes_inline( + kernel.id, + &cur[..2usize.strict_mul(pairs)], + key_words, + flags, + &mut next[..pairs], + ); cur[..pairs].copy_from_slice(&next[..pairs]); cur_len = pairs; } @@ -918,7 +770,7 @@ fn reduce_subtree_to_pair( let mut in_next = false; // false = current data is in cvs, true = in next while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if !in_next { kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cvs[..cur_len], key_words, flags, &mut next[..pairs]); } else { @@ -956,8 +808,8 @@ fn reduce_power_of_two_chunk_cvs_any( let threads_total = threads_total.max(1); - let mut buf0 = alloc::vec![[0u32; 8]; cvs.len() / 2]; - let mut buf1 = alloc::vec![[0u32; 8]; cvs.len() / 2]; + let mut buf0 = alloc::vec![[0u32; 8]; cvs.len().strict_div(2)]; + let mut buf1 = alloc::vec![[0u32; 8]; cvs.len().strict_div(2)]; enum Cur<'a> { Input(&'a [[u32; 8]]), @@ -969,7 +821,7 @@ fn reduce_power_of_two_chunk_cvs_any( let mut cur_len = cvs.len(); let result = loop { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); debug_assert!(pairs != 0); // Parallelize only levels admitted by the explicit parent-fold threshold. @@ -998,21 +850,21 @@ fn reduce_power_of_two_chunk_cvs_any( let out0: [u32; 8] = match cur { Cur::Input(children) => { let out = &mut buf0[..pairs]; - let children = &children[..2 * pairs]; + let children = &children[..2usize.strict_mul(pairs)]; fold_level!(children, out); cur = Cur::Buf0; out[0] } Cur::Buf0 => { let out = &mut buf1[..pairs]; - let children = &buf0[..2 * pairs]; + let children = &buf0[..2usize.strict_mul(pairs)]; fold_level!(children, out); cur = Cur::Buf1; out[0] } Cur::Buf1 => { let out = &mut buf0[..pairs]; - let children = &buf1[..2 * pairs]; + let children = &buf1[..2usize.strict_mul(pairs)]; fold_level!(children, out); cur = Cur::Buf0; out[0] @@ -1053,12 +905,12 @@ fn add_chunk_cvs_batched( #[inline] fn push_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, cv: [u32; 8]) { stack[*len].write(cv); - *len += 1; + *len = len.strict_add(1); } #[inline] fn pop_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize) -> [u32; 8] { - *len -= 1; + *len = len.strict_sub(1); // SAFETY: `len` tracks the number of initialized entries. unsafe { stack[*len].assume_init_read() } } @@ -1067,7 +919,7 @@ fn add_chunk_cvs_batched( let mut chunk_counter = base_counter; while offset < cvs.len() { - let remaining = cvs.len() - offset; + let remaining = cvs.len().strict_sub(offset); let mut size = pow2_floor(remaining); let aligned_max = if chunk_counter == 0 { @@ -1077,14 +929,14 @@ fn add_chunk_cvs_batched( if tz >= (usize::BITS as usize) { usize::MAX } else { - 1usize << tz + 1usize.strict_shl(u32::try_from(tz).expect("BLAKE3 tree level fits in u32")) } }; size = size.min(aligned_max).min(remaining); debug_assert!(size.is_power_of_two()); - let subtree_cv = reduce_power_of_two_chunk_cvs(kernel, key_words, flags, &cvs[offset..offset + size]); + let subtree_cv = reduce_power_of_two_chunk_cvs(kernel, key_words, flags, &cvs[offset..offset.strict_add(size)]); chunk_counter = chunk_counter.wrapping_add(size as u64); // Merge this subtree into the global stack. Because `size` is a power of two @@ -1099,7 +951,7 @@ fn add_chunk_cvs_batched( } push_stack(stack, stack_len, cv); - offset += size; + offset = offset.strict_add(size); } } @@ -1123,12 +975,12 @@ fn add_chunk_cvs_batched_bytes( #[inline] fn push_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, cv: [u32; 8]) { stack[*len].write(cv); - *len += 1; + *len = len.strict_add(1); } #[inline] fn pop_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize) -> [u32; 8] { - *len -= 1; + *len = len.strict_sub(1); // SAFETY: `len` tracks the number of initialized entries. unsafe { stack[*len].assume_init_read() } } @@ -1137,7 +989,7 @@ fn add_chunk_cvs_batched_bytes( let mut chunk_counter = base_counter; while offset < cvs.len() { - let remaining = cvs.len() - offset; + let remaining = cvs.len().strict_sub(offset); let mut size = pow2_floor(remaining); let aligned_max = if chunk_counter == 0 { @@ -1147,14 +999,15 @@ fn add_chunk_cvs_batched_bytes( if tz >= (usize::BITS as usize) { usize::MAX } else { - 1usize << tz + 1usize.strict_shl(u32::try_from(tz).expect("BLAKE3 tree level fits in u32")) } }; size = size.min(aligned_max).min(remaining); debug_assert!(size.is_power_of_two()); - let subtree_cv_bytes = reduce_power_of_two_chunk_cvs_bytes(kernel, key_words, flags, &cvs[offset..offset + size]); + let subtree_cv_bytes = + reduce_power_of_two_chunk_cvs_bytes(kernel, key_words, flags, &cvs[offset..offset.strict_add(size)]); let subtree_cv = words8_from_le_bytes_32(&subtree_cv_bytes); chunk_counter = chunk_counter.wrapping_add(size as u64); @@ -1170,7 +1023,7 @@ fn add_chunk_cvs_batched_bytes( } push_stack(stack, stack_len, cv); - offset += size; + offset = offset.strict_add(size); } } @@ -1193,8 +1046,9 @@ fn compress(chaining_value: &[u32; 8], block_words: &[u32; 16], counter: u64, bl let m14 = block_words[14]; let m15 = block_words[15]; - let counter_low = counter as u32; - let counter_high = (counter >> 32) as u32; + let [c0, c1, c2, c3, c4, c5, c6, c7] = counter.to_le_bytes(); + let counter_low = u32::from_le_bytes([c0, c1, c2, c3]); + let counter_high = u32::from_le_bytes([c4, c5, c6, c7]); let mut v0 = chaining_value[0]; let mut v1 = chaining_value[1]; let mut v2 = chaining_value[2]; @@ -1290,8 +1144,8 @@ fn words8_to_le_bytes(words: &[u32; 8]) -> [u8; OUT_LEN] { unsafe { ptr::copy_nonoverlapping(words.as_ptr().cast::(), out.as_mut_ptr(), OUT_LEN) }; } else { for (i, word) in words.iter().copied().enumerate() { - let offset = i * 4; - out[offset..offset + 4].copy_from_slice(&word.to_le_bytes()); + let offset = i.strict_mul(4); + out[offset..offset.strict_add(4)].copy_from_slice(&word.to_le_bytes()); } } out @@ -1355,15 +1209,13 @@ impl OutputState { #[inline] fn into_root_emit_state(self) -> RootEmitState { - debug_assert!(self.block_len <= u8::MAX as u32); - debug_assert!(self.flags <= u8::MAX as u32); RootEmitState { kernel_id: self.kernel_id, input_chaining_value: self.input_chaining_value, block_bytes: words16_to_le_bytes_64(&self.block_words), counter: self.counter, - block_len: self.block_len as u8, - flags: self.flags as u8, + block_len: u8::try_from(self.block_len).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(self.flags).expect("BLAKE3 flags fit in u8"), } } } @@ -1406,8 +1258,8 @@ impl RootEmitState { input_chaining_value: key_words, block_bytes, counter: 0, - block_len: BLOCK_LEN as u8, - flags: (PARENT | flags) as u8, + block_len: u8::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(PARENT | flags).expect("BLAKE3 flags fit in u8"), }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut left_child_cv); @@ -1439,11 +1291,11 @@ impl RootEmitState { "digest prefix fast path only applies to the first output block" ); debug_assert!(offset <= OUT_LEN); - debug_assert!(offset + out.len() <= OUT_LEN); + debug_assert!(offset.strict_add(out.len()) <= OUT_LEN); let mut block = [0u8; OUTPUT_BLOCK_LEN]; self.emit_one_block(&mut block); - out.copy_from_slice(&block[offset..offset + out.len()]); + out.copy_from_slice(&block[offset..offset.strict_add(out.len())]); if u32::from(self.flags) & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize(&mut block); } @@ -1531,7 +1383,9 @@ impl ChunkState { #[inline] fn len(&self) -> usize { - BLOCK_LEN * self.blocks_compressed as usize + self.block_len as usize + BLOCK_LEN + .strict_mul(usize::from(self.blocks_compressed)) + .strict_add(usize::from(self.block_len)) } #[inline] @@ -1551,11 +1405,11 @@ impl ChunkState { self.chunk_counter, self.flags, &mut self.blocks_compressed, - &input[..CHUNK_LEN - BLOCK_LEN], + &input[..CHUNK_LEN.strict_sub(BLOCK_LEN)], ); debug_assert_eq!(self.blocks_compressed, 15); - self.block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); - self.block_len = BLOCK_LEN as u8; + self.block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); + self.block_len = u8::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u8"); } #[inline] @@ -1567,7 +1421,7 @@ impl ChunkState { let (cv, last_block) = absorb_exact_one_chunk_state(self.kernel_id, input, key, self.chunk_counter, self.flags); self.chaining_value = cv; self.block = last_block; - self.block_len = BLOCK_LEN as u8; + self.block_len = u8::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u8"); self.blocks_compressed = 15; } @@ -1579,7 +1433,7 @@ impl ChunkState { if self.block_len == 0 { if input.len() <= BLOCK_LEN { self.block[..input.len()].copy_from_slice(input); - self.block_len = input.len() as u8; + self.block_len = u8::try_from(input.len()).expect("BLAKE3 buffered block length fits in u8"); return; } @@ -1600,7 +1454,7 @@ impl ChunkState { // Calls assembly directly, bypassing the multi-block wrappers and their // #[target_feature] boundaries. This matches the official blake3 crate's // call depth: one match + one extern "C" assembly call. - if self.block_len as usize == BLOCK_LEN && self.blocks_compressed < 15 && input.len() == BLOCK_LEN { + if usize::from(self.block_len) == BLOCK_LEN && self.blocks_compressed < 15 && input.len() == BLOCK_LEN { let start = if self.blocks_compressed == 0 { CHUNK_START } else { 0 }; // SAFETY: kernel_id was validated at construction to match available CPU features. unsafe { @@ -1612,7 +1466,7 @@ impl ChunkState { self.flags | start, ); } - self.blocks_compressed = self.blocks_compressed.wrapping_add(1); + self.blocks_compressed = self.blocks_compressed.strict_add(1); self.block.copy_from_slice(input); // block_len stays BLOCK_LEN — no zero-fill needed return; @@ -1626,10 +1480,12 @@ impl ChunkState { // (or, if it's already full, compress it) before touching the caller // slice. This keeps the hot "many full blocks" path branch-light. if self.block_len != 0 { - let want = BLOCK_LEN - self.block_len as usize; + let want = BLOCK_LEN.strict_sub(usize::from(self.block_len)); let take = min(want, input.len()); - self.block[self.block_len as usize..][..take].copy_from_slice(&input[..take]); - self.block_len = self.block_len.strict_add(take as u8); + self.block[usize::from(self.block_len)..][..take].copy_from_slice(&input[..take]); + self.block_len = self + .block_len + .strict_add(u8::try_from(take).expect("BLAKE3 buffered byte count fits in u8")); input = &input[take..]; // If the caller ended mid-block, we're done. Note that this also covers @@ -1640,7 +1496,7 @@ impl ChunkState { return; } - if self.block_len as usize == BLOCK_LEN { + if usize::from(self.block_len) == BLOCK_LEN { debug_assert!( self.blocks_compressed < 15, "last chunk block stays buffered until output()" @@ -1656,7 +1512,7 @@ impl ChunkState { self.flags | start, ); } - self.blocks_compressed = self.blocks_compressed.wrapping_add(1); + self.blocks_compressed = self.blocks_compressed.strict_add(1); self.block_len = 0; self.block = [0u8; BLOCK_LEN]; } @@ -1671,13 +1527,13 @@ impl ChunkState { if self.blocks_compressed == 15 { debug_assert!(input.len() <= BLOCK_LEN); self.block[..input.len()].copy_from_slice(input); - self.block_len = input.len() as u8; + self.block_len = u8::try_from(input.len()).expect("BLAKE3 buffered block length fits in u8"); return; } - let full_blocks = input.len() / BLOCK_LEN; + let full_blocks = input.len().strict_div(BLOCK_LEN); if full_blocks != 0 { - let max_blocks = 15usize - self.blocks_compressed as usize; + let max_blocks = 15usize.strict_sub(usize::from(self.blocks_compressed)); let mut blocks_to_compress = full_blocks.min(max_blocks); // If we'd consume the entire input as full blocks, leave one block @@ -1701,7 +1557,7 @@ impl ChunkState { self.flags | start, ); } - self.blocks_compressed = self.blocks_compressed.wrapping_add(1); + self.blocks_compressed = self.blocks_compressed.strict_add(1); } input = &input[bytes..]; continue; @@ -1711,7 +1567,7 @@ impl ChunkState { // Remainder: buffer <= 64 bytes and return. let take = min(BLOCK_LEN, input.len()); self.block[..take].copy_from_slice(&input[..take]); - self.block_len = take as u8; + self.block_len = u8::try_from(take).expect("BLAKE3 buffered block length fits in u8"); return; } } @@ -1723,7 +1579,7 @@ impl ChunkState { self.chaining_value, &self.block, self.chunk_counter, - self.block_len as u32, + u32::from(self.block_len), self.flags | self.start_flag() | CHUNK_END, ) } @@ -1740,7 +1596,7 @@ impl ChunkState { block_bytes: self.block, counter: self.chunk_counter, block_len: self.block_len, - flags: (self.flags | self.start_flag() | CHUNK_END) as u8, + flags: u8::try_from(self.flags | self.start_flag() | CHUNK_END).expect("BLAKE3 flags fit in u8"), } } @@ -1751,7 +1607,7 @@ impl ChunkState { input_chaining_value: self.chaining_value, block_words: words16_from_le_bytes_64(&self.block), counter: self.chunk_counter, - block_len: self.block_len as u32, + block_len: u32::from(self.block_len), flags: self.flags | self.start_flag() | CHUNK_END, } } @@ -1769,7 +1625,7 @@ fn absorb_exact_one_chunk_state( #[cfg(target_arch = "x86_64")] { - let (prefix_blocks, remainder) = input[..CHUNK_LEN - BLOCK_LEN].as_chunks::(); + let (prefix_blocks, remainder) = input[..CHUNK_LEN.strict_sub(BLOCK_LEN)].as_chunks::(); debug_assert!(remainder.is_empty()); match kernel_id { @@ -1780,11 +1636,17 @@ fn absorb_exact_one_chunk_state( // SAFETY: dispatch validates SSE4.1 for both kernels, and `block` is a // readable 64-byte buffer. unsafe { - x86_64::compress_in_place_sse41_bytes(&mut cv, block.as_ptr(), counter, BLOCK_LEN as u32, block_flags) + x86_64::compress_in_place_sse41_bytes( + &mut cv, + block.as_ptr(), + counter, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + ) }; } let mut last_block = [0u8; BLOCK_LEN]; - last_block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); + last_block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); return (cv, last_block); } kernels::Blake3KernelId::X86Avx512 => { @@ -1796,7 +1658,13 @@ fn absorb_exact_one_chunk_state( // SAFETY: dispatch validates AVX-512 availability, and `block` is a // readable 64-byte buffer. unsafe { - x86_64::asm::compress_in_place_avx512_mut(&mut cv, block.as_ptr(), counter, BLOCK_LEN as u32, block_flags) + x86_64::asm::compress_in_place_avx512_mut( + &mut cv, + block.as_ptr(), + counter, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + ) }; } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] @@ -1804,12 +1672,18 @@ fn absorb_exact_one_chunk_state( // SAFETY: dispatch validates AVX-512 availability, and `block` is a // readable 64-byte buffer. unsafe { - x86_64::compress_in_place_avx512_bytes(&mut cv, block.as_ptr(), counter, BLOCK_LEN as u32, block_flags) + x86_64::compress_in_place_avx512_bytes( + &mut cv, + block.as_ptr(), + counter, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + ) }; } } let mut last_block = [0u8; BLOCK_LEN]; - last_block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); + last_block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); return (cv, last_block); } _ => {} @@ -1847,11 +1721,11 @@ fn absorb_exact_one_chunk_state( counter, flags, &mut blocks_compressed, - &input[..CHUNK_LEN - BLOCK_LEN], + &input[..CHUNK_LEN.strict_sub(BLOCK_LEN)], ); debug_assert_eq!(blocks_compressed, 15); let mut last_block = [0u8; BLOCK_LEN]; - last_block.copy_from_slice(&input[CHUNK_LEN - BLOCK_LEN..]); + last_block.copy_from_slice(&input[CHUNK_LEN.strict_sub(BLOCK_LEN)..]); (cv, last_block) } @@ -1893,7 +1767,7 @@ fn parent_output( input_chaining_value: key_words, block_words, counter: 0, - block_len: BLOCK_LEN as u32, + block_len: u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), flags: PARENT | flags, }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -1945,7 +1819,7 @@ fn single_chunk_output( input_chaining_value: cv_words, block_words, counter: chunk_counter, - block_len: BLOCK_LEN as u32, + block_len: u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), flags: flags | CHUNK_END, }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -1965,14 +1839,14 @@ fn single_chunk_output( let (full_blocks, last_len) = if input.is_empty() { (0usize, 0usize) } else if input.len().is_multiple_of(BLOCK_LEN) { - (blocks - 1, BLOCK_LEN) + (blocks.strict_sub(1), BLOCK_LEN) } else { - (blocks - 1, input.len() % BLOCK_LEN) + (blocks.strict_sub(1), input.len().strict_rem(BLOCK_LEN)) }; let mut chaining_value = key_words; let mut blocks_compressed: u8 = 0; - let full_bytes = full_blocks * BLOCK_LEN; + let full_bytes = full_blocks.strict_mul(BLOCK_LEN); (kernel.chunk_compress_blocks)( &mut chaining_value, chunk_counter, @@ -1984,7 +1858,7 @@ fn single_chunk_output( let mut block_words = if cfg!(target_endian = "little") { let mut out = [0u32; 16]; if !input.is_empty() { - let offset = full_blocks * BLOCK_LEN; + let offset = full_blocks.strict_mul(BLOCK_LEN); // SAFETY: `out` is 64 bytes, and `last_len <= 64`. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), out.as_mut_ptr().cast::(), last_len); @@ -1994,8 +1868,8 @@ fn single_chunk_output( } else { let mut last_block = [0u8; BLOCK_LEN]; if !input.is_empty() { - let offset = full_blocks * BLOCK_LEN; - last_block[..last_len].copy_from_slice(&input[offset..offset + last_len]); + let offset = full_blocks.strict_mul(BLOCK_LEN); + last_block[..last_len].copy_from_slice(&input[offset..offset.strict_add(last_len)]); } let block_words = words16_from_le_bytes_64(&last_block); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -2010,7 +1884,7 @@ fn single_chunk_output( input_chaining_value: chaining_value, block_words, counter: chunk_counter, - block_len: last_len as u32, + block_len: u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), flags: flags | start | CHUNK_END, }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -2039,8 +1913,8 @@ fn root_output_oneshot( return output; } - let full_chunks = input.len() / CHUNK_LEN; - let remainder = input.len() % CHUNK_LEN; + let full_chunks = input.len().strict_div(CHUNK_LEN); + let remainder = input.len().strict_rem(CHUNK_LEN); const MAX_SIMD_DEGREE: usize = 16; const FAST_TREE_MAX_CHUNKS: usize = 128; @@ -2053,7 +1927,11 @@ fn root_output_oneshot( // // This is intentionally conservative to avoid overhead on latency-critical // small inputs (including keyed/derive). - let commit_full_chunks = if remainder == 0 { full_chunks - 1 } else { full_chunks }; + let commit_full_chunks = if remainder == 0 { + full_chunks.strict_sub(1) + } else { + full_chunks + }; if let Some(threads) = control::parallel_policy_threads_with_admission(mode, input.len(), full_chunks, commit_full_chunks) { @@ -2072,7 +1950,7 @@ fn root_output_oneshot( #[cfg(target_endian = "little")] { let mut cur = [[0u32; 8]; MAX_SIMD_DEGREE]; - let mut next = [[0u32; 8]; MAX_SIMD_DEGREE / 2]; + let mut next = [[0u32; 8]; MAX_SIMD_DEGREE.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2090,7 +1968,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cur[..cur_len], key_words, flags, &mut next[..pairs]); } else { @@ -2116,7 +1994,7 @@ fn root_output_oneshot( #[cfg(not(target_endian = "little"))] { let mut cur = [[0u8; OUT_LEN]; MAX_SIMD_DEGREE]; - let mut next = [[0u8; OUT_LEN]; MAX_SIMD_DEGREE / 2]; + let mut next = [[0u8; OUT_LEN]; MAX_SIMD_DEGREE.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2134,7 +2012,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_bytes_inline( kernel.id, @@ -2177,7 +2055,7 @@ fn root_output_oneshot( #[cfg(target_endian = "little")] { let mut cur = [[0u32; 8]; FAST_TREE_MAX_CHUNKS]; - let mut next = [[0u32; 8]; FAST_TREE_MAX_CHUNKS / 2]; + let mut next = [[0u32; 8]; FAST_TREE_MAX_CHUNKS.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2195,7 +2073,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_cvs_inline(kernel.id, &cur[..cur_len], key_words, flags, &mut next[..pairs]); } else { @@ -2221,7 +2099,7 @@ fn root_output_oneshot( #[cfg(not(target_endian = "little"))] { let mut cur = [[0u8; OUT_LEN]; FAST_TREE_MAX_CHUNKS]; - let mut next = [[0u8; OUT_LEN]; FAST_TREE_MAX_CHUNKS / 2]; + let mut next = [[0u8; OUT_LEN]; FAST_TREE_MAX_CHUNKS.strict_div(2)]; // SAFETY: input has exactly `full_chunks * CHUNK_LEN` bytes and `cur` has // `full_chunks` CV slots (`full_chunks * OUT_LEN` bytes). @@ -2239,7 +2117,7 @@ fn root_output_oneshot( let mut cur_len = full_chunks; let mut cur_is_cur = true; while cur_len > 2 { - let pairs = cur_len / 2; + let pairs = cur_len.strict_div(2); if cur_is_cur { kernels::parent_cvs_many_from_bytes_inline( kernel.id, @@ -2291,7 +2169,7 @@ fn root_output_oneshot( let mut chunk_counter = 0u64; let mut offset = 0usize; while chunk_counter < full_chunks as u64 { - let remaining = (full_chunks as u64).strict_sub(chunk_counter) as usize; + let remaining = full_chunks.strict_sub(offset.strict_div(CHUNK_LEN)); let batch = core::cmp::min(remaining, MAX_SIMD_DEGREE); debug_assert!(batch != 0); @@ -2310,8 +2188,8 @@ fn root_output_oneshot( let mut commit = batch; if remainder == 0 && chunk_counter.strict_add(batch as u64) == full_chunks as u64 { - last_full_chunk_cv = Some(cvs[batch - 1]); - commit -= 1; + last_full_chunk_cv = Some(cvs[batch.strict_sub(1)]); + commit = commit.strict_sub(1); } if commit != 0 { @@ -2331,7 +2209,7 @@ fn root_output_oneshot( } let right_cv = if remainder != 0 { - let chunk_bytes = &input[full_chunks * CHUNK_LEN..]; + let chunk_bytes = &input[full_chunks.strict_mul(CHUNK_LEN)..]; single_chunk_output(kernel, key_words, full_chunks as u64, flags, chunk_bytes).chaining_value() } else { last_full_chunk_cv @@ -2357,7 +2235,7 @@ fn root_output_oneshot( let mut chunk_counter = 0u64; let mut offset = 0usize; while chunk_counter < full_chunks as u64 { - let remaining = (full_chunks as u64).strict_sub(chunk_counter) as usize; + let remaining = full_chunks.strict_sub(offset.strict_div(CHUNK_LEN)); let batch = core::cmp::min(remaining, MAX_SIMD_DEGREE); debug_assert!(batch != 0); @@ -2375,8 +2253,8 @@ fn root_output_oneshot( let mut commit = batch; if remainder == 0 && chunk_counter.strict_add(batch as u64) == full_chunks as u64 { - last_full_chunk_cv = Some(cvs[batch - 1]); - commit -= 1; + last_full_chunk_cv = Some(cvs[batch.strict_sub(1)]); + commit = commit.strict_sub(1); } if commit != 0 { @@ -2396,7 +2274,7 @@ fn root_output_oneshot( } let right_cv = if remainder != 0 { - let chunk_bytes = &input[full_chunks * CHUNK_LEN..]; + let chunk_bytes = &input[full_chunks.strict_mul(CHUNK_LEN)..]; single_chunk_output(kernel, key_words, full_chunks as u64, flags, chunk_bytes).chaining_value() } else if let Some(cv) = last_full_chunk_cv.as_ref() { words8_from_le_bytes_32(cv) @@ -2416,7 +2294,7 @@ fn root_output_oneshot( let mut parent_nodes_remaining = cv_stack_len; debug_assert!(parent_nodes_remaining > 0); - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { cv_stack[parent_nodes_remaining].assume_init_read() }; let mut output = parent_output(kernel.id, left, right_cv, key_words, flags); @@ -2425,7 +2303,7 @@ fn root_output_oneshot( ct::zeroize_words_no_fence(&mut right_cv); } while parent_nodes_remaining > 0 { - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { cv_stack[parent_nodes_remaining].assume_init_read() }; let mut right = output.chaining_value(); @@ -2458,8 +2336,8 @@ fn recompute_last_full_chunk_cv( debug_assert!(full_chunks != 0); debug_assert_eq!(input.len() % CHUNK_LEN, 0); - let last_chunk_index = full_chunks.saturating_sub(1); - let offset = input.len().saturating_sub(CHUNK_LEN); + let last_chunk_index = full_chunks.strict_sub(1); + let offset = input.len().strict_sub(CHUNK_LEN); let cv = single_chunk_output(kernel, key_words, last_chunk_index as u64, flags, &input[offset..]).chaining_value(); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut key_words); @@ -2502,7 +2380,7 @@ fn hash_full_chunks_cvs_scoped( threads_total: usize, ) { let out_len = out.len(); - debug_assert_eq!(input.len(), out_len * CHUNK_LEN); + debug_assert_eq!(input.len(), out_len.strict_mul(CHUNK_LEN)); #[cfg(feature = "parallel")] { @@ -2628,6 +2506,7 @@ fn digest_public_oneshot(mut key_words: [u32; 8], flags: u32, input: &[u8]) -> [ } #[cfg(feature = "diag")] +/// Computes the constant-time evidence digest with the portable kernel. #[must_use] pub fn diag_blake3_keyed_digest_portable(key: &[u8; KEY_LEN]) -> Blake3KeyedHash { let mut key_words = words8_from_le_bytes_32(key); @@ -2682,34 +2561,47 @@ pub fn diag_zeroize_blake3_xof_consume(mut reader: Blake3XofReader) -> u8 { } #[cfg(feature = "diag")] +/// BLAKE3 kernels exposed for diagnostic comparison. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Blake3DiagKernel { + /// Portable Rust kernel. Portable, #[cfg(target_arch = "x86_64")] + /// x86-64 SSE4.1 kernel. X86Sse41, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel. X86Avx2, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel with the owned hash-many implementation. X86Avx2OwnedHashMany, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel with paired chunk-tail processing. X86Avx2PairChunkTail, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX2 kernel with paired parent-tail processing. X86Avx2PairParentTail, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel. X86Avx512, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel forced through exact-block assembly. X86Avx512ExactBlockAsm, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel with the owned hash-many implementation. X86Avx512OwnedHashMany, #[cfg(target_arch = "x86_64")] + /// x86-64 AVX-512 kernel with the owned compression implementation. X86Avx512OwnedCompress, #[cfg(target_arch = "aarch64")] + /// AArch64 NEON kernel. Aarch64Neon, } #[cfg(feature = "diag")] impl Blake3DiagKernel { #[inline] + /// Returns the stable diagnostic label for this kernel. #[must_use] pub const fn label(self) -> &'static str { match self { @@ -2738,6 +2630,7 @@ impl Blake3DiagKernel { } #[inline] + /// Returns whether this kernel supports streaming diagnostics. #[must_use] pub const fn supports_streaming(self) -> bool { #[cfg(target_arch = "x86_64")] @@ -2825,6 +2718,7 @@ impl Blake3DiagKernel { #[cfg(feature = "diag")] #[inline] +/// Returns whether the current CPU supports the diagnostic kernel. #[must_use] pub fn diag_blake3_kernel_available(kernel: Blake3DiagKernel) -> bool { #[cfg(target_arch = "x86_64")] @@ -2870,6 +2764,7 @@ fn diag_blake3_kernel(kernel: Blake3DiagKernel) -> Option { } #[cfg(feature = "diag")] +/// Computes a digest with a selected diagnostic kernel. #[must_use] pub fn diag_blake3_digest_with_kernel(kernel: Blake3DiagKernel, data: &[u8]) -> Option<[u8; OUT_LEN]> { let kernel = diag_blake3_kernel(kernel)?; @@ -2877,6 +2772,7 @@ pub fn diag_blake3_digest_with_kernel(kernel: Blake3DiagKernel, data: &[u8]) -> } #[cfg(feature = "diag")] +/// Computes a keyed digest with a selected diagnostic kernel. #[must_use] pub fn diag_blake3_keyed_digest_with_kernel( kernel: Blake3DiagKernel, @@ -2891,6 +2787,7 @@ pub fn diag_blake3_keyed_digest_with_kernel( } #[cfg(feature = "diag")] +/// Fills XOF output with a selected diagnostic kernel. pub fn diag_blake3_xof_with_kernel(kernel: Blake3DiagKernel, data: &[u8], out: &mut [u8]) -> Option<()> { let kernel = diag_blake3_kernel(kernel)?; let mut reader = if data.len() <= CHUNK_LEN { @@ -2909,6 +2806,7 @@ pub fn diag_blake3_xof_with_kernel(kernel: Blake3DiagKernel, data: &[u8], out: & } #[cfg(feature = "diag")] +/// Computes a streaming digest with a selected diagnostic kernel and update size. #[must_use] pub fn diag_blake3_streaming_digest_with_kernel( kernel: Blake3DiagKernel, @@ -2931,14 +2829,15 @@ pub fn diag_blake3_streaming_digest_with_kernel( } #[cfg(feature = "diag")] +/// Writes full-chunk chaining values with a selected diagnostic kernel. pub fn diag_blake3_chunk_cvs_with_kernel(kernel: Blake3DiagKernel, data: &[u8], out: &mut [u8]) -> Option<()> { #[cfg(target_arch = "x86_64")] let requested_kernel = kernel; if data.is_empty() || !data.len().is_multiple_of(CHUNK_LEN) { return None; } - let chunks = data.len() / CHUNK_LEN; - if out.len() != chunks * OUT_LEN { + let chunks = data.len().strict_div(CHUNK_LEN); + if out.len() != chunks.strict_mul(OUT_LEN) { return None; } @@ -2959,6 +2858,7 @@ pub fn diag_blake3_chunk_cvs_with_kernel(kernel: Blake3DiagKernel, data: &[u8], } #[cfg(feature = "diag")] +/// Writes parent chaining values with a selected diagnostic kernel. pub fn diag_blake3_parent_cvs_with_kernel(kernel: Blake3DiagKernel, children: &[u8], out: &mut [u8]) -> Option<()> { #[cfg(target_arch = "x86_64")] let requested_kernel = kernel; @@ -2967,7 +2867,7 @@ pub fn diag_blake3_parent_cvs_with_kernel(kernel: Blake3DiagKernel, children: &[ return None; } - let parent_count = children.len() / 2; + let parent_count = children.len().strict_div(2); let (out, out_remainder) = out.as_chunks_mut::(); if !out_remainder.is_empty() || out.len() != parent_count { return None; @@ -3314,13 +3214,17 @@ impl Blake3 { }; let keep_last_full_chunk = input.len().is_multiple_of(CHUNK_LEN) && batch == full_chunks; - let commit = if keep_last_full_chunk { batch - 1 } else { batch }; + let commit = if keep_last_full_chunk { + batch.strict_sub(1) + } else { + batch + }; if commit != 0 { // SAFETY: `out_buf` stores `batch` contiguous CV outputs, and // `commit <= batch <= MAX_SIMD_DEGREE`. let cvs_bytes: &[[u8; OUT_LEN]] = unsafe { slice::from_raw_parts(out_buf.as_ptr().cast::<[u8; OUT_LEN]>(), commit) }; - let mut stack_len = self.cv_stack_len as usize; + let mut stack_len = usize::from(self.cv_stack_len); add_chunk_cvs_batched_bytes( kernels::kernel(self.bulk_kernel_id), &mut self.cv_stack, @@ -3330,10 +3234,10 @@ impl Blake3 { self.key_words, self.chunk_state.flags, ); - self.cv_stack_len = stack_len as u8; + self.cv_stack_len = u8::try_from(stack_len).expect("BLAKE3 CV stack length is at most 64"); } - let new_counter = base_counter.strict_add(batch as u64); + let new_counter = base_counter.strict_add(u64::try_from(batch).expect("BLAKE3 SIMD batch size fits in u64")); self.chunk_state = ChunkState::new( self.key_words, new_counter, @@ -3344,7 +3248,7 @@ impl Blake3 { let offset = batch.strict_sub(1).strict_mul(OUT_LEN); // SAFETY: `out_buf` is `OUT_LEN * MAX_SIMD_DEGREE`, and `offset` // is `(batch - 1) * OUT_LEN` with `batch <= MAX_SIMD_DEGREE`. - let mut cv = unsafe { words8_from_le_bytes_32(&*(out_buf.as_ptr().add(offset) as *const [u8; OUT_LEN])) }; + let mut cv = unsafe { words8_from_le_bytes_32(&*out_buf.as_ptr().add(offset).cast::<[u8; OUT_LEN]>()) }; self.pending_chunk_cv = Some(cv); self.pending_cv_chunks = 1; if self.chunk_state.flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -3412,7 +3316,7 @@ impl Blake3 { continue; } - let want = CHUNK_LEN - self.chunk_state.len(); + let want = CHUNK_LEN.strict_sub(self.chunk_state.len()); let take = min(want, input.len()); self.chunk_state.update(&input[..take]); input = &input[take..]; @@ -3445,8 +3349,8 @@ impl Blake3 { #[inline] fn push_stack(&mut self, mut cv: [u32; 8]) { - self.cv_stack[self.cv_stack_len as usize].write(cv); - self.cv_stack_len = self.cv_stack_len.wrapping_add(1); + self.cv_stack[usize::from(self.cv_stack_len)].write(cv); + self.cv_stack_len = self.cv_stack_len.strict_add(1); if self.chunk_state.flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut cv); } @@ -3454,13 +3358,13 @@ impl Blake3 { #[inline] fn pop_stack(&mut self) -> [u32; 8] { - self.cv_stack_len = self.cv_stack_len.wrapping_sub(1); + self.cv_stack_len = self.cv_stack_len.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. - let cv = unsafe { self.cv_stack[self.cv_stack_len as usize].assume_init_read() }; + let cv = unsafe { self.cv_stack[usize::from(self.cv_stack_len)].assume_init_read() }; if self.chunk_state.flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { // SAFETY: the slot held the `Copy` CV read above and remains initialized // until the next push overwrites it. - ct::zeroize_words(unsafe { self.cv_stack[self.cv_stack_len as usize].assume_init_mut() }); + ct::zeroize_words(unsafe { self.cv_stack[usize::from(self.cv_stack_len)].assume_init_mut() }); } cv } @@ -3593,7 +3497,7 @@ impl Blake3 { self.advance_full_chunk(); } - if self.chunk_state.len() + input.len() <= CHUNK_LEN { + if self.chunk_state.len().strict_add(input.len()) <= CHUNK_LEN { self.chunk_state.update(input); return; } @@ -3630,7 +3534,7 @@ impl Blake3 { }; while parent_nodes_remaining > 0 { - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { *self.cv_stack[parent_nodes_remaining].assume_init_ref() }; let mut right = output.chaining_value(); @@ -3702,7 +3606,7 @@ impl Blake3 { }; while parent_nodes_remaining > 0 { - parent_nodes_remaining -= 1; + parent_nodes_remaining = parent_nodes_remaining.strict_sub(1); // SAFETY: `cv_stack_len` tracks the number of initialized entries. let mut left = unsafe { *self.cv_stack[parent_nodes_remaining].assume_init_ref() }; if parent_nodes_remaining == 0 { @@ -3920,8 +3824,8 @@ fn xof_oneshot_single_chunk(kernel: Kernel, mut key_words: [u32; 8], flags: u32, input_chaining_value: key_words, block_bytes, counter: 0, - block_len: input.len() as u8, - flags: (flags | CHUNK_START | CHUNK_END) as u8, + block_len: u8::try_from(input.len()).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(flags | CHUNK_START | CHUNK_END).expect("BLAKE3 flags fit in u8"), }); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut block_bytes); @@ -3946,24 +3850,29 @@ fn xof_oneshot_single_chunk(kernel: Kernel, mut key_words: [u32; 8], flags: u32, } // Single-chunk input (65–1024B): compress full blocks, store last block. - let rem = input.len() % BLOCK_LEN; + let rem = input.len().strict_rem(BLOCK_LEN); let (full_blocks, last_len) = if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + (input.len().strict_div(BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { - (input.len() / BLOCK_LEN, rem) + (input.len().strict_div(BLOCK_LEN), rem) }; let mut cv = key_words; + let (input_blocks, _) = input.as_chunks::(); // Compress full non-final blocks using the kernel's compress function // pointer directly. Single-chunk XOF is latency-sensitive, and the bulk // wrapper setup costs more than it saves in the 256B–1KiB range. - for i in 0..full_blocks { - let offset = i * BLOCK_LEN; + for (i, block) in input_blocks[..full_blocks].iter().enumerate() { let block_flags = flags | if i == 0 { CHUNK_START } else { 0 }; - // SAFETY: `offset + BLOCK_LEN <= input.len()` by construction. - let mut block_words = unsafe { words16_from_le_bytes_64(&*input.as_ptr().add(offset).cast::<[u8; BLOCK_LEN]>()) }; - cv = first_8_words((kernel.compress)(&cv, &block_words, 0, BLOCK_LEN as u32, block_flags)); + let mut block_words = words16_from_le_bytes_64(block); + cv = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut block_words); } @@ -3972,16 +3881,16 @@ fn xof_oneshot_single_chunk(kernel: Kernel, mut key_words: [u32; 8], flags: u32, // Stage the final block bytes for the root-output reader. let start = if full_blocks == 0 { CHUNK_START } else { 0 }; let mut block_bytes = [0u8; BLOCK_LEN]; - let offset = full_blocks * BLOCK_LEN; - block_bytes[..last_len].copy_from_slice(&input[offset..offset + last_len]); + let offset = full_blocks.strict_mul(BLOCK_LEN); + block_bytes[..last_len].copy_from_slice(&input[offset..offset.strict_add(last_len)]); let reader = Blake3XofReader::new(RootEmitState { kernel_id: kernel.id, input_chaining_value: cv, block_bytes, counter: 0, - block_len: last_len as u8, - flags: (flags | start | CHUNK_END) as u8, + block_len: u8::try_from(last_len).expect("BLAKE3 block length fits in u8"), + flags: u8::try_from(flags | start | CHUNK_END).expect("BLAKE3 flags fit in u8"), }); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut cv); @@ -4036,17 +3945,21 @@ impl Blake3XofReader { #[inline] fn fill_root_hash_prefix(&mut self, out: &mut &mut [u8]) { - let offset = self.position_within_block as usize; - let take = min(out.len(), OUT_LEN - offset); + let offset = usize::from(self.position_within_block); + let take = min(out.len(), OUT_LEN.strict_sub(offset)); self.root.emit_root_hash_prefix(offset, &mut out[..take]); - self.position_within_block += take as u8; + self.position_within_block = self + .position_within_block + .strict_add(u8::try_from(take).expect("BLAKE3 output position fits in u8")); *out = &mut core::mem::take(out)[take..]; } #[inline] fn fill_one_block(&mut self, out: &mut &mut [u8]) { - if self.root.counter == 0 && self.position_within_block < OUT_LEN as u8 { - let digest_bytes_remaining = OUT_LEN - self.position_within_block as usize; + if self.root.counter == 0 + && self.position_within_block < u8::try_from(OUT_LEN).expect("BLAKE3 digest length fits in u8") + { + let digest_bytes_remaining = OUT_LEN.strict_sub(usize::from(self.position_within_block)); if out.len() <= digest_bytes_remaining { self.fill_root_hash_prefix(out); return; @@ -4055,11 +3968,13 @@ impl Blake3XofReader { let mut block = [0u8; OUTPUT_BLOCK_LEN]; self.root.emit_one_block(&mut block); - let output_bytes = &block[self.position_within_block as usize..]; + let output_bytes = &block[usize::from(self.position_within_block)..]; let take = min(out.len(), output_bytes.len()); out[..take].copy_from_slice(&output_bytes[..take]); - self.position_within_block += take as u8; - if self.position_within_block == OUTPUT_BLOCK_LEN as u8 { + self.position_within_block = self + .position_within_block + .strict_add(u8::try_from(take).expect("BLAKE3 output position fits in u8")); + if self.position_within_block == u8::try_from(OUTPUT_BLOCK_LEN).expect("BLAKE3 output block length fits in u8") { self.root.counter = self.root.counter.wrapping_add(1); self.position_within_block = 0; } @@ -4080,11 +3995,14 @@ impl Blake3XofReader { self.fill_one_block(out); } - let full = out.len() / OUTPUT_BLOCK_LEN * OUTPUT_BLOCK_LEN; + let blocks = out.len().strict_div(OUTPUT_BLOCK_LEN); + let full = blocks.strict_mul(OUTPUT_BLOCK_LEN); if full != 0 { - let blocks = (full / OUTPUT_BLOCK_LEN) as u64; self.root.emit_blocks_into(&mut out[..full]); - self.root.counter = self.root.counter.wrapping_add(blocks); + self.root.counter = self + .root + .counter + .wrapping_add(u64::try_from(blocks).expect("BLAKE3 output block count fits in u64")); *out = &mut core::mem::take(out)[full..]; } @@ -4169,7 +4087,15 @@ fn compress_chunk_tail_to_root_words( kernels::Blake3KernelId::X86Sse41 | kernels::Blake3KernelId::X86Avx2 | kernels::Blake3KernelId::X86Avx512 => { // SAFETY: dispatch validates required CPU features before selecting // each x86 kernel; `block` is a readable 64-byte buffer. - let output = unsafe { (kernel.x86_compress_cv_bytes)(&cv, block.as_ptr(), 0, block_len as u32, final_flags) }; + let output = unsafe { + (kernel.x86_compress_cv_bytes)( + &cv, + block.as_ptr(), + 0, + u32::try_from(block_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ) + }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut cv); } @@ -4183,7 +4109,15 @@ fn compress_chunk_tail_to_root_words( { if kernel.id == kernels::Blake3KernelId::Aarch64Neon { // SAFETY: NEON availability validated by dispatch - let output = unsafe { aarch64::compress_cv_neon_bytes(&cv, block.as_ptr(), 0, block_len as u32, final_flags) }; + let output = unsafe { + aarch64::compress_cv_neon_bytes( + &cv, + block.as_ptr(), + 0, + u32::try_from(block_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ) + }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut cv); } @@ -4193,7 +4127,13 @@ fn compress_chunk_tail_to_root_words( // Portable fallback let mut block_words = words16_from_le_bytes_64(block); - let output = first_8_words((kernel.compress)(&cv, &block_words, 0, block_len as u32, final_flags)); + let output = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(block_len).expect("BLAKE3 block length fits in u32"), + final_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut cv); ct::zeroize_words_no_fence(&mut block_words); @@ -4240,26 +4180,31 @@ fn digest_one_chunk_root_hash_words_generic( let (full_blocks, last_len) = if input.is_empty() { (0usize, 0usize) } else { - let rem = input.len() % BLOCK_LEN; + let rem = input.len().strict_rem(BLOCK_LEN); if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + (input.len().strict_div(BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { - (input.len() / BLOCK_LEN, rem) + (input.len().strict_div(BLOCK_LEN), rem) } }; let mut cv = key_words; + let (input_blocks, _) = input.as_chunks::(); // Process full non-final blocks. Use the compress function pointer directly // instead of chunk_compress_blocks_inline to avoid the kernel-ID match // dispatch on every call — the Kernel struct already carries the resolved // function pointer. - for i in 0..full_blocks { - let offset = i * BLOCK_LEN; + for (i, block) in input_blocks[..full_blocks].iter().enumerate() { let block_flags = flags | if i == 0 { CHUNK_START } else { 0 }; - // SAFETY: `offset + BLOCK_LEN <= input.len()` by construction. - let mut block_words = unsafe { words16_from_le_bytes_64(&*input.as_ptr().add(offset).cast::<[u8; BLOCK_LEN]>()) }; - cv = first_8_words((kernel.compress)(&cv, &block_words, 0, BLOCK_LEN as u32, block_flags)); + let mut block_words = words16_from_le_bytes_64(block); + cv = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + block_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words(&mut block_words); } @@ -4269,10 +4214,14 @@ fn digest_one_chunk_root_hash_words_generic( let final_flags = flags | start | CHUNK_END | ROOT; if last_len == BLOCK_LEN && !input.is_empty() { - let offset = full_blocks * BLOCK_LEN; - // SAFETY: `offset + BLOCK_LEN <= input.len()` by construction. - let mut block_words = unsafe { words16_from_le_bytes_64(&*input.as_ptr().add(offset).cast::<[u8; BLOCK_LEN]>()) }; - let output = first_8_words((kernel.compress)(&cv, &block_words, 0, BLOCK_LEN as u32, final_flags)); + let mut block_words = words16_from_le_bytes_64(&input_blocks[full_blocks]); + let output = first_8_words((kernel.compress)( + &cv, + &block_words, + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + final_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_words_no_fence(&mut block_words); ct::zeroize_words_no_fence(&mut cv); @@ -4284,13 +4233,18 @@ fn digest_one_chunk_root_hash_words_generic( let mut final_block = [0u8; BLOCK_LEN]; if last_len != 0 { - let offset = full_blocks * BLOCK_LEN; - // SAFETY: `last_len < BLOCK_LEN` here, and source range is in-bounds. - unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), final_block.as_mut_ptr(), last_len) }; + let offset = full_blocks.strict_mul(BLOCK_LEN); + final_block[..last_len].copy_from_slice(&input[offset..offset.strict_add(last_len)]); } let mut final_words = words16_from_le_bytes_64(&final_block); - let output = first_8_words((kernel.compress)(&cv, &final_words, 0, last_len as u32, final_flags)); + let output = first_8_words((kernel.compress)( + &cv, + &final_words, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + )); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut final_block); ct::zeroize_words_no_fence(&mut final_words); @@ -4310,7 +4264,10 @@ fn use_avx2_hash_many_one_chunk_fast_path() -> bool { dispatch::avx2_hash_many_one_chunk_fast_path() } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(target_os = "linux", target_os = "macos", target_os = "windows") +))] #[inline] #[must_use] fn use_avx512_four_block_avx2_fast_path() -> bool { @@ -4343,6 +4300,12 @@ fn use_x86_hash_many_exact_block_one_chunk_fast_path(kernel: Kernel, input_len: #[cfg(target_arch = "x86_64")] #[inline] +/// Compresses an exact-block one-chunk prefix with AVX2. +/// +/// # Safety +/// +/// The current CPU must support AVX2, SSE4.1, and SSSE3. `input` must be +/// nonempty, no longer than one chunk, and contain only complete blocks. unsafe fn avx2_owned_exact_block_chain( mut key_words: [u32; 8], flags: u32, @@ -4360,7 +4323,7 @@ unsafe fn avx2_owned_exact_block_chain( if block_idx == 0 { block_flags |= CHUNK_START; } - if block_idx + 1 == blocks { + if block_idx.strict_add(1) == blocks { block_flags |= final_extra_flags; } @@ -4372,9 +4335,9 @@ unsafe fn avx2_owned_exact_block_chain( cv = unsafe { x86_64::compress_cv_avx2_bytes( &cv, - input.as_ptr().add(block_idx * BLOCK_LEN), + input.as_ptr().add(block_idx.strict_mul(BLOCK_LEN)), 0, - BLOCK_LEN as u32, + BLOCK_LEN_U32, block_flags, ) }; @@ -4390,6 +4353,13 @@ unsafe fn avx2_owned_exact_block_chain( #[cfg(all(feature = "diag", target_arch = "x86_64"))] #[inline] +/// Hashes one exact-block input through every AVX-512 lane. +/// +/// # Safety +/// +/// The current CPU must support the AVX-512 and AVX2 features required by +/// `hash16_owned`. `input` must be nonempty, no longer than one chunk, and +/// contain only complete blocks. unsafe fn avx512_owned_exact_block_hash_many( mut key_words: [u32; 8], flags: u32, @@ -4410,17 +4380,17 @@ unsafe fn avx512_owned_exact_block_hash_many( // 2. Every duplicated lane points at `input`, which is readable for `blocks * BLOCK_LEN` bytes. // 3. `out` is writable for the full 16-lane output contract; only lane 0 is returned. unsafe { - x86_64::avx512::hash16_owned( - &inputs, + x86_64::avx512::hash16_owned(x86_64::HashManyRequest { + inputs: &inputs, blocks, - &key_words, - 0, - false, + key: &key_words, + counter: 0, + increment_counter: false, flags, - CHUNK_START, - final_extra_flags, - out.as_mut_ptr(), - ); + flags_start: CHUNK_START, + flags_end: final_extra_flags, + out: out.as_mut_ptr(), + }); } let mut lane0 = [0u8; OUT_LEN]; @@ -4437,6 +4407,12 @@ unsafe fn avx512_owned_exact_block_hash_many( #[cfg(target_arch = "x86_64")] #[inline] +/// Builds an XOF reader for an exact-block x86 one-chunk input. +/// +/// # Safety +/// +/// The caller must have validated the target features required by `kernel`. +/// `input` must contain two through sixteen complete blocks. unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( kernel: Kernel, mut key_words: [u32; 8], @@ -4454,15 +4430,20 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( return None; } - let prefix_blocks = input.len() / BLOCK_LEN - 1; + let prefix_blocks = (input.len() / BLOCK_LEN).strict_sub(1); debug_assert!(prefix_blocks != 0); - debug_assert!(flags <= u8::MAX as u32); - debug_assert!((flags | CHUNK_START) <= u8::MAX as u32); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] + let flags_u8 = u8::try_from(flags).expect("BLAKE3 flags fit in u8"); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] + let flags_start_u8 = u8::try_from(flags | CHUNK_START).expect("BLAKE3 start flags fit in u8"); + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] let mut cv_bytes = [0u8; OUT_LEN]; + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + let mut cv_bytes: [u8; OUT_LEN]; match kernel.id { kernels::Blake3KernelId::X86Avx2 => { - let prefix_len = prefix_blocks * BLOCK_LEN; + let prefix_len = prefix_blocks.strict_mul(BLOCK_LEN); // SAFETY: AVX2 dispatch selected this kernel. `prefix_len` is non-zero, // exact-block aligned, and within `input`. let mut cv = unsafe { avx2_owned_exact_block_chain(key_words, flags, &input[..prefix_len], 0) }; @@ -4474,7 +4455,7 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( kernels::Blake3KernelId::X86Avx512 => { #[cfg(feature = "diag")] if kernel.owned_x86_hash_many { - let prefix_len = prefix_blocks * BLOCK_LEN; + let prefix_len = prefix_blocks.strict_mul(BLOCK_LEN); // SAFETY: Diagnostic availability checked the owned AVX-512 hash-many feature set, and the prefix // is exact-block aligned. let mut cv = unsafe { avx512_owned_exact_block_hash_many(key_words, flags, &input[..prefix_len], 0) }; @@ -4483,23 +4464,24 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( ct::zeroize_words(&mut cv); } } else { + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] let input_ptrs = [input.as_ptr()]; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] // SAFETY: this branch only runs for the AVX-512 kernel on supported OSes, // with one in-bounds exact-block input lane and a valid 32-byte output buffer. unsafe { - x86_64::asm::hash_many_avx512( - input_ptrs.as_ptr(), - 1, - prefix_blocks, - key_words.as_ptr(), - 0, - false, - flags as u8, - (flags | CHUNK_START) as u8, - flags as u8, - cv_bytes.as_mut_ptr(), - ); + x86_64::asm::hash_many_avx512(x86_64::asm::HashManyRequest { + inputs: input_ptrs.as_ptr(), + num_inputs: 1, + blocks: prefix_blocks, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: flags_u8, + flags_start: flags_start_u8, + flags_end: flags_u8, + out: cv_bytes.as_mut_ptr(), + }); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] { @@ -4511,23 +4493,24 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( } #[cfg(not(feature = "diag"))] { + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] let input_ptrs = [input.as_ptr()]; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] // SAFETY: this branch only runs for the AVX-512 kernel on supported OSes, // with one in-bounds exact-block input lane and a valid 32-byte output buffer. unsafe { - x86_64::asm::hash_many_avx512( - input_ptrs.as_ptr(), - 1, - prefix_blocks, - key_words.as_ptr(), - 0, - false, - flags as u8, - (flags | CHUNK_START) as u8, - flags as u8, - cv_bytes.as_mut_ptr(), - ); + x86_64::asm::hash_many_avx512(x86_64::asm::HashManyRequest { + inputs: input_ptrs.as_ptr(), + num_inputs: 1, + blocks: prefix_blocks, + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: flags_u8, + flags_start: flags_start_u8, + flags_end: flags_u8, + out: cv_bytes.as_mut_ptr(), + }); } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] { @@ -4547,7 +4530,7 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( } let mut block_bytes = [0u8; BLOCK_LEN]; - let last_offset = prefix_blocks * BLOCK_LEN; + let last_offset = prefix_blocks.strict_mul(BLOCK_LEN); // SAFETY: `last_offset + BLOCK_LEN <= input.len()` by construction. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(last_offset), block_bytes.as_mut_ptr(), BLOCK_LEN); @@ -4558,8 +4541,8 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( input_chaining_value: words8_from_le_bytes_32(&cv_bytes), block_bytes, counter: 0, - block_len: BLOCK_LEN as u8, - flags: (flags | CHUNK_END) as u8, + block_len: BLOCK_LEN_U8, + flags: u8::try_from(flags | CHUNK_END).expect("BLAKE3 end flags fit in u8"), }); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut cv_bytes); @@ -4572,6 +4555,12 @@ unsafe fn xof_oneshot_single_chunk_x86_exact_blocks( #[cfg(target_arch = "x86_64")] #[inline] +/// Hashes a one-chunk input with the selected x86 kernel. +/// +/// # Safety +/// +/// The caller must have validated the target features required by `kernel`, +/// and `input` must not exceed one chunk. unsafe fn digest_one_chunk_root_hash_words_x86( kernel: Kernel, mut key_words: [u32; 8], @@ -4587,9 +4576,6 @@ unsafe fn digest_one_chunk_root_hash_words_x86( { let blocks = input.len() / BLOCK_LEN; debug_assert!((1..=CHUNK_LEN / BLOCK_LEN).contains(&blocks)); - debug_assert!(flags <= u8::MAX as u32); - debug_assert!((flags | CHUNK_START) <= u8::MAX as u32); - debug_assert!((flags | CHUNK_END | ROOT) <= u8::MAX as u32); // SAFETY: AVX2 dispatch selected this kernel, and this branch is restricted // to non-empty exact-block one-chunk input. let output = unsafe { avx2_owned_exact_block_chain(key_words, flags, input, CHUNK_END | ROOT) }; @@ -4607,9 +4593,9 @@ unsafe fn digest_one_chunk_root_hash_words_x86( debug_assert!((1..=CHUNK_LEN / BLOCK_LEN).contains(&blocks)); let flags_start = flags | CHUNK_START; let flags_end = flags | CHUNK_END | ROOT; - debug_assert!(flags <= u8::MAX as u32); - debug_assert!(flags_start <= u8::MAX as u32); - debug_assert!(flags_end <= u8::MAX as u32); + let flags_u8 = u8::try_from(flags).expect("BLAKE3 flags fit in u8"); + let flags_start_u8 = u8::try_from(flags_start).expect("BLAKE3 start flags fit in u8"); + let flags_end_u8 = u8::try_from(flags_end).expect("BLAKE3 root flags fit in u8"); #[cfg(feature = "diag")] if kernel.owned_x86_hash_many { // SAFETY: Diagnostic availability checked the owned AVX-512 hash-many feature set, and `input` is @@ -4641,18 +4627,18 @@ unsafe fn digest_one_chunk_root_hash_words_x86( // SAFETY: AVX-512 dispatch selected this kernel; input is one contiguous // full-chunk-or-less buffer; output points to one OUT_LEN digest lane. unsafe { - x86_64::asm::hash_many_avx512( - input_ptrs.as_ptr(), - 1, + x86_64::asm::hash_many_avx512(x86_64::asm::HashManyRequest { + inputs: input_ptrs.as_ptr(), + num_inputs: 1, blocks, - key_words.as_ptr(), - 0, - false, - flags as u8, - flags_start as u8, - flags_end as u8, - out.as_mut_ptr(), - ); + key: key_words.as_ptr(), + counter: 0, + increment_counter: false, + flags: flags_u8, + flags_start: flags_start_u8, + flags_end: flags_end_u8, + out: out.as_mut_ptr(), + }); } let output = words8_from_le_bytes_32(&out); if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { @@ -4670,7 +4656,7 @@ unsafe fn digest_one_chunk_root_hash_words_x86( } else { let rem = input.len() % BLOCK_LEN; if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + ((input.len() / BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { (input.len() / BLOCK_LEN, rem) } @@ -4679,7 +4665,7 @@ unsafe fn digest_one_chunk_root_hash_words_x86( // Hash all full blocks except the final block, updating the CV. This keeps // ROOT out of the dependency chain until the last compress. let mut cv = key_words; - let full_bytes = full_blocks * BLOCK_LEN; + let full_bytes = full_blocks.strict_mul(BLOCK_LEN); if full_blocks != 0 { let first_block_ptr = input.as_ptr(); let first_flags = flags | CHUNK_START; @@ -4688,13 +4674,13 @@ unsafe fn digest_one_chunk_root_hash_words_x86( unsafe { match kernel.id { kernels::Blake3KernelId::X86Sse41 => { - x86_64::compress_in_place_sse41_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN as u32, first_flags); + x86_64::compress_in_place_sse41_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN_U32, first_flags); } kernels::Blake3KernelId::X86Avx2 => { - x86_64::compress_in_place_avx2_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN as u32, first_flags); + x86_64::compress_in_place_avx2_bytes(&mut cv, first_block_ptr, 0, BLOCK_LEN_U32, first_flags); } kernels::Blake3KernelId::X86Avx512 => { - cv = (kernel.x86_compress_cv_bytes)(&cv, first_block_ptr, 0, BLOCK_LEN as u32, first_flags); + cv = (kernel.x86_compress_cv_bytes)(&cv, first_block_ptr, 0, BLOCK_LEN_U32, first_flags); } _ => { let output = digest_one_chunk_root_hash_words_generic(kernel, key_words, flags, input); @@ -4719,18 +4705,18 @@ unsafe fn digest_one_chunk_root_hash_words_x86( if last_len == BLOCK_LEN && !input.is_empty() { // SAFETY: `full_blocks * BLOCK_LEN + BLOCK_LEN <= input.len()`. - let block_ptr = unsafe { input.as_ptr().add(full_blocks * BLOCK_LEN) }; + let block_ptr = unsafe { input.as_ptr().add(full_blocks.strict_mul(BLOCK_LEN)) }; // SAFETY: x86 dispatch selected this function only for x86 SIMD kernels. unsafe { match kernel.id { kernels::Blake3KernelId::X86Sse41 => { - x86_64::compress_in_place_sse41_bytes(&mut cv, block_ptr, 0, BLOCK_LEN as u32, final_flags); + x86_64::compress_in_place_sse41_bytes(&mut cv, block_ptr, 0, BLOCK_LEN_U32, final_flags); } kernels::Blake3KernelId::X86Avx2 => { - x86_64::compress_in_place_avx2_bytes(&mut cv, block_ptr, 0, BLOCK_LEN as u32, final_flags); + x86_64::compress_in_place_avx2_bytes(&mut cv, block_ptr, 0, BLOCK_LEN_U32, final_flags); } kernels::Blake3KernelId::X86Avx512 => { - cv = (kernel.x86_compress_cv_bytes)(&cv, block_ptr, 0, BLOCK_LEN as u32, final_flags); + cv = (kernel.x86_compress_cv_bytes)(&cv, block_ptr, 0, BLOCK_LEN_U32, final_flags); } _ => { let output = digest_one_chunk_root_hash_words_generic(kernel, key_words, flags, input); @@ -4755,7 +4741,7 @@ unsafe fn digest_one_chunk_root_hash_words_x86( // Partial final block (including empty): pad to 64 bytes. let mut padded = [0u8; BLOCK_LEN]; if last_len != 0 { - let offset = full_blocks * BLOCK_LEN; + let offset = full_blocks.strict_mul(BLOCK_LEN); // SAFETY: `padded` is 64 bytes, and `last_len < 64` here. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), padded.as_mut_ptr(), last_len) }; } @@ -4765,13 +4751,31 @@ unsafe fn digest_one_chunk_root_hash_words_x86( unsafe { match kernel.id { kernels::Blake3KernelId::X86Sse41 => { - x86_64::compress_in_place_sse41_bytes(&mut cv, block_ptr, 0, last_len as u32, final_flags); + x86_64::compress_in_place_sse41_bytes( + &mut cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ); } kernels::Blake3KernelId::X86Avx2 => { - x86_64::compress_in_place_avx2_bytes(&mut cv, block_ptr, 0, last_len as u32, final_flags); + x86_64::compress_in_place_avx2_bytes( + &mut cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ); } kernels::Blake3KernelId::X86Avx512 => { - cv = (kernel.x86_compress_cv_bytes)(&cv, block_ptr, 0, last_len as u32, final_flags); + cv = (kernel.x86_compress_cv_bytes)( + &cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ); } _ => { let output = digest_one_chunk_root_hash_words_generic(kernel, key_words, flags, input); @@ -4799,6 +4803,12 @@ unsafe fn digest_one_chunk_root_hash_words_x86( #[cfg(target_arch = "aarch64")] #[inline] +/// Hashes a one-chunk input with the selected AArch64 NEON kernel. +/// +/// # Safety +/// +/// The caller must ensure that the runtime CPU supports the features required +/// by `kernel` and that `kernel` is the AArch64 NEON implementation. unsafe fn digest_one_chunk_root_hash_words_aarch64( kernel: Kernel, mut key_words: [u32; 8], @@ -4813,22 +4823,30 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( let (full_blocks, last_len) = if input.is_empty() { (0usize, 0usize) } else { - let rem = input.len() % BLOCK_LEN; + let rem = input.len().strict_rem(BLOCK_LEN); if rem == 0 { - (input.len() / BLOCK_LEN - 1, BLOCK_LEN) + (input.len().strict_div(BLOCK_LEN).strict_sub(1), BLOCK_LEN) } else { - (input.len() / BLOCK_LEN, rem) + (input.len().strict_div(BLOCK_LEN), rem) } }; // Hash all full blocks except the final block, updating the CV. This keeps // ROOT out of the dependency chain until the last compress. let mut cv = key_words; - let full_bytes = full_blocks * BLOCK_LEN; + let full_bytes = full_blocks.strict_mul(BLOCK_LEN); if full_blocks != 0 { // SAFETY: `input` covers at least one full 64-byte block here, and this // helper is only selected when NEON support has already been validated. - cv = unsafe { aarch64::compress_cv_neon_bytes(&cv, input.as_ptr(), 0, BLOCK_LEN as u32, flags | CHUNK_START) }; + cv = unsafe { + aarch64::compress_cv_neon_bytes( + &cv, + input.as_ptr(), + 0, + u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), + flags | CHUNK_START, + ) + }; if full_blocks > 1 { let mut blocks_compressed: u8 = 1; kernels::chunk_compress_blocks_inline( @@ -4849,10 +4867,10 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( let mut padded = [0u8; BLOCK_LEN]; let block_ptr = if last_len == BLOCK_LEN && !input.is_empty() { // SAFETY: `full_blocks * BLOCK_LEN + BLOCK_LEN <= input.len()`. - unsafe { input.as_ptr().add(full_blocks * BLOCK_LEN) } + unsafe { input.as_ptr().add(full_blocks.strict_mul(BLOCK_LEN)) } } else { if last_len != 0 { - let offset = full_blocks * BLOCK_LEN; + let offset = full_blocks.strict_mul(BLOCK_LEN); // SAFETY: `padded` is 64 bytes, and `last_len < 64` here. unsafe { ptr::copy_nonoverlapping(input.as_ptr().add(offset), padded.as_mut_ptr(), last_len) }; } @@ -4861,7 +4879,15 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( // SAFETY: `block_ptr` points to 64 bytes (either into `input` or `padded`), // and dispatch only selects NEON when the required CPU features are present. - let output = unsafe { aarch64::compress_cv_neon_bytes(&cv, block_ptr, 0, last_len as u32, final_flags) }; + let output = unsafe { + aarch64::compress_cv_neon_bytes( + &cv, + block_ptr, + 0, + u32::try_from(last_len).expect("BLAKE3 block length fits in u32"), + final_flags, + ) + }; if flags & (KEYED_HASH | DERIVE_KEY_MATERIAL) != 0 { ct::zeroize_no_fence(&mut padded); ct::zeroize_words_no_fence(&mut cv); @@ -4873,7 +4899,9 @@ unsafe fn digest_one_chunk_root_hash_words_aarch64( #[cfg(test)] mod tests { - use super::{Blake3, Blake3KeyedHash, CHUNK_LEN, OUT_LEN}; + #[cfg(feature = "diag")] + use super::CHUNK_LEN; + use super::{Blake3, Blake3KeyedHash, OUT_LEN}; use crate::traits::{Digest, VerificationError, Xof}; #[test] @@ -4940,14 +4968,16 @@ mod tests { for len in [0usize, 1, 64, 256, 1024, 4096, 8192, 16384] { let input = input_pattern(len); - let digest = diag_blake3_digest_with_kernel(kernel, &input).unwrap(); + let digest = + diag_blake3_digest_with_kernel(kernel, &input).expect("available diagnostic kernel returns a digest"); assert_eq!(digest, *blake3::hash(&input).as_bytes(), "diag digest mismatch"); - let keyed = diag_blake3_keyed_digest_with_kernel(kernel, KEY, &input).unwrap(); + let keyed = diag_blake3_keyed_digest_with_kernel(kernel, KEY, &input) + .expect("available diagnostic kernel returns a digest"); assert_eq!(keyed.to_bytes(), *blake3::keyed_hash(KEY, &input).as_bytes()); let mut xof = [0u8; 64]; - diag_blake3_xof_with_kernel(kernel, &input, &mut xof).unwrap(); + diag_blake3_xof_with_kernel(kernel, &input, &mut xof).expect("available diagnostic kernel returns XOF output"); let mut expected_xof = [0u8; 64]; let mut official = blake3::Hasher::new(); official.update(&input); @@ -4955,7 +4985,8 @@ mod tests { assert_eq!(xof, expected_xof, "diag xof mismatch"); if kernel.supports_streaming() { - let streaming = diag_blake3_streaming_digest_with_kernel(kernel, &input, 64).unwrap(); + let streaming = diag_blake3_streaming_digest_with_kernel(kernel, &input, 64) + .expect("streaming-capable diagnostic kernel returns a digest"); assert_eq!(streaming, *blake3::hash(&input).as_bytes(), "diag streaming mismatch"); } } @@ -4997,16 +5028,18 @@ mod tests { ]; for &chunks in RAW_CV_COUNTS { - let input = input_pattern(chunks * CHUNK_LEN); - let mut expected = alloc::vec![0u8; chunks * OUT_LEN]; - diag_blake3_chunk_cvs_with_kernel(Blake3DiagKernel::Portable, &input, &mut expected).unwrap(); + let input = input_pattern(chunks.strict_mul(CHUNK_LEN)); + let mut expected = alloc::vec![0u8; chunks.strict_mul(OUT_LEN)]; + diag_blake3_chunk_cvs_with_kernel(Blake3DiagKernel::Portable, &input, &mut expected) + .expect("portable diagnostic kernel returns chunk chaining values"); for &kernel in KERNELS { if !diag_blake3_kernel_available(kernel) { continue; } - let mut actual = alloc::vec![0u8; chunks * OUT_LEN]; - diag_blake3_chunk_cvs_with_kernel(kernel, &input, &mut actual).unwrap(); + let mut actual = alloc::vec![0u8; chunks.strict_mul(OUT_LEN)]; + diag_blake3_chunk_cvs_with_kernel(kernel, &input, &mut actual) + .expect("available diagnostic kernel returns chunk chaining values"); assert_eq!( actual, expected, "diag chunk CV mismatch kernel={kernel:?} chunks={chunks}" @@ -5015,16 +5048,18 @@ mod tests { } for &parents in RAW_CV_COUNTS { - let children = input_pattern(parents * 2 * OUT_LEN); - let mut expected = alloc::vec![0u8; parents * OUT_LEN]; - diag_blake3_parent_cvs_with_kernel(Blake3DiagKernel::Portable, &children, &mut expected).unwrap(); + let children = input_pattern(parents.strict_mul(2).strict_mul(OUT_LEN)); + let mut expected = alloc::vec![0u8; parents.strict_mul(OUT_LEN)]; + diag_blake3_parent_cvs_with_kernel(Blake3DiagKernel::Portable, &children, &mut expected) + .expect("portable diagnostic kernel returns parent chaining values"); for &kernel in KERNELS { if !diag_blake3_kernel_available(kernel) { continue; } - let mut actual = alloc::vec![0u8; parents * OUT_LEN]; - diag_blake3_parent_cvs_with_kernel(kernel, &children, &mut actual).unwrap(); + let mut actual = alloc::vec![0u8; parents.strict_mul(OUT_LEN)]; + diag_blake3_parent_cvs_with_kernel(kernel, &children, &mut actual) + .expect("available diagnostic kernel returns parent chaining values"); assert_eq!( actual, expected, "diag parent CV mismatch kernel={kernel:?} parents={parents}" @@ -5037,18 +5072,24 @@ mod tests { const CONTEXT: &str = "BLAKE3 2019-12-27 16:29:52 test vectors context"; fn hex_to_bytes(hex: &str, out: &mut [u8]) { - assert_eq!(hex.len(), out.len() * 2); - for (i, chunk) in hex.as_bytes().chunks_exact(2).enumerate() { - let hi = (chunk[0] as char).to_digit(16).unwrap(); - let lo = (chunk[1] as char).to_digit(16).unwrap(); - out[i] = ((hi << 4) | lo) as u8; + assert_eq!(hex.len(), out.len().strict_mul(2)); + let (hex_bytes, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty()); + for (i, chunk) in hex_bytes.iter().enumerate() { + let hi = char::from(chunk[0]) + .to_digit(16) + .expect("test vector contains hexadecimal digits"); + let lo = char::from(chunk[1]) + .to_digit(16) + .expect("test vector contains hexadecimal digits"); + out[i] = u8::try_from(hi.strict_shl(4) | lo).expect("two hexadecimal digits fit in u8"); } } fn input_pattern(len: usize) -> alloc::vec::Vec { let mut v = alloc::vec::Vec::with_capacity(len); for i in 0..len { - v.push((i % 251) as u8); + v.push(u8::try_from(i.strict_rem(251)).expect("input pattern byte fits in u8")); } v } diff --git a/src/hashes/crypto/blake3/parallel.rs b/src/hashes/crypto/blake3/parallel.rs index 1324a24d..d83fe213 100644 --- a/src/hashes/crypto/blake3/parallel.rs +++ b/src/hashes/crypto/blake3/parallel.rs @@ -47,7 +47,7 @@ impl Drop for ParallelBatchScratch { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_blake3_parallel_scratch(input: [u32; 8]) -> u32 { +pub(super) fn diag_zeroize_blake3_parallel_scratch(input: [u32; 8]) -> u32 { let mut scratch = ParallelBatchScratch::default(); scratch.zeroize = true; scratch.roots.push(input); @@ -76,7 +76,7 @@ impl Blake3 { #[inline] fn push_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, mut cv: [u32; 8], zeroize: bool) { stack[*len].write(cv); - *len += 1; + *len = len.strict_add(1); if zeroize { ct::zeroize_words(&mut cv); } @@ -84,7 +84,7 @@ impl Blake3 { #[inline] fn pop_stack(stack: &mut [MaybeUninit<[u32; 8]>; CV_STACK_LEN], len: &mut usize, zeroize: bool) -> [u32; 8] { - *len -= 1; + *len = len.strict_sub(1); // SAFETY: `len` tracks the number of initialized entries. let cv = unsafe { stack[*len].assume_init_read() }; if zeroize { @@ -125,22 +125,28 @@ impl Blake3 { size = size.min(aligned_max).min(remaining_commit); debug_assert!(size.is_power_of_two()); - let bytes_base = offset_chunks * CHUNK_LEN; - let subtree_bytes = size * CHUNK_LEN; - let subtree_input = &batch_input[bytes_base..bytes_base + subtree_bytes]; + let bytes_base = offset_chunks.strict_mul(CHUNK_LEN); + let subtree_bytes = size.strict_mul(CHUNK_LEN); + let subtree_end = bytes_base.strict_add(subtree_bytes); + let subtree_input = &batch_input[bytes_base..subtree_end]; let bulk_kernel = kernels::kernel(self.bulk_kernel_id); - let mut subtree_cv = if size >= threads && (counter == 0 || (counter & (size as u64 - 1)) == 0) { + let size_u64 = u64::try_from(size).expect("a BLAKE3 subtree size fits in u64"); + let mut subtree_cv = if size >= threads && (counter == 0 || (counter & size_u64.strict_sub(1)) == 0) { const MAX_SUBTREE_CHUNKS: usize = 1 << 12; - let mut subtree_chunks = size / threads; + let mut subtree_chunks = size + .checked_div(threads) + .expect("parallel BLAKE3 thread count is nonzero"); subtree_chunks = subtree_chunks.max(1); subtree_chunks = pow2_floor(subtree_chunks); subtree_chunks = subtree_chunks.min(MAX_SUBTREE_CHUNKS).min(size); - let roots_len = size / subtree_chunks; + let roots_len = size + .checked_div(subtree_chunks) + .expect("parallel BLAKE3 subtree size is nonzero"); debug_assert!(roots_len.is_power_of_two()); - debug_assert_eq!(roots_len * subtree_chunks, size); + debug_assert_eq!(roots_len.strict_mul(subtree_chunks), size); scratch.roots.resize(roots_len, [0u32; 8]); hash_power_of_two_subtree_roots_parallel_rayon(super::SubtreeRootsRequest { @@ -181,7 +187,7 @@ impl Blake3 { ) }; - counter = counter.wrapping_add(size as u64); + counter = counter.strict_add(size_u64); let level = size.trailing_zeros(); let mut total = counter >> level; let mut cv = subtree_cv; @@ -208,12 +214,13 @@ impl Blake3 { } offset_chunks = offset_chunks.strict_add(size); - remaining_commit -= size; + remaining_commit = remaining_commit.strict_sub(size); } - self.cv_stack_len = stack_len as u8; + self.cv_stack_len = u8::try_from(stack_len).expect("BLAKE3 CV stack length fits in u8"); - let new_counter = base_counter.strict_add(batch.batch_chunks as u64); + let batch_chunks = u64::try_from(batch.batch_chunks).expect("a BLAKE3 batch size fits in u64"); + let new_counter = base_counter.strict_add(batch_chunks); self.chunk_state = ChunkState::new( self.key_words, new_counter, @@ -253,12 +260,16 @@ impl Blake3 { let base_counter = self.chunk_state.chunk_counter; let keep_last_full_chunk = input.len().is_multiple_of(CHUNK_LEN) && batch == full_chunks; - let commit = if keep_last_full_chunk { batch - 1 } else { batch }; + let commit = if keep_last_full_chunk { + batch.strict_sub(1) + } else { + batch + }; if commit == 0 { return None; } - let bytes = batch * CHUNK_LEN; + let bytes = batch.strict_mul(CHUNK_LEN); let threads = self.streaming_parallel_threads(bytes, batch, commit)?; if threads <= 1 { return None; @@ -310,9 +321,15 @@ fn compress_parents_parallel_bytes( debug_assert!(out.len() >= child_cvs.len().div_ceil(2)); let pairs = child_cvs.len() / 2; - kernels::parent_cvs_many_from_bytes_inline(kernel.id, &child_cvs[..pairs * 2], key_words, flags, &mut out[..pairs]); + kernels::parent_cvs_many_from_bytes_inline( + kernel.id, + &child_cvs[..pairs.strict_mul(2)], + key_words, + flags, + &mut out[..pairs], + ); if (child_cvs.len() & 1) == 1 { - out[pairs] = child_cvs[child_cvs.len() - 1]; + out[pairs] = child_cvs[child_cvs.len().strict_sub(1)]; pairs.strict_add(1) } else { pairs @@ -334,10 +351,10 @@ fn compress_subtree_wide_bytes( let simd_degree = kernel.id.simd_degree(); let max_leaf_bytes = simd_degree.strict_mul(CHUNK_LEN); if input.len() <= max_leaf_bytes { - let chunks_exact = input.chunks_exact(CHUNK_LEN); - let full_chunks = chunks_exact.len(); + let (chunks, remainder) = input.as_chunks::(); + let full_chunks = chunks.len(); debug_assert!(full_chunks <= simd_degree); - debug_assert!(out.len() >= full_chunks.strict_add(usize::from(!chunks_exact.remainder().is_empty()))); + debug_assert!(out.len() >= full_chunks.strict_add(usize::from(!remainder.is_empty()))); if full_chunks != 0 { // SAFETY: `input` has at least `full_chunks * CHUNK_LEN` bytes and @@ -355,12 +372,12 @@ fn compress_subtree_wide_bytes( } let mut out_len = full_chunks; - let rem = chunks_exact.remainder(); + let rem = remainder; if !rem.is_empty() { let mut cv_words = single_chunk_output( kernel, key_words, - chunk_counter.strict_add(full_chunks as u64), + chunk_counter.strict_add(u64::try_from(full_chunks).expect("a BLAKE3 chunk count fits in u64")), flags, rem, ) @@ -378,7 +395,8 @@ fn compress_subtree_wide_bytes( debug_assert!(out.len() >= simd_degree.max(2)); let left_len = left_subtree_len_bytes(input.len()); let (left, right) = input.split_at(left_len); - let right_chunk_counter = chunk_counter.strict_add((left.len() / CHUNK_LEN) as u64); + let left_chunks = u64::try_from(left.len() / CHUNK_LEN).expect("a BLAKE3 subtree chunk count fits in u64"); + let right_chunk_counter = chunk_counter.strict_add(left_chunks); const MAX_SIMD_DEGREE: usize = 16; let mut cv_array = [[0u8; OUT_LEN]; 2 * MAX_SIMD_DEGREE]; @@ -494,7 +512,7 @@ pub(super) fn root_output_oneshot_join_parallel( debug_assert!(threads > 1); // Cap Rayon recursion depth to approximately match `threads` leaves. - let depth = (usize::BITS - 1 - threads.leading_zeros()) as usize; + let depth = usize::try_from(threads.ilog2()).expect("parallel recursion depth fits in usize"); let budget = depth.max(1); let mut parent_block = @@ -505,7 +523,7 @@ pub(super) fn root_output_oneshot_join_parallel( input_chaining_value: key_words, block_words, counter: 0, - block_len: BLOCK_LEN as u32, + block_len: u32::try_from(BLOCK_LEN).expect("BLAKE3 block length fits in u32"), flags: super::PARENT | flags, }; if flags & (super::KEYED_HASH | super::DERIVE_KEY_MATERIAL) != 0 { diff --git a/src/hashes/crypto/blake3/x86_64.rs b/src/hashes/crypto/blake3/x86_64.rs index 55ee44fd..e7e64421 100644 --- a/src/hashes/crypto/blake3/x86_64.rs +++ b/src/hashes/crypto/blake3/x86_64.rs @@ -8,13 +8,8 @@ //! //! # Safety //! -//! All functions in this module are marked `unsafe` and require specific CPU -//! features to be present. Callers must verify CPU capabilities before calling. - -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] +//! Unsafe entry points require specific CPU features to be present. Callers +//! must verify CPU capabilities before calling them. #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -25,25 +20,55 @@ pub(crate) mod avx2; pub(crate) mod avx512; pub(crate) mod sse41; -use super::{BLOCK_LEN, CHUNK_START, IV, PARENT}; +use super::{BLOCK_LEN, BLOCK_LEN_U32, CHUNK_START, IV, PARENT}; + +/// Inputs shared by the x86 intrinsic hash-many kernels. +pub(crate) struct HashManyRequest<'a, const DEGREE: usize> { + /// One pointer per SIMD lane. + pub(crate) inputs: &'a [*const u8; DEGREE], + /// Complete BLAKE3 blocks in every input. + pub(crate) blocks: usize, + /// Eight-word chaining key. + pub(crate) key: &'a [u32; 8], + /// Initial chunk counter. + pub(crate) counter: u64, + /// Whether to advance the counter for each lane. + pub(crate) increment_counter: bool, + /// Flags applied to every block. + pub(crate) flags: u32, + /// Flags added to the first block. + pub(crate) flags_start: u32, + /// Flags added to the last block. + pub(crate) flags_end: u32, + /// Destination for one chaining value per lane. + pub(crate) out: *mut u8, +} // Shared helpers for SIMD kernels. #[inline(always)] pub(crate) const fn counter_low(counter: u64) -> u32 { - counter as u32 + let [c0, c1, c2, c3, _, _, _, _] = counter.to_le_bytes(); + u32::from_le_bytes([c0, c1, c2, c3]) } #[inline(always)] pub(crate) const fn counter_high(counter: u64) -> u32 { - (counter >> 32) as u32 + let [_, _, _, _, c4, c5, c6, c7] = counter.to_le_bytes(); + u32::from_le_bytes([c4, c5, c6, c7]) } // CV-only compression helpers (avoid `[u32; 16]` materialization) #[cfg(target_arch = "x86_64")] #[inline(always)] -unsafe fn load_msg_vecs(block: *const u8) -> (__m128i, __m128i, __m128i, __m128i) { +/// Loads one unaligned BLAKE3 block into four SIMD registers. +/// +/// # Safety +/// +/// The current CPU must support SSE2, and `block` must be readable for +/// `BLOCK_LEN` bytes. +unsafe fn load_msg_vecs(block: *const u8) -> [__m128i; 4] { // SAFETY: Caller guarantees block pointer is valid for 64 bytes. Intrinsics require SSSE3 via // caller's #[target_feature]. unsafe { @@ -51,12 +76,18 @@ unsafe fn load_msg_vecs(block: *const u8) -> (__m128i, __m128i, __m128i, __m128i let m1 = _mm_loadu_si128(block.add(16).cast()); let m2 = _mm_loadu_si128(block.add(32).cast()); let m3 = _mm_loadu_si128(block.add(48).cast()); - (m0, m1, m2, m3) + [m0, m1, m2, m3] } } #[cfg(target_arch = "x86_64")] #[target_feature(enable = "sse4.1,ssse3")] +/// Compresses one byte-oriented block in place with SSE4.1. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3, and `block` must be readable +/// for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_in_place_sse41_bytes( chaining_value: &mut [u32; 8], block: *const u8, @@ -66,8 +97,8 @@ pub(crate) unsafe fn compress_in_place_sse41_bytes( ) { // SAFETY: SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let message = load_msg_vecs(block); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); _mm_storeu_si128(chaining_value.as_mut_ptr().cast(), _mm_xor_si128(row0, row2)); _mm_storeu_si128(chaining_value.as_mut_ptr().add(4).cast(), _mm_xor_si128(row1, row3)); } @@ -75,6 +106,12 @@ pub(crate) unsafe fn compress_in_place_sse41_bytes( #[cfg(target_arch = "x86_64")] #[target_feature(enable = "sse4.1,ssse3")] +/// Compresses one byte-oriented block to a chaining value with SSE4.1. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3, and `block` must be readable +/// for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_cv_sse41_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -84,8 +121,8 @@ pub(crate) unsafe fn compress_cv_sse41_bytes( ) -> [u32; 8] { // SAFETY: SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let message = load_msg_vecs(block); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); @@ -96,6 +133,12 @@ pub(crate) unsafe fn compress_cv_sse41_bytes( } } +/// Compresses one byte-oriented block to a chaining value with AVX2. +/// +/// # Safety +/// +/// The current CPU must support AVX2, SSE4.1, and SSSE3, and `block` must be +/// readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_cv_avx2_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -105,13 +148,19 @@ pub(crate) unsafe fn compress_cv_avx2_bytes( ) -> [u32; 8] { // SAFETY: AVX2/SSE4.1/SSSE3 intrinsics are available via caller's target_feature guarantee. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - compress_cv_avx2(chaining_value, m0, m1, m2, m3, counter, block_len, flags) + let message = load_msg_vecs(block); + compress_cv_avx2(chaining_value, message, counter, block_len, flags) } } #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2,sse4.1,ssse3")] +/// Compresses one byte-oriented block in place with AVX2. +/// +/// # Safety +/// +/// The current CPU must support AVX2, SSE4.1, and SSSE3, and `block` must be +/// readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_in_place_avx2_bytes( chaining_value: &mut [u32; 8], block: *const u8, @@ -128,12 +177,17 @@ pub(crate) unsafe fn compress_in_place_avx2_bytes( // On ASM-supported platforms, we prefer the handwritten assembly. This intrinsics // version is kept as fallback for other x86_64 platforms (e.g., FreeBSD, illumos). -#[cfg(target_arch = "x86_64")] -#[cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code) -)] +#[cfg(any( + feature = "diag", + not(any(target_os = "linux", target_os = "macos", target_os = "windows")) +))] #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] +/// Compresses one byte-oriented block in place with AVX-512. +/// +/// # Safety +/// +/// The current CPU must support the declared AVX-512, AVX2, SSE4.1, and SSSE3 +/// features, and `block` must be readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_in_place_avx512_bytes( chaining_value: &mut [u32; 8], block: *const u8, @@ -144,19 +198,24 @@ pub(crate) unsafe fn compress_in_place_avx512_bytes( // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let message = load_msg_vecs(block); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); _mm_storeu_si128(chaining_value.as_mut_ptr().cast(), _mm_xor_si128(row0, row2)); _mm_storeu_si128(chaining_value.as_mut_ptr().add(4).cast(), _mm_xor_si128(row1, row3)); } } -#[cfg(target_arch = "x86_64")] -#[cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code) -)] +#[cfg(any( + feature = "diag", + not(any(target_os = "linux", target_os = "macos", target_os = "windows")) +))] #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] +/// Compresses one byte-oriented block to a chaining value with AVX-512. +/// +/// # Safety +/// +/// The current CPU must support the declared AVX-512, AVX2, SSE4.1, and SSSE3 +/// features, and `block` must be readable for `BLOCK_LEN` bytes. pub(crate) unsafe fn compress_cv_avx512_bytes( chaining_value: &[u32; 8], block: *const u8, @@ -167,8 +226,8 @@ pub(crate) unsafe fn compress_cv_avx512_bytes( // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let (m0, m1, m2, m3) = load_msg_vecs(block); - compress_cv_avx512(chaining_value, m0, m1, m2, m3, counter, block_len, flags) + let message = load_msg_vecs(block); + compress_cv_avx512(chaining_value, message, counter, block_len, flags) } } @@ -182,6 +241,9 @@ pub(crate) unsafe fn compress_cv_avx512_bytes( #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot16_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 16), _mm_slli_epi32(a, 16)) } @@ -189,6 +251,9 @@ unsafe fn rot16_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot12_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 12), _mm_slli_epi32(a, 20)) } @@ -196,6 +261,9 @@ unsafe fn rot12_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot8_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 8), _mm_slli_epi32(a, 24)) } @@ -203,6 +271,9 @@ unsafe fn rot8_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn rot7_sse41(a: __m128i) -> __m128i { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { _mm_or_si128(_mm_srli_epi32(a, 7), _mm_slli_epi32(a, 25)) } @@ -210,6 +281,11 @@ unsafe fn rot7_sse41(a: __m128i) -> __m128i { #[cfg(target_arch = "x86_64")] #[inline(always)] +/// Executes the first half of the BLAKE3 mixing function. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3. unsafe fn g1_sse41(row0: &mut __m128i, row1: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i, m: __m128i) { // SAFETY: SSE4.1/SSSE3 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -224,6 +300,11 @@ unsafe fn g1_sse41(row0: &mut __m128i, row1: &mut __m128i, row2: &mut __m128i, r #[cfg(target_arch = "x86_64")] #[inline(always)] +/// Executes the second half of the BLAKE3 mixing function. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3. unsafe fn g2_sse41(row0: &mut __m128i, row1: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i, m: __m128i) { // SAFETY: SSE4.1/SSSE3 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -251,6 +332,9 @@ macro_rules! shuffle2 { // Leave row1 unrotated and diagonalize the other rows. #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn diagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -262,6 +346,9 @@ unsafe fn diagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut _ #[cfg(target_arch = "x86_64")] #[inline(always)] +/// # Safety +/// +/// The current CPU must support SSE2. unsafe fn undiagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut __m128i) { // SAFETY: SSE2 intrinsics are available via caller's #[target_feature] attribute. unsafe { @@ -273,18 +360,21 @@ unsafe fn undiagonalize_sse41(row0: &mut __m128i, row2: &mut __m128i, row3: &mut #[cfg(target_arch = "x86_64")] #[inline(always)] +/// Runs the shared SSE4.1 compression schedule without the output transform. +/// +/// # Safety +/// +/// The current CPU must support SSE4.1 and SSSE3. unsafe fn compress_pre_sse41_impl( chaining_value: &[u32; 8], - mut m0: __m128i, - mut m1: __m128i, - mut m2: __m128i, - mut m3: __m128i, + message: [__m128i; 4], counter: u64, block_len: u32, flags: u32, ) -> [__m128i; 4] { // SAFETY: SSE4.1/SSSE3 intrinsics are available via caller's #[target_feature] attribute. unsafe { + let [mut m0, mut m1, mut m2, mut m3] = message; let mut row0 = _mm_loadu_si128(chaining_value.as_ptr().cast()); let mut row1 = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); let mut row2 = _mm_setr_epi32( @@ -294,8 +384,8 @@ unsafe fn compress_pre_sse41_impl( IV[3].cast_signed(), ); let mut row3 = _mm_setr_epi32( - (counter as u32).cast_signed(), - ((counter >> 32) as u32).cast_signed(), + counter_low(counter).cast_signed(), + counter_high(counter).cast_signed(), block_len.cast_signed(), flags.cast_signed(), ); @@ -396,10 +486,7 @@ unsafe fn compress_pre_sse41_impl( #[target_feature(enable = "avx2,sse4.1,ssse3")] unsafe fn compress_cv_avx2( chaining_value: &[u32; 8], - m0: __m128i, - m1: __m128i, - m2: __m128i, - m3: __m128i, + message: [__m128i; 4], counter: u64, block_len: u32, flags: u32, @@ -407,7 +494,7 @@ unsafe fn compress_cv_avx2( // SAFETY: AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); let mut out = [0u32; 8]; @@ -425,10 +512,7 @@ unsafe fn compress_cv_avx2( #[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] unsafe fn compress_cv_avx512( chaining_value: &[u32; 8], - m0: __m128i, - m1: __m128i, - m2: __m128i, - m3: __m128i, + message: [__m128i; 4], counter: u64, block_len: u32, flags: u32, @@ -436,7 +520,7 @@ unsafe fn compress_cv_avx512( // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's #[target_feature] // attribute. unsafe { - let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + let [row0, row1, row2, row3] = compress_pre_sse41_impl(chaining_value, message, counter, block_len, flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); let mut out = [0u32; 8]; @@ -472,7 +556,7 @@ pub(crate) unsafe fn compress_avx2( let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); let [mut row0, mut row1, mut row2, mut row3] = - compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + compress_pre_sse41_impl(chaining_value, [m0, m1, m2, m3], counter, block_len, flags); let cv_lo = _mm_loadu_si128(chaining_value.as_ptr().cast()); let cv_hi = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); @@ -515,7 +599,7 @@ pub(crate) unsafe fn compress_sse41( let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); let [mut row0, mut row1, mut row2, mut row3] = - compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + compress_pre_sse41_impl(chaining_value, [m0, m1, m2, m3], counter, block_len, flags); let cv_lo = _mm_loadu_si128(chaining_value.as_ptr().cast()); let cv_hi = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); @@ -558,10 +642,10 @@ pub(crate) unsafe fn chunk_compress_blocks_sse41( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -573,10 +657,10 @@ pub(crate) unsafe fn chunk_compress_blocks_sse41( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -600,7 +684,7 @@ pub(crate) unsafe fn parent_cv_sse41( let m2 = _mm_loadu_si128(right_child_cv.as_ptr().cast()); let m3 = _mm_loadu_si128(right_child_cv.as_ptr().add(4).cast()); let [row0, row1, row2, row3] = - compress_pre_sse41_impl(&key_words, m0, m1, m2, m3, 0, BLOCK_LEN as u32, PARENT | flags); + compress_pre_sse41_impl(&key_words, [m0, m1, m2, m3], 0, BLOCK_LEN_U32, PARENT | flags); let row0 = _mm_xor_si128(row0, row2); let row1 = _mm_xor_si128(row1, row3); let mut out = [0u32; 8]; @@ -636,10 +720,10 @@ pub(crate) unsafe fn chunk_compress_blocks_avx2( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -651,10 +735,10 @@ pub(crate) unsafe fn chunk_compress_blocks_avx2( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -678,7 +762,7 @@ pub(crate) unsafe fn parent_cv_avx2( let m1 = _mm_loadu_si128(left_child_cv.as_ptr().add(4).cast()); let m2 = _mm_loadu_si128(right_child_cv.as_ptr().cast()); let m3 = _mm_loadu_si128(right_child_cv.as_ptr().add(4).cast()); - compress_cv_avx2(&key_words, m0, m1, m2, m3, 0, BLOCK_LEN as u32, PARENT | flags) + compress_cv_avx2(&key_words, [m0, m1, m2, m3], 0, BLOCK_LEN_U32, PARENT | flags) } } @@ -730,7 +814,7 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } @@ -740,11 +824,11 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, blocks.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); return; } @@ -758,7 +842,7 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } @@ -768,11 +852,11 @@ pub(crate) unsafe fn chunk_compress_blocks_avx512( chaining_value, block_bytes.as_ptr(), chunk_counter, - BLOCK_LEN as u32, + BLOCK_LEN_U32, flags | start, ); } - *blocks_compressed = blocks_compressed.wrapping_add(1); + *blocks_compressed = blocks_compressed.strict_add(1); } } } @@ -796,6 +880,6 @@ pub(crate) unsafe fn parent_cv_avx512( let m1 = _mm_loadu_si128(left_child_cv.as_ptr().add(4).cast()); let m2 = _mm_loadu_si128(right_child_cv.as_ptr().cast()); let m3 = _mm_loadu_si128(right_child_cv.as_ptr().add(4).cast()); - compress_cv_avx512(&key_words, m0, m1, m2, m3, 0, BLOCK_LEN as u32, PARENT | flags) + compress_cv_avx512(&key_words, [m0, m1, m2, m3], 0, BLOCK_LEN_U32, PARENT | flags) } } diff --git a/src/hashes/crypto/blake3/x86_64/asm.rs b/src/hashes/crypto/blake3/x86_64/asm.rs index 7b58649d..c29a4a34 100644 --- a/src/hashes/crypto/blake3/x86_64/asm.rs +++ b/src/hashes/crypto/blake3/x86_64/asm.rs @@ -8,8 +8,6 @@ //! preprocessor conditionals removed and symbols renamed. //! - The implementation is compiled via `global_asm!` (no external objects). -#![allow(unsafe_code)] - use core::arch::global_asm; #[cfg(target_os = "linux")] @@ -33,7 +31,7 @@ global_asm!(include_str!("asm/rscrypto_blake3_avx512_x86-64_windows_msvc.s")); // entrypoints are internal to rscrypto and are only called from our own code. #[cfg(target_os = "windows")] unsafe extern "sysv64" { - pub fn rscrypto_blake3_hash_many_avx2( + fn rscrypto_blake3_hash_many_avx2( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -46,7 +44,7 @@ unsafe extern "sysv64" { out: *mut u8, ); - pub fn rscrypto_blake3_hash_many_avx512( + fn rscrypto_blake3_hash_many_avx512( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -59,7 +57,7 @@ unsafe extern "sysv64" { out: *mut u8, ); - pub fn rscrypto_blake3_xof_many_avx512( + fn rscrypto_blake3_xof_many_avx512( cv: *const u32, block: *const u8, block_len: u8, @@ -69,19 +67,13 @@ unsafe extern "sysv64" { outblocks: usize, ); - pub fn rscrypto_blake3_compress_in_place_avx512( - cv: *mut u32, - block: *const u8, - counter: u64, - block_len: u8, - flags: u8, - ); + fn rscrypto_blake3_compress_in_place_avx512(cv: *mut u32, block: *const u8, counter: u64, block_len: u8, flags: u8); } #[cfg(not(target_os = "windows"))] unsafe extern "C" { - pub fn rscrypto_blake3_hash_many_avx2( + fn rscrypto_blake3_hash_many_avx2( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -94,7 +86,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_hash_many_avx512( + fn rscrypto_blake3_hash_many_avx512( inputs: *const *const u8, num_inputs: usize, blocks: usize, @@ -107,7 +99,7 @@ unsafe extern "C" { out: *mut u8, ); - pub fn rscrypto_blake3_xof_many_avx512( + fn rscrypto_blake3_xof_many_avx512( cv: *const u32, block: *const u8, block_len: u8, @@ -117,16 +109,34 @@ unsafe extern "C" { outblocks: usize, ); - pub fn rscrypto_blake3_compress_in_place_avx512( - cv: *mut u32, - block: *const u8, - counter: u64, - block_len: u8, - flags: u8, - ); + fn rscrypto_blake3_compress_in_place_avx512(cv: *mut u32, block: *const u8, counter: u64, block_len: u8, flags: u8); } +/// Raw parameters for an x86 BLAKE3 hash-many assembly call. +pub(crate) struct HashManyRequest { + /// Input pointer array. + pub(crate) inputs: *const *const u8, + /// Number of input pointers. + pub(crate) num_inputs: usize, + /// Complete BLAKE3 blocks in each input. + pub(crate) blocks: usize, + /// Eight-word chaining key. + pub(crate) key: *const u32, + /// Initial chunk counter. + pub(crate) counter: u64, + /// Whether to advance the counter for each input. + pub(crate) increment_counter: bool, + /// Flags applied to every block. + pub(crate) flags: u8, + /// Flags added to the first block. + pub(crate) flags_start: u8, + /// Flags added to the last block. + pub(crate) flags_end: u8, + /// Destination for one chaining value per input. + pub(crate) out: *mut u8, +} + /// AVX2 `hash_many` assembly entrypoint. /// /// # Safety @@ -140,18 +150,19 @@ unsafe extern "C" { /// 6. The output range does not alias any input range or `key`. /// 7. `num_inputs`, `blocks`, counters, and flags are public values. #[inline(always)] -pub(crate) unsafe fn hash_many_avx2( - inputs: *const *const u8, - num_inputs: usize, - blocks: usize, - key: *const u32, - counter: u64, - increment_counter: bool, - flags: u8, - flags_start: u8, - flags_end: u8, - out: *mut u8, -) { +pub(crate) unsafe fn hash_many_avx2(request: HashManyRequest) { + let HashManyRequest { + inputs, + num_inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + } = request; // SAFETY: AVX2 `hash_many` FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. // 2. The caller upholds the input pointer, key, output, and aliasing contracts. @@ -185,18 +196,19 @@ pub(crate) unsafe fn hash_many_avx2( /// 6. The output range does not alias any input range or `key`. /// 7. `num_inputs`, `blocks`, counters, and flags are public values. #[inline(always)] -pub(crate) unsafe fn hash_many_avx512( - inputs: *const *const u8, - num_inputs: usize, - blocks: usize, - key: *const u32, - counter: u64, - increment_counter: bool, - flags: u8, - flags_start: u8, - flags_end: u8, - out: *mut u8, -) { +pub(crate) unsafe fn hash_many_avx512(request: HashManyRequest) { + let HashManyRequest { + inputs, + num_inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + } = request; // SAFETY: AVX-512 `hash_many` FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. // 2. The caller upholds the input pointer, key, output, and aliasing contracts. @@ -265,8 +277,8 @@ pub(crate) unsafe fn compress_in_place_avx512( block_len: u32, flags: u32, ) -> [u32; 8] { - debug_assert!(block_len <= u8::MAX as u32); - debug_assert!(flags <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length fits the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags fit the assembly ABI"); let mut cv_out = *cv; // SAFETY: AVX-512 single-block FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. @@ -274,7 +286,7 @@ pub(crate) unsafe fn compress_in_place_avx512( // 3. The caller guarantees `block` is readable for one BLAKE3 block. // 4. Debug assertions document the ABI narrowing for `block_len` and `flags`. unsafe { - rscrypto_blake3_compress_in_place_avx512(cv_out.as_mut_ptr(), block, counter, block_len as u8, flags as u8); + rscrypto_blake3_compress_in_place_avx512(cv_out.as_mut_ptr(), block, counter, block_len, flags); } cv_out } @@ -297,14 +309,14 @@ pub(crate) unsafe fn compress_in_place_avx512_mut( block_len: u32, flags: u32, ) { - debug_assert!(block_len <= u8::MAX as u32); - debug_assert!(flags <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length fits the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags fit the assembly ABI"); // SAFETY: AVX-512 in-place single-block FFI call because: // 1. The caller upholds this wrapper's CPU-feature contract. // 2. `cv` is writable for 8 words and the mutable borrow prevents another Rust alias. // 3. The caller guarantees `block` is readable for one BLAKE3 block and does not alias `cv`. // 4. Debug assertions document the ABI narrowing for `block_len` and `flags`. unsafe { - rscrypto_blake3_compress_in_place_avx512(cv.as_mut_ptr(), block, counter, block_len as u8, flags as u8); + rscrypto_blake3_compress_in_place_avx512(cv.as_mut_ptr(), block, counter, block_len, flags); } } diff --git a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s index 80cfa159..1d01f25e 100644 --- a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s +++ b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx2_x86-64_apple_darwin.s @@ -5,7 +5,6 @@ // Symbols are renamed into the rscrypto namespace and the Darwin-specific // symbol spelling is kept explicit for Rust global_asm!. -.intel_syntax noprefix diff --git a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s index 00177327..449a5f91 100644 --- a/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s +++ b/src/hashes/crypto/blake3/x86_64/asm/rscrypto_blake3_avx512_x86-64_apple_darwin.s @@ -5,7 +5,6 @@ // Symbols are renamed into the rscrypto namespace and the Darwin-specific // symbol spelling is kept explicit for Rust global_asm!. -.intel_syntax noprefix diff --git a/src/hashes/crypto/blake3/x86_64/avx2.rs b/src/hashes/crypto/blake3/x86_64/avx2.rs index e0fb67c6..4401abc3 100644 --- a/src/hashes/crypto/blake3/x86_64/avx2.rs +++ b/src/hashes/crypto/blake3/x86_64/avx2.rs @@ -1,26 +1,23 @@ //! BLAKE3 x86_64 AVX2 throughput kernel (8-way). -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] // Supported OS builds still keep the assembly ABI around for sub-degree tails // and parent reductions. The owned intrinsic body is also used directly for // full contiguous batches and by diagnostic benches. -#![cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code, unused_imports) -)] use core::arch::x86_64::*; +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] +use super::super::{CHUNK_END, CHUNK_LEN, CHUNK_START, OUT_LEN, PARENT}; use super::{ - super::{BLOCK_LEN, CHUNK_END, CHUNK_LEN, CHUNK_START, IV, MSG_SCHEDULE, OUT_LEN, PARENT}, - counter_high, counter_low, + super::{BLOCK_LEN, BLOCK_LEN_U32, IV, MSG_SCHEDULE}, + HashManyRequest, counter_high, counter_low, }; -pub const DEGREE: usize = 8; +pub(crate) const DEGREE: usize = 8; +/// # Safety +/// +/// AVX2 must be available and `src` must be readable for 32 bytes. #[inline(always)] unsafe fn loadu(src: *const u8) -> __m256i { // SAFETY: Unaligned AVX2 load because: @@ -30,6 +27,9 @@ unsafe fn loadu(src: *const u8) -> __m256i { unsafe { _mm256_loadu_si256(src.cast()) } } +/// # Safety +/// +/// AVX2 must be available and `dest` must be writable for 32 bytes. #[inline(always)] unsafe fn storeu(src: __m256i, dest: *mut u8) { // SAFETY: Unaligned AVX2 store because: @@ -39,6 +39,9 @@ unsafe fn storeu(src: __m256i, dest: *mut u8) { unsafe { _mm256_storeu_si256(dest.cast(), src) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn add(a: __m256i, b: __m256i) -> __m256i { // SAFETY: AVX2 lane add because: @@ -47,6 +50,9 @@ unsafe fn add(a: __m256i, b: __m256i) -> __m256i { unsafe { _mm256_add_epi32(a, b) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn xor(a: __m256i, b: __m256i) -> __m256i { // SAFETY: AVX2 lane xor because: @@ -55,6 +61,9 @@ unsafe fn xor(a: __m256i, b: __m256i) -> __m256i { unsafe { _mm256_xor_si256(a, b) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn set1(x: u32) -> __m256i { // SAFETY: AVX2 scalar broadcast because: @@ -63,25 +72,31 @@ unsafe fn set1(x: u32) -> __m256i { unsafe { _mm256_set1_epi32(x.cast_signed()) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] -unsafe fn set8(a: u32, b: u32, c: u32, d: u32, e: u32, f: u32, g: u32, h: u32) -> __m256i { +unsafe fn set8(lanes: [u32; DEGREE]) -> __m256i { // SAFETY: AVX2 lane construction because: // 1. Any `u32` bit pattern is valid lane data after reinterpretation as `i32`. // 2. The caller only reaches this helper with AVX2 available. unsafe { _mm256_setr_epi32( - a.cast_signed(), - b.cast_signed(), - c.cast_signed(), - d.cast_signed(), - e.cast_signed(), - f.cast_signed(), - g.cast_signed(), - h.cast_signed(), + lanes[0].cast_signed(), + lanes[1].cast_signed(), + lanes[2].cast_signed(), + lanes[3].cast_signed(), + lanes[4].cast_signed(), + lanes[5].cast_signed(), + lanes[6].cast_signed(), + lanes[7].cast_signed(), ) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn rot12(x: __m256i) -> __m256i { // SAFETY: AVX2 rotate-right-by-12 sequence because: @@ -90,6 +105,9 @@ unsafe fn rot12(x: __m256i) -> __m256i { unsafe { _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 20)) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn rot7(x: __m256i) -> __m256i { // SAFETY: AVX2 rotate-right-by-7 sequence because: @@ -98,6 +116,9 @@ unsafe fn rot7(x: __m256i) -> __m256i { unsafe { _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 25)) } } +/// # Safety +/// +/// AVX2 must be available and `r` must be in `0..7`. #[inline(always)] unsafe fn round(v: &mut [__m256i; 16], m: &[__m256i; 16], r: usize, rot16_mask: __m256i, rot8_mask: __m256i) { // SAFETY: One AVX2 BLAKE3 round because: @@ -223,6 +244,9 @@ unsafe fn round(v: &mut [__m256i; 16], m: &[__m256i; 16], r: usize, rot16_mask: } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn interleave128(a: __m256i, b: __m256i) -> (__m256i, __m256i) { // SAFETY: AVX2 128-bit lane interleave because: @@ -237,6 +261,9 @@ unsafe fn interleave128(a: __m256i, b: __m256i) -> (__m256i, __m256i) { } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] pub(super) unsafe fn transpose8x8(vecs: &mut [__m256i; 8]) { // SAFETY: AVX2 8x8 register transpose because: @@ -278,6 +305,10 @@ pub(super) unsafe fn transpose8x8(vecs: &mut [__m256i; 8]) { } } +/// # Safety +/// +/// AVX2 must be available and every input must be readable for a complete +/// block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m256i; 16] { // SAFETY: AVX2 message transpose because: @@ -287,7 +318,7 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) // dereferenced by Rust. // 4. The transpose operates only on local vector registers. unsafe { - let stride = 4 * DEGREE; + let stride = 4usize.strict_mul(DEGREE); let mut half0 = [ loadu(inputs[0].add(block_offset)), loadu(inputs[1].add(block_offset)), @@ -299,18 +330,21 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) loadu(inputs[7].add(block_offset)), ]; let mut half1 = [ - loadu(inputs[0].add(block_offset + stride)), - loadu(inputs[1].add(block_offset + stride)), - loadu(inputs[2].add(block_offset + stride)), - loadu(inputs[3].add(block_offset + stride)), - loadu(inputs[4].add(block_offset + stride)), - loadu(inputs[5].add(block_offset + stride)), - loadu(inputs[6].add(block_offset + stride)), - loadu(inputs[7].add(block_offset + stride)), + loadu(inputs[0].add(block_offset.strict_add(stride))), + loadu(inputs[1].add(block_offset.strict_add(stride))), + loadu(inputs[2].add(block_offset.strict_add(stride))), + loadu(inputs[3].add(block_offset.strict_add(stride))), + loadu(inputs[4].add(block_offset.strict_add(stride))), + loadu(inputs[5].add(block_offset.strict_add(stride))), + loadu(inputs[6].add(block_offset.strict_add(stride))), + loadu(inputs[7].add(block_offset.strict_add(stride))), ]; for &input in inputs.iter() { - _mm_prefetch(input.wrapping_add(block_offset + 256).cast::(), _MM_HINT_T0); + _mm_prefetch( + input.wrapping_add(block_offset.strict_add(256)).cast::(), + _MM_HINT_T0, + ); } transpose8x8(&mut half0); @@ -323,6 +357,9 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) } } +/// # Safety +/// +/// AVX2 must be available. #[inline(always)] unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m256i) { let mask = if increment_counter { !0u64 } else { 0u64 }; @@ -332,7 +369,7 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m2 // 3. The caller only reaches this helper with AVX2 available. unsafe { ( - set8( + set8([ counter_low(counter), counter_low(counter.wrapping_add(mask & 1)), counter_low(counter.wrapping_add(mask & 2)), @@ -341,8 +378,8 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m2 counter_low(counter.wrapping_add(mask & 5)), counter_low(counter.wrapping_add(mask & 6)), counter_low(counter.wrapping_add(mask & 7)), - ), - set8( + ]), + set8([ counter_high(counter), counter_high(counter.wrapping_add(mask & 1)), counter_high(counter.wrapping_add(mask & 2)), @@ -351,23 +388,30 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m256i, __m2 counter_high(counter.wrapping_add(mask & 5)), counter_high(counter.wrapping_add(mask & 6)), counter_high(counter.wrapping_add(mask & 7)), - ), + ]), ) } } +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] macro_rules! avx2_shuffle { ($z:expr, $y:expr, $x:expr, $w:expr) => { ($z << 6) | ($y << 4) | ($x << 2) | $w }; } +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] macro_rules! shuffle2 { ($a:expr, $b:expr, $c:expr) => { _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps($a), _mm256_castsi256_ps($b), $c)) }; } +/// # Safety +/// +/// AVX2 must be available, and both pointers must be readable for 16 bytes +/// starting at `offset`. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn load2x128(lo: *const u8, hi: *const u8, offset: usize) -> __m256i { // SAFETY: Loading two 128-bit halves into one YMM register because: @@ -381,6 +425,10 @@ unsafe fn load2x128(lo: *const u8, hi: *const u8, offset: usize) -> __m256i { } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn set2x128(row: __m128i) -> __m256i { // SAFETY: Duplicating one 128-bit row into both halves because: @@ -389,6 +437,10 @@ unsafe fn set2x128(row: __m128i) -> __m256i { unsafe { _mm256_broadcastsi128_si256(row) } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn g1_2( row0: &mut __m256i, @@ -412,6 +464,10 @@ unsafe fn g1_2( } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn g2_2( row0: &mut __m256i, @@ -435,6 +491,10 @@ unsafe fn g2_2( } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn diagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m256i) { // SAFETY: Two-lane AVX2 diagonalization because: @@ -447,6 +507,10 @@ unsafe fn diagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m25 } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn undiagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m256i) { // SAFETY: Two-lane AVX2 undiagonalization because: @@ -459,16 +523,14 @@ unsafe fn undiagonalize_2(row0: &mut __m256i, row2: &mut __m256i, row3: &mut __m } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn compress2_pre( - mut row0: __m256i, - mut row1: __m256i, - mut row2: __m256i, - mut row3: __m256i, - mut m0: __m256i, - mut m1: __m256i, - mut m2: __m256i, - mut m3: __m256i, + [mut row0, mut row1, mut row2, mut row3]: [__m256i; 4], + [mut m0, mut m1, mut m2, mut m3]: [__m256i; 4], ) -> [__m256i; 4] { // SAFETY: Two-lane AVX2 compression preimage because: // 1. The caller only reaches this helper with AVX2 available. @@ -565,6 +627,10 @@ unsafe fn compress2_pre( } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn iv_row2x128() -> __m256i { // SAFETY: Duplicating the fixed BLAKE3 IV row because: @@ -580,6 +646,10 @@ unsafe fn iv_row2x128() -> __m256i { } } +/// # Safety +/// +/// AVX2 must be available and `out` must be writable for two chaining values. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn store2_cvs(row0: __m256i, row1: __m256i, out: *mut u8) { // SAFETY: Storing two 32-byte CVs from two independent 128-bit lanes because: @@ -590,10 +660,17 @@ unsafe fn store2_cvs(row0: __m256i, row1: __m256i, out: *mut u8) { _mm_storeu_si128(out.cast(), _mm256_castsi256_si128(row0)); _mm_storeu_si128(out.add(16).cast(), _mm256_castsi256_si128(row1)); _mm_storeu_si128(out.add(OUT_LEN).cast(), _mm256_extracti128_si256(row0, 1)); - _mm_storeu_si128(out.add(OUT_LEN + 16).cast(), _mm256_extracti128_si256(row1, 1)); + _mm_storeu_si128( + out.add(OUT_LEN.strict_add(16)).cast(), + _mm256_extracti128_si256(row1, 1), + ); } } +/// # Safety +/// +/// AVX2 must be available. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[inline(always)] unsafe fn compress2_parent_pre( key: &[u32; 8], @@ -609,19 +686,18 @@ unsafe fn compress2_parent_pre( // 3. `m0..m3` are the two 64-byte parent blocks. unsafe { compress2_pre( - set2x128(_mm_loadu_si128(key.as_ptr().cast())), - set2x128(_mm_loadu_si128(key.as_ptr().add(4).cast())), - iv_row2x128(), - set2x128(_mm_setr_epi32( - 0, - 0, - (BLOCK_LEN as u32).cast_signed(), - (PARENT | flags).cast_signed(), - )), - m0, - m1, - m2, - m3, + [ + set2x128(_mm_loadu_si128(key.as_ptr().cast())), + set2x128(_mm_loadu_si128(key.as_ptr().add(4).cast())), + iv_row2x128(), + set2x128(_mm_setr_epi32( + 0, + 0, + BLOCK_LEN_U32.cast_signed(), + (PARENT | flags).cast_signed(), + )), + ], + [m0, m1, m2, m3], ) } } @@ -634,6 +710,7 @@ unsafe fn compress2_parent_pre( /// 1. AVX2 is available on the current CPU. /// 2. `parents[0]` and `parents[1]` are each readable for one 64-byte parent block. /// 3. `out` is writable for two 32-byte CV outputs. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[target_feature(enable = "avx2")] pub(crate) unsafe fn parent_cv2_owned(parents: &[*const u8; 2], key: &[u32; 8], flags: u32, out: *mut u8) { // SAFETY: Two-parent AVX2 CV reduction because: @@ -663,6 +740,7 @@ pub(crate) unsafe fn parent_cv2_owned(parents: &[*const u8; 2], key: &[u32; 8], /// 1. AVX2 is available on the current CPU. /// 2. `input` is readable for two full BLAKE3 chunks. /// 3. `out` is writable for two 32-byte CV outputs. +#[cfg(any(feature = "diag", target_os = "linux", target_os = "macos", target_os = "windows"))] #[target_feature(enable = "avx2")] pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { // SAFETY: Two-chunk AVX2 CV reduction because: @@ -680,14 +758,14 @@ pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counte let counter1 = counter.wrapping_add(1); let counter1_low = counter_low(counter1).cast_signed(); let counter1_high = counter_high(counter1).cast_signed(); - let block_len = (BLOCK_LEN as u32).cast_signed(); + let block_len = BLOCK_LEN_U32.cast_signed(); for block_idx in 0..(CHUNK_LEN / BLOCK_LEN) { let mut block_flags = flags; if block_idx == 0 { block_flags |= CHUNK_START; } - if block_idx + 1 == CHUNK_LEN / BLOCK_LEN { + if block_idx.strict_add(1) == CHUNK_LEN / BLOCK_LEN { block_flags |= CHUNK_END; } @@ -702,16 +780,15 @@ pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counte _mm_setr_epi32(counter1_low, counter1_high, block_len, block_flags.cast_signed()), 1, ); - let offset = block_idx * BLOCK_LEN; + let offset = block_idx.strict_mul(BLOCK_LEN); let [mut v0, mut v1, v2, v3] = compress2_pre( - row0, - row1, - row2, - row3, - load2x128(input, input1, offset), - load2x128(input, input1, offset + 16), - load2x128(input, input1, offset + 32), - load2x128(input, input1, offset + 48), + [row0, row1, row2, row3], + [ + load2x128(input, input1, offset), + load2x128(input, input1, offset.strict_add(16)), + load2x128(input, input1, offset.strict_add(32)), + load2x128(input, input1, offset.strict_add(48)), + ], ); v0 = xor(v0, v2); v1 = xor(v1, v3); @@ -735,15 +812,17 @@ pub(crate) unsafe fn hash2_chunks_owned(input: *const u8, key: &[u32; 8], counte /// `DEGREE * OUT_LEN` writable bytes. #[target_feature(enable = "avx2")] pub(crate) unsafe fn hash8_owned( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, + HashManyRequest { + inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + }: HashManyRequest<'_, DEGREE>, ) { // SAFETY: 8-way AVX2 BLAKE3 hash-many because: // 1. The caller guarantees AVX2 availability for this target-feature function. @@ -758,7 +837,7 @@ pub(crate) unsafe fn hash8_owned( 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12, ); - let block_len_vec = set1(BLOCK_LEN as u32); + let block_len_vec = set1(BLOCK_LEN_U32); let iv0 = set1(IV[0]); let iv1 = set1(IV[1]); let iv2 = set1(IV[2]); @@ -782,12 +861,12 @@ pub(crate) unsafe fn hash8_owned( if block == 0 { block_flags |= flags_start; } - if block + 1 == blocks { + if block.strict_add(1) == blocks { block_flags |= flags_end; } let block_flags_vec = set1(block_flags); - let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN); + let msg_vecs = transpose_msg_vecs(inputs, block.strict_mul(BLOCK_LEN)); let mut v = [ h_vecs[0], @@ -829,15 +908,15 @@ pub(crate) unsafe fn hash8_owned( // Unlike SSE4.1, this transpose yields output vecs already ordered by word. transpose8x8(&mut h_vecs); - let stride = 4 * DEGREE; + let stride = 4usize.strict_mul(DEGREE); storeu(h_vecs[0], out); storeu(h_vecs[1], out.add(stride)); - storeu(h_vecs[2], out.add(2 * stride)); - storeu(h_vecs[3], out.add(3 * stride)); - storeu(h_vecs[4], out.add(4 * stride)); - storeu(h_vecs[5], out.add(5 * stride)); - storeu(h_vecs[6], out.add(6 * stride)); - storeu(h_vecs[7], out.add(7 * stride)); + storeu(h_vecs[2], out.add(2usize.strict_mul(stride))); + storeu(h_vecs[3], out.add(3usize.strict_mul(stride))); + storeu(h_vecs[4], out.add(4usize.strict_mul(stride))); + storeu(h_vecs[5], out.add(5usize.strict_mul(stride))); + storeu(h_vecs[6], out.add(6usize.strict_mul(stride))); + storeu(h_vecs[7], out.add(7usize.strict_mul(stride))); } } @@ -849,33 +928,11 @@ pub(crate) unsafe fn hash8_owned( /// for `blocks * BLOCK_LEN` bytes. #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn hash8( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, -) { +pub(crate) unsafe fn hash8(request: HashManyRequest<'_, DEGREE>) { // SAFETY: Forwarding to the owned AVX2 implementation because: // 1. This function has the same AVX2 target-feature requirement. // 2. The caller's pointer/output contract is identical to `hash8_owned`. - unsafe { - hash8_owned( - inputs, - blocks, - key, - counter, - increment_counter, - flags, - flags_start, - flags_end, - out, - ) - } + unsafe { hash8_owned(request) } } /// Generate 8 root output blocks (64 bytes each) in parallel. @@ -999,7 +1056,7 @@ pub(crate) unsafe fn root_output_blocks8( transpose8x8(&mut out_hi); for lane in 0..DEGREE { - let base = out.add(lane * 64); + let base = out.add(lane.strict_mul(64)); storeu(out_lo[lane], base); storeu(out_hi[lane], base.add(32)); } diff --git a/src/hashes/crypto/blake3/x86_64/avx512.rs b/src/hashes/crypto/blake3/x86_64/avx512.rs index 8388d138..aee01992 100644 --- a/src/hashes/crypto/blake3/x86_64/avx512.rs +++ b/src/hashes/crypto/blake3/x86_64/avx512.rs @@ -1,57 +1,64 @@ //! BLAKE3 x86_64 AVX-512 throughput kernel (16-way). -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] -// On Linux we currently prefer the upstream asm implementation; keep the -// intrinsic fallback compiled but don't let `-D warnings` turn it into a build -// failure. -#![cfg_attr( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - allow(dead_code, unused_imports) -)] - use core::arch::x86_64::*; use super::{ - super::{BLOCK_LEN, CHUNK_LEN, IV, MSG_SCHEDULE, OUT_LEN}, + super::{BLOCK_LEN, BLOCK_LEN_U32, CHUNK_LEN, IV, MSG_SCHEDULE, OUT_LEN}, + HashManyRequest, avx2::transpose8x8, counter_high, counter_low, }; -pub const DEGREE: usize = 16; +pub(crate) const DEGREE: usize = 16; +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn add(a: __m512i, b: __m512i) -> __m512i { // SAFETY: Caller guarantees the required AVX-512 feature set for this backend. unsafe { _mm512_add_epi32(a, b) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn xor(a: __m512i, b: __m512i) -> __m512i { // SAFETY: Caller guarantees the required AVX-512 feature set for this backend. unsafe { _mm512_xor_si512(a, b) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn set1(x: u32) -> __m512i { // SAFETY: Caller guarantees the required AVX-512 feature set for this backend. unsafe { _mm512_set1_epi32(x.cast_signed()) } } +/// # Safety +/// +/// AVX2 must be available and `src` must be readable for 32 bytes. #[inline(always)] unsafe fn loadu256(src: *const u8) -> __m256i { // SAFETY: Caller guarantees `src` is valid to read 32 bytes and has enabled this AVX-512 backend. unsafe { _mm256_loadu_si256(src.cast()) } } +/// # Safety +/// +/// AVX2 must be available and `dest` must be writable for 32 bytes. #[inline(always)] unsafe fn storeu256(src: __m256i, dest: *mut u8) { // SAFETY: Caller guarantees `dest` is valid to write 32 bytes and has enabled this AVX-512 backend. unsafe { _mm256_storeu_si256(dest.cast(), src) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot16(x: __m512i) -> __m512i { // 32-bit rotate by 16. Prefer `vprold` over shift/or. @@ -59,6 +66,9 @@ unsafe fn rot16(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 16) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot12(x: __m512i) -> __m512i { // Rotate right by 12 == rotate left by 20. @@ -66,6 +76,9 @@ unsafe fn rot12(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 20) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot8(x: __m512i) -> __m512i { // Rotate right by 8 == rotate left by 24. @@ -73,6 +86,9 @@ unsafe fn rot8(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 24) } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn rot7(x: __m512i) -> __m512i { // Rotate right by 7 == rotate left by 25. @@ -80,6 +96,9 @@ unsafe fn rot7(x: __m512i) -> __m512i { unsafe { _mm512_rol_epi32(x, 25) } } +/// # Safety +/// +/// AVX-512F must be available and `r` must be in `0..7`. #[inline(always)] unsafe fn round(v: &mut [__m512i; 16], m: &[__m512i; 16], r: usize) { // SAFETY: Caller guarantees this AVX-512 backend is active; all vector lanes are local registers. @@ -200,6 +219,9 @@ unsafe fn round(v: &mut [__m512i; 16], m: &[__m512i; 16], r: usize) { } } +/// # Safety +/// +/// AVX-512F must be available. #[inline(always)] unsafe fn counter_vec(counter: u64, increment_counter: bool) -> (__m512i, __m512i) { let mask = if increment_counter { !0u64 } else { 0u64 }; @@ -245,11 +267,15 @@ unsafe fn counter_vec(counter: u64, increment_counter: bool) -> (__m512i, __m512 } } +/// # Safety +/// +/// AVX2 must be available and every input must be readable for a complete +/// block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs8(inputs: &[*const u8; 8], block_offset: usize) -> [__m256i; 16] { // SAFETY: Caller guarantees each input points to at least one full block at `block_offset`. unsafe { - let stride = 4 * 8; + let stride = 4usize.strict_mul(8); let mut half0 = [ loadu256(inputs[0].add(block_offset)), loadu256(inputs[1].add(block_offset)), @@ -261,18 +287,21 @@ unsafe fn transpose_msg_vecs8(inputs: &[*const u8; 8], block_offset: usize) -> [ loadu256(inputs[7].add(block_offset)), ]; let mut half1 = [ - loadu256(inputs[0].add(block_offset + stride)), - loadu256(inputs[1].add(block_offset + stride)), - loadu256(inputs[2].add(block_offset + stride)), - loadu256(inputs[3].add(block_offset + stride)), - loadu256(inputs[4].add(block_offset + stride)), - loadu256(inputs[5].add(block_offset + stride)), - loadu256(inputs[6].add(block_offset + stride)), - loadu256(inputs[7].add(block_offset + stride)), + loadu256(inputs[0].add(block_offset.strict_add(stride))), + loadu256(inputs[1].add(block_offset.strict_add(stride))), + loadu256(inputs[2].add(block_offset.strict_add(stride))), + loadu256(inputs[3].add(block_offset.strict_add(stride))), + loadu256(inputs[4].add(block_offset.strict_add(stride))), + loadu256(inputs[5].add(block_offset.strict_add(stride))), + loadu256(inputs[6].add(block_offset.strict_add(stride))), + loadu256(inputs[7].add(block_offset.strict_add(stride))), ]; for &input in inputs.iter() { - _mm_prefetch(input.wrapping_add(block_offset + 256).cast::(), _MM_HINT_T0); + _mm_prefetch( + input.wrapping_add(block_offset.strict_add(256)).cast::(), + _MM_HINT_T0, + ); } transpose8x8(&mut half0); @@ -285,6 +314,10 @@ unsafe fn transpose_msg_vecs8(inputs: &[*const u8; 8], block_offset: usize) -> [ } } +/// # Safety +/// +/// AVX-512F, AVX-512DQ, and AVX2 must be available. Every input must be +/// readable for a complete block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs16(inputs: &[*const u8; 16], block_offset: usize) -> [__m512i; 16] { // SAFETY: Caller guarantees each input points to at least one full block at `block_offset`. @@ -310,63 +343,6 @@ unsafe fn transpose_msg_vecs16(inputs: &[*const u8; 16], block_offset: usize) -> } } -/// Hash 16 contiguous independent inputs in parallel. -/// -/// This is optimized for the contiguous chunk hashing hot path, where inputs -/// are arranged as `CHUNK_LEN`-byte blocks back-to-back. -/// -/// # Safety -/// Caller must ensure AVX-512 is available, and `input`/`out` are valid for -/// `DEGREE * CHUNK_LEN` and `DEGREE * OUT_LEN` bytes respectively. -#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] -// Match upstream: AVX-512 detection is based on `avx512f` + `avx512vl`. -// The Linux backend delegates to upstream-grade asm, so we intentionally do -// not require BW/DQ here. -#[target_feature(enable = "avx512f,avx512vl,avx2")] -pub(crate) unsafe fn hash16_contiguous(input: *const u8, key: &[u32; 8], counter: u64, flags: u32, out: *mut u8) { - // Delegate to the upstream-grade AVX-512 asm implementation on Linux. - // - // The upstream function accepts an array of input pointers, so we build it - // from the contiguous layout. - debug_assert!(flags <= u8::MAX as u32); - // SAFETY: Caller guarantees AVX-512F/VL+AVX2 availability, `input` readable for 16 chunks, and - // `out` writable for `DEGREE * OUT_LEN`. The asm entrypoint shares that contract. - unsafe { - let inputs = [ - input, - input.add(CHUNK_LEN), - input.add(2 * CHUNK_LEN), - input.add(3 * CHUNK_LEN), - input.add(4 * CHUNK_LEN), - input.add(5 * CHUNK_LEN), - input.add(6 * CHUNK_LEN), - input.add(7 * CHUNK_LEN), - input.add(8 * CHUNK_LEN), - input.add(9 * CHUNK_LEN), - input.add(10 * CHUNK_LEN), - input.add(11 * CHUNK_LEN), - input.add(12 * CHUNK_LEN), - input.add(13 * CHUNK_LEN), - input.add(14 * CHUNK_LEN), - input.add(15 * CHUNK_LEN), - ]; - let flags_start = (flags | super::super::CHUNK_START) as u8; - let flags_end = (flags | super::super::CHUNK_END) as u8; - super::asm::rscrypto_blake3_hash_many_avx512( - inputs.as_ptr(), - DEGREE, - CHUNK_LEN / BLOCK_LEN, - key.as_ptr(), - counter, - true, - flags as u8, - flags_start, - flags_end, - out, - ); - } -} - /// Owned Rust-intrinsic implementation of the 16-way contiguous hash-many kernel. /// /// This stays callable on platforms where production dispatch still prefers @@ -379,15 +355,17 @@ pub(crate) unsafe fn hash16_contiguous(input: *const u8, key: &[u32; 8], counter /// `out` is valid for `DEGREE * OUT_LEN` writable bytes. #[target_feature(enable = "avx512f,avx512vl,avx512dq,avx2")] pub(crate) unsafe fn hash16_owned( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, + HashManyRequest { + inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + }: HashManyRequest<'_, DEGREE>, ) { // SAFETY: 16-way AVX-512 BLAKE3 contiguous hash-many because: // 1. The caller guarantees AVX-512F/VL/DQ plus AVX2 availability. @@ -395,7 +373,7 @@ pub(crate) unsafe fn hash16_owned( // 3. `out` is writable for `DEGREE * OUT_LEN` bytes. // 4. All lane pointers and stores are bounded by fixed-size local arrays. unsafe { - let block_len_vec = set1(BLOCK_LEN as u32); + let block_len_vec = set1(BLOCK_LEN_U32); let iv0 = set1(IV[0]); let iv1 = set1(IV[1]); let iv2 = set1(IV[2]); @@ -419,13 +397,13 @@ pub(crate) unsafe fn hash16_owned( if block == 0 { block_flags |= flags_start; } - if block + 1 == blocks { + if block.strict_add(1) == blocks { block_flags |= flags_end; } let block_flags_vec = set1(block_flags); - let m = transpose_msg_vecs16(inputs, block * BLOCK_LEN); + let m = transpose_msg_vecs16(inputs, block.strict_mul(BLOCK_LEN)); let mut v = [ h_vecs[0], @@ -476,8 +454,8 @@ pub(crate) unsafe fn hash16_owned( transpose8x8(&mut hi); for chunk in 0..8 { - storeu256(lo[chunk], out.add(chunk * OUT_LEN)); - storeu256(hi[chunk], out.add((chunk + 8) * OUT_LEN)); + storeu256(lo[chunk], out.add(chunk.strict_mul(OUT_LEN))); + storeu256(hi[chunk], out.add(chunk.strict_add(8).strict_mul(OUT_LEN))); } } } @@ -518,17 +496,17 @@ pub(crate) unsafe fn hash16_contiguous_owned(input: *const u8, key: &[u32; 8], c input.add(14 * CHUNK_LEN), input.add(15 * CHUNK_LEN), ]; - hash16_owned( - &inputs, - CHUNK_LEN / BLOCK_LEN, + hash16_owned(HashManyRequest { + inputs: &inputs, + blocks: CHUNK_LEN / BLOCK_LEN, key, counter, - true, + increment_counter: true, flags, - super::super::CHUNK_START, - super::super::CHUNK_END, + flags_start: super::super::CHUNK_START, + flags_end: super::super::CHUNK_END, out, - ); + }); } } @@ -569,21 +547,21 @@ pub(crate) unsafe fn root_output_blocks16( ) { #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] { - debug_assert!(flags <= u8::MAX as u32); - debug_assert!(block_len <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length must fit the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags must fit the assembly ABI"); // SAFETY: AVX-512 XOF assembly call because: // 1. This target-feature function requires the AVX-512 features used by the wrapper. // 2. `chaining_value` and `block_words` are fixed-size readable arrays. // 3. The caller guarantees `out` is writable for `16 * 64` bytes. - // 4. `block_len` and `flags` are debug-checked to fit the assembly ABI. + // 4. `block_len` and `flags` were converted without loss to the assembly ABI types. // 5. Counters, block length, flags, and output block count are public values. unsafe { super::asm::xof_many_avx512( chaining_value.as_ptr(), block_words.as_ptr().cast(), - block_len as u8, + block_len, counter, - flags as u8, + flags, out, 16, ); @@ -591,6 +569,9 @@ pub(crate) unsafe fn root_output_blocks16( } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + // SAFETY: Running the intrinsic AVX-512 root-output implementation because this function's + // target-feature contract provides AVX-512F/VL and AVX2, and the caller provides 16 writable + // output blocks. unsafe { let cv_vecs = [ set1(chaining_value[0]), @@ -685,8 +666,8 @@ pub(crate) unsafe fn root_output_blocks16( for i in 0..8 { lo0[i] = _mm512_castsi512_si256(out_words[i]); hi0[i] = _mm512_extracti64x4_epi64(out_words[i], 1); - lo1[i] = _mm512_castsi512_si256(out_words[i + 8]); - hi1[i] = _mm512_extracti64x4_epi64(out_words[i + 8], 1); + lo1[i] = _mm512_castsi512_si256(out_words[i.strict_add(8)]); + hi1[i] = _mm512_extracti64x4_epi64(out_words[i.strict_add(8)], 1); } transpose8x8(&mut lo0); @@ -694,13 +675,13 @@ pub(crate) unsafe fn root_output_blocks16( transpose8x8(&mut lo1); transpose8x8(&mut hi1); - for lane in 0..8 { - let base = out.add(lane * 64); + for lane in 0usize..8 { + let base = out.add(lane.strict_mul(64)); storeu256(lo0[lane], base); storeu256(lo1[lane], base.add(32)); } - for lane in 0..8 { - let base = out.add((lane + 8) * 64); + for lane in 0usize..8 { + let base = out.add(lane.strict_add(8).strict_mul(64)); storeu256(hi0[lane], base); storeu256(hi1[lane], base.add(32)); } @@ -723,21 +704,21 @@ pub(crate) unsafe fn root_output_blocks( blocks: usize, ) { debug_assert!(blocks != 0); - debug_assert!(flags <= u8::MAX as u32); - debug_assert!(block_len <= u8::MAX as u32); + let block_len = u8::try_from(block_len).expect("BLAKE3 block length must fit the assembly ABI"); + let flags = u8::try_from(flags).expect("BLAKE3 flags must fit the assembly ABI"); // SAFETY: AVX-512 XOF assembly call because: // 1. Dispatch only selects this function for the AVX-512 kernel. // 2. The caller guarantees `out` is writable for `blocks * 64` bytes. // 3. `block_words` is a readable 64-byte block and `chaining_value` has 8 words. - // 4. `block_len` and `flags` are debug-checked to fit the assembly ABI. + // 4. `block_len` and `flags` were converted without loss to the assembly ABI types. // 5. Counters, block length, flags, and output block count are public values. unsafe { super::asm::xof_many_avx512( chaining_value.as_ptr(), block_words.as_ptr().cast(), - block_len as u8, + block_len, counter, - flags as u8, + flags, out, blocks, ); @@ -806,7 +787,7 @@ pub(crate) unsafe fn compress_block( let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); let [mut row0, mut row1, mut row2, mut row3] = - super::compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); + super::compress_pre_sse41_impl(chaining_value, [m0, m1, m2, m3], counter, block_len, flags); let cv_lo = _mm_loadu_si128(chaining_value.as_ptr().cast()); let cv_hi = _mm_loadu_si128(chaining_value.as_ptr().add(4).cast()); @@ -824,33 +805,3 @@ pub(crate) unsafe fn compress_block( out } } - -/// Compress one BLAKE3 block and return only the chaining value (8 words). -/// -/// # Safety -/// Caller must ensure AVX-512F + AVX-512VL + AVX2 + SSE4.1 + SSSE3 are available. -#[target_feature(enable = "avx512f,avx512vl,avx2,sse4.1,ssse3")] -pub(crate) unsafe fn compress_cv_block( - chaining_value: &[u32; 8], - block_words: &[u32; 16], - counter: u64, - block_len: u32, - flags: u32, -) -> [u32; 8] { - // SAFETY: AVX-512/AVX2/SSE4.1/SSSE3 intrinsics are available via this function's - // #[target_feature] attribute. Pointer accesses are to valid fixed-size array references. - unsafe { - let m0 = _mm_loadu_si128(block_words.as_ptr().cast()); - let m1 = _mm_loadu_si128(block_words.as_ptr().add(4).cast()); - let m2 = _mm_loadu_si128(block_words.as_ptr().add(8).cast()); - let m3 = _mm_loadu_si128(block_words.as_ptr().add(12).cast()); - let [row0, row1, row2, row3] = - super::compress_pre_sse41_impl(chaining_value, m0, m1, m2, m3, counter, block_len, flags); - let row0 = _mm_xor_si128(row0, row2); - let row1 = _mm_xor_si128(row1, row3); - let mut out = [0u32; 8]; - _mm_storeu_si128(out.as_mut_ptr().cast(), row0); - _mm_storeu_si128(out.as_mut_ptr().add(4).cast(), row1); - out - } -} diff --git a/src/hashes/crypto/blake3/x86_64/sse41.rs b/src/hashes/crypto/blake3/x86_64/sse41.rs index d3f36d33..d680af40 100644 --- a/src/hashes/crypto/blake3/x86_64/sse41.rs +++ b/src/hashes/crypto/blake3/x86_64/sse41.rs @@ -1,67 +1,89 @@ //! BLAKE3 x86_64 SSE4.1 throughput kernel (4-way). -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::too_many_arguments)] -#![allow(clippy::many_single_char_names)] - use core::arch::x86_64::*; use super::{ - super::{BLOCK_LEN, IV, MSG_SCHEDULE}, - counter_high, counter_low, + super::{BLOCK_LEN, BLOCK_LEN_U32, IV, MSG_SCHEDULE}, + HashManyRequest, counter_high, counter_low, }; -pub const DEGREE: usize = 4; +pub(crate) const DEGREE: usize = 4; +/// # Safety +/// +/// SSE2 must be available and `src` must be readable for 16 bytes. #[inline(always)] unsafe fn loadu(src: *const u8) -> __m128i { // SAFETY: Caller guarantees `src` is valid to read 16 bytes and has enabled this SSE backend. unsafe { _mm_loadu_si128(src.cast()) } } +/// # Safety +/// +/// SSE2 must be available and `dest` must be writable for 16 bytes. #[inline(always)] unsafe fn storeu(src: __m128i, dest: *mut u8) { // SAFETY: Caller guarantees `dest` is valid to write 16 bytes and has enabled this SSE backend. unsafe { _mm_storeu_si128(dest.cast(), src) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn add(a: __m128i, b: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_add_epi32(a, b) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn xor(a: __m128i, b: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_xor_si128(a, b) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn set1(x: u32) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_set1_epi32(x.cast_signed()) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn set4(a: u32, b: u32, c: u32, d: u32) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_setr_epi32(a.cast_signed(), b.cast_signed(), c.cast_signed(), d.cast_signed()) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn rot12(a: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_or_si128(_mm_srli_epi32(a, 12), _mm_slli_epi32(a, 20)) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn rot7(a: __m128i) -> __m128i { // SAFETY: Caller guarantees the required SSE4.1/SSSE3 feature set for this backend. unsafe { _mm_or_si128(_mm_srli_epi32(a, 7), _mm_slli_epi32(a, 25)) } } +/// # Safety +/// +/// SSE4.1 and SSSE3 must be available, and `r` must be in `0..7`. #[inline(always)] unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize, rot16_mask: __m128i, rot8_mask: __m128i) { // SAFETY: Caller guarantees this SSE4.1/SSSE3 backend is active; all vector lanes are local @@ -183,6 +205,9 @@ unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize, rot16_mask: } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn transpose_vecs(vecs: &mut [__m128i; DEGREE]) { // SAFETY: Caller guarantees this SSE4.1/SSSE3 backend is active; `vecs` is a valid fixed-size @@ -205,11 +230,15 @@ unsafe fn transpose_vecs(vecs: &mut [__m128i; DEGREE]) { } } +/// # Safety +/// +/// SSE4.1 and SSSE3 must be available. Every input must be readable for a +/// complete block starting at `block_offset`. #[inline(always)] unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) -> [__m128i; 16] { // SAFETY: Caller guarantees each input points to at least one full block at `block_offset`. unsafe { - let stride = 4 * DEGREE; + let stride = 4usize.strict_mul(DEGREE); let mut quarter0 = [ loadu(inputs[0].add(block_offset)), loadu(inputs[1].add(block_offset)), @@ -217,26 +246,29 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) loadu(inputs[3].add(block_offset)), ]; let mut quarter1 = [ - loadu(inputs[0].add(block_offset + stride)), - loadu(inputs[1].add(block_offset + stride)), - loadu(inputs[2].add(block_offset + stride)), - loadu(inputs[3].add(block_offset + stride)), + loadu(inputs[0].add(block_offset.strict_add(stride))), + loadu(inputs[1].add(block_offset.strict_add(stride))), + loadu(inputs[2].add(block_offset.strict_add(stride))), + loadu(inputs[3].add(block_offset.strict_add(stride))), ]; let mut quarter2 = [ - loadu(inputs[0].add(block_offset + 2 * stride)), - loadu(inputs[1].add(block_offset + 2 * stride)), - loadu(inputs[2].add(block_offset + 2 * stride)), - loadu(inputs[3].add(block_offset + 2 * stride)), + loadu(inputs[0].add(block_offset.strict_add(2usize.strict_mul(stride)))), + loadu(inputs[1].add(block_offset.strict_add(2usize.strict_mul(stride)))), + loadu(inputs[2].add(block_offset.strict_add(2usize.strict_mul(stride)))), + loadu(inputs[3].add(block_offset.strict_add(2usize.strict_mul(stride)))), ]; let mut quarter3 = [ - loadu(inputs[0].add(block_offset + 3 * stride)), - loadu(inputs[1].add(block_offset + 3 * stride)), - loadu(inputs[2].add(block_offset + 3 * stride)), - loadu(inputs[3].add(block_offset + 3 * stride)), + loadu(inputs[0].add(block_offset.strict_add(3usize.strict_mul(stride)))), + loadu(inputs[1].add(block_offset.strict_add(3usize.strict_mul(stride)))), + loadu(inputs[2].add(block_offset.strict_add(3usize.strict_mul(stride)))), + loadu(inputs[3].add(block_offset.strict_add(3usize.strict_mul(stride)))), ]; for &input in inputs.iter() { - _mm_prefetch(input.wrapping_add(block_offset + 256).cast::(), _MM_HINT_T0); + _mm_prefetch( + input.wrapping_add(block_offset.strict_add(256)).cast::(), + _MM_HINT_T0, + ); } transpose_vecs(&mut quarter0); @@ -265,6 +297,9 @@ unsafe fn transpose_msg_vecs(inputs: &[*const u8; DEGREE], block_offset: usize) } } +/// # Safety +/// +/// SSE2 must be available. #[inline(always)] unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m128i, __m128i) { let mask = if increment_counter { !0u64 } else { 0u64 }; @@ -294,15 +329,17 @@ unsafe fn load_counters(counter: u64, increment_counter: bool) -> (__m128i, __m1 /// for `blocks * BLOCK_LEN` bytes. #[target_feature(enable = "sse4.1,ssse3")] pub(crate) unsafe fn hash4( - inputs: &[*const u8; DEGREE], - blocks: usize, - key: &[u32; 8], - counter: u64, - increment_counter: bool, - flags: u32, - flags_start: u32, - flags_end: u32, - out: *mut u8, + HashManyRequest { + inputs, + blocks, + key, + counter, + increment_counter, + flags, + flags_start, + flags_end, + out, + }: HashManyRequest<'_, DEGREE>, ) { // SAFETY: Caller guarantees SSE4.1/SSSE3 availability, valid input pointers for `blocks * // BLOCK_LEN`, and `out` writable for `DEGREE * OUT_LEN`. @@ -310,7 +347,7 @@ pub(crate) unsafe fn hash4( let rot16_mask = _mm_setr_epi8(2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13); let rot8_mask = _mm_setr_epi8(1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12); - let block_len_vec = set1(BLOCK_LEN as u32); + let block_len_vec = set1(BLOCK_LEN_U32); let iv0 = set1(IV[0]); let iv1 = set1(IV[1]); let iv2 = set1(IV[2]); @@ -334,12 +371,12 @@ pub(crate) unsafe fn hash4( if block == 0 { block_flags |= flags_start; } - if block + 1 == blocks { + if block.strict_add(1) == blocks { block_flags |= flags_end; } let block_flags_vec = set1(block_flags); - let msg_vecs = transpose_msg_vecs(inputs, block * BLOCK_LEN); + let msg_vecs = transpose_msg_vecs(inputs, block.strict_mul(BLOCK_LEN)); let mut v = [ h_vecs[0], @@ -515,7 +552,7 @@ pub(crate) unsafe fn root_output_blocks4( transpose_vecs(&mut g3); for lane in 0..DEGREE { - let base = out.add(lane * 64); + let base = out.add(lane.strict_mul(64)); storeu(g0[lane], base); storeu(g1[lane], base.add(16)); storeu(g2[lane], base.add(32)); @@ -560,8 +597,8 @@ unsafe fn root_output_blocks1_from_ptr( let mut row3 = _mm_set_epi32( flags.cast_signed(), block_len.cast_signed(), - (counter >> 32) as i32, - counter as i32, + counter_high(counter).cast_signed(), + counter_low(counter).cast_signed(), ); // Load message words row-wise diff --git a/src/hashes/crypto/cshake.rs b/src/hashes/crypto/cshake.rs index cbbb0ac8..c2d293c5 100644 --- a/src/hashes/crypto/cshake.rs +++ b/src/hashes/crypto/cshake.rs @@ -1,7 +1,5 @@ //! cSHAKE128 and cSHAKE256 (SP 800-185). -#![allow(clippy::indexing_slicing)] // Fixed-width prefix encodings and rate-sized zero padding. - use super::{ keccak::{KeccakCore, KeccakXof}, sp800185::{RATE_128, RATE_256, absorb_bytepad, encoded_string_len, left_encode}, @@ -151,7 +149,7 @@ mod tests { let mut hex = String::new(); for byte in out { - write!(&mut hex, "{byte:02x}").unwrap(); + write!(&mut hex, "{byte:02x}").expect("writing hexadecimal to String must succeed"); } hex } diff --git a/src/hashes/crypto/dispatch_util.rs b/src/hashes/crypto/dispatch_util.rs index a14c49ba..b95f7e30 100644 --- a/src/hashes/crypto/dispatch_util.rs +++ b/src/hashes/crypto/dispatch_util.rs @@ -35,20 +35,12 @@ impl SizeClassDispatch { #[must_use] #[cfg(feature = "sha2")] pub(crate) fn len_hint_from_u64(v: u64) -> usize { - if (v as usize) as u64 == v { - v as usize - } else { - usize::MAX - } + usize::try_from(v).unwrap_or(usize::MAX) } #[inline] #[must_use] #[cfg(feature = "sha2")] pub(crate) fn len_hint_from_u128(v: u128) -> usize { - if (v as usize) as u128 == v { - v as usize - } else { - usize::MAX - } + usize::try_from(v).unwrap_or(usize::MAX) } diff --git a/src/hashes/crypto/keccak.rs b/src/hashes/crypto/keccak.rs index ec54dc6e..74dc4ea6 100644 --- a/src/hashes/crypto/keccak.rs +++ b/src/hashes/crypto/keccak.rs @@ -3,14 +3,12 @@ //! This module intentionally exposes only the minimum surface needed by SHA-3, //! SHAKE, and SP800-185 derived constructions. -#![allow(clippy::indexing_slicing)] // Keccak state is fixed-size; indexing is audited - #[cfg(all(target_arch = "aarch64", not(miri)))] pub(crate) mod aarch64; -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[doc(hidden)] pub(crate) mod dispatch; -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[doc(hidden)] pub(crate) mod dispatch_tables; #[cfg(test)] @@ -73,16 +71,17 @@ const RC: [u64; KECCAKF_ROUNDS] = [ /// all rounds. #[cfg(not(target_arch = "aarch64"))] #[inline] -#[allow(unused_assignments)] // final ρ+π iteration assigns `last` which is intentionally unused pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { // ρ+π chain: swap state[PI] with rotated last, hardcoded for constant folding. macro_rules! rho_pi { - ($state:ident, $last:ident, $(($pi:expr, $rho:expr)),+ $(,)?) => { - $( - let tmp = $state[$pi]; - $state[$pi] = $last.rotate_left($rho); - $last = tmp; - )+ + ($state:ident, $last:ident, ($pi:expr, $rho:expr) $(,)?) => { + $state[$pi] = $last.rotate_left($rho); + }; + ($state:ident, $last:ident, ($pi:expr, $rho:expr), $(($rest_pi:expr, $rest_rho:expr)),+ $(,)?) => { + let tmp = $state[$pi]; + $state[$pi] = $last.rotate_left($rho); + $last = tmp; + rho_pi!($state, $last, $(($rest_pi, $rest_rho)),+); }; } @@ -416,6 +415,16 @@ pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { state[24] = a24; } +#[inline(always)] +fn read_unaligned_block_lane(block: &[u8; RATE], lane: usize) -> u64 { + assert!(lane < RATE / 8, "Keccak lane must be inside the rate block"); + let offset = lane.strict_mul(8); + + // SAFETY: The assertion establishes `lane * 8 + 8 <= RATE`; the pointer retains `block`'s provenance, and + // `read_unaligned` accepts the byte array's alignment. + u64::from_le(unsafe { core::ptr::read_unaligned(block.as_ptr().add(offset).cast::()) }) +} + /// aarch64: fused absorb + Keccak-f[1600] — XOR block data during register load. /// /// Loads `state[i] ^ block_lane_i` directly into named register variables for @@ -427,20 +436,10 @@ pub(crate) fn keccakf_portable(state: &mut [u64; 25]) { /// eliminates all `if lane < lanes` branches — the result is straight-line code. #[cfg(target_arch = "aarch64")] #[inline] -#[allow(dead_code)] // Only used by non-Miri platform permuters with absorb-block overrides. fn keccakf_absorb_portable(state: &mut [u64; 25], block: &[u8; RATE]) { debug_assert_eq!(RATE % 8, 0); let lanes = RATE / 8; - #[inline(always)] - fn read_block_lane(block: *const u8, i: usize) -> u64 { - // SAFETY: caller guarantees `i < block.len() / 8`, so the read is in bounds. - // `read_unaligned` supports the 1-byte alignment of `[u8; RATE]`. - u64::from_le(unsafe { core::ptr::read_unaligned(block.cast::().add(i)) }) - } - - let ptr = block.as_ptr(); - // Fused load: `state[i] ^ block_lane_i` for absorbed (rate) lanes, // plain `state[i]` for capacity lanes. Each `$i` is a literal, so // `$i < lanes` (where `lanes = RATE / 8`, a compile-time constant) @@ -448,7 +447,7 @@ fn keccakf_absorb_portable(state: &mut [u64; 25], block: &[u8 macro_rules! fused_load { ($i:expr) => { if $i < lanes { - state[$i] ^ read_block_lane(ptr, $i) + state[$i] ^ read_unaligned_block_lane(block, $i) } else { state[$i] } @@ -548,8 +547,8 @@ pub(crate) trait Permuter: Copy { /// Permute three independent states in parallel. /// Default: one paired permutation and one single-state permutation. + #[cfg(feature = "ml-kem")] #[inline(always)] - #[allow(dead_code)] fn permute_x3(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25], len_hint: usize) { self.permute_x2(state_a, state_b, len_hint); self.permute(state_c, len_hint); @@ -557,8 +556,8 @@ pub(crate) trait Permuter: Copy { /// Permute four independent states in parallel. /// Default: two paired permutations. + #[cfg(feature = "ml-kem")] #[inline(always)] - #[allow(dead_code)] fn permute_x4( self, state_a: &mut [u64; 25], @@ -584,10 +583,17 @@ pub(crate) trait Permuter: Copy { /// Direct-call permuter using the portable scalar kernel. No function pointer /// indirection — LLVM can inline `keccakf_portable` into the absorb loop. +#[cfg(any( + miri, + not(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "s390x")) +))] #[derive(Clone, Copy, Default)] -#[allow(dead_code)] // Reference/test permuter is target- and feature-combination dependent. pub(crate) struct InlinePermuter; +#[cfg(any( + miri, + not(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "s390x")) +))] impl Permuter for InlinePermuter { #[inline(always)] fn permute(self, state: &mut [u64; 25], _len_hint: usize) { @@ -636,6 +642,7 @@ impl Permuter for X86Permuter { } } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x4( self, @@ -672,7 +679,7 @@ pub(crate) struct Aarch64Permuter; #[derive(Clone, Copy)] pub(crate) struct Aarch64Permuter { has_sha3: bool, - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] has_sve2_sha3: bool, } @@ -684,7 +691,7 @@ impl Default for Aarch64Permuter { let caps = crate::platform::caps(); Self { has_sha3: caps.has(aarch64_caps::SHA3), - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] has_sve2_sha3: caps.has(aarch64_caps::SVE2_SHA3), } } @@ -720,20 +727,22 @@ impl Permuter for Aarch64Permuter { aarch64::keccakf_aarch64_sha3_x2(state_a, state_b); } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x3(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25], _len_hint: usize) { - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); } - #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"))))] + #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem")))] { self.permute_x2(state_a, state_b, _len_hint); self.permute(state_c, _len_hint); } } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x4( self, @@ -743,7 +752,7 @@ impl Permuter for Aarch64Permuter { state_d: &mut [u64; 25], _len_hint: usize, ) { - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] { use crate::platform::caps::aarch64 as aarch64_caps; if crate::platform::caps().has(aarch64_caps::SVE2_SHA3) @@ -753,13 +762,13 @@ impl Permuter for Aarch64Permuter { } } - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); keccakf_portable(state_d); } - #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"))))] + #[cfg(not(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem")))] { self.permute_x2(state_a, state_b, _len_hint); self.permute_x2(state_c, state_d, _len_hint); @@ -820,9 +829,10 @@ impl Permuter for Aarch64Permuter { } } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x3(self, state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25], len_hint: usize) { - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] if self.has_sha3 { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); return; @@ -832,6 +842,7 @@ impl Permuter for Aarch64Permuter { self.permute(state_c, len_hint); } + #[cfg(feature = "ml-kem")] #[inline(always)] fn permute_x4( self, @@ -841,12 +852,12 @@ impl Permuter for Aarch64Permuter { state_d: &mut [u64; 25], len_hint: usize, ) { - #[cfg(target_os = "linux")] + #[cfg(all(target_os = "linux", feature = "ml-kem"))] if self.has_sve2_sha3 && aarch64::keccakf_aarch64_sve2_sha3_x4(state_a, state_b, state_c, state_d) { return; } - #[cfg(all(target_os = "linux", not(target_vendor = "apple")))] + #[cfg(all(target_os = "linux", not(target_vendor = "apple"), feature = "ml-kem"))] if self.has_sha3 { aarch64::keccakf_aarch64_sha3_x3_hybrid(state_a, state_b, state_c); keccakf_portable(state_d); @@ -960,25 +971,6 @@ impl Default for KeccakCoreImpl Default for KeccakCoreImpl { - #[inline] - fn default() -> Self { - Self { - state: [0u64; 25], - buf_len: 0, - permuter: InlinePermuter, - } - } -} - impl Drop for KeccakCoreImpl { fn drop(&mut self) { if ZEROIZE { @@ -997,7 +989,8 @@ impl KeccakCoreImpl(&mut self.state, self.buf_len, data); self.buf_len = self.buf_len.strict_add(data.len()); return; @@ -1005,7 +998,7 @@ impl KeccakCoreImpl(&mut self.state, self.buf_len, &data[..take]); self.buf_len = self.buf_len.strict_add(take); data = &data[take..]; @@ -1076,21 +1069,16 @@ impl KeccakCoreImpl KeccakCoreImpl { - pub(crate) fn into_xof(self, ds: u8) -> KeccakXofImpl { - let mut this = core::mem::ManuallyDrop::new(self); - let inner = &mut *this; - let permuter = inner.permuter; - let buf_len = inner.buf_len; + pub(crate) fn into_xof(mut self, ds: u8) -> KeccakXofImpl { + let permuter = self.permuter; + let buf_len = self.buf_len; debug_assert!(buf_len < RATE, "buf_len={} should be < RATE={}", buf_len, RATE); - pad_absorbed_state::(&mut inner.state, buf_len, ds); - permuter.permute(&mut inner.state, 0); + pad_absorbed_state::(&mut self.state, buf_len, ds); + permuter.permute(&mut self.state, 0); KeccakXofImpl { - // SAFETY: this public Keccak core is non-zeroizing, and `this` is - // `ManuallyDrop`, so moving the finalized state into the reader does not - // bypass any required cleanup. - state: unsafe { core::ptr::read(&inner.state) }, + state: self.state, pos: 0, permuter, } @@ -1147,14 +1135,10 @@ pub(crate) fn oneshot_fixed(ds: u8, data: & fn xor_block_into(state: &mut [u64; 25], block: &[u8; RATE]) { debug_assert_eq!(RATE % 8, 0); let lanes = RATE / 8; - let ptr = block.as_ptr() as *const u64; let mut i = 0usize; while i < lanes { - // SAFETY: `RATE % 8 == 0` and `i < lanes == RATE / 8`, so this reads within `block`; - // `read_unaligned` supports the 1-byte alignment of `[u8; RATE]`. - let v = unsafe { core::ptr::read_unaligned(ptr.add(i)) }; - state[i] ^= u64::from_le(v); - i += 1; + state[i] ^= read_unaligned_block_lane(block, i); + i = i.strict_add(1); } } @@ -1163,7 +1147,7 @@ fn xor_block_into(state: &mut [u64; 25], block: &[u8; RATE]) fn xor_bytes_into_state(state: &mut [u64; 25], mut offset: usize, mut data: &[u8]) { debug_assert_eq!(RATE % 8, 0); debug_assert!(offset <= RATE); - debug_assert!(data.len() <= RATE - offset); + debug_assert!(data.len() <= RATE.strict_sub(offset)); while !data.is_empty() && (offset & 7) != 0 { state[offset / 8] ^= (data[0] as u64) << ((offset & 7).strict_mul(8)); @@ -1195,7 +1179,7 @@ fn pad_absorbed_state(state: &mut [u64; 25], pos: usize, ds: state[pos / 8] ^= (ds as u64) << ((pos & 7).strict_mul(8)); - let last = RATE - 1; + let last = RATE.strict_sub(1); state[last / 8] ^= 0x80_u64 << ((last & 7).strict_mul(8)); } @@ -1310,7 +1294,7 @@ fn xof_seeded_32_2_base_state(ds: u8, seed: &[u8; 32]) -> [u6 state[3] = u64::from_le_bytes(seed_words[3]); state[4] = u64::from(ds) << 16; - let last = RATE - 1; + let last = RATE.strict_sub(1); state[last / 8] ^= 0x80_u64 << ((last & 7).strict_mul(8)); state } @@ -1342,7 +1326,7 @@ fn xof_seeded_32_1_state(ds: u8, seed: &[u8; 32], x: u8) -> [ state[3] = u64::from_le_bytes(seed_words[3]); state[4] = u64::from(x) | (u64::from(ds) << 8); - let last = RATE - 1; + let last = RATE.strict_sub(1); state[last / 8] ^= 0x80_u64 << ((last & 7).strict_mul(8)); state } @@ -1680,13 +1664,13 @@ impl KeccakXofImpl KeccakXofImpl KeccakXofImpl KeccakXofImpl KeccakXofImpl KeccakXofImpl(state: &mut [u64; 25], block: &[u8; RATE]) { @@ -431,7 +432,7 @@ pub(crate) fn keccakf_aarch64_sha3_absorb_single(state: &mut #[target_feature(enable = "sha3")] unsafe fn keccakf_sha3_absorb_blocks_impl(state: &mut [u64; 25], blocks: &[u8]) { debug_assert_eq!(RATE % 8, 0); - debug_assert_eq!(blocks.len() % RATE, 0); + debug_assert!(blocks.len().is_multiple_of(RATE)); let lanes = RATE / 8; let z = vcreate_u64(0); @@ -545,7 +546,17 @@ unsafe fn keccakf_sha3_absorb_blocks_impl(state: &mut [u64; 2 #[cfg(target_arch = "aarch64")] #[inline] pub(crate) fn keccakf_aarch64_sha3_absorb_blocks(state: &mut [u64; 25], blocks: &[u8]) { - // SAFETY: Dispatch verifies aarch64::SHA3 capability before calling. + assert!( + RATE != 0 && RATE.is_multiple_of(8), + "Keccak rate must be a positive whole number of lanes" + ); + assert!( + blocks.len().is_multiple_of(RATE), + "Keccak batch must contain complete rate blocks" + ); + + // SAFETY: Dispatch verifies aarch64::SHA3 capability before calling. The assertions above establish the nonzero, + // whole-lane rate and complete-block partition required by the pointer loop. unsafe { keccakf_sha3_absorb_blocks_impl::(state, blocks) } } @@ -553,6 +564,10 @@ pub(crate) fn keccakf_aarch64_sha3_absorb_blocks(state: &mut /// Combine lane 0 from `state_a[i]` and lane 1 from `state_b[i]` into one /// uint64x2_t register. +/// +/// # Safety +/// +/// The caller must ensure the current CPU supports NEON. #[cfg(target_arch = "aarch64")] #[inline(always)] unsafe fn combine_lanes(a: u64, b: u64) -> uint64x2_t { @@ -689,7 +704,7 @@ pub(crate) fn keccakf_aarch64_sha3_x2(state_a: &mut [u64; 25], state_b: &mut [u6 /// # Safety /// /// Caller must ensure `sha3` target feature is available. -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem"))] #[target_feature(enable = "sha3")] unsafe fn keccakf_sha3_x3_hybrid_impl(state_a: &mut [u64; 25], state_b: &mut [u64; 25], state_c: &mut [u64; 25]) { // SAFETY: NEON + SHA3 CE intrinsics are available because: @@ -840,7 +855,7 @@ unsafe fn keccakf_sha3_x3_hybrid_impl(state_a: &mut [u64; 25], state_b: &mut [u6 } } -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem"))] #[inline] pub(crate) fn keccakf_aarch64_sha3_x3_hybrid( state_a: &mut [u64; 25], @@ -853,7 +868,7 @@ pub(crate) fn keccakf_aarch64_sha3_x3_hybrid( unsafe { keccakf_sha3_x3_hybrid_impl(state_a, state_b, state_c) } } -#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri), feature = "ml-kem"))] unsafe extern "C" { fn rscrypto_keccakf1600_aarch64_sve2_sha3_x4( state_a: *mut u64, @@ -867,7 +882,7 @@ unsafe extern "C" { /// /// Returns `false` when the runtime SVE vector length is too small for four /// 64-bit lanes. Callers must still gate this on `aarch64::SVE2_SHA3`. -#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] +#[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri), feature = "ml-kem"))] #[inline] pub(crate) fn keccakf_aarch64_sve2_sha3_x4( state_a: &mut [u64; 25], diff --git a/src/hashes/crypto/keccak/dispatch.rs b/src/hashes/crypto/keccak/dispatch.rs index 41eeb6a6..23526478 100644 --- a/src/hashes/crypto/keccak/dispatch.rs +++ b/src/hashes/crypto/keccak/dispatch.rs @@ -1,5 +1,3 @@ -#![cfg_attr(test, allow(dead_code))] - use super::{ dispatch_tables::DispatchTable, kernels::{Keccakf1600KernelId, permute_fn, required_caps}, @@ -96,7 +94,7 @@ fn select(d: &ActiveDispatch, len: usize) -> (PermuteFn, &'static str) { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] -pub fn kernel_name_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { #[cfg(target_arch = "s390x")] { use crate::platform::caps::s390x; diff --git a/src/hashes/crypto/keccak/dispatch_tables.rs b/src/hashes/crypto/keccak/dispatch_tables.rs index 4047b11e..dfde8615 100644 --- a/src/hashes/crypto/keccak/dispatch_tables.rs +++ b/src/hashes/crypto/keccak/dispatch_tables.rs @@ -1,17 +1,15 @@ -#![cfg_attr(test, allow(dead_code))] - //! Tuned dispatch tables for Keccak-f[1600]. //! //! This table controls which permutation kernel is used by SHA-3/SHAKE and //! SP800-185 derived constructions. -pub use super::kernels::Keccakf1600KernelId as KernelId; +pub(crate) use super::kernels::Keccakf1600KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -19,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -31,7 +29,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { // across both NEON lanes. SHA3 CE is used by the two-state interleaved path, // where each lane carries an independent state. #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -41,7 +39,7 @@ pub static AARCH64_SHA3_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "aarch64")] { use crate::platform::caps::aarch64; @@ -50,6 +48,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D } } - #[allow(unreachable_code)] + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/keccak/kernel_test.rs b/src/hashes/crypto/keccak/kernel_test.rs index 1472e019..1d0f4c5c 100644 --- a/src/hashes/crypto/keccak/kernel_test.rs +++ b/src/hashes/crypto/keccak/kernel_test.rs @@ -3,7 +3,7 @@ use alloc::vec::Vec; use super::kernels::{ALL, permute_fn, required_caps}; #[derive(Clone, Debug)] -pub struct KernelResult { +pub(crate) struct KernelResult { pub name: &'static str, pub state: [u64; 25], } @@ -24,7 +24,7 @@ fn state_from_bytes(data: &[u8]) -> [u64; 25] { } #[must_use] -pub fn run_all_keccakf1600_kernels(data: &[u8]) -> Vec { +pub(crate) fn run_all_keccakf1600_kernels(data: &[u8]) -> Vec { let caps = crate::platform::caps(); let mut out = Vec::with_capacity(ALL.len()); let init = state_from_bytes(data); @@ -44,7 +44,7 @@ pub fn run_all_keccakf1600_kernels(data: &[u8]) -> Vec { out } -pub fn verify_keccakf1600_kernels(data: &[u8]) -> Result<(), &'static str> { +pub(crate) fn verify_keccakf1600_kernels(data: &[u8]) -> Result<(), &'static str> { let results = run_all_keccakf1600_kernels(data); let Some(first) = results.first() else { return Ok(()); @@ -167,11 +167,11 @@ mod tests { for seed in 0u8..16 { let mut input_a = [0u8; 200]; let mut input_b = [0u8; 200]; - for (i, byte) in input_a.iter_mut().enumerate() { - *byte = seed.wrapping_add((i as u8).wrapping_mul(17)); + for (i, byte) in (0u8..).zip(&mut input_a) { + *byte = seed.wrapping_add(i.wrapping_mul(17)); } - for (i, byte) in input_b.iter_mut().enumerate() { - *byte = seed.wrapping_mul(3).wrapping_add((i as u8).wrapping_mul(29)); + for (i, byte) in (0u8..).zip(&mut input_b) { + *byte = seed.wrapping_mul(3).wrapping_add(i.wrapping_mul(29)); } let mut state_a = state_from_bytes(&input_a); @@ -219,7 +219,7 @@ mod tests { /// Verify the production aarch64 x3 dispatch matches three independent portable runs. #[test] - #[cfg(all(target_arch = "aarch64", not(miri)))] + #[cfg(all(feature = "ml-kem", target_arch = "aarch64", not(miri)))] fn keccakf1600_aarch64_platform_x3_matches_portable() { use super::super::Permuter; @@ -260,7 +260,7 @@ mod tests { /// Verify Linux-targeted hybrid x3 kernel matches three independent portable runs. #[test] - #[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] + #[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem", not(miri)))] fn keccakf1600_sha3_x3_hybrid_matches_portable() { let caps = crate::platform::caps(); if !caps.has(crate::platform::caps::aarch64::SHA3) { @@ -287,7 +287,7 @@ mod tests { /// Verify SVE2-SHA3 4-state kernel matches four independent portable runs. #[test] - #[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] + #[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "ml-kem", not(miri)))] fn keccakf1600_sve2_sha3_x4_matches_portable() { let caps = crate::platform::caps(); if !caps.has(crate::platform::caps::aarch64::SVE2_SHA3) { @@ -331,7 +331,7 @@ mod tests { let len = RATE.strict_mul(4); let mut blocks = Vec::with_capacity(len); for i in 0..len { - blocks.push(((i.strict_mul(37).strict_add(11)) & 0xff) as u8); + blocks.push(i.strict_mul(37).strict_add(11).to_le_bytes()[0]); } let mut expected = state_from_bytes(b"batch-absorb-reference-state"); @@ -354,4 +354,21 @@ mod tests { assert_rate::<144>(); assert_rate::<168>(); } + + #[test] + #[cfg(all(target_arch = "aarch64", not(miri)))] + #[should_panic(expected = "Keccak rate must be a positive whole number of lanes")] + fn keccakf1600_absorb_blocks_rejects_zero_rate() { + let mut state = [0u64; 25]; + super::super::aarch64::keccakf_aarch64_sha3_absorb_blocks::<0>(&mut state, &[]); + } + + #[test] + #[cfg(all(target_arch = "aarch64", not(miri)))] + #[should_panic(expected = "Keccak batch must contain complete rate blocks")] + fn keccakf1600_absorb_blocks_rejects_partial_block() { + let mut state = [0u64; 25]; + let blocks = [0u8; 73]; + super::super::aarch64::keccakf_aarch64_sha3_absorb_blocks::<72>(&mut state, &blocks); + } } diff --git a/src/hashes/crypto/keccak/kernels.rs b/src/hashes/crypto/keccak/kernels.rs index a842ee7a..2a3b1dc8 100644 --- a/src/hashes/crypto/keccak/kernels.rs +++ b/src/hashes/crypto/keccak/kernels.rs @@ -4,14 +4,14 @@ use crate::platform::Caps; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Keccakf1600KernelId { +pub(crate) enum Keccakf1600KernelId { Portable = 0, } impl Keccakf1600KernelId { #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", } @@ -19,7 +19,7 @@ impl Keccakf1600KernelId { } #[must_use] -pub fn permute_fn(id: Keccakf1600KernelId) -> fn(&mut [u64; 25]) { +pub(crate) fn permute_fn(id: Keccakf1600KernelId) -> fn(&mut [u64; 25]) { match id { Keccakf1600KernelId::Portable => keccakf_portable, } @@ -27,11 +27,11 @@ pub fn permute_fn(id: Keccakf1600KernelId) -> fn(&mut [u64; 25]) { /// All kernel IDs for agreement testing. #[cfg(test)] -pub const ALL: &[Keccakf1600KernelId] = &[Keccakf1600KernelId::Portable]; +pub(crate) const ALL: &[Keccakf1600KernelId] = &[Keccakf1600KernelId::Portable]; #[inline] #[must_use] -pub const fn required_caps(id: Keccakf1600KernelId) -> Caps { +pub(crate) const fn required_caps(id: Keccakf1600KernelId) -> Caps { match id { Keccakf1600KernelId::Portable => Caps::NONE, } diff --git a/src/hashes/crypto/keccak/s390x.rs b/src/hashes/crypto/keccak/s390x.rs index 3611d22d..2854c527 100644 --- a/src/hashes/crypto/keccak/s390x.rs +++ b/src/hashes/crypto/keccak/s390x.rs @@ -25,8 +25,6 @@ //! //! Requires MSA8 facility (CPACF, z14+). Caller must verify `s390x::MSA8`. -#![allow(unsafe_code)] - /// Map Keccak rate (bytes) to KIMD function code. /// /// For rate=136 (shared by SHA3-256 and SHAKE-256), returns SHA3-256's code (33). diff --git a/src/hashes/crypto/keccak/x86_64.rs b/src/hashes/crypto/keccak/x86_64.rs index 2d6fcbe4..54873b0a 100644 --- a/src/hashes/crypto/keccak/x86_64.rs +++ b/src/hashes/crypto/keccak/x86_64.rs @@ -11,6 +11,16 @@ #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; +#[inline(always)] +const fn u64_to_i64_bits(value: u64) -> i64 { + i64::from_ne_bytes(value.to_ne_bytes()) +} + +#[inline(always)] +const fn i64_to_u64_bits(value: i64) -> u64 { + u64::from_ne_bytes(value.to_ne_bytes()) +} + #[cfg(target_arch = "x86_64")] macro_rules! rol { ($value:expr, $left:literal) => {{ _mm_rol_epi64::<$left>($value) }}; @@ -21,12 +31,12 @@ macro_rules! chi { ($a:expr, $b:expr, $c:expr) => {{ _mm_ternarylogic_epi64($a, $b, $c, 0xD2) }}; } -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] macro_rules! rol4 { ($value:expr, $left:literal) => {{ _mm256_rol_epi64::<$left>($value) }}; } -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] macro_rules! chi4 { ($a:expr, $b:expr, $c:expr) => {{ _mm256_ternarylogic_epi64($a, $b, $c, 0xD2) }}; } @@ -42,14 +52,14 @@ macro_rules! chi4 { unsafe fn keccakf_x86_avx512_x2_impl(state_a: &mut [u64; 25], state_b: &mut [u64; 25]) { macro_rules! load { ($i:literal) => { - _mm_set_epi64x(state_b[$i] as i64, state_a[$i] as i64) + _mm_set_epi64x(u64_to_i64_bits(state_b[$i]), u64_to_i64_bits(state_a[$i])) }; } macro_rules! store { ($i:literal, $value:expr) => {{ - state_a[$i] = _mm_extract_epi64::<0>($value) as u64; - state_b[$i] = _mm_extract_epi64::<1>($value) as u64; + state_a[$i] = i64_to_u64_bits(_mm_extract_epi64::<0>($value)); + state_b[$i] = i64_to_u64_bits(_mm_extract_epi64::<1>($value)); }}; } @@ -186,7 +196,7 @@ unsafe fn keccakf_x86_avx512_x2_impl(state_a: &mut [u64; 25], state_b: &mut [u64 a23 = chi!(b23, b24, b20); a24 = chi!(b24, b20, b21); - a0 = xor!(a0, _mm_set1_epi64x(rc as i64)); + a0 = xor!(a0, _mm_set1_epi64x(u64_to_i64_bits(rc))); } store!(0, a0); @@ -235,7 +245,7 @@ pub(crate) unsafe fn keccakf_x86_avx512_x2(state_a: &mut [u64; 25], state_b: &mu /// # Safety /// /// Caller must ensure `avx512f`, `avx512vl`, and `sse4.1` CPU features are available. -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] #[target_feature(enable = "avx512f,avx512vl,sse4.1")] #[inline] unsafe fn keccakf_x86_avx512_x4_impl( @@ -247,20 +257,20 @@ unsafe fn keccakf_x86_avx512_x4_impl( macro_rules! load { ($i:literal) => { _mm256_set_epi64x( - state_d[$i] as i64, - state_c[$i] as i64, - state_b[$i] as i64, - state_a[$i] as i64, + u64_to_i64_bits(state_d[$i]), + u64_to_i64_bits(state_c[$i]), + u64_to_i64_bits(state_b[$i]), + u64_to_i64_bits(state_a[$i]), ) }; } macro_rules! store { ($i:literal, $value:expr) => {{ - state_a[$i] = _mm256_extract_epi64::<0>($value) as u64; - state_b[$i] = _mm256_extract_epi64::<1>($value) as u64; - state_c[$i] = _mm256_extract_epi64::<2>($value) as u64; - state_d[$i] = _mm256_extract_epi64::<3>($value) as u64; + state_a[$i] = i64_to_u64_bits(_mm256_extract_epi64::<0>($value)); + state_b[$i] = i64_to_u64_bits(_mm256_extract_epi64::<1>($value)); + state_c[$i] = i64_to_u64_bits(_mm256_extract_epi64::<2>($value)); + state_d[$i] = i64_to_u64_bits(_mm256_extract_epi64::<3>($value)); }}; } @@ -397,7 +407,7 @@ unsafe fn keccakf_x86_avx512_x4_impl( a23 = chi4!(b23, b24, b20); a24 = chi4!(b24, b20, b21); - a0 = xor!(a0, _mm256_set1_epi64x(rc as i64)); + a0 = xor!(a0, _mm256_set1_epi64x(u64_to_i64_bits(rc))); } store!(0, a0); @@ -432,7 +442,7 @@ unsafe fn keccakf_x86_avx512_x4_impl( /// # Safety /// /// Caller must ensure `avx512f`, `avx512vl`, and `sse4.1` CPU features are available. -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "ml-kem"))] #[inline] pub(crate) unsafe fn keccakf_x86_avx512_x4( state_a: &mut [u64; 25], diff --git a/src/hashes/crypto/mod.rs b/src/hashes/crypto/mod.rs index 2c02f9e2..bee0e22f 100644 --- a/src/hashes/crypto/mod.rs +++ b/src/hashes/crypto/mod.rs @@ -1,9 +1,5 @@ //! Cryptographic hash functions. -#![cfg_attr(not(test), deny(clippy::unwrap_used))] -#![cfg_attr(not(test), deny(clippy::expect_used))] -#![cfg_attr(not(test), deny(clippy::indexing_slicing))] - #[cfg(any(feature = "blake2b", feature = "blake2s"))] use core::fmt; diff --git a/src/hashes/crypto/sha224.rs b/src/hashes/crypto/sha224.rs index 0c05e7f8..64350c42 100644 --- a/src/hashes/crypto/sha224.rs +++ b/src/hashes/crypto/sha224.rs @@ -3,8 +3,6 @@ //! SHA-224 is identical to SHA-256 except for initial hash values (H0) and //! output truncation (28 bytes / 7 words). The compression function is shared. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays in finalization - use self::kernels::CompressBlocksFn; use crate::{ hashes::crypto::dispatch_util::{SizeClassDispatch, len_hint_from_u64}, @@ -181,7 +179,7 @@ impl Sha224 { compress_blocks(&mut state, &block); let mut out = [0u8; 28]; - for (chunk, &word) in out.chunks_exact_mut(4).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<4>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } out @@ -196,9 +194,9 @@ impl Drop for Sha224 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -257,7 +255,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{:02x}", b).expect("writing hexadecimal to String must succeed"); } s } diff --git a/src/hashes/crypto/sha224/dispatch_tables.rs b/src/hashes/crypto/sha224/dispatch_tables.rs index 181c0766..2a2924c4 100644 --- a/src/hashes/crypto/sha224/dispatch_tables.rs +++ b/src/hashes/crypto/sha224/dispatch_tables.rs @@ -2,13 +2,13 @@ //! //! Mirrors SHA-256 — same compression function, same HW accel for all sizes. -pub use super::kernels::Sha224KernelId as KernelId; +pub(crate) use super::kernels::Sha224KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -16,7 +16,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -25,7 +25,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha, s: KernelId::X86Sha, @@ -34,7 +34,7 @@ pub static X86_SHA_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha2, s: KernelId::Aarch64Sha2, @@ -43,7 +43,7 @@ pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { }; #[cfg(any(target_arch = "riscv64", target_arch = "riscv32"))] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::RiscvZknh, s: KernelId::RiscvZknh, @@ -52,7 +52,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -61,7 +61,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -71,7 +71,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "x86_64")] { use crate::platform::caps::x86; @@ -107,5 +107,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha224/kernel_test.rs b/src/hashes/crypto/sha224/kernel_test.rs index 53fb42e9..c8201594 100644 --- a/src/hashes/crypto/sha224/kernel_test.rs +++ b/src/hashes/crypto/sha224/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 28], -} - fn hasher_for_kernel(id: Sha224KernelId) -> Sha224 { let compress = compress_blocks_fn(id); Sha224 { @@ -34,41 +27,16 @@ fn digest_with_kernel(id: Sha224KernelId, data: &[u8]) -> [u8; 28] { h.finalize() } -#[must_use] -pub fn run_all_sha224_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -pub fn verify_sha224_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha224_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha224 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(17).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(17).wrapping_add(bytes[1]) + }) .collect() } @@ -77,7 +45,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, + 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 55, 56, 57, 63, 64, 65, 127, 128, 129]; @@ -140,10 +108,4 @@ mod tests { } } } - - #[test] - fn run_all_agree() { - verify_sha224_kernels(b"abc").expect("kernels should agree"); - verify_sha224_kernels(&pattern(4096)).expect("kernels should agree"); - } } diff --git a/src/hashes/crypto/sha224/kernels.rs b/src/hashes/crypto/sha224/kernels.rs index 3a28f3f4..13386de4 100644 --- a/src/hashes/crypto/sha224/kernels.rs +++ b/src/hashes/crypto/sha224/kernels.rs @@ -16,7 +16,7 @@ use crate::{hashes::crypto::sha256::Sha256, platform::Caps}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha224KernelId { +pub(crate) enum Sha224KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Sha = 1, @@ -34,7 +34,7 @@ impl Sha224KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -52,7 +52,7 @@ impl Sha224KernelId { } #[cfg(test)] -pub const ALL: &[Sha224KernelId] = &[ +pub(crate) const ALL: &[Sha224KernelId] = &[ Sha224KernelId::Portable, #[cfg(target_arch = "x86_64")] Sha224KernelId::X86Sha, @@ -96,7 +96,7 @@ pub(crate) fn compress_blocks_fn(id: Sha224KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha224KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha224KernelId) -> Caps { match id { Sha224KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] diff --git a/src/hashes/crypto/sha256/aarch64.rs b/src/hashes/crypto/sha256/aarch64.rs index 82fd0c87..030e216d 100644 --- a/src/hashes/crypto/sha256/aarch64.rs +++ b/src/hashes/crypto/sha256/aarch64.rs @@ -7,9 +7,6 @@ //! All functions require the `sha2` target feature. //! Callers must verify CPU capabilities before calling. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] - #[cfg(target_arch = "aarch64")] use core::arch::aarch64::*; @@ -258,19 +255,19 @@ pub(crate) unsafe fn compress_blocks_aarch64_sha2(state: &mut [u32; 8], blocks: efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); s1 = vsha256su1q_u32(vsha256su0q_u32(s1, s2), s3, s0); - tmp = vaddq_u32(s1, vld1q_u32(kp.add(t + 4))); + tmp = vaddq_u32(s1, vld1q_u32(kp.add(t.strict_add(4)))); abcd_prev = abcd; abcd = vsha256hq_u32(abcd_prev, efgh, tmp); efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); s2 = vsha256su1q_u32(vsha256su0q_u32(s2, s3), s0, s1); - tmp = vaddq_u32(s2, vld1q_u32(kp.add(t + 8))); + tmp = vaddq_u32(s2, vld1q_u32(kp.add(t.strict_add(8)))); abcd_prev = abcd; abcd = vsha256hq_u32(abcd_prev, efgh, tmp); efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); s3 = vsha256su1q_u32(vsha256su0q_u32(s3, s0), s1, s2); - tmp = vaddq_u32(s3, vld1q_u32(kp.add(t + 12))); + tmp = vaddq_u32(s3, vld1q_u32(kp.add(t.strict_add(12)))); abcd_prev = abcd; abcd = vsha256hq_u32(abcd_prev, efgh, tmp); efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); diff --git a/src/hashes/crypto/sha256/dispatch_tables.rs b/src/hashes/crypto/sha256/dispatch_tables.rs index d28808d1..06a5d45e 100644 --- a/src/hashes/crypto/sha256/dispatch_tables.rs +++ b/src/hashes/crypto/sha256/dispatch_tables.rs @@ -6,13 +6,13 @@ //! POWER uses portable compression because the retired hybrid `vshasigmaw` //! kernel lost to portable Rust on POWER10. -pub use super::kernels::Sha256KernelId as KernelId; +pub(crate) use super::kernels::Sha256KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -20,7 +20,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -29,7 +29,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha, s: KernelId::X86Sha, @@ -38,7 +38,7 @@ pub static X86_SHA_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha2, s: KernelId::Aarch64Sha2, @@ -47,7 +47,7 @@ pub static AARCH64_SHA2_TABLE: DispatchTable = DispatchTable { }; #[cfg(any(target_arch = "riscv64", target_arch = "riscv32"))] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::RiscvZknh, s: KernelId::RiscvZknh, @@ -56,7 +56,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -65,7 +65,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -75,7 +75,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { #[cfg(target_arch = "x86_64")] { use crate::platform::caps::x86; @@ -111,5 +111,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha256/kernel_test.rs b/src/hashes/crypto/sha256/kernel_test.rs index 7df8f1ce..0394d050 100644 --- a/src/hashes/crypto/sha256/kernel_test.rs +++ b/src/hashes/crypto/sha256/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::hashes::crypto::dispatch_util::SizeClassDispatch; -#[derive(Clone, Debug)] -#[allow(dead_code)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 32], -} - fn hasher_for_kernel(id: Sha256KernelId) -> Sha256 { let compress = compress_blocks_fn(id); Sha256 { @@ -34,41 +27,16 @@ fn digest_with_kernel(id: Sha256KernelId, data: &[u8]) -> [u8; 32] { h.finalize() } -#[must_use] -pub fn run_all_sha256_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -pub fn verify_sha256_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha256_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha256 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(31).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(31).wrapping_add(bytes[1]) + }) .collect() } @@ -91,7 +59,7 @@ mod tests { #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, + 0usize, 1, 2, 3, 55, 56, 57, 63, 64, 65, 119, 120, 121, 127, 128, 129, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 55, 56, 57, 63, 64, 65, 127, 128, 129]; @@ -152,12 +120,6 @@ mod tests { } } - #[test] - fn run_all_agree() { - verify_sha256_kernels(b"abc").expect("kernels should agree"); - verify_sha256_kernels(&pattern(4096)).expect("kernels should agree"); - } - #[test] #[cfg(target_arch = "x86_64")] fn x86_sha_kernel_requires_sha_and_sse41() { diff --git a/src/hashes/crypto/sha256/kernels.rs b/src/hashes/crypto/sha256/kernels.rs index 25482e79..cf761bf3 100644 --- a/src/hashes/crypto/sha256/kernels.rs +++ b/src/hashes/crypto/sha256/kernels.rs @@ -16,7 +16,7 @@ pub(crate) type CompressBlocksFn = fn(&mut [u32; 8], &[u8]); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha256KernelId { +pub(crate) enum Sha256KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] X86Sha = 1, @@ -34,7 +34,7 @@ impl Sha256KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -52,7 +52,7 @@ impl Sha256KernelId { } #[cfg(test)] -pub const ALL: &[Sha256KernelId] = &[ +pub(crate) const ALL: &[Sha256KernelId] = &[ Sha256KernelId::Portable, #[cfg(target_arch = "x86_64")] Sha256KernelId::X86Sha, @@ -120,7 +120,7 @@ pub(crate) fn compress_blocks_fn(id: Sha256KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha256KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha256KernelId) -> Caps { match id { Sha256KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] @@ -171,11 +171,11 @@ pub(crate) const COMPILE_TIME_HW: bool = cfg!(not(miri)) pub(crate) fn compile_time_best() -> CompressBlocksFn { #[cfg(miri)] { - return Sha256::compress_blocks_portable; + Sha256::compress_blocks_portable } #[cfg(all(not(miri), target_arch = "x86_64", target_feature = "sha", target_feature = "sse4.1"))] { - return compress_blocks_x86_sha; + compress_blocks_x86_sha } #[cfg(all( not(miri), @@ -183,7 +183,7 @@ pub(crate) fn compile_time_best() -> CompressBlocksFn { any(target_os = "macos", target_feature = "sha2") ))] { - return compress_blocks_aarch64_sha2; + compress_blocks_aarch64_sha2 } #[cfg(all( not(miri), @@ -191,18 +191,28 @@ pub(crate) fn compile_time_best() -> CompressBlocksFn { target_feature = "zknh" ))] { - return compress_blocks_riscv_zknh; + compress_blocks_riscv_zknh } #[cfg(all(not(miri), target_arch = "wasm32", target_feature = "simd128"))] { - return compress_blocks_wasm_simd128; + compress_blocks_wasm_simd128 + } + #[cfg(all( + not(miri), + not(any( + all(target_arch = "x86_64", target_feature = "sha", target_feature = "sse4.1"), + all(target_arch = "aarch64", any(target_os = "macos", target_feature = "sha2")), + all(any(target_arch = "riscv64", target_arch = "riscv32"), target_feature = "zknh"), + all(target_arch = "wasm32", target_feature = "simd128") + )) + ))] + { + Sha256::compress_blocks_portable } - #[allow(unreachable_code)] - Sha256::compress_blocks_portable } /// Kernel name for the compile-time-best path (introspection). -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] pub(crate) const COMPILE_TIME_NAME: &str = if cfg!(miri) { "portable" } else if cfg!(all( diff --git a/src/hashes/crypto/sha256/mod.rs b/src/hashes/crypto/sha256/mod.rs index c67d238d..5a24ba47 100644 --- a/src/hashes/crypto/sha256/mod.rs +++ b/src/hashes/crypto/sha256/mod.rs @@ -1,7 +1,5 @@ //! SHA-256 (FIPS 180-4). -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - use self::kernels::CompressBlocksFn; use crate::{ hashes::{ @@ -387,7 +385,7 @@ impl Sha256Prefix { unsafe { core::ptr::write_volatile(word, 0) }; } // SAFETY: bytes_hashed is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -453,14 +451,11 @@ impl Sha256 { #[inline] pub(crate) fn compress_blocks_portable(state: &mut [u32; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + for block in chunks { Self::compress_block(state, block); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } #[inline] @@ -622,7 +617,7 @@ impl Sha256 { compress_blocks(&mut state, &block); let mut out = [0u8; 32]; - for (chunk, &word) in out.chunks_exact_mut(4).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<4>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } if ZEROIZE { @@ -703,9 +698,8 @@ impl Sha256 { #[cfg(all(feature = "hmac", not(target_arch = "x86_64")))] fn reset_update_mode_to_aligned_prefix(&mut self, _prefix: Sha256Prefix) {} - #[cfg(all(feature = "hmac", any(test, feature = "diag")))] + #[cfg(all(feature = "hmac", any(feature = "diag", all(test, feature = "hkdf"))))] #[inline] - #[allow(dead_code)] pub(crate) fn new_with_compress_for_test(compress_blocks: CompressBlocksFn) -> Self { Self { state: H0, @@ -744,9 +738,9 @@ impl Drop for Sha256 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -794,7 +788,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{:02x}", b).expect("writing hexadecimal to String must succeed"); } s } diff --git a/src/hashes/crypto/sha256/riscv64.rs b/src/hashes/crypto/sha256/riscv64.rs index 12ee02ac..c47798c4 100644 --- a/src/hashes/crypto/sha256/riscv64.rs +++ b/src/hashes/crypto/sha256/riscv64.rs @@ -8,8 +8,6 @@ //! //! All functions require the `zknh` target feature. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "riscv32")] use core::arch::riscv32::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; #[cfg(target_arch = "riscv64")] @@ -17,10 +15,6 @@ use core::arch::riscv64::{sha256sig0, sha256sig1, sha256sum0, sha256sum1}; use super::BLOCK_LEN; -// Safe wrappers for the unsafe Zknh intrinsics. -// SAFETY: only called from within `compress_blocks_zknh` which has -// `#[target_feature(enable = "zknh")]`, guaranteeing the feature is available. - #[inline(always)] fn sum0(x: u32) -> u32 { // SAFETY: `compress_blocks_zknh` calls this wrapper only inside its Zknh target-feature scope. @@ -52,12 +46,9 @@ fn sig1(x: u32) -> u32 { /// Caller must ensure `zknh` CPU feature is available. #[target_feature(enable = "zknh")] pub(crate) unsafe fn compress_blocks_zknh(state: &mut [u32; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); + for block in chunks { super::compress_block_with(state, block, sum0, sum1, sig0, sig1); } - debug_assert!(chunks.remainder().is_empty()); } diff --git a/src/hashes/crypto/sha256/s390x.rs b/src/hashes/crypto/sha256/s390x.rs index 9f14fc3f..21dbf7c7 100644 --- a/src/hashes/crypto/sha256/s390x.rs +++ b/src/hashes/crypto/sha256/s390x.rs @@ -13,9 +13,6 @@ //! //! Requires the MSA facility (CPACF). Caller must verify `s390x::MSA`. -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] - use super::BLOCK_LEN; /// SHA-256 block compression via KIMD instruction. diff --git a/src/hashes/crypto/sha256/wasm.rs b/src/hashes/crypto/sha256/wasm.rs index a074102e..ff4a1e15 100644 --- a/src/hashes/crypto/sha256/wasm.rs +++ b/src/hashes/crypto/sha256/wasm.rs @@ -3,8 +3,6 @@ //! Vectorizes the message schedule computation using 128-bit SIMD (4 × u32 //! lanes). Compression rounds remain scalar (sequential data dependency). -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; @@ -12,12 +10,20 @@ use super::{BLOCK_LEN, K, ch, maj}; use crate::hashes::util::rotr32; /// Load 4 big-endian message words from `ptr` into a v128, byte-swapping each. +/// +/// # Safety +/// +/// `ptr` must be valid for an unaligned 16-byte read. The caller must execute +/// this function only when WebAssembly SIMD128 is enabled. #[cfg(target_arch = "wasm32")] #[inline(always)] unsafe fn load_be(ptr: *const u8) -> v128 { - // SAFETY: caller guarantees `ptr` is valid for a 16-byte aligned read - // and the simd128 target feature is enabled. - let raw = unsafe { v128_load(ptr as *const v128) }; + // SAFETY: the caller guarantees that `ptr..ptr+16` is readable. The byte + // array has alignment one, and `read_unaligned` accepts any address. + let bytes = unsafe { ptr.cast::<[u8; 16]>().read_unaligned() }; + // SAFETY: `[u8; 16]` and `v128` have the same size, and every SIMD bit + // pattern is valid. + let raw = unsafe { core::mem::transmute::<[u8; 16], v128>(bytes) }; i8x16_shuffle::<3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12>(raw, raw) } @@ -68,36 +74,36 @@ fn schedule_4(w: &mut [v128; 16], i: usize) { let t = i << 2; let mut slot = u32x4_splat(0); - let w16_0 = schedule_word(w, t - 16); - let w15_0 = schedule_word(w, t - 15); - let w7_0 = schedule_word(w, t - 7); - let w2_0 = schedule_word(w, t - 2); + let w16_0 = schedule_word(w, t.strict_sub(16)); + let w15_0 = schedule_word(w, t.strict_sub(15)); + let w7_0 = schedule_word(w, t.strict_sub(7)); + let w2_0 = schedule_word(w, t.strict_sub(2)); let w0 = small_sigma1(w2_0) .wrapping_add(w7_0) .wrapping_add(small_sigma0(w15_0)) .wrapping_add(w16_0); - let w16_1 = schedule_word(w, t - 15); - let w15_1 = schedule_word(w, t - 14); - let w7_1 = schedule_word(w, t - 6); - let w2_1 = schedule_word(w, t - 1); + let w16_1 = schedule_word(w, t.strict_sub(15)); + let w15_1 = schedule_word(w, t.strict_sub(14)); + let w7_1 = schedule_word(w, t.strict_sub(6)); + let w2_1 = schedule_word(w, t.strict_sub(1)); let w1 = small_sigma1(w2_1) .wrapping_add(w7_1) .wrapping_add(small_sigma0(w15_1)) .wrapping_add(w16_1); - let w16_2 = schedule_word(w, t - 14); - let w15_2 = schedule_word(w, t - 13); - let w7_2 = schedule_word(w, t - 5); + let w16_2 = schedule_word(w, t.strict_sub(14)); + let w15_2 = schedule_word(w, t.strict_sub(13)); + let w7_2 = schedule_word(w, t.strict_sub(5)); let w2_2 = w0; let w2 = small_sigma1(w2_2) .wrapping_add(w7_2) .wrapping_add(small_sigma0(w15_2)) .wrapping_add(w16_2); - let w16_3 = schedule_word(w, t - 13); - let w15_3 = schedule_word(w, t - 12); - let w7_3 = schedule_word(w, t - 4); + let w16_3 = schedule_word(w, t.strict_sub(13)); + let w15_3 = schedule_word(w, t.strict_sub(12)); + let w7_3 = schedule_word(w, t.strict_sub(4)); let w2_3 = w1; let w3 = small_sigma1(w2_3) .wrapping_add(w7_3) @@ -117,6 +123,11 @@ fn schedule_4(w: &mut [v128; 16], i: usize) { /// /// The message schedule is computed with SIMD (4 words per v128). /// Compression rounds are scalar (sequential dependency chain). +/// +/// # Safety +/// +/// The caller must ensure WebAssembly SIMD128 is available and `blocks` +/// contains only complete SHA-256 blocks. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u32; 8], blocks: &[u8]) { @@ -202,9 +213,9 @@ pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u32; 8], blocks: &[u schedule_4(&mut wv, r / 4); let sched = wv[(r / 4) & 0xF]; sha_round!(K[r], u32x4_extract_lane::<0>(sched)); - sha_round!(K[r + 1], u32x4_extract_lane::<1>(sched)); - sha_round!(K[r + 2], u32x4_extract_lane::<2>(sched)); - sha_round!(K[r + 3], u32x4_extract_lane::<3>(sched)); + sha_round!(K[r.strict_add(1)], u32x4_extract_lane::<1>(sched)); + sha_round!(K[r.strict_add(2)], u32x4_extract_lane::<2>(sched)); + sha_round!(K[r.strict_add(3)], u32x4_extract_lane::<3>(sched)); } state[0] = state[0].wrapping_add(a); diff --git a/src/hashes/crypto/sha256/x86_64.rs b/src/hashes/crypto/sha256/x86_64.rs index 6aa3469d..6f2e426a 100644 --- a/src/hashes/crypto/sha256/x86_64.rs +++ b/src/hashes/crypto/sha256/x86_64.rs @@ -8,9 +8,6 @@ //! All functions require `sha` and `sse4.1` target features. //! Callers must verify CPU capabilities before calling. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; diff --git a/src/hashes/crypto/sha3.rs b/src/hashes/crypto/sha3.rs index b71a3b4e..ca02f464 100644 --- a/src/hashes/crypto/sha3.rs +++ b/src/hashes/crypto/sha3.rs @@ -75,26 +75,17 @@ impl MlKemShake256XofReader { } #[inline] - #[allow(clippy::too_many_arguments)] - pub(crate) fn squeeze_quad( - a: &mut Self, + pub(crate) fn squeeze_quad( + &mut self, b: &mut Self, c: &mut Self, d: &mut Self, - out_a: &mut [u8], - out_b: &mut [u8], - out_c: &mut [u8], - out_d: &mut [u8], + outputs: &mut [[u8; N]; 4], ) { + let [out_a, out_b, out_c, out_d] = outputs; KeccakXof::<136>::squeeze_quad_into( - &mut a.inner, - &mut b.inner, - &mut c.inner, - &mut d.inner, - out_a, - out_b, - out_c, - out_d, + [&mut self.inner, &mut b.inner, &mut c.inner, &mut d.inner], + [out_a, out_b, out_c, out_d], ); } } @@ -148,13 +139,9 @@ pub fn diag_zeroize_mlkem_shake256_pair(mut seed: [u8; 32]) -> u8 { pub fn diag_zeroize_mlkem_shake256_quad(mut seed: [u8; 32]) -> u8 { let (mut a, mut b, mut c, mut d) = MlKemShake256XofReader::seeded_32_1_quad(&seed, 1, 2, 3, 4); crate::traits::ct::zeroize(&mut seed); - let mut out_a = [0u8; 192]; - let mut out_b = [0u8; 192]; - let mut out_c = [0u8; 192]; - let mut out_d = [0u8; 192]; - MlKemShake256XofReader::squeeze_quad( - &mut a, &mut b, &mut c, &mut d, &mut out_a, &mut out_b, &mut out_c, &mut out_d, - ); + let mut outputs = [[0u8; 192]; 4]; + a.squeeze_quad(&mut b, &mut c, &mut d, &mut outputs); + let [mut out_a, mut out_b, mut out_c, mut out_d] = outputs; let observed = out_a[0] ^ out_b[0] ^ out_c[0] ^ out_d[0]; crate::traits::ct::zeroize(&mut out_a); crate::traits::ct::zeroize(&mut out_b); @@ -497,12 +484,14 @@ impl core::fmt::Debug for Shake128 { } impl Shake128 { + /// Create an empty SHAKE128 state. #[inline] #[must_use] pub fn new() -> Self { Self::default() } + /// Absorb `data` and return a SHAKE128 output reader. #[inline] #[must_use] pub fn xof(data: &[u8]) -> Shake128XofReader { @@ -511,11 +500,13 @@ impl Shake128 { h.finalize_xof() } + /// Absorb more input into this state. #[inline] pub fn update(&mut self, data: &[u8]) { self.core.update(data); } + /// Finalize absorption and return an output reader. #[inline] #[must_use] pub fn finalize_xof(self) -> Shake128XofReader { @@ -524,6 +515,7 @@ impl Shake128 { } } + /// Reset this state to its empty value. #[inline] pub fn reset(&mut self) { *self = Self::default(); @@ -619,6 +611,7 @@ impl Shake128 { } } +/// Streaming reader for SHAKE128 output. #[derive(Clone)] pub struct Shake128XofReader { inner: PublicKeccakXof<168>, @@ -678,26 +671,17 @@ impl Shake128XofReader { )) ) ))] - #[allow(clippy::too_many_arguments)] - pub(crate) fn squeeze_quad( - a: &mut Self, + pub(crate) fn squeeze_quad( + &mut self, b: &mut Self, c: &mut Self, d: &mut Self, - out_a: &mut [u8], - out_b: &mut [u8], - out_c: &mut [u8], - out_d: &mut [u8], + outputs: &mut [[u8; N]; 4], ) { + let [out_a, out_b, out_c, out_d] = outputs; PublicKeccakXof::<168>::squeeze_quad_into( - &mut a.inner, - &mut b.inner, - &mut c.inner, - &mut d.inner, - out_a, - out_b, - out_c, - out_d, + [&mut self.inner, &mut b.inner, &mut c.inner, &mut d.inner], + [out_a, out_b, out_c, out_d], ); } @@ -740,12 +724,14 @@ impl Shake128XofReader { impl_xof_read!(Shake128XofReader); impl Shake256 { + /// Create an empty SHAKE256 state. #[inline] #[must_use] pub fn new() -> Self { Self::default() } + /// Absorb `data` and return a SHAKE256 output reader. #[inline] #[must_use] pub fn xof(data: &[u8]) -> Shake256XofReader { @@ -754,11 +740,13 @@ impl Shake256 { h.finalize_xof() } + /// Absorb more input into this state. #[inline] pub fn update(&mut self, data: &[u8]) { self.core.update(data); } + /// Finalize absorption and return an output reader. #[inline] #[must_use] pub fn finalize_xof(self) -> Shake256XofReader { @@ -767,6 +755,7 @@ impl Shake256 { } } + /// Reset this state to its empty value. #[inline] pub fn reset(&mut self) { *self = Self::default(); @@ -782,6 +771,7 @@ impl Shake256 { } } +/// Streaming reader for SHAKE256 output. #[derive(Clone)] pub struct Shake256XofReader { inner: PublicKeccakXof<136>, @@ -804,8 +794,6 @@ impl_xof_read!(Shake256XofReader); #[cfg(test)] mod tests { - #[cfg(feature = "ml-kem")] - use super::Shake128XofReader; use super::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; use crate::traits::{Digest, Xof}; @@ -814,7 +802,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{b:02x}").expect("writing to a String should not fail"); } s } @@ -881,30 +869,8 @@ mod tests { let msg_d = &[0xa5u8; 409]; let (mut quad_a, mut quad_b, mut quad_c, mut quad_d) = Shake128::xof_quad(msg_a, msg_b, msg_c, msg_d); - let mut actual_a = [0u8; 320]; - let mut actual_b = [0u8; 320]; - let mut actual_c = [0u8; 320]; - let mut actual_d = [0u8; 320]; - Shake128XofReader::squeeze_quad( - &mut quad_a, - &mut quad_b, - &mut quad_c, - &mut quad_d, - &mut actual_a[..168], - &mut actual_b[..168], - &mut actual_c[..168], - &mut actual_d[..168], - ); - Shake128XofReader::squeeze_quad( - &mut quad_a, - &mut quad_b, - &mut quad_c, - &mut quad_d, - &mut actual_a[168..], - &mut actual_b[168..], - &mut actual_c[168..], - &mut actual_d[168..], - ); + let mut actual = [[0u8; 320]; 4]; + quad_a.squeeze_quad(&mut quad_b, &mut quad_c, &mut quad_d, &mut actual); let mut expected_a = [0u8; 320]; let mut expected_b = [0u8; 320]; @@ -915,10 +881,10 @@ mod tests { Shake128::xof(msg_c).squeeze(&mut expected_c); Shake128::xof(msg_d).squeeze(&mut expected_d); - assert_eq!(actual_a, expected_a, "lane 0"); - assert_eq!(actual_b, expected_b, "lane 1"); - assert_eq!(actual_c, expected_c, "lane 2"); - assert_eq!(actual_d, expected_d, "lane 3"); + assert_eq!(actual[0], expected_a, "lane 0"); + assert_eq!(actual[1], expected_b, "lane 1"); + assert_eq!(actual[2], expected_c, "lane 2"); + assert_eq!(actual[3], expected_d, "lane 3"); } /// Test inputs of length `RATE - 1` for each SHA-3 variant. diff --git a/src/hashes/crypto/sha384.rs b/src/hashes/crypto/sha384.rs index 53720dea..44b1feb2 100644 --- a/src/hashes/crypto/sha384.rs +++ b/src/hashes/crypto/sha384.rs @@ -3,8 +3,6 @@ //! SHA-384 is identical to SHA-512 except for initial hash values (H0) and //! output truncation (48 bytes / 6 words). The compression function is shared. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays in finalization - use self::kernels::CompressBlocksFn; use super::sha512::Sha512; use crate::{ @@ -82,7 +80,7 @@ impl Sha384Prefix { unsafe { core::ptr::write_volatile(word, 0) }; } // SAFETY: bytes_hashed is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -255,7 +253,6 @@ impl Sha384 { #[cfg(all(feature = "hmac", any(test, feature = "diag")))] #[inline] - #[allow(dead_code)] pub(crate) fn new_with_compress_for_test(compress_blocks: CompressBlocksFn) -> Self { Self { state: H0, @@ -282,9 +279,9 @@ impl Drop for Sha384 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } diff --git a/src/hashes/crypto/sha384/dispatch_tables.rs b/src/hashes/crypto/sha384/dispatch_tables.rs index 90de8339..e2eae2e7 100644 --- a/src/hashes/crypto/sha384/dispatch_tables.rs +++ b/src/hashes/crypto/sha384/dispatch_tables.rs @@ -3,13 +3,13 @@ //! SHA-384 uses identical compression to SHA-512, so the same hardware kernels //! and cascade order apply. -pub use super::kernels::Sha384KernelId as KernelId; +pub(crate) use super::kernels::Sha384KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -17,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -26,7 +26,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha512, s: KernelId::Aarch64Sha512, @@ -35,7 +35,7 @@ pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha512, s: KernelId::X86Sha512, @@ -44,7 +44,7 @@ pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vl, s: KernelId::X86Avx512vl, @@ -53,7 +53,7 @@ pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx2Decoupled, s: KernelId::X86Avx2Decoupled, @@ -62,7 +62,7 @@ pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vlDecoupled, s: KernelId::X86Avx512vlDecoupled, @@ -71,7 +71,7 @@ pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "riscv64")] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Riscv64Zknh, s: KernelId::Riscv64Zknh, @@ -80,7 +80,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -89,7 +89,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -99,7 +99,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { // x86_64 cascade: SHA-512 NI > vendor-aware AVX2/AVX-512VL > Portable // AMD: AVX2 decoupled > AVX-512VL; Intel: AVX-512VL decoupled > AVX2 decoupled. // See sha512/dispatch_tables.rs for full rationale. @@ -157,5 +157,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha384/kernel_test.rs b/src/hashes/crypto/sha384/kernel_test.rs index fe5a45f0..798388a3 100644 --- a/src/hashes/crypto/sha384/kernel_test.rs +++ b/src/hashes/crypto/sha384/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 48], -} - fn hasher_for_kernel(id: Sha384KernelId) -> Sha384 { let compress = compress_blocks_fn(id); Sha384 { @@ -63,43 +56,16 @@ fn digest_oneshot_with_kernel(id: Sha384KernelId, data: &[u8]) -> [u8; 48] { out } -#[allow(dead_code)] -#[must_use] -pub fn run_all_sha384_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -#[allow(dead_code)] -pub fn verify_sha384_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha384_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha384 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(19).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(19).wrapping_add(bytes[1]) + }) .collect() } @@ -108,7 +74,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, + 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 111, 112, 113, 127, 128, 129, 255, 256, 257]; diff --git a/src/hashes/crypto/sha384/kernels.rs b/src/hashes/crypto/sha384/kernels.rs index eef50d09..4ef7b0e2 100644 --- a/src/hashes/crypto/sha384/kernels.rs +++ b/src/hashes/crypto/sha384/kernels.rs @@ -6,7 +6,7 @@ use crate::{hashes::crypto::sha512::Sha512, platform::Caps}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha384KernelId { +pub(crate) enum Sha384KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Sha512 = 1, @@ -30,7 +30,7 @@ impl Sha384KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -110,13 +110,13 @@ pub(crate) fn compress_blocks_fn(id: Sha384KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha384KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha384KernelId) -> Caps { crate::hashes::crypto::sha512::kernels::required_caps(to_sha512_kernel_id(id)) } // Keep kernel tests focused on backends that runtime dispatch can actually pick. #[cfg(test)] -pub const ALL: &[Sha384KernelId] = &[ +pub(crate) const ALL: &[Sha384KernelId] = &[ Sha384KernelId::Portable, #[cfg(target_arch = "aarch64")] Sha384KernelId::Aarch64Sha512, diff --git a/src/hashes/crypto/sha512/aarch64.rs b/src/hashes/crypto/sha512/aarch64.rs index d36f94c8..dd185705 100644 --- a/src/hashes/crypto/sha512/aarch64.rs +++ b/src/hashes/crypto/sha512/aarch64.rs @@ -6,10 +6,6 @@ //! //! Available on Apple Silicon (M1+), Graviton2+, Ampere Altra, Cortex-A76+. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::indexing_slicing)] - #[cfg(target_arch = "aarch64")] use core::arch::aarch64::*; @@ -163,49 +159,49 @@ pub(crate) unsafe fn compress_blocks_aarch64_sha512(state: &mut [u64; 8], blocks cd = vaddq_u64(cd, intermed); s1 = vsha512su1q_u64(vsha512su0q_u64(s1, s2), s0, vextq_u64(s5, s6, 1)); - initial_sum = vaddq_u64(s1, k(pair + 1)); + initial_sum = vaddq_u64(s1, k(pair.strict_add(1))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); ef = vsha512h2q_u64(intermed, ab, gh); ab = vaddq_u64(ab, intermed); s2 = vsha512su1q_u64(vsha512su0q_u64(s2, s3), s1, vextq_u64(s6, s7, 1)); - initial_sum = vaddq_u64(s2, k(pair + 2)); + initial_sum = vaddq_u64(s2, k(pair.strict_add(2))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); cd = vsha512h2q_u64(intermed, gh, ef); gh = vaddq_u64(gh, intermed); s3 = vsha512su1q_u64(vsha512su0q_u64(s3, s4), s2, vextq_u64(s7, s0, 1)); - initial_sum = vaddq_u64(s3, k(pair + 3)); + initial_sum = vaddq_u64(s3, k(pair.strict_add(3))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); ab = vsha512h2q_u64(intermed, ef, cd); ef = vaddq_u64(ef, intermed); s4 = vsha512su1q_u64(vsha512su0q_u64(s4, s5), s3, vextq_u64(s0, s1, 1)); - initial_sum = vaddq_u64(s4, k(pair + 4)); + initial_sum = vaddq_u64(s4, k(pair.strict_add(4))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh); intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); gh = vsha512h2q_u64(intermed, cd, ab); cd = vaddq_u64(cd, intermed); s5 = vsha512su1q_u64(vsha512su0q_u64(s5, s6), s4, vextq_u64(s1, s2, 1)); - initial_sum = vaddq_u64(s5, k(pair + 5)); + initial_sum = vaddq_u64(s5, k(pair.strict_add(5))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); ef = vsha512h2q_u64(intermed, ab, gh); ab = vaddq_u64(ab, intermed); s6 = vsha512su1q_u64(vsha512su0q_u64(s6, s7), s5, vextq_u64(s2, s3, 1)); - initial_sum = vaddq_u64(s6, k(pair + 6)); + initial_sum = vaddq_u64(s6, k(pair.strict_add(6))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); cd = vsha512h2q_u64(intermed, gh, ef); gh = vaddq_u64(gh, intermed); s7 = vsha512su1q_u64(vsha512su0q_u64(s7, s0), s6, vextq_u64(s3, s4, 1)); - initial_sum = vaddq_u64(s7, k(pair + 7)); + initial_sum = vaddq_u64(s7, k(pair.strict_add(7))); sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); ab = vsha512h2q_u64(intermed, ef, cd); diff --git a/src/hashes/crypto/sha512/dispatch_tables.rs b/src/hashes/crypto/sha512/dispatch_tables.rs index f5bd5fb6..2f95a47f 100644 --- a/src/hashes/crypto/sha512/dispatch_tables.rs +++ b/src/hashes/crypto/sha512/dispatch_tables.rs @@ -3,13 +3,13 @@ //! SHA-512 NI, ARM SHA512 CE, and Zknh have negligible setup cost — use HW //! accel for all size classes when available. -pub use super::kernels::Sha512KernelId as KernelId; +pub(crate) use super::kernels::Sha512KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -17,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -26,7 +26,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha512, s: KernelId::Aarch64Sha512, @@ -35,7 +35,7 @@ pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha512, s: KernelId::X86Sha512, @@ -44,7 +44,7 @@ pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vl, s: KernelId::X86Avx512vl, @@ -53,7 +53,7 @@ pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx2Decoupled, s: KernelId::X86Avx2Decoupled, @@ -62,7 +62,7 @@ pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vlDecoupled, s: KernelId::X86Avx512vlDecoupled, @@ -71,7 +71,7 @@ pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "riscv64")] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Riscv64Zknh, s: KernelId::Riscv64Zknh, @@ -80,7 +80,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -89,7 +89,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -99,11 +99,11 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { // x86_64 cascade: SHA-512 NI, then vendor-aware AVX2/AVX-512VL, then portable. // - // The AVX2 kernel falls back to portable for odd block counts, while - // AVX-512VL handles a trailing single block. Vendor-specific ordering is a + // Both AVX2 and AVX-512VL handle a trailing single block inside their SIMD + // kernels. Vendor-specific ordering between the implementations remains a // manually maintained dispatch policy. #[cfg(target_arch = "x86_64")] { @@ -164,5 +164,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha512/kernel_test.rs b/src/hashes/crypto/sha512/kernel_test.rs index 1a4dcd2e..63275d98 100644 --- a/src/hashes/crypto/sha512/kernel_test.rs +++ b/src/hashes/crypto/sha512/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 64], -} - fn hasher_for_kernel(id: Sha512KernelId) -> Sha512 { let compress = compress_blocks_fn(id); Sha512 { @@ -67,43 +60,16 @@ fn digest_64_byte_prefix_with_kernel(id: Sha512KernelId, prefix: &[u8; 64], data Sha512::digest_64_byte_prefix_with(prefix, data, compress_blocks_fn(id)) } -#[allow(dead_code)] -#[must_use] -pub fn run_all_sha512_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -#[allow(dead_code)] -pub fn verify_sha512_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha512_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha512 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(13).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(13).wrapping_add(bytes[1]) + }) .collect() } @@ -112,7 +78,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, + 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 111, 112, 113, 127, 128, 129, 255, 256, 257]; @@ -195,7 +161,10 @@ mod tests { let mut prefix = [0u8; 64]; for (i, byte) in prefix.iter_mut().enumerate() { - *byte = (i as u8).wrapping_mul(17).wrapping_add(0x5a); + *byte = u8::try_from(i) + .expect("SHA-512 prefix index fits in u8") + .wrapping_mul(17) + .wrapping_add(0x5a); } for &id in ALL { @@ -222,10 +191,4 @@ mod tests { } } } - - #[test] - fn run_all_agree() { - verify_sha512_kernels(b"abc").expect("kernels should agree"); - verify_sha512_kernels(&pattern(4096)).expect("kernels should agree"); - } } diff --git a/src/hashes/crypto/sha512/kernels.rs b/src/hashes/crypto/sha512/kernels.rs index 2395f469..941f6c8a 100644 --- a/src/hashes/crypto/sha512/kernels.rs +++ b/src/hashes/crypto/sha512/kernels.rs @@ -16,7 +16,7 @@ pub(crate) type CompressBlocksFn = fn(&mut [u64; 8], &[u8]); #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha512KernelId { +pub(crate) enum Sha512KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Sha512 = 1, @@ -40,7 +40,7 @@ impl Sha512KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -141,7 +141,7 @@ pub(crate) fn compress_blocks_fn(id: Sha512KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha512KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha512KernelId) -> Caps { match id { Sha512KernelId::Portable => Caps::NONE, #[cfg(target_arch = "aarch64")] @@ -165,7 +165,7 @@ pub const fn required_caps(id: Sha512KernelId) -> Caps { // Keep kernel tests focused on backends that runtime dispatch can actually pick. #[cfg(test)] -pub const ALL: &[Sha512KernelId] = &[ +pub(crate) const ALL: &[Sha512KernelId] = &[ Sha512KernelId::Portable, #[cfg(target_arch = "aarch64")] Sha512KernelId::Aarch64Sha512, diff --git a/src/hashes/crypto/sha512/mod.rs b/src/hashes/crypto/sha512/mod.rs index 65459e27..20ff59d4 100644 --- a/src/hashes/crypto/sha512/mod.rs +++ b/src/hashes/crypto/sha512/mod.rs @@ -1,7 +1,5 @@ //! SHA-512 (FIPS 180-4). -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - use self::kernels::CompressBlocksFn; use crate::{ hashes::{ @@ -216,7 +214,7 @@ impl Sha512Prefix { unsafe { core::ptr::write_volatile(word, 0) }; } // SAFETY: bytes_hashed is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -280,14 +278,11 @@ impl Sha512 { #[inline] pub(crate) fn compress_blocks_portable(state: &mut [u64; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + for block in chunks { Self::compress_block(state, block); } - debug_assert!(chunks.remainder().is_empty()); + debug_assert!(remainder.is_empty()); } #[inline] @@ -465,7 +460,7 @@ impl Sha512 { #[inline(always)] fn state_to_digest(state: &[u64; 8]) -> [u8; 64] { let mut out = [0u8; 64]; - for (chunk, &word) in out.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } out @@ -528,7 +523,6 @@ impl Sha512 { #[cfg(all(feature = "hmac", any(test, feature = "diag")))] #[inline] - #[allow(dead_code)] pub(crate) fn new_with_compress_for_test(compress_blocks: CompressBlocksFn) -> Self { Self { state: H0, @@ -818,9 +812,9 @@ impl Drop for Sha512 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } @@ -871,7 +865,7 @@ mod tests { use core::fmt::Write; let mut s = String::new(); for &b in bytes { - write!(&mut s, "{:02x}", b).unwrap(); + write!(&mut s, "{:02x}", b).expect("writing hexadecimal to String must succeed"); } s } diff --git a/src/hashes/crypto/sha512/riscv64.rs b/src/hashes/crypto/sha512/riscv64.rs index 811d62fa..64a1d277 100644 --- a/src/hashes/crypto/sha512/riscv64.rs +++ b/src/hashes/crypto/sha512/riscv64.rs @@ -8,17 +8,11 @@ //! //! All functions require the `zknh` target feature. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "riscv64")] use core::arch::riscv64::{sha512sig0, sha512sig1, sha512sum0, sha512sum1}; use super::BLOCK_LEN; -// Safe wrappers for the unsafe Zknh intrinsics. -// SAFETY: only called from within `compress_blocks_zknh` which has -// `#[target_feature(enable = "zknh")]`, guaranteeing the feature is available. - #[inline(always)] fn sum0(x: u64) -> u64 { // SAFETY: `compress_blocks_zknh` calls this wrapper only inside its Zknh target-feature scope. @@ -50,12 +44,9 @@ fn sig1(x: u64) -> u64 { /// Caller must ensure `zknh` CPU feature is available. #[target_feature(enable = "zknh")] pub(crate) unsafe fn compress_blocks_zknh(state: &mut [u64; 8], blocks: &[u8]) { - debug_assert_eq!(blocks.len() % BLOCK_LEN, 0); - let mut chunks = blocks.chunks_exact(BLOCK_LEN); - for chunk in &mut chunks { - // SAFETY: `chunks_exact(BLOCK_LEN)` yields slices of exactly `BLOCK_LEN` bytes. - let block = unsafe { &*(chunk.as_ptr() as *const [u8; BLOCK_LEN]) }; + let (chunks, remainder) = blocks.as_chunks::(); + debug_assert!(remainder.is_empty()); + for block in chunks { super::compress_block_with(state, block, sum0, sum1, sig0, sig1); } - debug_assert!(chunks.remainder().is_empty()); } diff --git a/src/hashes/crypto/sha512/s390x.rs b/src/hashes/crypto/sha512/s390x.rs index 4d50c97f..dbaf1b15 100644 --- a/src/hashes/crypto/sha512/s390x.rs +++ b/src/hashes/crypto/sha512/s390x.rs @@ -13,9 +13,6 @@ //! //! Requires the MSA facility (CPACF). Caller must verify `s390x::MSA`. -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] - use super::BLOCK_LEN; /// SHA-512 block compression via KIMD instruction. diff --git a/src/hashes/crypto/sha512/wasm.rs b/src/hashes/crypto/sha512/wasm.rs index d752c9fb..4f6920ee 100644 --- a/src/hashes/crypto/sha512/wasm.rs +++ b/src/hashes/crypto/sha512/wasm.rs @@ -7,8 +7,6 @@ //! (they depend on W[t-2]/W[t-1] respectively, both from prior rounds), so //! we get true 2-wide parallelism for sigma computations and additions. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays + compression schedule - #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; @@ -38,12 +36,20 @@ fn small_sigma1_v(x: v128) -> v128 { // Message schedule helpers /// Load 2 big-endian u64 message words from `ptr`, byte-swapping each. +/// +/// # Safety +/// +/// `ptr` must be valid for an unaligned 16-byte read. The caller must execute +/// this function only when WebAssembly SIMD128 is enabled. #[cfg(target_arch = "wasm32")] #[inline(always)] unsafe fn load_be(ptr: *const u8) -> v128 { - // SAFETY: caller guarantees `ptr` is valid for a 16-byte read - // and the simd128 target feature is enabled. - let raw = unsafe { v128_load(ptr as *const v128) }; + // SAFETY: the caller guarantees that `ptr..ptr+16` is readable. The byte + // array has alignment one, and `read_unaligned` accepts any address. + let bytes = unsafe { ptr.cast::<[u8; 16]>().read_unaligned() }; + // SAFETY: `[u8; 16]` and `v128` have the same size, and every SIMD bit + // pattern is valid. + let raw = unsafe { core::mem::transmute::<[u8; 16], v128>(bytes) }; i8x16_shuffle::<7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8>(raw, raw) } @@ -82,6 +88,11 @@ fn schedule_pair(w: &mut [v128; 8], i: usize) { /// /// Message schedule is computed with SIMD (2 u64 words per v128). /// Compression rounds are scalar (sequential dependency chain). +/// +/// # Safety +/// +/// The caller must ensure WebAssembly SIMD128 is available and `blocks` +/// contains only complete SHA-512 blocks. #[cfg(target_arch = "wasm32")] #[target_feature(enable = "simd128")] pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u64; 8], blocks: &[u8]) { @@ -136,24 +147,23 @@ pub(crate) unsafe fn compress_blocks_wasm_simd(state: &mut [u64; 8], blocks: &[u } // Rounds 0-15: extract from loaded vectors. - for pair in 0..8 { - let v = wv[pair]; - let w_lo = i64x2_extract_lane::<0>(v) as u64; - let w_hi = i64x2_extract_lane::<1>(v) as u64; + for (pair, &v) in wv.iter().enumerate() { + let w_lo = i64x2_extract_lane::<0>(v).cast_unsigned(); + let w_hi = i64x2_extract_lane::<1>(v).cast_unsigned(); let r = pair.strict_mul(2); sha_round!(K[r], w_lo); - sha_round!(K[r + 1], w_hi); + sha_round!(K[r.strict_add(1)], w_hi); } // Rounds 16-79: expand schedule with SIMD, then extract for scalar rounds. for pair in 8..40 { schedule_pair(&mut wv, pair); let v = wv[pair & 7]; - let w_lo = i64x2_extract_lane::<0>(v) as u64; - let w_hi = i64x2_extract_lane::<1>(v) as u64; + let w_lo = i64x2_extract_lane::<0>(v).cast_unsigned(); + let w_hi = i64x2_extract_lane::<1>(v).cast_unsigned(); let r = pair.strict_mul(2); sha_round!(K[r], w_lo); - sha_round!(K[r + 1], w_hi); + sha_round!(K[r.strict_add(1)], w_hi); } state[0] = state[0].wrapping_add(a); diff --git a/src/hashes/crypto/sha512/x86_64.rs b/src/hashes/crypto/sha512/x86_64.rs index 5d6f422d..74e0d038 100644 --- a/src/hashes/crypto/sha512/x86_64.rs +++ b/src/hashes/crypto/sha512/x86_64.rs @@ -15,9 +15,6 @@ //! All functions require `sha512` and `avx2` target features. //! Callers must verify CPU capabilities before calling. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -160,10 +157,14 @@ static BSWAP64_MASK: [u8; 32] = [ /// /// Given a = [W[n], W[n+1], W[n+2], W[n+3]] and b = [W[n+4], ...], /// returns [W[n+1], W[n+2], W[n+3], W[n+4]]. +/// +/// # Safety +/// +/// The caller must ensure that AVX2 is available on the executing CPU. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_w_tm7(a: __m256i, b: __m256i) -> __m256i { - // SAFETY: AVX2 intrinsics are available via this function's #[target_feature] attribute. + // SAFETY: The caller guarantees AVX2 support. unsafe { // Swap halves: [a_hi, b_lo] let t = _mm256_permute2x128_si256(a, b, 0x21); diff --git a/src/hashes/crypto/sha512/x86_64_avx2.rs b/src/hashes/crypto/sha512/x86_64_avx2.rs index 993da189..1951db42 100644 --- a/src/hashes/crypto/sha512/x86_64_avx2.rs +++ b/src/hashes/crypto/sha512/x86_64_avx2.rs @@ -22,10 +22,6 @@ //! //! All functions require `avx2` and `bmi2` target features. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::indexing_slicing)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -34,6 +30,9 @@ use super::{BLOCK_LEN, K, big_sigma0, big_sigma1, ch, maj}; // 256-bit SIMD sigma (dual-block schedule, 2 × u64 per 128-bit lane) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 256-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_256(x: __m256i) -> __m256i { @@ -46,6 +45,9 @@ unsafe fn small_sigma0_256(x: __m256i) -> __m256i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 256-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_256(x: __m256i) -> __m256i { @@ -60,6 +62,9 @@ unsafe fn small_sigma1_256(x: __m256i) -> __m256i { // 128-bit SIMD sigma (single-block schedule, 2 × u64 per register) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 128-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_128(x: __m128i) -> __m128i { @@ -72,6 +77,9 @@ unsafe fn small_sigma0_128(x: __m128i) -> __m128i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 128-bit, 3-op rotates. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_128(x: __m128i) -> __m128i { @@ -90,6 +98,10 @@ unsafe fn small_sigma1_128(x: __m128i) -> __m128i { static BSWAP64_128: [u8; 16] = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]; /// Load 2 big-endian u64 words from each of two blocks into a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. Both block +/// pointers must remain valid for 16 readable bytes starting at `offset`. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap: __m128i) -> __m256i { @@ -102,33 +114,53 @@ unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap } /// Extract two u64 words from the lower 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_lo(v: __m256i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. unsafe { let lo128 = _mm256_castsi256_si128(v); - (_mm_extract_epi64(lo128, 0) as u64, _mm_extract_epi64(lo128, 1) as u64) + ( + _mm_extract_epi64(lo128, 0).cast_unsigned(), + _mm_extract_epi64(lo128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from the upper 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_hi(v: __m256i) -> (u64, u64) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. unsafe { let hi128 = _mm256_extracti128_si256(v, 1); - (_mm_extract_epi64(hi128, 0) as u64, _mm_extract_epi64(hi128, 1) as u64) + ( + _mm_extract_epi64(hi128, 0).cast_unsigned(), + _mm_extract_epi64(hi128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from a __m128i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_128(v: __m128i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. - unsafe { (_mm_extract_epi64(v, 0) as u64, _mm_extract_epi64(v, 1) as u64) } + unsafe { + ( + _mm_extract_epi64(v, 0).cast_unsigned(), + _mm_extract_epi64(v, 1).cast_unsigned(), + ) + } } // Rotation-based schedule (eliminates ring-buffer index computation) @@ -138,6 +170,9 @@ unsafe fn extract_128(v: __m128i) -> (u64, u64) { /// Same approach as [`schedule_rotate_256`] but for 128-bit registers. /// Eliminates ring-buffer index computation (`wrapping_sub`, `& 7`) in favour /// of fixed-offset array accesses and physical rotation. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { @@ -173,6 +208,9 @@ unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { /// Physically rotates the `x[]` array so that adjacent schedule words are /// always in adjacent registers. This lets `_mm256_alignr_epi8` extract /// cross-register values without `_mm256_permute2x128_si256`. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX2 target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { diff --git a/src/hashes/crypto/sha512/x86_64_avx512vl.rs b/src/hashes/crypto/sha512/x86_64_avx512vl.rs index 9d29c714..4062783e 100644 --- a/src/hashes/crypto/sha512/x86_64_avx512vl.rs +++ b/src/hashes/crypto/sha512/x86_64_avx512vl.rs @@ -28,10 +28,6 @@ //! //! All functions require `avx512f`, `avx512vl`, and `bmi2` target features. -#![allow(unsafe_code)] -#![allow(clippy::inline_always)] -#![allow(clippy::indexing_slicing)] - #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; @@ -40,6 +36,9 @@ use super::{BLOCK_LEN, K, big_sigma0, big_sigma1, ch, maj}; // 256-bit SIMD sigma with VPRORQ (dual-block schedule) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 256-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_256(x: __m256i) -> __m256i { @@ -53,6 +52,9 @@ unsafe fn small_sigma0_256(x: __m256i) -> __m256i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 256-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_256(x: __m256i) -> __m256i { @@ -68,6 +70,9 @@ unsafe fn small_sigma1_256(x: __m256i) -> __m256i { // 128-bit SIMD sigma with VPRORQ (single-block schedule) /// σ0(x) = ROTR(1) ^ ROTR(8) ^ SHR(7) — 128-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma0_128(x: __m128i) -> __m128i { @@ -81,6 +86,9 @@ unsafe fn small_sigma0_128(x: __m128i) -> __m128i { } /// σ1(x) = ROTR(19) ^ ROTR(61) ^ SHR(6) — 128-bit, VPRORQ. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn small_sigma1_128(x: __m128i) -> __m128i { @@ -96,6 +104,9 @@ unsafe fn small_sigma1_128(x: __m128i) -> __m128i { // Dual-block message schedule helpers (256-bit) /// Cross-register extraction (256-bit): [a[1], a[2], a[3], b[0]]. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn cross_lanes_256(a: __m256i, b: __m256i) -> __m256i { @@ -107,6 +118,9 @@ unsafe fn cross_lanes_256(a: __m256i, b: __m256i) -> __m256i { } /// Compute 2 schedule words for both blocks (256-bit ring buffer, VPRORQ). +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_pair_256(w: &mut [__m256i; 8], i: usize) { @@ -127,6 +141,9 @@ unsafe fn schedule_pair_256(w: &mut [__m256i; 8], i: usize) { // Single-block message schedule helpers (128-bit) /// Cross-register extraction (128-bit): [a[1], b[0]]. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn cross_lanes_128(a: __m128i, b: __m128i) -> __m128i { @@ -135,6 +152,9 @@ unsafe fn cross_lanes_128(a: __m128i, b: __m128i) -> __m128i { } /// Compute 2 schedule words for a single block (128-bit ring buffer, VPRORQ). +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_pair_128(w: &mut [__m128i; 8], i: usize) { @@ -159,6 +179,10 @@ unsafe fn schedule_pair_128(w: &mut [__m128i; 8], i: usize) { static BSWAP64_128: [u8; 16] = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]; /// Load 2 big-endian u64 words from each of two blocks into a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. Both +/// block pointers must remain valid for 16 readable bytes starting at `offset`. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap: __m128i) -> __m256i { @@ -171,33 +195,53 @@ unsafe fn load_two_blocks(blk1: *const u8, blk2: *const u8, offset: usize, bswap } /// Extract two u64 words from the lower 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_lo(v: __m256i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. unsafe { let lo128 = _mm256_castsi256_si128(v); - (_mm_extract_epi64(lo128, 0) as u64, _mm_extract_epi64(lo128, 1) as u64) + ( + _mm_extract_epi64(lo128, 0).cast_unsigned(), + _mm_extract_epi64(lo128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from the upper 128-bit lane of a __m256i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_hi(v: __m256i) -> (u64, u64) { // SAFETY: AVX2 intrinsics are available via the caller's #[target_feature] attribute. unsafe { let hi128 = _mm256_extracti128_si256(v, 1); - (_mm_extract_epi64(hi128, 0) as u64, _mm_extract_epi64(hi128, 1) as u64) + ( + _mm_extract_epi64(hi128, 0).cast_unsigned(), + _mm_extract_epi64(hi128, 1).cast_unsigned(), + ) } } /// Extract two u64 words from a __m128i. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn extract_128(v: __m128i) -> (u64, u64) { // SAFETY: SSE intrinsics are available via the caller's #[target_feature] attribute. - unsafe { (_mm_extract_epi64(v, 0) as u64, _mm_extract_epi64(v, 1) as u64) } + unsafe { + ( + _mm_extract_epi64(v, 0).cast_unsigned(), + _mm_extract_epi64(v, 1).cast_unsigned(), + ) + } } // Entry point @@ -418,6 +462,9 @@ pub(crate) unsafe fn compress_blocks_avx512vl(state: &mut [u64; 8], blocks: &[u8 /// /// Same approach as [`schedule_rotate_256_vl`] but for 128-bit registers. /// Uses VPRORQ native rotates for sigma functions. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { @@ -454,6 +501,9 @@ unsafe fn schedule_rotate_128(x: &mut [__m128i; 8], k: __m128i) -> __m128i { /// shift-shift-or of the AVX2 path. Combined with array rotation, this also /// eliminates `_mm256_permute2x128_si256`, which the ring-buffer schedule uses /// for cross-lane extraction. +/// +/// # Safety +/// Caller must execute this helper only from this module's AVX-512VL target-feature scope. #[cfg(target_arch = "x86_64")] #[inline(always)] unsafe fn schedule_rotate_256(x: &mut [__m256i; 8], k: __m256i) -> __m256i { diff --git a/src/hashes/crypto/sha512_256.rs b/src/hashes/crypto/sha512_256.rs index baa1d8e9..11f3ffb1 100644 --- a/src/hashes/crypto/sha512_256.rs +++ b/src/hashes/crypto/sha512_256.rs @@ -3,8 +3,6 @@ //! SHA-512/256 is identical to SHA-512 except for initial hash values (H0) and //! output truncation (32 bytes / 4 words). The compression function is shared. -#![allow(clippy::indexing_slicing)] // Fixed-size arrays in finalization - use self::kernels::CompressBlocksFn; use super::sha512::Sha512; use crate::{ @@ -153,7 +151,7 @@ impl Sha512_256 { let state = Sha512::finalize_state_words(self.state, self.block, self.block_len, total_len, compress_blocks); let mut out = [0u8; 32]; - for (chunk, &word) in out.chunks_exact_mut(8).zip(state.iter()) { + for (chunk, &word) in out.as_chunks_mut::<8>().0.iter_mut().zip(state.iter()) { chunk.copy_from_slice(&word.to_be_bytes()); } out @@ -168,9 +166,9 @@ impl Drop for Sha512_256 { } crate::traits::ct::zeroize(&mut self.block); // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.bytes_hashed, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.bytes_hashed, 0) }; // SAFETY: field is a valid, aligned, dereferenceable pointer to initialized memory. - unsafe { core::ptr::write_volatile(&mut self.block_len, 0) }; + unsafe { core::ptr::write_volatile(&raw mut self.block_len, 0) }; core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } } diff --git a/src/hashes/crypto/sha512_256/dispatch_tables.rs b/src/hashes/crypto/sha512_256/dispatch_tables.rs index 0b5c5586..0d391f11 100644 --- a/src/hashes/crypto/sha512_256/dispatch_tables.rs +++ b/src/hashes/crypto/sha512_256/dispatch_tables.rs @@ -3,13 +3,13 @@ //! SHA-512/256 uses identical compression to SHA-512, so the same hardware kernels //! and cascade order apply. -pub use super::kernels::Sha512_256KernelId as KernelId; +pub(crate) use super::kernels::Sha512_256KernelId as KernelId; use crate::platform::Caps; -pub const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; +pub(crate) const DEFAULT_BOUNDARIES: [usize; 3] = [64, 256, 4096]; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub boundaries: [usize; 3], pub xs: KernelId, pub s: KernelId, @@ -17,7 +17,7 @@ pub struct DispatchTable { pub l: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Portable, s: KernelId::Portable, @@ -26,7 +26,7 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "aarch64")] -pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Aarch64Sha512, s: KernelId::Aarch64Sha512, @@ -35,7 +35,7 @@ pub static AARCH64_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_SHA512_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Sha512, s: KernelId::X86Sha512, @@ -44,7 +44,7 @@ pub static X86_SHA512_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vl, s: KernelId::X86Avx512vl, @@ -53,7 +53,7 @@ pub static X86_AVX512VL_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx2Decoupled, s: KernelId::X86Avx2Decoupled, @@ -62,7 +62,7 @@ pub static X86_AVX2_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "x86_64")] -pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { +pub(crate) static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::X86Avx512vlDecoupled, s: KernelId::X86Avx512vlDecoupled, @@ -71,7 +71,7 @@ pub static X86_AVX512VL_DECOUPLED_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "riscv64")] -pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { +pub(crate) static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::Riscv64Zknh, s: KernelId::Riscv64Zknh, @@ -80,7 +80,7 @@ pub static RISCV_ZKNH_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "wasm32")] -pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { +pub(crate) static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::WasmSimd128, s: KernelId::WasmSimd128, @@ -89,7 +89,7 @@ pub static WASM_SIMD128_TABLE: DispatchTable = DispatchTable { }; #[cfg(target_arch = "s390x")] -pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { +pub(crate) static S390X_KIMD_TABLE: DispatchTable = DispatchTable { boundaries: DEFAULT_BOUNDARIES, xs: KernelId::S390xKimd, s: KernelId::S390xKimd, @@ -99,7 +99,7 @@ pub static S390X_KIMD_TABLE: DispatchTable = DispatchTable { #[inline] #[must_use] -pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { // x86_64 cascade: SHA-512 NI > vendor-aware AVX2/AVX-512VL > Portable // AMD: AVX2 decoupled > AVX-512VL; Intel: AVX-512VL decoupled > AVX2 decoupled. // See sha512/dispatch_tables.rs for full rationale. @@ -157,5 +157,6 @@ pub fn select_runtime_table(#[allow(unused_variables)] caps: Caps) -> &'static D return &S390X_KIMD_TABLE; } } + let _ = caps; &DEFAULT_TABLE } diff --git a/src/hashes/crypto/sha512_256/kernel_test.rs b/src/hashes/crypto/sha512_256/kernel_test.rs index d86b3024..1d13ac90 100644 --- a/src/hashes/crypto/sha512_256/kernel_test.rs +++ b/src/hashes/crypto/sha512_256/kernel_test.rs @@ -6,13 +6,6 @@ use super::{ }; use crate::{hashes::crypto::dispatch_util::SizeClassDispatch, traits::Digest as _}; -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct KernelResult { - pub name: &'static str, - pub digest: [u8; 32], -} - fn hasher_for_kernel(id: Sha512_256KernelId) -> Sha512_256 { let compress = compress_blocks_fn(id); Sha512_256 { @@ -63,43 +56,16 @@ fn digest_oneshot_with_kernel(id: Sha512_256KernelId, data: &[u8]) -> [u8; 32] { out } -#[allow(dead_code)] -#[must_use] -pub fn run_all_sha512_256_kernels(data: &[u8]) -> Vec { - let caps = crate::platform::caps(); - let mut out = Vec::with_capacity(ALL.len()); - for &id in ALL { - if caps.has(required_caps(id)) { - out.push(KernelResult { - name: id.as_str(), - digest: digest_with_kernel(id, data), - }); - } - } - out -} - -#[allow(dead_code)] -pub fn verify_sha512_256_kernels(data: &[u8]) -> Result<(), &'static str> { - let results = run_all_sha512_256_kernels(data); - let Some(first) = results.first() else { - return Ok(()); - }; - for r in &results[1..] { - if r.digest != first.digest { - return Err("sha512-256 kernel mismatch"); - } - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; fn pattern(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(29).wrapping_add((i >> 8) as u8)) + .map(|i| { + let bytes = i.to_le_bytes(); + bytes[0].wrapping_mul(29).wrapping_add(bytes[1]) + }) .collect() } @@ -108,7 +74,7 @@ mod tests { let caps = crate::platform::caps(); #[cfg(not(miri))] let lens = [ - 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, + 0usize, 1, 2, 3, 111, 112, 113, 127, 128, 129, 239, 240, 241, 255, 256, 257, 1000, 4096, ]; #[cfg(miri)] let lens = [0usize, 1, 111, 112, 113, 127, 128, 129, 255, 256, 257]; diff --git a/src/hashes/crypto/sha512_256/kernels.rs b/src/hashes/crypto/sha512_256/kernels.rs index 869c0661..e92032e7 100644 --- a/src/hashes/crypto/sha512_256/kernels.rs +++ b/src/hashes/crypto/sha512_256/kernels.rs @@ -6,7 +6,7 @@ use crate::{hashes::crypto::sha512::Sha512, platform::Caps}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] #[non_exhaustive] -pub enum Sha512_256KernelId { +pub(crate) enum Sha512_256KernelId { Portable = 0, #[cfg(target_arch = "aarch64")] Aarch64Sha512 = 1, @@ -30,7 +30,7 @@ impl Sha512_256KernelId { #[cfg(any(test, feature = "diag"))] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "aarch64")] @@ -120,13 +120,13 @@ pub(crate) fn compress_blocks_fn(id: Sha512_256KernelId) -> CompressBlocksFn { #[inline] #[must_use] -pub const fn required_caps(id: Sha512_256KernelId) -> Caps { +pub(crate) const fn required_caps(id: Sha512_256KernelId) -> Caps { crate::hashes::crypto::sha512::kernels::required_caps(to_sha512_kernel_id(id)) } // Keep kernel tests focused on backends that runtime dispatch can actually pick. #[cfg(test)] -pub const ALL: &[Sha512_256KernelId] = &[ +pub(crate) const ALL: &[Sha512_256KernelId] = &[ Sha512_256KernelId::Portable, #[cfg(target_arch = "aarch64")] Sha512_256KernelId::Aarch64Sha512, diff --git a/src/hashes/crypto/sp800185.rs b/src/hashes/crypto/sp800185.rs index 0e45cbaf..53b2be62 100644 --- a/src/hashes/crypto/sp800185.rs +++ b/src/hashes/crypto/sp800185.rs @@ -1,7 +1,5 @@ //! SP 800-185 framing helpers shared by cSHAKE and KMAC. -#![allow(clippy::indexing_slicing)] // Fixed-width 9-byte encodings and rate-sized padding buffers. - use super::keccak::KeccakCore; /// cSHAKE128 / KMAC128 bitrate in bytes. @@ -18,10 +16,10 @@ fn encode_u64_be(value: u64, out: &mut [u8; 9], right: bool) -> usize { if right { out[..width].copy_from_slice(&bytes[first..]); - out[width] = width as u8; + out[width] = u8::try_from(width).expect("SP 800-185 integer width is at most eight bytes"); width.strict_add(1) } else { - out[0] = width as u8; + out[0] = u8::try_from(width).expect("SP 800-185 integer width is at most eight bytes"); out[1..=width].copy_from_slice(&bytes[first..]); width.strict_add(1) } @@ -48,14 +46,15 @@ pub(crate) fn encoded_string_len(data: &[u8]) -> usize { } pub(crate) fn absorb_bytepad(core: &mut KeccakCore, segments: &[&[u8]], payload_len: usize) { - let (prefix, prefix_len) = left_encode(RATE as u64); + let rate = u64::try_from(RATE).expect("Keccak rate must fit u64"); + let (prefix, prefix_len) = left_encode(rate); core.update(&prefix[..prefix_len]); for segment in segments { core.update(segment); } let total_len = prefix_len.strict_add(payload_len); - let rem = total_len % RATE; + let rem = total_len.strict_rem(RATE); if rem != 0 { core.update(&[0u8; RATE][..RATE.strict_sub(rem)]); } diff --git a/src/hashes/fast/rapidhash.rs b/src/hashes/fast/rapidhash.rs index dad39ee1..d83b016f 100644 --- a/src/hashes/fast/rapidhash.rs +++ b/src/hashes/fast/rapidhash.rs @@ -2,8 +2,6 @@ //! //! The portable, safe Rust implementation is the sole semantic authority. -#![allow(clippy::indexing_slicing)] - use crate::traits::FastHash; mod stream; @@ -89,32 +87,49 @@ impl FastHash for RapidHash64 { #[inline(always)] const fn read_u32_le(input: &[u8], offset: usize) -> u32 { - let (_, tail) = input.split_at(offset); + let Some((_, tail)) = input.split_at_checked(offset) else { + return 0; + }; let Some(bytes) = tail.first_chunk::<4>() else { - panic!("RapidHash u32 read exceeds input"); + return 0; }; u32::from_le_bytes(*bytes) } #[inline(always)] const fn read_u64_le(input: &[u8], offset: usize) -> u64 { - let (_, tail) = input.split_at(offset); + let Some((_, tail)) = input.split_at_checked(offset) else { + return 0; + }; let Some(bytes) = tail.first_chunk::<8>() else { - panic!("RapidHash u64 read exceeds input"); + return 0; }; u64::from_le_bytes(*bytes) } +#[inline(always)] +const fn u128_words(value: u128) -> (u64, u64) { + let bytes = value.to_le_bytes(); + let low = u64::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]); + let high = u64::from_le_bytes([ + bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15], + ]); + (low, high) +} + #[inline(always)] const fn rapid_mum(a: u64, b: u64) -> (u64, u64) { let product = (a as u128).wrapping_mul(b as u128); - (product as u64, (product >> 64) as u64) + u128_words(product) } #[inline(always)] const fn rapid_mix(a: u64, b: u64) -> u64 { let product = (a as u128).wrapping_mul(b as u128); - (product as u64) ^ ((product >> 64) as u64) + let (low, high) = u128_words(product); + low ^ high } #[inline(always)] @@ -192,12 +207,7 @@ const fn rapidhash_core_large(data: &[u8], mut seed: u64, secrets: &[u64; 7]) -> // Validate the span once so the fixed-size safe reads need no per-load bounds checks. let (_, remaining) = data.split_at(offset); let (block, _) = remaining.split_at(224); - let Some(first) = block.first_chunk::<112>() else { - panic!("RapidHash block is shorter than 112 bytes"); - }; - let Some(second) = block.last_chunk::<112>() else { - panic!("RapidHash block is shorter than 224 bytes"); - }; + let (first, second) = block.split_at(112); seed = rapid_mix(read_u64_le(first, 0) ^ secrets[0], read_u64_le(first, 8) ^ seed); see1 = rapid_mix(read_u64_le(first, 16) ^ secrets[1], read_u64_le(first, 24) ^ see1); see2 = rapid_mix(read_u64_le(first, 32) ^ secrets[2], read_u64_le(first, 40) ^ see2); @@ -218,9 +228,7 @@ const fn rapidhash_core_large(data: &[u8], mut seed: u64, secrets: &[u64; 7]) -> if data.len().strict_sub(offset) > 112 { let (_, remaining) = data.split_at(offset); - let Some(block) = remaining.first_chunk::<112>() else { - panic!("RapidHash block is shorter than 112 bytes"); - }; + let (block, _) = remaining.split_at(112); seed = rapid_mix(read_u64_le(block, 0) ^ secrets[0], read_u64_le(block, 8) ^ seed); see1 = rapid_mix(read_u64_le(block, 16) ^ secrets[1], read_u64_le(block, 24) ^ see1); see2 = rapid_mix(read_u64_le(block, 32) ^ secrets[2], read_u64_le(block, 40) ^ see2); @@ -303,7 +311,9 @@ mod tests { const CONST_SEEDED: u64 = RapidHash64::hash_with_seed(42, b"const RapidHash"); fn data(len: usize) -> Vec { - (0..len).map(|i| i.wrapping_mul(131).wrapping_add(17) as u8).collect() + (0..len) + .map(|i| i.wrapping_mul(131).wrapping_add(17).to_le_bytes()[0]) + .collect() } fn reference(seed: u64, data: &[u8]) -> u64 { diff --git a/src/hashes/fast/rapidhash/stream.rs b/src/hashes/fast/rapidhash/stream.rs index 568ea334..079dcc7a 100644 --- a/src/hashes/fast/rapidhash/stream.rs +++ b/src/hashes/fast/rapidhash/stream.rs @@ -2,7 +2,7 @@ use core::hash::{BuildHasher, Hasher}; use super::{ DEFAULT_SECRETS, RapidSecrets, premix_seed, rapid_mix, rapid_mum, rapidhash_core, rapidhash_seed_cpp, read_u32_le, - read_u64_le, + read_u64_le, u128_words, }; const CHUNK_SIZE: usize = 112; @@ -76,7 +76,11 @@ impl RapidStreamHasher { &data[copy_len..] }; - let stop = remaining.len().saturating_sub(1) / CHUNK_SIZE * CHUNK_SIZE; + let stop = remaining + .len() + .saturating_sub(1) + .strict_div(CHUNK_SIZE) + .strict_mul(CHUNK_SIZE); let mut last_chunk = None; let mut offset = 0usize; while offset < stop { @@ -242,10 +246,8 @@ impl RapidHasher { #[inline(always)] fn flush_sponge(&mut self) { if self.sponge_bits != 0 { - self.seed = rapid_mix( - self.sponge as u64 ^ self.seed, - (self.sponge >> 64) as u64 ^ self.word0(), - ); + let (low, high) = u128_words(self.sponge); + self.seed = rapid_mix(low ^ self.seed, high ^ self.word0()); self.sponge = 0; self.sponge_bits = 0; } @@ -256,10 +258,8 @@ impl RapidHasher { if self.sponge_bits == 0 { self.seed } else { - rapid_mix( - self.sponge as u64 ^ self.seed, - (self.sponge >> 64) as u64 ^ self.word0(), - ) + let (low, high) = u128_words(self.sponge); + rapid_mix(low ^ self.seed, high ^ self.word0()) } } } @@ -284,20 +284,26 @@ fn hash_deterministic(bytes: &[u8], seed: u64) -> u64 { } macro_rules! write_integer { - ($($method:ident, $ty:ty, $unsigned:ty),+ $(,)?) => { + (@convert unsigned, $value:expr) => { + $value as u128 + }; + (@convert signed, $value:expr) => { + $value.cast_unsigned() as u128 + }; + ($kind:ident; $($method:ident, $ty:ty),+ $(,)?) => { $( #[inline(always)] fn $method(&mut self, value: $ty) { - const BITS: u8 = core::mem::size_of::<$ty>() as u8 * 8; - let value = (value as $unsigned) as u128; - let next_bits = self.sponge_bits.strict_add(BITS); + let bits = u8::try_from(<$ty>::BITS).expect("Rust integer width should fit in u8"); + let value = write_integer!(@convert $kind, value); + let next_bits = self.sponge_bits.strict_add(bits); if next_bits <= 128 { self.sponge |= value << self.sponge_bits; self.sponge_bits = next_bits; } else { self.flush_sponge(); self.sponge = value; - self.sponge_bits = BITS; + self.sponge_bits = bits; } } )+ @@ -324,36 +330,20 @@ impl Hasher for RapidHasher { } write_integer!( - write_u8, - u8, - u8, - write_u16, - u16, - u16, - write_u32, - u32, - u32, - write_u64, - u64, - u64, - write_usize, - usize, - usize, - write_i8, - i8, - u8, - write_i16, - i16, - u16, - write_i32, - i32, - u32, - write_i64, - i64, - u64, - write_isize, - isize, - usize, + unsigned; + write_u8, u8, + write_u16, u16, + write_u32, u32, + write_u64, u64, + write_usize, usize, + ); + write_integer!( + signed; + write_i8, i8, + write_i16, i16, + write_i32, i32, + write_i64, i64, + write_isize, isize, ); #[inline(always)] @@ -365,7 +355,7 @@ impl Hasher for RapidHasher { #[inline(always)] fn write_i128(&mut self, value: i128) { - self.write_u128(value as u128); + self.write_u128(value.cast_unsigned()); } } @@ -452,7 +442,9 @@ mod tests { use super::*; fn data(len: usize) -> Vec { - (0..len).map(|i| i.wrapping_mul(131).wrapping_add(17) as u8).collect() + (0..len) + .map(|i| i.wrapping_mul(131).wrapping_add(17).to_le_bytes()[0]) + .collect() } fn seeded_hasher(seed: u64) -> RapidHasher { @@ -571,21 +563,22 @@ mod tests { seed.copy_from_slice(&1u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("caller-provided entropy should initialize the first state"); let same = RapidRandomState::try_new_with(|seed| { seed.copy_from_slice(&1u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("caller-provided entropy should initialize the matching state"); let second = RapidRandomState::try_new_with(|seed| { seed.copy_from_slice(&2u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("caller-provided entropy should initialize the second state"); assert_eq!(first.hash_one(b"collection key"), same.hash_one(b"collection key")); assert_ne!(first.hash_one(b"collection key"), second.hash_one(b"collection key")); - assert!(RapidRandomState::try_new_with(|_| Err::<(), _>("entropy unavailable")).is_err()); + RapidRandomState::try_new_with(|_| Err::<(), _>("entropy unavailable")) + .expect_err("entropy-source failures should be returned"); } #[test] diff --git a/src/hashes/fast/xxh3.rs b/src/hashes/fast/xxh3.rs index 402990d3..8aaeec53 100644 --- a/src/hashes/fast/xxh3.rs +++ b/src/hashes/fast/xxh3.rs @@ -3,8 +3,6 @@ //! Hardware-accelerated on x86-64 (AVX2, AVX-512), aarch64 (NEON), POWER //! (VSX), and s390x (z/Vector), with a portable scalar fallback. -#![allow(clippy::indexing_slicing)] // Tight block parsing + fixed-size arrays - use core::mem; use crate::traits::FastHash; @@ -30,9 +28,11 @@ pub(crate) mod x86_64_avx2; pub(crate) mod x86_64_avx512; #[derive(Clone, Debug, Default)] +/// XXH3 64-bit non-cryptographic hash. pub struct Xxh3_64; #[derive(Clone, Debug, Default)] +/// XXH3 128-bit non-cryptographic hash. pub struct Xxh3_128; // xxh32 primes (used in scramble/mix) @@ -92,10 +92,10 @@ const INITIAL_ACC: [u64; ACC_NB] = [ /// Caller must ensure `offset + 4 <= input.len()`. #[inline(always)] unsafe fn read_u32_le(input: &[u8], offset: usize) -> u32 { - debug_assert!(offset + 4 <= input.len()); + debug_assert!(offset.strict_add(4) <= input.len()); // SAFETY: caller ensures `offset + 4 <= input.len()`, and `read_unaligned` supports unaligned // loads. - let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset) as *const u32) }; + let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset).cast::()) }; u32::from_le(v) } @@ -104,10 +104,10 @@ unsafe fn read_u32_le(input: &[u8], offset: usize) -> u32 { /// Caller must ensure `offset + 8 <= input.len()`. #[inline(always)] unsafe fn read_u64_le(input: &[u8], offset: usize) -> u64 { - debug_assert!(offset + 8 <= input.len()); + debug_assert!(offset.strict_add(8) <= input.len()); // SAFETY: caller ensures `offset + 8 <= input.len()`, and `read_unaligned` supports unaligned // loads. - let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset) as *const u64) }; + let v = unsafe { core::ptr::read_unaligned(input.as_ptr().add(offset).cast::()) }; u64::from_le(v) } @@ -120,9 +120,9 @@ unsafe fn read_u64_le(input: &[u8], offset: usize) -> u64 { /// Caller must ensure `offset + 16 <= data.len()`. #[inline(always)] unsafe fn chunk16(data: &[u8], offset: usize) -> &[[u8; 8]; 2] { - debug_assert!(offset + 16 <= data.len()); + debug_assert!(offset.strict_add(16) <= data.len()); // SAFETY: caller ensures bounds. The resulting reference is valid for 16 bytes. - unsafe { &*(data.as_ptr().add(offset) as *const [[u8; 8]; 2]) } + unsafe { &*data.as_ptr().add(offset).cast::<[[u8; 8]; 2]>() } } /// Extract two consecutive 16-byte chunks (32 bytes) at `offset`. @@ -135,14 +135,29 @@ unsafe fn chunk16(data: &[u8], offset: usize) -> &[[u8; 8]; 2] { /// Caller must ensure `offset + 32 <= data.len()`. #[inline(always)] unsafe fn chunk32(data: &[u8], offset: usize) -> &[[[u8; 8]; 2]; 2] { - debug_assert!(offset + 32 <= data.len()); + debug_assert!(offset.strict_add(32) <= data.len()); // SAFETY: caller ensures bounds. The resulting reference is valid for 32 bytes. - unsafe { &*(data.as_ptr().add(offset) as *const [[[u8; 8]; 2]; 2]) } + unsafe { &*data.as_ptr().add(offset).cast::<[[[u8; 8]; 2]; 2]>() } } #[inline(always)] const fn mult32_to64(left: u32, right: u32) -> u64 { - (left as u64).wrapping_mul(right as u64) + (left as u64).strict_mul(right as u64) +} + +#[inline(always)] +const fn low_u32(value: u64) -> u32 { + let [b0, b1, b2, b3, ..] = value.to_le_bytes(); + u32::from_le_bytes([b0, b1, b2, b3]) +} + +#[inline(always)] +const fn split_u128(value: u128) -> (u64, u64) { + let [b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15] = value.to_le_bytes(); + ( + u64::from_le_bytes([b0, b1, b2, b3, b4, b5, b6, b7]), + u64::from_le_bytes([b8, b9, b10, b11, b12, b13, b14, b15]), + ) } #[inline(always)] @@ -168,8 +183,7 @@ const fn strong_avalanche(mut value: u64, len: u64) -> u64 { #[inline(always)] const fn mul64_to128(left: u64, right: u64) -> (u64, u64) { - let product = (left as u128).wrapping_mul(right as u128); - (product as u64, (product >> 64) as u64) + split_u128((left as u128).strict_mul(right as u128)) } #[inline(always)] @@ -229,7 +243,7 @@ fn xxh3_64_9to16(input: &[u8], seed: u64, secret: &[u8]) -> u64 { let flip2 = (read_u64_le(secret, 40) ^ read_u64_le(secret, 48)).wrapping_sub(seed); let input_lo = read_u64_le(input, 0) ^ flip1; - let input_hi = read_u64_le(input, input.len() - 8) ^ flip2; + let input_hi = read_u64_le(input, input.len().strict_sub(8)) ^ flip2; let acc = (input.len() as u64) .wrapping_add(input_lo.swap_bytes()) @@ -242,12 +256,12 @@ fn xxh3_64_9to16(input: &[u8], seed: u64, secret: &[u8]) -> u64 { #[inline(always)] fn xxh3_64_4to8(input: &[u8], mut seed: u64, secret: &[u8]) -> u64 { - seed ^= ((seed as u32).swap_bytes() as u64) << 32; + seed ^= u64::from(low_u32(seed).swap_bytes()) << 32; // SAFETY: input.len() is 4..=8, secret.len() >= SECRET_SIZE_MIN (136). unsafe { let input1 = read_u32_le(input, 0); - let input2 = read_u32_le(input, input.len() - 4); + let input2 = read_u32_le(input, input.len().strict_sub(4)); let flip = (read_u64_le(secret, 8) ^ read_u64_le(secret, 16)).wrapping_sub(seed); let input64 = (input2 as u64).wrapping_add((input1 as u64) << 32); @@ -264,7 +278,8 @@ fn xxh3_64_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u64 { let c1 = *input.get_unchecked(0); let c2 = *input.get_unchecked(input.len() >> 1); let c3 = *input.get_unchecked(input.len().strict_sub(1)); - let combo = ((c1 as u32) << 16) | ((c2 as u32) << 24) | (c3 as u32) | ((input.len() as u32) << 8); + let input_len = u32::try_from(input.len()).expect("XXH3 1-to-3-byte path length must fit u32"); + let combo = (u32::from(c1) << 16) | (u32::from(c2) << 24) | u32::from(c3) | (input_len << 8); let flip = ((read_u32_le(secret, 0) ^ read_u32_le(secret, 4)) as u64).wrapping_add(seed); xxh64_avalanche((combo as u64) ^ flip) @@ -296,19 +311,35 @@ fn xxh3_64_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u64 { if input.len() > 64 { if input.len() > 96 { acc = acc.wrapping_add(mix16_b(chunk16(input, 48), chunk16(secret, 96), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 64), chunk16(secret, 112), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(64)), + chunk16(secret, 112), + seed, + )); } acc = acc.wrapping_add(mix16_b(chunk16(input, 32), chunk16(secret, 64), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 48), chunk16(secret, 80), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(48)), + chunk16(secret, 80), + seed, + )); } acc = acc.wrapping_add(mix16_b(chunk16(input, 16), chunk16(secret, 32), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 32), chunk16(secret, 48), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(32)), + chunk16(secret, 48), + seed, + )); } acc = acc.wrapping_add(mix16_b(chunk16(input, 0), chunk16(secret, 0), seed)); - acc = acc.wrapping_add(mix16_b(chunk16(input, input.len() - 16), chunk16(secret, 16), seed)); + acc = acc.wrapping_add(mix16_b( + chunk16(input, input.len().strict_sub(16)), + chunk16(secret, 16), + seed, + )); xxh3_avalanche(acc) } @@ -361,22 +392,25 @@ fn xxh3_64_129to240(input: &[u8], seed: u64, secret: &[u8]) -> u64 { let mut idx = 0usize; while idx < 8 { - acc = acc.wrapping_add(mix16_b(chunk16(input, 16 * idx), chunk16(secret, 16 * idx), seed)); - idx += 1; + let offset = idx.strict_mul(16); + acc = acc.wrapping_add(mix16_b(chunk16(input, offset), chunk16(secret, offset), seed)); + idx = idx.strict_add(1); } acc = xxh3_avalanche(acc); while idx < nb_rounds { + let input_offset = idx.strict_mul(16); + let secret_offset = idx.strict_sub(8).strict_mul(16).strict_add(START_OFFSET); acc = acc.wrapping_add(mix16_b( - chunk16(input, 16 * idx), - chunk16(secret, 16 * (idx - 8) + START_OFFSET), + chunk16(input, input_offset), + chunk16(secret, secret_offset), seed, )); - idx += 1; + idx = idx.strict_add(1); } acc = acc.wrapping_add(mix16_b( - chunk16(input, input.len() - 16), + chunk16(input, input.len().strict_sub(16)), chunk16(secret, SECRET_SIZE_MIN - LAST_OFFSET), seed, )); @@ -391,7 +425,7 @@ fn mix_two_accs(acc: &[u64], acc_offset: usize, secret: &[u8], secret_offset: us unsafe { mul128_fold64( acc[acc_offset] ^ read_u64_le(secret, secret_offset), - acc[acc_offset + 1] ^ read_u64_le(secret, secret_offset + 8), + acc[acc_offset.strict_add(1)] ^ read_u64_le(secret, secret_offset.strict_add(8)), ) } } @@ -400,8 +434,10 @@ fn mix_two_accs(acc: &[u64], acc_offset: usize, secret: &[u8], secret_offset: us fn merge_accs(acc: &[u64], secret: &[u8], secret_offset: usize, mut result: u64) -> u64 { let mut idx = 0usize; while idx < 4 { - result = result.wrapping_add(mix_two_accs(acc, idx * 2, secret, secret_offset + idx * 16)); - idx += 1; + let acc_offset = idx.strict_mul(2); + let key_offset = secret_offset.strict_add(idx.strict_mul(16)); + result = result.wrapping_add(mix_two_accs(acc, acc_offset, secret, key_offset)); + idx = idx.strict_add(1); } xxh3_avalanche(result) @@ -409,7 +445,8 @@ fn merge_accs(acc: &[u64], secret: &[u8], secret_offset: usize, mut result: u64) #[inline(always)] fn scramble_acc(mut acc: [u64; ACC_NB], secret: &[u8], secret_offset: usize) -> [u64; ACC_NB] { - let secret_stripe = &secret[secret_offset..secret_offset + STRIPE_LEN]; + let secret_end = secret_offset.strict_add(STRIPE_LEN); + let secret_stripe = &secret[secret_offset..secret_end]; let (secret_chunks, _) = secret_stripe.as_chunks::<8>(); let mut idx = 0usize; @@ -419,7 +456,7 @@ fn scramble_acc(mut acc: [u64; ACC_NB], secret: &[u8], secret_offset: usize) -> acc_val ^= key; acc[idx] = acc_val.wrapping_mul(PRIME32_1 as u64); - idx += 1; + idx = idx.strict_add(1); } acc @@ -440,7 +477,7 @@ fn accumulate_512(mut acc: [u64; ACC_NB], stripe: &[u8], secret_stripe: &[u8]) - acc[idx ^ 1] = acc[idx ^ 1].wrapping_add(data_val); acc[idx] = acc[idx].wrapping_add(mult32_to64((data_key & 0xFFFF_FFFF) as u32, (data_key >> 32) as u32)); - idx += 1; + idx = idx.strict_add(1); } acc @@ -462,16 +499,18 @@ fn accumulate_loop( let mut idx = 0usize; while idx < nb_stripes { - debug_assert!(stripe_offset + STRIPE_LEN <= input_len); - debug_assert!(secret_stripe_offset + STRIPE_LEN <= secret_len); + let stripe_end = stripe_offset.strict_add(STRIPE_LEN); + let secret_stripe_end = secret_stripe_offset.strict_add(STRIPE_LEN); + debug_assert!(stripe_end <= input_len); + debug_assert!(secret_stripe_end <= secret_len); acc = accumulate_512( acc, - &input[stripe_offset..stripe_offset + STRIPE_LEN], - &secret[secret_stripe_offset..secret_stripe_offset + STRIPE_LEN], + &input[stripe_offset..stripe_end], + &secret[secret_stripe_offset..secret_stripe_end], ); - stripe_offset += STRIPE_LEN; - secret_stripe_offset += SECRET_CONSUME_RATE; - idx += 1; + stripe_offset = stripe_end; + secret_stripe_offset = secret_stripe_offset.strict_add(SECRET_CONSUME_RATE); + idx = idx.strict_add(1); } acc @@ -496,31 +535,48 @@ fn stream_accumulate_portable( } #[inline(never)] +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { let mut acc = INITIAL_ACC; - let nb_stripes = (secret.len() - STRIPE_LEN) / SECRET_CONSUME_RATE; - let block_len = STRIPE_LEN * nb_stripes; - let nb_blocks = (input.len() - 1) / block_len; + let nb_stripes = secret.len().strict_sub(STRIPE_LEN) / SECRET_CONSUME_RATE; + let block_len = STRIPE_LEN.strict_mul(nb_stripes); + let remaining = input.len().strict_sub(1); + let nb_blocks = remaining.strict_div(block_len); let mut idx = 0usize; while idx < nb_blocks { - acc = accumulate_loop(acc, input, idx * block_len, secret, 0, nb_stripes); - acc = scramble_acc(acc, secret, secret.len() - STRIPE_LEN); - idx += 1; + let block_offset = idx.strict_mul(block_len); + acc = accumulate_loop(acc, input, block_offset, secret, 0, nb_stripes); + acc = scramble_acc(acc, secret, secret.len().strict_sub(STRIPE_LEN)); + idx = idx.strict_add(1); } - let nb_stripes = ((input.len() - 1) - (block_len * nb_blocks)) / STRIPE_LEN; - acc = accumulate_loop(acc, input, nb_blocks * block_len, secret, 0, nb_stripes); + let block_offset = nb_blocks.strict_mul(block_len); + let nb_stripes = remaining.strict_sub(block_offset) / STRIPE_LEN; + acc = accumulate_loop(acc, input, block_offset, secret, 0, nb_stripes); - accumulate_512( - acc, - &input[input.len() - STRIPE_LEN..], - &secret[secret.len() - STRIPE_LEN - SECRET_LASTACC_START..secret.len() - SECRET_LASTACC_START], - ) + let input_start = input.len().strict_sub(STRIPE_LEN); + let secret_end = secret.len().strict_sub(SECRET_LASTACC_START); + let secret_start = secret_end.strict_sub(STRIPE_LEN); + + accumulate_512(acc, &input[input_start..], &secret[secret_start..secret_end]) } #[inline(never)] +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] fn xxh3_64_long_impl(input: &[u8], secret: &[u8]) -> u64 { let acc = hash_long_internal_loop(input, secret); merge_accs( @@ -532,6 +588,13 @@ fn xxh3_64_long_impl(input: &[u8], secret: &[u8]) -> u64 { } /// Long-path default-seed entry point (>240B) — no seed branch. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { xxh3_64_long_impl(input, &DEFAULT_SECRET) } @@ -543,23 +606,19 @@ fn custom_default_secret(seed: u64) -> [u8; DEFAULT_SECRET_SIZE] { } let mut result = [0u8; DEFAULT_SECRET_SIZE]; - const NB_ROUNDS: usize = DEFAULT_SECRET_SIZE / 16; - - let mut idx = 0usize; - while idx < NB_ROUNDS { - // SAFETY: idx < NB_ROUNDS = DEFAULT_SECRET_SIZE/16, so idx*16+16 <= DEFAULT_SECRET_SIZE. - let lo = unsafe { read_u64_le(&DEFAULT_SECRET, idx * 16).wrapping_add(seed).to_le_bytes() }; - // SAFETY: idx < NB_ROUNDS = DEFAULT_SECRET_SIZE/16, so idx*16+8+8 <= DEFAULT_SECRET_SIZE. - let hi = unsafe { - read_u64_le(&DEFAULT_SECRET, idx * 16 + 8) - .wrapping_sub(seed) - .to_le_bytes() - }; - - result[idx * 16..idx * 16 + 8].copy_from_slice(&lo); - result[idx * 16 + 8..idx * 16 + 16].copy_from_slice(&hi); - - idx += 1; + let (source_chunks, source_tail) = DEFAULT_SECRET.as_chunks::<16>(); + let (destination_chunks, destination_tail) = result.as_chunks_mut::<16>(); + debug_assert!(source_tail.is_empty()); + debug_assert!(destination_tail.is_empty()); + + for (source, destination) in source_chunks.iter().zip(destination_chunks) { + let (source_words, source_tail) = source.as_chunks::<8>(); + let (destination_words, destination_tail) = destination.as_chunks_mut::<8>(); + debug_assert!(source_tail.is_empty()); + debug_assert!(destination_tail.is_empty()); + + destination_words[0] = u64::from_le_bytes(source_words[0]).wrapping_add(seed).to_le_bytes(); + destination_words[1] = u64::from_le_bytes(source_words[1]).wrapping_sub(seed).to_le_bytes(); } result @@ -568,6 +627,13 @@ fn custom_default_secret(seed: u64) -> [u8; DEFAULT_SECRET_SIZE] { /// Long-path entry point (>240B) — no ≤240B branches. /// /// Called from compile-time dispatch when the caller already knows `input.len() > MID_SIZE_MAX`. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) @@ -584,7 +650,8 @@ fn xxh3_128_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u128 { let c1 = *input.get_unchecked(0); let c2 = *input.get_unchecked(input.len() >> 1); let c3 = *input.get_unchecked(input.len().strict_sub(1)); - let input_lo = (c1 as u32) << 16 | (c2 as u32) << 24 | c3 as u32 | (input.len() as u32) << 8; + let input_len = u32::try_from(input.len()).expect("XXH3 1-to-3-byte path length must fit u32"); + let input_lo = (u32::from(c1) << 16) | (u32::from(c2) << 24) | u32::from(c3) | (input_len << 8); let input_hi = input_lo.swap_bytes().rotate_left(13); let flip_lo = (read_u32_le(secret, 0) as u64 ^ read_u32_le(secret, 4) as u64).wrapping_add(seed); @@ -598,12 +665,12 @@ fn xxh3_128_1to3(input: &[u8], seed: u64, secret: &[u8]) -> u128 { #[inline(always)] fn xxh3_128_4to8(input: &[u8], mut seed: u64, secret: &[u8]) -> u128 { - seed ^= ((seed as u32).swap_bytes() as u64) << 32; + seed ^= u64::from(low_u32(seed).swap_bytes()) << 32; // SAFETY: input.len() is 4..=8, secret.len() >= SECRET_SIZE_MIN (136). unsafe { let lo = read_u32_le(input, 0); - let hi = read_u32_le(input, input.len() - 4); + let hi = read_u32_le(input, input.len().strict_sub(4)); let input_64 = (lo as u64).wrapping_add((hi as u64) << 32); let flip = (read_u64_le(secret, 16) ^ read_u64_le(secret, 24)).wrapping_add(seed); @@ -629,13 +696,13 @@ fn xxh3_128_9to16(input: &[u8], seed: u64, secret: &[u8]) -> u128 { let flip_lo = (read_u64_le(secret, 32) ^ read_u64_le(secret, 40)).wrapping_sub(seed); let flip_hi = (read_u64_le(secret, 48) ^ read_u64_le(secret, 56)).wrapping_add(seed); let input_lo = read_u64_le(input, 0); - let mut input_hi = read_u64_le(input, input.len() - 8); + let mut input_hi = read_u64_le(input, input.len().strict_sub(8)); let (mut mul_low, mut mul_high) = mul64_to128(input_lo ^ input_hi ^ flip_lo, PRIME64_1); - mul_low = mul_low.wrapping_add(((input.len() as u64) - 1) << 54); + mul_low = mul_low.wrapping_add((input.len() as u64).strict_sub(1) << 54); input_hi ^= flip_hi; - mul_high = mul_high.wrapping_add(input_hi.wrapping_add(mult32_to64(input_hi as u32, PRIME32_2 - 1))); + mul_high = mul_high.wrapping_add(input_hi.wrapping_add(mult32_to64(low_u32(input_hi), PRIME32_2.strict_sub(1)))); mul_low ^= mul_high.swap_bytes(); @@ -679,7 +746,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 48), - chunk16(input, input.len() - 64), + chunk16(input, input.len().strict_sub(64)), chunk32(secret, 96), seed, ); @@ -689,7 +756,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 32), - chunk16(input, input.len() - 48), + chunk16(input, input.len().strict_sub(48)), chunk32(secret, 64), seed, ); @@ -699,7 +766,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 16), - chunk16(input, input.len() - 32), + chunk16(input, input.len().strict_sub(32)), chunk32(secret, 32), seed, ); @@ -709,7 +776,7 @@ fn xxh3_128_7to128(input: &[u8], seed: u64, secret: &[u8]) -> u128 { &mut lo, &mut hi, chunk16(input, 0), - chunk16(input, input.len() - 16), + chunk16(input, input.len().strict_sub(16)), chunk32(secret, 0), seed, ); @@ -808,38 +875,40 @@ fn xxh3_128_129to240(input: &[u8], seed: u64, secret: &[u8]) -> u128 { let mut idx = 0usize; while idx < 4 { + let offset = idx.strict_mul(32); mix32_b( &mut lo, &mut hi, - chunk16(input, 32 * idx), - chunk16(input, (32 * idx) + 16), - chunk32(secret, 32 * idx), + chunk16(input, offset), + chunk16(input, offset.strict_add(16)), + chunk32(secret, offset), seed, ); - idx += 1; + idx = idx.strict_add(1); } lo = xxh3_avalanche(lo); hi = xxh3_avalanche(hi); while idx < nb_rounds { - let sec_off = START_OFFSET.wrapping_add(32 * (idx - 4)); + let input_offset = idx.strict_mul(32); + let secret_offset = START_OFFSET.strict_add(idx.strict_sub(4).strict_mul(32)); mix32_b( &mut lo, &mut hi, - chunk16(input, 32 * idx), - chunk16(input, (32 * idx) + 16), - chunk32(secret, sec_off), + chunk16(input, input_offset), + chunk16(input, input_offset.strict_add(16)), + chunk32(secret, secret_offset), seed, ); - idx += 1; + idx = idx.strict_add(1); } mix32_b( &mut lo, &mut hi, - chunk16(input, input.len() - 16), - chunk16(input, input.len() - 32), + chunk16(input, input.len().strict_sub(16)), + chunk16(input, input.len().strict_sub(32)), chunk32(secret, SECRET_SIZE_MIN - LAST_OFFSET - 16), 0u64.wrapping_sub(seed), ); @@ -855,6 +924,13 @@ fn xxh3_128_129to240(input: &[u8], seed: u64, secret: &[u8]) -> u128 { } #[inline(never)] +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] fn xxh3_128_long_impl(input: &[u8], secret: &[u8]) -> u128 { let acc = hash_long_internal_loop(input, secret); @@ -867,7 +943,10 @@ fn xxh3_128_long_impl(input: &[u8], secret: &[u8]) -> u128 { let hi = merge_accs( &acc, secret, - secret.len() - (ACC_NB * mem::size_of::()) - SECRET_MERGEACCS_START, + secret + .len() + .strict_sub(ACC_NB.strict_mul(mem::size_of::())) + .strict_sub(SECRET_MERGEACCS_START), !(input.len() as u64).wrapping_mul(PRIME64_2), ); @@ -875,11 +954,25 @@ fn xxh3_128_long_impl(input: &[u8], secret: &[u8]) -> u128 { } /// Long-path default-seed entry point (>240B) — no seed branch. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { xxh3_128_long_impl(input, &DEFAULT_SECRET) } /// Long-path entry point (>240B) — no ≤240B branches. +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon") + )) +))] pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) @@ -1008,6 +1101,18 @@ mod tests { xxhash_rust::xxh3::xxh3_128_with_seed(&data, seed), "xxh3_128 mismatch (seed={seed}, len={len})" ); + if len > super::MID_SIZE_MAX { + assert_eq!( + super::xxh3_64_long(&data, seed), + xxhash_rust::xxh3::xxh3_64_with_seed(&data, seed), + "portable xxh3_64 long-path mismatch (seed={seed}, len={len})" + ); + assert_eq!( + super::xxh3_128_long(&data, seed), + xxhash_rust::xxh3::xxh3_128_with_seed(&data, seed), + "portable xxh3_128 long-path mismatch (seed={seed}, len={len})" + ); + } } } } diff --git a/src/hashes/fast/xxh3/aarch64_neon.rs b/src/hashes/fast/xxh3/aarch64_neon.rs index d9f0b4b9..8e06cbce 100644 --- a/src/hashes/fast/xxh3/aarch64_neon.rs +++ b/src/hashes/fast/xxh3/aarch64_neon.rs @@ -15,8 +15,6 @@ //! //! Uses `unsafe` for NEON intrinsics. Callers must ensure NEON is available //! (always true on aarch64 — it is the baseline ISA). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::arch::aarch64::*; @@ -32,6 +30,9 @@ const LAST_ACC_SECRET_OFFSET: usize = DEFAULT_SECRET_SIZE - STRIPE_LEN - SECRET_ // SIMD accumulate + scramble +/// # Safety +/// +/// The caller must ensure NEON is available, as guaranteed by AArch64. #[inline] #[target_feature(enable = "neon")] unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [uint64x2_t; 4] { @@ -46,6 +47,9 @@ unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [uint64x2_t; 4] { } } +/// # Safety +/// +/// The caller must ensure NEON is available, as guaranteed by AArch64. #[inline] #[target_feature(enable = "neon")] unsafe fn store_acc(acc: &[uint64x2_t; 4]) -> [u64; ACC_NB] { @@ -67,6 +71,10 @@ unsafe fn store_acc(acc: &[uint64x2_t; 4]) -> [u64; ACC_NB] { /// (vs 2 separate `vmovn` + `vshrn`) /// - `vmlal_high_u32` for the second lane (avoids `vget_high_u32` extraction) /// - `vmlal(data_swap, ...)` base to break the acc→vmlal dependency chain +/// +/// # Safety +/// +/// The caller must ensure NEON is available and both pointers are valid for 64 readable bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn accumulate_512(acc: &mut [uint64x2_t; 4], stripe: *const u8, secret: *const u8) { @@ -112,6 +120,10 @@ unsafe fn accumulate_512(acc: &mut [uint64x2_t; 4], stripe: *const u8, secret: * } /// Scramble the accumulator at block boundaries. +/// +/// # Safety +/// +/// The caller must ensure NEON is available and `secret` is valid for 64 readable bytes. #[inline] #[target_feature(enable = "neon")] unsafe fn scramble_acc(acc: &mut [uint64x2_t; 4], secret: *const u8) { @@ -154,11 +166,11 @@ unsafe fn scramble_acc(acc: &mut [uint64x2_t; 4], secret: *const u8) { /// buffer — `prfm` silently ignores invalid addresses. #[inline(always)] #[cfg(miri)] -unsafe fn prefetch_stripe(_input_ptr: *const u8) {} +fn prefetch_stripe(_input_ptr: *const u8) {} #[inline(always)] #[cfg(not(miri))] -unsafe fn prefetch_stripe(input_ptr: *const u8) { +fn prefetch_stripe(input_ptr: *const u8) { // SAFETY: PRFM is a CPU hint; invalid addresses are silently ignored. unsafe { core::arch::asm!( @@ -171,6 +183,10 @@ unsafe fn prefetch_stripe(input_ptr: *const u8) { // Long-path loop (SIMD inner, scalar merge) +/// # Safety +/// +/// The caller must ensure NEON is available, `stripes` is nonzero, and the +/// requested input and secret stripe ranges are in bounds. #[target_feature(enable = "neon")] unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], @@ -246,8 +262,15 @@ pub(crate) unsafe fn stream_accumulate( } } +/// # Safety +/// +/// The caller must ensure NEON is available and `input` contains an XXH3 long +/// input of more than 240 bytes. #[target_feature(enable = "neon")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop( + input: &[u8], + secret: &[u8; DEFAULT_SECRET_SIZE], +) -> [u64; ACC_NB] { // SAFETY: NEON available via target_feature. Input/secret bounds checked by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); @@ -300,8 +323,12 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[ } #[inline] +/// # Safety +/// +/// The caller must ensure NEON is available and `input` contains an XXH3 long +/// input of more than 240 bytes. #[target_feature(enable = "neon")] -unsafe fn hash_long_internal_loop_for_len(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop_for_len(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // Small long inputs have too few stripes to amortize software prefetch. // Keep PRFM for larger streaming inputs where it helps Apple/Neoverse throughput. // SAFETY: caller upholds the NEON and input/secret bounds contracts. @@ -319,7 +346,7 @@ unsafe fn hash_long_internal_loop_for_len(input: &[u8], secret: &[u8]) -> [u64; /// Long-path entry point (>240B) — no ≤240B branches. /// /// Called from compile-time dispatch when the caller already knows `input.len() > MID_SIZE_MAX`. -pub fn xxh3_64_long_default(input: &[u8]) -> u64 { +pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { // SAFETY: NEON always available on aarch64. let acc = unsafe { if input.len() <= 1024 { @@ -336,7 +363,7 @@ pub fn xxh3_64_long_default(input: &[u8]) -> u64 { ) } -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) } else { @@ -359,13 +386,13 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long_default(input: &[u8]) -> u128 { +pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: NEON always available on aarch64. let acc = unsafe { hash_long_internal_loop_for_len(input, &DEFAULT_SECRET) }; xxh3_128_long_finalize(&acc, &DEFAULT_SECRET, input.len()) } -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) } else { diff --git a/src/hashes/fast/xxh3/dispatch.rs b/src/hashes/fast/xxh3/dispatch.rs index 0c1b4be4..f0690add 100644 --- a/src/hashes/fast/xxh3/dispatch.rs +++ b/src/hashes/fast/xxh3/dispatch.rs @@ -1,25 +1,46 @@ -#![cfg_attr(test, allow(dead_code))] - +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +use super::kernels::{hash64_long_fn, hash128_long_fn}; use super::{ dispatch_tables::DispatchTable, - kernels::{ - StreamAccumulateFn, Xxh3KernelId, hash64_long_fn, hash128_long_fn, required_caps, - stream_accumulate_fn as kernel_stream_accumulate_fn, - }, + kernels::{StreamAccumulateFn, Xxh3KernelId, required_caps, stream_accumulate_fn as kernel_stream_accumulate_fn}, }; use crate::{backend::cache::OnceCache, platform::Caps}; +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] type Hash64Fn = fn(&[u8], u64) -> u64; +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] type Hash128Fn = fn(&[u8], u64) -> u128; #[derive(Clone, Copy)] struct ActiveDispatch { /// Long-path-only entry for 64-bit hash (>240B, no redundant length checks). + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long64: Hash64Fn, /// Long-path-only entry for 128-bit hash (>240B, no redundant length checks). + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long128: Hash128Fn, stream_accumulate: StreamAccumulateFn, - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] long_id: Xxh3KernelId, } @@ -57,10 +78,20 @@ fn active() -> ActiveDispatch { let long_id = resolve(table.long, caps); ActiveDispatch { + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long64: hash64_long_fn(long_id), + #[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )))] long128: hash128_long_fn(long_id), stream_accumulate: kernel_stream_accumulate_fn(long_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] long_id, } }) @@ -71,7 +102,7 @@ pub(crate) fn stream_accumulate_fn() -> StreamAccumulateFn { active().stream_accumulate } -#[cfg(any(test, feature = "diag"))] +#[cfg(any(feature = "diag", all(test, target_arch = "x86_64")))] #[inline] #[must_use] fn kernel_id64_for_len(long_id: Xxh3KernelId, caps: Caps, len: usize) -> Xxh3KernelId { @@ -90,18 +121,18 @@ fn kernel_id64_for_len(long_id: Xxh3KernelId, caps: Caps, len: usize) -> Xxh3Ker long_id } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name64_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name64_for_len(len: usize) -> &'static str { let d = active(); kernel_id64_for_len(d.long_id, crate::platform::caps(), len).as_str() } -#[cfg(any(test, feature = "diag"))] +#[cfg(feature = "diag")] #[inline] #[must_use] -pub fn kernel_name128_for_len(len: usize) -> &'static str { +pub(crate) fn kernel_name128_for_len(len: usize) -> &'static str { if len <= super::MID_SIZE_MAX { Xxh3KernelId::Portable.as_str() } else { @@ -119,7 +150,7 @@ pub fn kernel_name128_for_len(len: usize) -> &'static str { /// stays out-of-line so ≤240B paths do not carry the cache/feature lookup. #[inline(always)] #[must_use] -pub fn hash64(data: &[u8]) -> u64 { +pub(crate) fn hash64(data: &[u8]) -> u64 { let len = data.len(); if len == 0 { return super::XXH3_64_EMPTY_DEFAULT; @@ -144,7 +175,7 @@ pub fn hash64(data: &[u8]) -> u64 { #[inline(always)] #[must_use] -pub fn hash128(data: &[u8]) -> u128 { +pub(crate) fn hash128(data: &[u8]) -> u128 { let len = data.len(); if len == 0 { return super::XXH3_128_EMPTY_DEFAULT; @@ -169,7 +200,7 @@ pub fn hash128(data: &[u8]) -> u128 { #[inline(always)] #[must_use] -pub fn hash64_with_seed(seed: u64, data: &[u8]) -> u64 { +pub(crate) fn hash64_with_seed(seed: u64, data: &[u8]) -> u64 { let len = data.len(); if len == 0 && seed == 0 { return super::XXH3_64_EMPTY_DEFAULT; @@ -192,40 +223,49 @@ pub fn hash64_with_seed(seed: u64, data: &[u8]) -> u64 { hash64_long(seed, data) } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] #[inline(always)] fn hash64_long_default(data: &[u8]) -> u64 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] - { - if use_zen5_xxh3_64_avx2_short_long(data.len()) { - return super::x86_64_avx2::xxh3_64_long_default(data); - } + if use_zen5_xxh3_64_avx2_short_long(data.len()) { + super::x86_64_avx2::xxh3_64_long_default(data) + } else { + super::x86_64_avx512::xxh3_64_long_default(data) } +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_64_long_default(data); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(target_feature = "avx2")))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::x86_64_avx512::xxh3_64_long_default(data) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_64_long_default(data); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::x86_64_avx2::xxh3_64_long_default(data) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_64_long_default(data); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::aarch64_neon::xxh3_64_long_default(data) +} - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_64_long_default(data); - } +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + super::xxh3_64_long_default(data) +} - #[allow(unreachable_code)] - { - let d = active(); - (d.long64)(data, 0) - } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash64_long_default(data: &[u8]) -> u64 { + let d = active(); + (d.long64)(data, 0) } /// Long-path dispatch (>240B). @@ -238,45 +278,57 @@ fn hash64_long_default(data: &[u8]) -> u64 { /// Falls back to runtime dispatch when features are unknown at compile time, /// using the dedicated long-path entry point that skips redundant ≤240B length /// checks in the kernel. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] #[inline(always)] fn hash64_long(seed: u64, data: &[u8]) -> u64 { - // Tier 1: compile-time dispatch — dedicated long entry points skip ≤240B - // branches that are guaranteed dead at this call site. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx2"))] - { - if use_zen5_xxh3_64_avx2_short_long(data.len()) { - return super::x86_64_avx2::xxh3_64_long(data, seed); - } + if use_zen5_xxh3_64_avx2_short_long(data.len()) { + super::x86_64_avx2::xxh3_64_long(data, seed) + } else { + super::x86_64_avx512::xxh3_64_long(data, seed) } +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_64_long(data, seed); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(target_feature = "avx2")))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::x86_64_avx512::xxh3_64_long(data, seed) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_64_long(data, seed); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::x86_64_avx2::xxh3_64_long(data, seed) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_64_long(data, seed); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::aarch64_neon::xxh3_64_long(data, seed) +} - // The retired RVV path lost to portable at 256 B–64 KiB on SpacemiT K1. - // Bypass runtime dispatch on RISC-V. - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_64_long(data, seed); - } +// The retired RVV path lost to portable at 256 B–64 KiB on SpacemiT K1. +// Bypass runtime dispatch on RISC-V. +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { + super::xxh3_64_long(data, seed) +} - // Tier 2: runtime dispatch — dedicated long-path fn pointer, no redundant - // length checks. - #[allow(unreachable_code)] +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash64_long(seed: u64, data: &[u8]) -> u64 { hash64_long_runtime(seed, data) } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] #[inline(never)] fn hash64_long_runtime(seed: u64, data: &[u8]) -> u64 { #[cfg(target_arch = "x86_64")] @@ -293,7 +345,7 @@ fn hash64_long_runtime(seed: u64, data: &[u8]) -> u64 { /// See [`hash64_with_seed`] for the dispatch rationale. #[inline(always)] #[must_use] -pub fn hash128_with_seed(seed: u64, data: &[u8]) -> u128 { +pub(crate) fn hash128_with_seed(seed: u64, data: &[u8]) -> u128 { let len = data.len(); if len == 0 && seed == 0 { return super::XXH3_128_EMPTY_DEFAULT; @@ -316,65 +368,82 @@ pub fn hash128_with_seed(seed: u64, data: &[u8]) -> u128 { hash128_long(seed, data) } +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] #[inline(always)] fn hash128_long_default(data: &[u8]) -> u128 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_128_long_default(data); - } + super::x86_64_avx512::xxh3_128_long_default(data) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_128_long_default(data); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + super::x86_64_avx2::xxh3_128_long_default(data) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_128_long_default(data); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + super::aarch64_neon::xxh3_128_long_default(data) +} - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_128_long_default(data); - } +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + super::xxh3_128_long_default(data) +} - #[allow(unreachable_code)] - { - let d = active(); - (d.long128)(data, 0) - } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash128_long_default(data: &[u8]) -> u128 { + let d = active(); + (d.long128)(data, 0) } /// See [`hash64_long`] for the compile-time dispatch rationale. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] #[inline(always)] fn hash128_long(seed: u64, data: &[u8]) -> u128 { - // Tier 1: compile-time dispatch (dedicated long entry points). - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - { - return super::x86_64_avx512::xxh3_128_long(data, seed); - } + super::x86_64_avx512::xxh3_128_long(data, seed) +} - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - { - return super::x86_64_avx2::xxh3_128_long(data, seed); - } +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { + super::x86_64_avx2::xxh3_128_long(data, seed) +} - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return super::aarch64_neon::xxh3_128_long(data, seed); - } +#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { + super::aarch64_neon::xxh3_128_long(data, seed) +} - // RISC-V: see hash64_long comment. - #[cfg(target_arch = "riscv64")] - { - return super::xxh3_128_long(data, seed); - } +// RISC-V: see hash64_long comment. +#[cfg(target_arch = "riscv64")] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { + super::xxh3_128_long(data, seed) +} - // Tier 2: runtime dispatch — dedicated long-path fn pointer. - #[allow(unreachable_code)] +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] +#[inline(always)] +fn hash128_long(seed: u64, data: &[u8]) -> u128 { hash128_long_runtime(seed, data) } +#[cfg(not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" +)))] #[inline(never)] fn hash128_long_runtime(seed: u64, data: &[u8]) -> u128 { let d = active(); diff --git a/src/hashes/fast/xxh3/dispatch_tables.rs b/src/hashes/fast/xxh3/dispatch_tables.rs index c600d01b..9643a885 100644 --- a/src/hashes/fast/xxh3/dispatch_tables.rs +++ b/src/hashes/fast/xxh3/dispatch_tables.rs @@ -2,15 +2,15 @@ //! //! This module is the checked-in runtime table used by capability-driven dispatch. -pub use super::kernels::Xxh3KernelId as KernelId; +pub(crate) use super::kernels::Xxh3KernelId as KernelId; use crate::platform::Caps; #[derive(Clone, Copy, Debug)] -pub struct DispatchTable { +pub(crate) struct DispatchTable { pub long: KernelId, } -pub static DEFAULT_TABLE: DispatchTable = DispatchTable { +pub(crate) static DEFAULT_TABLE: DispatchTable = DispatchTable { long: KernelId::Portable, }; @@ -18,27 +18,27 @@ pub static DEFAULT_TABLE: DispatchTable = DispatchTable { /// x86-64 with AVX-512F: single-iteration per stripe. #[cfg(target_arch = "x86_64")] -pub static AVX512_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx512 }; +pub(crate) static AVX512_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx512 }; /// x86-64 with AVX2 (no AVX-512): two iterations per stripe. #[cfg(target_arch = "x86_64")] -pub static AVX2_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx2 }; +pub(crate) static AVX2_TABLE: DispatchTable = DispatchTable { long: KernelId::Avx2 }; /// aarch64 with NEON: four iterations per stripe. #[cfg(target_arch = "aarch64")] -pub static NEON_TABLE: DispatchTable = DispatchTable { long: KernelId::Neon }; +pub(crate) static NEON_TABLE: DispatchTable = DispatchTable { long: KernelId::Neon }; /// POWER8+ with VSX: four iterations per stripe (128-bit vectors). #[cfg(all(target_arch = "powerpc64", target_endian = "little"))] -pub static VSX_TABLE: DispatchTable = DispatchTable { long: KernelId::Vsx }; +pub(crate) static VSX_TABLE: DispatchTable = DispatchTable { long: KernelId::Vsx }; /// s390x z13+ with z/Vector: four iterations per stripe (128-bit vectors). #[cfg(target_arch = "s390x")] -pub static ZVECTOR_TABLE: DispatchTable = DispatchTable { long: KernelId::Vector }; +pub(crate) static ZVECTOR_TABLE: DispatchTable = DispatchTable { long: KernelId::Vector }; #[inline] #[must_use] -pub fn select_runtime_table(caps: Caps) -> &'static DispatchTable { +pub(crate) fn select_runtime_table(caps: Caps) -> &'static DispatchTable { let _ = caps; #[cfg(target_arch = "x86_64")] { diff --git a/src/hashes/fast/xxh3/kernels.rs b/src/hashes/fast/xxh3/kernels.rs index 99f30202..f47c06f1 100644 --- a/src/hashes/fast/xxh3/kernels.rs +++ b/src/hashes/fast/xxh3/kernels.rs @@ -1,12 +1,12 @@ use super::{ACC_NB, DEFAULT_SECRET_SIZE}; use crate::platform::Caps; -pub type StreamAccumulateFn = +pub(crate) type StreamAccumulateFn = unsafe fn([u64; ACC_NB], &[u8], usize, &[u8; DEFAULT_SECRET_SIZE], usize, usize, bool) -> [u64; ACC_NB]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] -pub enum Xxh3KernelId { +pub(crate) enum Xxh3KernelId { Portable = 0, #[cfg(target_arch = "x86_64")] Avx2 = 1, @@ -21,10 +21,10 @@ pub enum Xxh3KernelId { } impl Xxh3KernelId { - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] #[inline] #[must_use] - pub const fn as_str(self) -> &'static str { + pub(crate) const fn as_str(self) -> &'static str { match self { Self::Portable => "portable", #[cfg(target_arch = "x86_64")] @@ -42,8 +42,16 @@ impl Xxh3KernelId { } /// Long-path-only entry for 64-bit hash (>240B, no ≤240B length checks). +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )) +))] #[must_use] -pub fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { +pub(crate) fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { match id { Xxh3KernelId::Portable => super::xxh3_64_long, #[cfg(target_arch = "x86_64")] @@ -60,8 +68,16 @@ pub fn hash64_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u64 { } /// Long-path-only entry for 128-bit hash (>240B, no ≤240B length checks). +#[cfg(any( + test, + not(any( + all(target_arch = "x86_64", any(target_feature = "avx512f", target_feature = "avx2")), + all(target_arch = "aarch64", target_feature = "neon"), + target_arch = "riscv64" + )) +))] #[must_use] -pub fn hash128_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u128 { +pub(crate) fn hash128_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u128 { match id { Xxh3KernelId::Portable => super::xxh3_128_long, #[cfg(target_arch = "x86_64")] @@ -78,7 +94,7 @@ pub fn hash128_long_fn(id: Xxh3KernelId) -> fn(&[u8], u64) -> u128 { } #[must_use] -pub fn stream_accumulate_fn(id: Xxh3KernelId) -> StreamAccumulateFn { +pub(crate) fn stream_accumulate_fn(id: Xxh3KernelId) -> StreamAccumulateFn { match id { Xxh3KernelId::Portable => super::stream_accumulate_portable, #[cfg(target_arch = "x86_64")] @@ -96,7 +112,7 @@ pub fn stream_accumulate_fn(id: Xxh3KernelId) -> StreamAccumulateFn { #[inline] #[must_use] -pub const fn required_caps(id: Xxh3KernelId) -> Caps { +pub(crate) const fn required_caps(id: Xxh3KernelId) -> Caps { match id { Xxh3KernelId::Portable => Caps::NONE, #[cfg(target_arch = "x86_64")] diff --git a/src/hashes/fast/xxh3/power.rs b/src/hashes/fast/xxh3/power.rs index bae33282..25b0f50b 100644 --- a/src/hashes/fast/xxh3/power.rs +++ b/src/hashes/fast/xxh3/power.rs @@ -8,8 +8,6 @@ //! Uses `unsafe` for POWER8+ VSX inline asm. Callers must ensure POWER8+ //! vector support before executing the accelerated path (the dispatcher //! does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::simd::i64x2; @@ -21,6 +19,10 @@ use super::{ // VSX primitive operations (inline asm, POWER8+) /// Add u64 lanes: `vaddudm`. +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vadd_u64(a: i64x2, b: i64x2) -> i64x2 { @@ -39,6 +41,10 @@ unsafe fn vadd_u64(a: i64x2, b: i64x2) -> i64x2 { } /// Logical shift right u64 lanes: `vsrd`. +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vshr_u64(a: i64x2, shift: i64x2) -> i64x2 { @@ -57,6 +63,10 @@ unsafe fn vshr_u64(a: i64x2, shift: i64x2) -> i64x2 { } /// Shift left u64 lanes: `vsld`. +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vshl_u64(a: i64x2, shift: i64x2) -> i64x2 { @@ -82,6 +92,10 @@ unsafe fn vshl_u64(a: i64x2, shift: i64x2) -> i64x2 { /// /// (Using `vmuleuw` here would multiply the ISA "even" = **high** 32 bits, /// which is wrong for the XXH3 MAC pattern on LE.) +/// +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn vmul_low32(a: i64x2, b: i64x2) -> i64x2 { @@ -102,17 +116,25 @@ unsafe fn vmul_low32(a: i64x2, b: i64x2) -> i64x2 { // Load, store, swap /// Load 128 bits from memory (unaligned). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 readable bytes. #[inline(always)] unsafe fn vload(ptr: *const u8) -> i64x2 { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::read_unaligned(ptr as *const i64x2) } + unsafe { core::ptr::read_unaligned(ptr.cast::()) } } /// Store 128 bits to memory (unaligned). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 writable bytes. #[inline(always)] unsafe fn vstore(ptr: *mut u8, val: i64x2) { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::write_unaligned(ptr as *mut i64x2, val) } + unsafe { core::ptr::write_unaligned(ptr.cast::(), val) } } /// Swap u64 lanes (idx ^ 1 effect). @@ -123,23 +145,29 @@ fn vswap(a: i64x2) -> i64x2 { // SIMD accumulate + scramble +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [i64x2; 4] { // SAFETY: POWER8+ VSX via target_feature. Pointer valid for 8 × u64. unsafe { - let p = initial.as_ptr() as *const u8; + let p = initial.as_ptr().cast::(); [vload(p), vload(p.add(16)), vload(p.add(32)), vload(p.add(48))] } } +/// # Safety +/// +/// The caller must ensure Altivec, VSX, and POWER8 vector instructions are available. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { // SAFETY: POWER8+ VSX via target_feature. unsafe { let mut out = [0u64; ACC_NB]; - let p = out.as_mut_ptr() as *mut u8; + let p = out.as_mut_ptr().cast::(); vstore(p, acc[0]); vstore(p.add(16), acc[1]); vstore(p.add(32), acc[2]); @@ -156,6 +184,11 @@ unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { /// 3. `vmuleuw`: low32(data_key) × high32(data_key) → u64 /// 4. Swap u64 lanes (idx ^ 1) and add data /// 5. Accumulate product + swapped data into acc +/// +/// # Safety +/// +/// The caller must ensure the POWER vector features are available and both +/// pointers are valid for 64 readable bytes. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const u8) { @@ -188,6 +221,11 @@ unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const /// /// Per element: `acc = (xorshift64(acc, 47) ^ secret) * PRIME32_1` /// The 64-bit multiply by a 32-bit prime is split into lo + hi halves. +/// +/// # Safety +/// +/// The caller must ensure the POWER vector features are available and `secret` +/// is valid for 64 readable bytes. #[inline] #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { @@ -222,6 +260,10 @@ unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) +/// # Safety +/// +/// The caller must ensure the POWER vector features are available, `stripes` is +/// nonzero, and the requested input and secret stripe ranges are in bounds. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], @@ -298,15 +340,19 @@ pub(crate) unsafe fn stream_accumulate( } } +/// # Safety +/// +/// The caller must ensure the POWER vector features are available and `input` +/// contains an XXH3 long input of more than 240 bytes. #[target_feature(enable = "altivec", enable = "vsx", enable = "power8-vector")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: POWER8+ VSX via target_feature. Input/secret bounds checked by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -349,7 +395,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { // SAFETY: Dispatcher verifies POWER8+ VSX before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; @@ -373,7 +419,7 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { // SAFETY: Dispatcher verifies POWER8+ VSX before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; diff --git a/src/hashes/fast/xxh3/s390x.rs b/src/hashes/fast/xxh3/s390x.rs index cb387664..b809ed7f 100644 --- a/src/hashes/fast/xxh3/s390x.rs +++ b/src/hashes/fast/xxh3/s390x.rs @@ -11,8 +11,6 @@ //! Uses `unsafe` for z/Vector inline asm. Callers must ensure z13+ //! vector facility before executing the accelerated path (the dispatcher //! does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::simd::i64x2; @@ -30,6 +28,10 @@ const BSWAP_MASK: [u8; 16] = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, // z/Vector primitive operations (inline asm, z13+) /// Add u64 lanes: `vag`. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { @@ -48,6 +50,10 @@ unsafe fn vag(a: i64x2, b: i64x2) -> i64x2 { } /// Logical shift right u64 lanes by immediate: `vesrlg`. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vesrlg(a: i64x2) -> i64x2 { @@ -66,6 +72,10 @@ unsafe fn vesrlg(a: i64x2) -> i64x2 { } /// Shift left u64 lanes by immediate: `veslg`. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn veslg(a: i64x2) -> i64x2 { @@ -87,6 +97,10 @@ unsafe fn veslg(a: i64x2) -> i64x2 { /// /// On s390x (big-endian), odd u32 elements are the low 32 bits of each /// u64 lane. This gives: `low32(a) × low32(b) → u64` per lane. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { @@ -108,6 +122,10 @@ unsafe fn vmlof(a: i64x2, b: i64x2) -> i64x2 { /// /// Selects bytes from the concatenation of `a:a` according to `mask`. /// Used to byte-reverse each u64 element (BE → LE). +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vperm(a: i64x2, mask: i64x2) -> i64x2 { @@ -128,6 +146,10 @@ unsafe fn vperm(a: i64x2, mask: i64x2) -> i64x2 { /// Swap u64 lanes (idx ^ 1 effect): `vpdi` with M3=4. /// /// Result element 0 = source element 1, result element 1 = source element 0. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn vpdi_swap(a: i64x2) -> i64x2 { @@ -147,20 +169,33 @@ unsafe fn vpdi_swap(a: i64x2) -> i64x2 { // Load / store helpers /// Load 128 bits from memory (unaligned, native byte order). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 readable bytes. #[inline(always)] unsafe fn vload_raw(ptr: *const u8) -> i64x2 { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::read_unaligned(ptr as *const i64x2) } + unsafe { core::ptr::read_unaligned(ptr.cast::()) } } /// Store 128 bits to memory (unaligned, native byte order). +/// +/// # Safety +/// +/// `ptr` must be valid for 16 writable bytes. #[inline(always)] unsafe fn vstore(ptr: *mut u8, val: i64x2) { // SAFETY: caller ensures ptr is valid for 16 bytes. - unsafe { core::ptr::write_unaligned(ptr as *mut i64x2, val) } + unsafe { core::ptr::write_unaligned(ptr.cast::(), val) } } /// Load 128 bits with per-element byte-reversal (BE → LE). +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and `ptr` is valid +/// for 16 readable bytes. #[inline] #[target_feature(enable = "vector")] unsafe fn vload_le(ptr: *const u8, bswap: i64x2) -> i64x2 { @@ -170,20 +205,23 @@ unsafe fn vload_le(ptr: *const u8, bswap: i64x2) -> i64x2 { /// Load the byte-swap permutation mask into a vector register. #[inline(always)] -unsafe fn load_bswap_mask() -> i64x2 { +fn load_bswap_mask() -> i64x2 { // SAFETY: BSWAP_MASK is a 16-byte constant. unsafe { vload_raw(BSWAP_MASK.as_ptr()) } } // SIMD accumulate + scramble +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [i64x2; 4] { // SAFETY: z13+ vector facility via target_feature. Pointer valid for 8 × u64. // Accumulator values are native u64s — no byte-swap needed. unsafe { - let p = initial.as_ptr() as *const u8; + let p = initial.as_ptr().cast::(); [ vload_raw(p), vload_raw(p.add(16)), @@ -193,6 +231,9 @@ unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [i64x2; 4] { } } +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available. #[inline] #[target_feature(enable = "vector")] unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { @@ -200,7 +241,7 @@ unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { // Accumulator values are native u64s — no byte-swap needed. unsafe { let mut out = [0u64; ACC_NB]; - let p = out.as_mut_ptr() as *mut u8; + let p = out.as_mut_ptr().cast::(); vstore(p, acc[0]); vstore(p.add(16), acc[1]); vstore(p.add(32), acc[2]); @@ -217,6 +258,11 @@ unsafe fn store_acc(acc: &[i64x2; 4]) -> [u64; ACC_NB] { /// 3. `vmlof`: low32(data_key) × high32(data_key) → u64 /// 4. Swap u64 lanes (idx ^ 1) and add data /// 5. Accumulate product + swapped data into acc +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and both pointers +/// are valid for 64 readable bytes. #[inline] #[target_feature(enable = "vector")] unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const u8) { @@ -250,6 +296,11 @@ unsafe fn accumulate_512(acc: &mut [i64x2; 4], stripe: *const u8, secret: *const /// /// Per element: `acc = (xorshift64(acc, 47) ^ secret) * PRIME32_1` /// The 64-bit multiply by a 32-bit prime is split into lo + hi halves. +/// +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and `secret` is +/// valid for 64 readable bytes. #[inline] #[target_feature(enable = "vector")] unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { @@ -283,6 +334,10 @@ unsafe fn scramble_acc(acc: &mut [i64x2; 4], secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available, `stripes` is +/// nonzero, and the requested input and secret stripe ranges are in bounds. #[target_feature(enable = "vector")] unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], @@ -358,8 +413,12 @@ pub(crate) unsafe fn stream_accumulate( } } +/// # Safety +/// +/// The caller must ensure the z/Vector facility is available and `input` +/// contains an XXH3 long input of more than 240 bytes. #[target_feature(enable = "vector")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: z13+ vector facility via target_feature. Input/secret bounds // checked by caller. unsafe { @@ -367,7 +426,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -410,7 +469,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { // SAFETY: Dispatcher verifies z13+ vector facility before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; @@ -434,7 +493,7 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { // SAFETY: Dispatcher verifies z13+ vector facility before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; diff --git a/src/hashes/fast/xxh3/stream.rs b/src/hashes/fast/xxh3/stream.rs index 60f76da2..dd81a380 100644 --- a/src/hashes/fast/xxh3/stream.rs +++ b/src/hashes/fast/xxh3/stream.rs @@ -203,14 +203,18 @@ impl Xxh3Hasher { *buffered = 0; } - let direct_len = input.len().saturating_sub(1) / INTERNAL_BUFFER_SIZE * INTERNAL_BUFFER_SIZE; + let direct_len = input + .len() + .saturating_sub(1) + .strict_div(INTERNAL_BUFFER_SIZE) + .strict_mul(INTERNAL_BUFFER_SIZE); let processed_direct = direct_len != 0; if processed_direct { let kernel = *stream_accumulate.get_or_insert_with(dispatch::stream_accumulate_fn); (*acc, *accumulated_stripes) = consume_stripes( kernel, *acc, - direct_len / STRIPE_LEN, + direct_len.strict_div(STRIPE_LEN), *accumulated_stripes, &input[..direct_len], secret, @@ -459,7 +463,11 @@ mod tests { use crate::traits::FastHash; fn data(len: usize) -> Vec { - (0..len).map(|i| i.wrapping_mul(131).wrapping_add(17) as u8).collect() + (u8::MIN..=u8::MAX) + .cycle() + .take(len) + .map(|i| i.wrapping_mul(131).wrapping_add(17)) + .collect() } #[test] @@ -675,7 +683,7 @@ mod tests { } #[test] - #[should_panic] + #[should_panic(expected = "assertion failed: count.strict_mul(STRIPE_LEN) <= input.len()")] fn consume_stripes_rejects_short_input_before_kernel_call() { use super::super::kernels::{Xxh3KernelId, stream_accumulate_fn}; diff --git a/src/hashes/fast/xxh3/x86_64_avx2.rs b/src/hashes/fast/xxh3/x86_64_avx2.rs index 42e2eedf..b3f26886 100644 --- a/src/hashes/fast/xxh3/x86_64_avx2.rs +++ b/src/hashes/fast/xxh3/x86_64_avx2.rs @@ -7,8 +7,6 @@ //! //! Uses `unsafe` for AVX2 intrinsics. Callers must ensure AVX2 is available //! before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::arch::x86_64::*; @@ -27,24 +25,32 @@ const SWAP32: i32 = 0x4E; #[inline] #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn load_acc(initial: &[u64; ACC_NB]) -> [__m256i; 2] { - // SAFETY: AVX2 available via target_feature. Pointer valid for 8 × u64. + // SAFETY: The function attribute enables the unaligned AVX2 loads, and + // `initial` contains the eight readable lanes loaded here. unsafe { [ - _mm256_loadu_si256(initial.as_ptr() as *const __m256i), - _mm256_loadu_si256(initial.as_ptr().add(4) as *const __m256i), + _mm256_loadu_si256(initial.as_ptr().cast()), + _mm256_loadu_si256(initial.as_ptr().add(4).cast()), ] } } #[inline] #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The current CPU must support AVX2. unsafe fn store_acc(acc: &[__m256i; 2]) -> [u64; ACC_NB] { - // SAFETY: AVX2 available via target_feature. + // SAFETY: The function attribute enables the unaligned AVX2 stores, and + // `out` contains the eight writable lanes stored here. unsafe { let mut out = [0u64; ACC_NB]; - _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, acc[0]); - _mm256_storeu_si256(out.as_mut_ptr().add(4) as *mut __m256i, acc[1]); + _mm256_storeu_si256(out.as_mut_ptr().cast(), acc[0]); + _mm256_storeu_si256(out.as_mut_ptr().add(4).cast(), acc[1]); out } } @@ -57,6 +63,11 @@ unsafe fn store_acc(acc: &[__m256i; 2]) -> [u64; ACC_NB] { /// 3. `vpmuludq`: multiply low32(data_key) × high32(data_key) → u64 /// 4. Shuffle data to swap u64 pairs (idx ^ 1) /// 5. Accumulate product + swapped data into acc +/// +/// # Safety +/// +/// The current CPU must support AVX2. `stripe` and `secret` must each remain +/// valid to read 64 initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx2")] unsafe fn accumulate_512(acc: &mut [__m256i; 2], stripe: *const u8, secret: *const u8) { @@ -65,8 +76,8 @@ unsafe fn accumulate_512(acc: &mut [__m256i; 2], stripe: *const u8, secret: *con unsafe { let mut i = 0usize; while i < 2 { - let data_vec = _mm256_loadu_si256(stripe.add(i.strict_mul(32)) as *const __m256i); - let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)) as *const __m256i); + let data_vec = _mm256_loadu_si256(stripe.add(i.strict_mul(32)).cast()); + let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)).cast()); let data_key = _mm256_xor_si256(data_vec, key_vec); // u32 × u32 → u64 multiply: low32(data_key) × high32(data_key) @@ -87,13 +98,18 @@ unsafe fn accumulate_512(acc: &mut [__m256i; 2], stripe: *const u8, secret: *con /// /// Per element: `acc = (xorshift64(acc, 47) ^ secret) * PRIME32_1` /// The 64-bit multiply by a 32-bit prime is split into lo + hi halves. +/// +/// # Safety +/// +/// The current CPU must support AVX2. `secret` must remain valid to read 64 +/// initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx2")] unsafe fn scramble_acc(acc: &mut [__m256i; 2], secret: *const u8) { // SAFETY: AVX2 available via target_feature. Caller ensures secret points to // ≥64 valid bytes. unsafe { - let prime32 = _mm256_set1_epi32(PRIME32_1 as i32); + let prime32 = _mm256_set1_epi32(PRIME32_1.cast_signed()); let mut i = 0usize; while i < 2 { @@ -101,7 +117,7 @@ unsafe fn scramble_acc(acc: &mut [__m256i; 2], secret: *const u8) { let shifted = _mm256_srli_epi64::<47>(acc_vec); let data_vec = _mm256_xor_si256(acc_vec, shifted); - let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)) as *const __m256i); + let key_vec = _mm256_loadu_si256(secret.add(i.strict_mul(32)).cast()); let data_key = _mm256_xor_si256(data_vec, key_vec); // 64-bit multiply by PRIME32_1: @@ -120,6 +136,10 @@ unsafe fn scramble_acc(acc: &mut [__m256i; 2], secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The current CPU must support AVX2. `stripes` must be nonzero, and the +/// requested input and secret stripes must be within their slices. unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], input: &[u8], @@ -139,7 +159,8 @@ unsafe fn stream_accumulate_inner( ); // SAFETY: Accumulating validated XXH3 stripes because: // 1. The dispatcher selects this function only when AVX2 is available. - // 2. The assertions above prove every 64-byte input and secret load is in bounds. + // 2. The caller contract keeps every 64-byte input and secret load in bounds; + // the assertions above check that contract in debug builds. // 3. `initial` and the returned array each contain exactly eight `u64` lanes. unsafe { let mut acc = load_acc(&initial); @@ -195,14 +216,18 @@ pub(crate) unsafe fn stream_accumulate( } #[target_feature(enable = "avx2")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +/// # Safety +/// +/// The current CPU must support AVX2, and `input` must contain more than 240 +/// bytes. +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: AVX2 available via target_feature. Input/secret bounds checked by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); - let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; + let nb_stripes = secret.len().strict_sub(STRIPE_LEN).strict_div(SECRET_CONSUME_RATE); let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -218,7 +243,11 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] } // Remaining stripes in final partial block - let nb_stripes_final = (input.len().strict_sub(1).strict_sub(block_len.strict_mul(nb_blocks))) / STRIPE_LEN; + let nb_stripes_final = input + .len() + .strict_sub(1) + .strict_sub(block_len.strict_mul(nb_blocks)) + .strict_div(STRIPE_LEN); let mut stripe = 0usize; while stripe < nb_stripes_final { let input_off = nb_blocks @@ -245,7 +274,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long_default(input: &[u8]) -> u64 { +pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { // SAFETY: Dispatcher verifies AVX2 before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; super::merge_accs( @@ -256,7 +285,7 @@ pub fn xxh3_64_long_default(input: &[u8]) -> u64 { ) } -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) } else { @@ -273,13 +302,13 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long_default(input: &[u8]) -> u128 { +pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: Dispatcher verifies AVX2 before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; xxh3_128_long_finalize(&acc, &DEFAULT_SECRET, input.len()) } -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) } else { diff --git a/src/hashes/fast/xxh3/x86_64_avx512.rs b/src/hashes/fast/xxh3/x86_64_avx512.rs index 13a95ac5..7f87ca4e 100644 --- a/src/hashes/fast/xxh3/x86_64_avx512.rs +++ b/src/hashes/fast/xxh3/x86_64_avx512.rs @@ -7,8 +7,6 @@ //! //! Uses `unsafe` for AVX-512F intrinsics. Callers must ensure AVX-512F is //! available before executing the accelerated path (the dispatcher does this). -#![allow(unsafe_code)] -#![allow(clippy::indexing_slicing)] use core::arch::x86_64::*; @@ -27,18 +25,26 @@ const SWAP32: i32 = 0x4E; #[inline] #[target_feature(enable = "avx512f")] +/// # Safety +/// +/// The current CPU must support AVX-512F. unsafe fn load_acc(initial: &[u64; ACC_NB]) -> __m512i { - // SAFETY: AVX-512F available via target_feature. Pointer valid for 8 × u64. - unsafe { _mm512_loadu_si512(initial.as_ptr() as *const __m512i) } + // SAFETY: The function attribute enables the unaligned AVX-512F load, and + // `initial` contains the eight readable lanes loaded here. + unsafe { _mm512_loadu_si512(initial.as_ptr().cast()) } } #[inline] #[target_feature(enable = "avx512f")] +/// # Safety +/// +/// The current CPU must support AVX-512F. unsafe fn store_acc(acc: __m512i) -> [u64; ACC_NB] { - // SAFETY: AVX-512F available via target_feature. + // SAFETY: The function attribute enables the unaligned AVX-512F store, and + // `out` contains the eight writable lanes stored here. unsafe { let mut out = [0u64; ACC_NB]; - _mm512_storeu_si512(out.as_mut_ptr() as *mut __m512i, acc); + _mm512_storeu_si512(out.as_mut_ptr().cast(), acc); out } } @@ -48,14 +54,19 @@ unsafe fn store_acc(acc: __m512i) -> [u64; ACC_NB] { /// Processes all 8 u64 accumulators simultaneously: /// 1. Load full 64 B stripe + 64 B secret in one shot /// 2. XOR → shift → multiply → shuffle-swap → accumulate +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. `stripe` and `secret` must each +/// remain valid to read 64 initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx512f")] unsafe fn accumulate_512(acc: &mut __m512i, stripe: *const u8, secret: *const u8) { // SAFETY: AVX-512F available via target_feature. Caller ensures stripe and // secret point to ≥64 valid bytes. unsafe { - let data_vec = _mm512_loadu_si512(stripe as *const __m512i); - let key_vec = _mm512_loadu_si512(secret as *const __m512i); + let data_vec = _mm512_loadu_si512(stripe.cast()); + let key_vec = _mm512_loadu_si512(secret.cast()); let data_key = _mm512_xor_si512(data_vec, key_vec); // u32 × u32 → u64 multiply: low32(data_key) × high32(data_key) @@ -70,18 +81,23 @@ unsafe fn accumulate_512(acc: &mut __m512i, stripe: *const u8, secret: *const u8 } /// Scramble the accumulator at block boundaries — single iteration. +/// +/// # Safety +/// +/// The current CPU must support AVX-512F. `secret` must remain valid to read +/// 64 initialized bytes from one allocation. #[inline] #[target_feature(enable = "avx512f")] unsafe fn scramble_acc(acc: &mut __m512i, secret: *const u8) { // SAFETY: AVX-512F available via target_feature. Caller ensures secret // points to ≥64 valid bytes. unsafe { - let prime32 = _mm512_set1_epi32(PRIME32_1 as i32); + let prime32 = _mm512_set1_epi32(PRIME32_1.cast_signed()); let shifted = _mm512_srli_epi64::<47>(*acc); let data_vec = _mm512_xor_si512(*acc, shifted); - let key_vec = _mm512_loadu_si512(secret as *const __m512i); + let key_vec = _mm512_loadu_si512(secret.cast()); let data_key = _mm512_xor_si512(data_vec, key_vec); // 64-bit multiply by PRIME32_1 (split into lo + hi halves) @@ -95,6 +111,10 @@ unsafe fn scramble_acc(acc: &mut __m512i, secret: *const u8) { // Long-path loop (SIMD inner, scalar merge) #[target_feature(enable = "avx512f")] +/// # Safety +/// +/// The current CPU must support AVX-512F. `stripes` must be nonzero, and the +/// requested input and secret stripes must be within their slices. unsafe fn stream_accumulate_inner( initial: [u64; ACC_NB], input: &[u8], @@ -114,7 +134,8 @@ unsafe fn stream_accumulate_inner( ); // SAFETY: Accumulating validated XXH3 stripes because: // 1. The dispatcher selects this function only when AVX-512F is available. - // 2. The assertions above prove every 64-byte input and secret load is in bounds. + // 2. The caller contract keeps every 64-byte input and secret load in bounds; + // the assertions above check that contract in debug builds. // 3. `initial` and the returned array each contain exactly eight `u64` lanes. unsafe { let mut acc = load_acc(&initial); @@ -170,15 +191,19 @@ pub(crate) unsafe fn stream_accumulate( } #[target_feature(enable = "avx512f")] -unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] { +/// # Safety +/// +/// The current CPU must support AVX-512F, and `input` must contain more than +/// 240 bytes. +unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8; DEFAULT_SECRET_SIZE]) -> [u64; ACC_NB] { // SAFETY: AVX-512F available via target_feature. Input/secret bounds checked // by caller. unsafe { let mut acc = load_acc(&INITIAL_ACC); - let nb_stripes = (secret.len().strict_sub(STRIPE_LEN)) / SECRET_CONSUME_RATE; + let nb_stripes = secret.len().strict_sub(STRIPE_LEN).strict_div(SECRET_CONSUME_RATE); let block_len = STRIPE_LEN.strict_mul(nb_stripes); - let nb_blocks = (input.len().strict_sub(1)) / block_len; + let nb_blocks = input.len().strict_sub(1).strict_div(block_len); let mut block = 0usize; while block < nb_blocks { @@ -194,7 +219,11 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] } // Remaining stripes in final partial block - let nb_stripes_final = (input.len().strict_sub(1).strict_sub(block_len.strict_mul(nb_blocks))) / STRIPE_LEN; + let nb_stripes_final = input + .len() + .strict_sub(1) + .strict_sub(block_len.strict_mul(nb_blocks)) + .strict_div(STRIPE_LEN); let mut stripe = 0usize; while stripe < nb_stripes_final { let input_off = nb_blocks @@ -221,7 +250,7 @@ unsafe fn hash_long_internal_loop(input: &[u8], secret: &[u8]) -> [u64; ACC_NB] // Top-level kernel functions (safe wrappers) /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_64_long_default(input: &[u8]) -> u64 { +pub(crate) fn xxh3_64_long_default(input: &[u8]) -> u64 { // SAFETY: Dispatcher verifies AVX-512F before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; super::merge_accs( @@ -232,7 +261,7 @@ pub fn xxh3_64_long_default(input: &[u8]) -> u64 { ) } -pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { +pub(crate) fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { if seed == 0 { xxh3_64_long_default(input) } else { @@ -249,13 +278,13 @@ pub fn xxh3_64_long(input: &[u8], seed: u64) -> u64 { } /// Long-path entry point (>240B) — no ≤240B branches. -pub fn xxh3_128_long_default(input: &[u8]) -> u128 { +pub(crate) fn xxh3_128_long_default(input: &[u8]) -> u128 { // SAFETY: Dispatcher verifies AVX-512F before selecting this kernel. let acc = unsafe { hash_long_internal_loop(input, &DEFAULT_SECRET) }; xxh3_128_long_finalize(&acc, &DEFAULT_SECRET, input.len()) } -pub fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { +pub(crate) fn xxh3_128_long(input: &[u8], seed: u64) -> u128 { if seed == 0 { xxh3_128_long_default(input) } else { diff --git a/src/hashes/mod.rs b/src/hashes/mod.rs index 4248c89e..dcc24822 100644 --- a/src/hashes/mod.rs +++ b/src/hashes/mod.rs @@ -45,16 +45,16 @@ //! ```toml //! [dependencies] //! # Smallest SHA-2-only build -//! rscrypto = { version = "0.7", default-features = false, features = ["sha2"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } //! //! # All cryptographic hashes -//! rscrypto = { version = "0.7", default-features = false, features = ["crypto-hashes"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["crypto-hashes"] } //! //! # Fast non-cryptographic hashes only -//! rscrypto = { version = "0.7", default-features = false, features = ["fast-hashes"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["fast-hashes"] } //! //! # Everything hash-related -//! rscrypto = { version = "0.7", default-features = false, features = ["hashes"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["hashes"] } //! ``` //! //! # API Conventions diff --git a/src/hashes/util.rs b/src/hashes/util.rs index f241ec5f..fbef6e85 100644 --- a/src/hashes/util.rs +++ b/src/hashes/util.rs @@ -1,5 +1,3 @@ -#![allow(clippy::indexing_slicing)] // Fixed-size array indexing and block parsing - #[repr(C, align(64))] pub(crate) struct Aligned64(pub T); @@ -13,11 +11,11 @@ impl core::ops::Deref for Aligned64 { } #[inline(always)] -pub const fn rotr32(x: u32, n: u32) -> u32 { +pub(super) const fn rotr32(x: u32, n: u32) -> u32 { x.rotate_right(n) } #[inline(always)] -pub const fn rotr64(x: u64, n: u32) -> u64 { +pub(super) const fn rotr64(x: u64, n: u32) -> u64 { x.rotate_right(n) } diff --git a/src/hex.rs b/src/hex.rs index 48f0bef3..97676f72 100644 --- a/src/hex.rs +++ b/src/hex.rs @@ -1,18 +1,3 @@ -#![allow(clippy::indexing_slicing)] -#![cfg_attr( - not(any( - feature = "aes-gcm", - feature = "aes-gcm-siv", - feature = "chacha20poly1305", - feature = "xchacha20poly1305", - feature = "aegis256", - feature = "ascon-aead", - feature = "ed25519", - feature = "x25519" - )), - allow(dead_code, unused_macros) -)] - //! Internal hex encoding, decoding, and formatting utilities. //! //! Provides zero-allocation, `no_std`-compatible hex formatting through @@ -22,6 +7,17 @@ use core::fmt; /// Hex decoding error. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] #[derive(Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum InvalidHexError { @@ -36,6 +32,17 @@ pub enum InvalidHexError { }, } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Debug for InvalidHexError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -48,6 +55,17 @@ impl fmt::Debug for InvalidHexError { } } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Display for InvalidHexError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -57,15 +75,37 @@ impl fmt::Display for InvalidHexError { } } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl core::error::Error for InvalidHexError {} /// Decode a single hex character to its 4-bit value. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] #[inline] const fn decode_nibble(byte: u8) -> Option { match byte { - b'0'..=b'9' => Some(byte - b'0'), - b'a'..=b'f' => Some(byte - b'a' + 10), - b'A'..=b'F' => Some(byte - b'A' + 10), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), _ => None, } } @@ -73,7 +113,18 @@ const fn decode_nibble(byte: u8) -> Option { /// Decode a hex string into `out`. Accepts mixed case, no `0x` prefix. /// /// Returns `InvalidHexError::InvalidLength` when `hex.len() != out.len() * 2`. -pub fn from_hex(hex: &str, out: &mut [u8]) -> Result<(), InvalidHexError> { +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] +pub(crate) fn from_hex(hex: &str, out: &mut [u8]) -> Result<(), InvalidHexError> { let hex = hex.as_bytes(); if hex.len() != out.len().strict_mul(2) { return Err(InvalidHexError::InvalidLength); @@ -107,7 +158,7 @@ pub fn from_hex(hex: &str, out: &mut [u8]) -> Result<(), InvalidHexError> { } /// Write each byte as two lowercase hex characters. -pub fn fmt_hex_lower(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { +pub(crate) fn fmt_hex_lower(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { for &b in bytes { write!(f, "{b:02x}")?; } @@ -115,7 +166,18 @@ pub fn fmt_hex_lower(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { } /// Write each byte as two uppercase hex characters. -pub fn fmt_hex_upper(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] +pub(crate) fn fmt_hex_upper(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { for &b in bytes { write!(f, "{b:02X}")?; } @@ -126,14 +188,53 @@ pub fn fmt_hex_upper(bytes: &[u8], f: &mut fmt::Formatter<'_>) -> fmt::Result { /// /// Returned by the `display_secret()` method on secret key types. Implements /// [`Display`](fmt::Display) so you can `format!("{}", key.display_secret())`. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] pub struct DisplaySecret<'a>(pub(crate) &'a [u8]); +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Display for DisplaySecret<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt_hex_lower(self.0, f) } } +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] impl fmt::Debug for DisplaySecret<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "DisplaySecret(\"")?; @@ -147,6 +248,17 @@ impl fmt::Debug for DisplaySecret<'_> { /// Implement `LowerHex`, `UpperHex`, `Display`, `Debug`, and `FromStr` for /// a public byte-array newtype that has `as_bytes()`, `from_bytes()`, and /// `LENGTH`. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] macro_rules! impl_hex_fmt { ($type:ty) => { impl core::fmt::LowerHex for $type { @@ -182,6 +294,17 @@ macro_rules! impl_hex_fmt { /// Implement masked `Debug`, `FromStr`, and `display_secret()` for a secret /// key newtype. Does **not** implement `Display`, `LowerHex`, or `UpperHex` /// to prevent accidental logging of key material. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" +))] macro_rules! impl_hex_fmt_secret { ($type:ty) => { impl core::str::FromStr for $type { @@ -211,7 +334,20 @@ macro_rules! impl_hex_fmt_secret { /// Implement `serde::Serialize` and `serde::Deserialize` for a byte-array /// newtype with `as_bytes() -> &[u8; N]`, `from_bytes([u8; N]) -> Self`, /// and `LENGTH`. -#[cfg(feature = "serde")] +#[cfg(all( + feature = "serde", + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_bytes_inner { ($type:ty, $feature:literal) => { #[cfg_attr(docsrs, doc(cfg(feature = $feature)))] @@ -255,7 +391,20 @@ macro_rules! impl_serde_bytes_inner { }; } -#[cfg(feature = "serde")] +#[cfg(all( + feature = "serde", + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_bytes { ($type:ty) => { impl_serde_bytes_inner!($type, "serde"); @@ -263,14 +412,40 @@ macro_rules! impl_serde_bytes { } // No-op when serde feature is disabled. -#[cfg(not(feature = "serde"))] +#[cfg(all( + not(feature = "serde"), + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_bytes { ($type:ty) => {}; } /// Implement `serde` for secret material behind the explicit `serde-secrets` /// feature. This keeps broad DTO serialization from silently exporting keys. -#[cfg(feature = "serde-secrets")] +#[cfg(all( + feature = "serde-secrets", + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_secret_bytes { ($type:ty) => { #[cfg_attr(docsrs, doc(cfg(feature = "serde-secrets")))] @@ -315,13 +490,36 @@ macro_rules! impl_serde_secret_bytes { }; } -#[cfg(not(feature = "serde-secrets"))] +#[cfg(all( + not(feature = "serde-secrets"), + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] macro_rules! impl_serde_secret_bytes { ($type:ty) => {}; } /// Generate a fallible constructor that fills `Self::LENGTH` bytes from the /// operating system CSPRNG via `getrandom`. +#[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "x25519", + feature = "xchacha20poly1305" +))] macro_rules! impl_getrandom { () => { /// Try to generate a random instance from the platform entropy source. @@ -344,7 +542,7 @@ macro_rules! impl_getrandom { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_hex_success() -> bool { +pub(crate) fn diag_zeroize_hex_success() -> bool { let parsed = core::hint::black_box("1111111111111111111111111111111111111111111111111111111111111111") .parse::(); core::hint::black_box(parsed.is_ok()) @@ -354,7 +552,7 @@ pub fn diag_zeroize_hex_success() -> bool { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_hex_error() -> bool { +pub(crate) fn diag_zeroize_hex_error() -> bool { let parsed = core::hint::black_box("11111111111111111111111111111111111111111111111111111111111111zz") .parse::(); core::hint::black_box(parsed.is_err()) @@ -362,7 +560,20 @@ pub fn diag_zeroize_hex_error() -> bool { // Tests -#[cfg(test)] +#[cfg(all( + test, + any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "chacha20poly1305", + feature = "ed25519", + feature = "ml-kem", + feature = "x25519", + feature = "xchacha20poly1305" + ) +))] mod tests { use super::*; @@ -373,7 +584,7 @@ mod tests { assert_eq!(hex, "deadbeef"); let mut out = [0u8; 4]; - from_hex(&hex, &mut out).unwrap(); + from_hex(&hex, &mut out).expect("lowercase hex fixture must decode"); assert_eq!(out, bytes); } @@ -391,7 +602,7 @@ mod tests { #[test] fn from_hex_mixed_case() { let mut out = [0u8; 3]; - from_hex("aAbBcC", &mut out).unwrap(); + from_hex("aAbBcC", &mut out).expect("mixed-case hex fixture must decode"); assert_eq!(out, [0xaa, 0xbb, 0xcc]); } @@ -404,7 +615,7 @@ mod tests { #[test] fn from_hex_invalid_char() { let mut out = [0u8; 2]; - let err = from_hex("abzz", &mut out).unwrap_err(); + let err = from_hex("abzz", &mut out).expect_err("non-hex characters must be rejected"); assert_eq!(err, InvalidHexError::InvalidChar { byte: b'z', index: 2 }); } @@ -416,7 +627,9 @@ mod tests { let mut encoded = alloc::string::String::from("11").repeat(Aes256GcmKey::LENGTH); encoded.replace_range(encoded.len().strict_sub(2).., "zz"); assert_eq!( - encoded.parse::().unwrap_err(), + encoded + .parse::() + .expect_err("invalid secret hex must be rejected"), InvalidHexError::InvalidChar { byte: b'z', index: encoded.len().strict_sub(2), diff --git a/src/lib.rs b/src/lib.rs index c583dc1d..6200286c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ //! //! ```toml //! [dependencies] -//! rscrypto = { version = "0.8.0", default-features = false, features = ["sha2"] } +//! rscrypto = { version = "0.8.1", default-features = false, features = ["sha2"] } //! ``` //! //! # Guides @@ -32,6 +32,8 @@ //! - XOFs: `Type::xof(data)` or `new` / `update` / `finalize_xof`. //! - MACs: `Type::mac(key, data)` and `Type::verify_tag(key, data, tag)`. //! - AEADs: typed keys and nonces, with combined, detached, and `alloc` Vec helpers. +#![deny(missing_debug_implementations, missing_docs)] +#![deny(clippy::print_stderr, clippy::print_stdout)] #![cfg_attr( feature = "sha2", doc = r#" @@ -125,9 +127,6 @@ assert!( //! evidence. See the security guidance for nonce lifecycle, PHC verification //! limits, and platform fallback notes. -#![cfg_attr(not(test), deny(clippy::unwrap_used))] -#![cfg_attr(not(test), deny(clippy::expect_used))] -#![cfg_attr(not(test), deny(clippy::indexing_slicing))] // Exotic-architecture backends require nightly-only features (inline asm + // portable_simd + unstable target-feature flags). Primary targets (x86_64, // aarch64, wasm) compile on stable Rust 1.91.0. @@ -153,7 +152,26 @@ assert!( )] // s390x VGFM/hash backends use vector asm; selected checksum/hash/AEAD/password // kernels also use portable SIMD. -#![cfg_attr(target_arch = "s390x", feature(asm_experimental_reg))] +#![cfg_attr( + all( + target_arch = "s390x", + any( + feature = "crc16", + feature = "crc24", + feature = "crc32", + feature = "crc64", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "aegis256", + feature = "xxh3", + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + feature = "ml-kem", + feature = "argon2" + ) + ), + feature(asm_experimental_reg) +)] #![cfg_attr( all( target_arch = "s390x", @@ -184,7 +202,6 @@ assert!( feature = "crc16", feature = "crc24", feature = "crc32", - feature = "crc64", feature = "blake2b", feature = "blake2s", feature = "blake3", @@ -205,7 +222,6 @@ assert!( feature = "crc16", feature = "crc24", feature = "crc32", - feature = "crc64", feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256" @@ -223,17 +239,16 @@ assert!( #![cfg_attr( all( target_arch = "riscv64", - any(feature = "blake3", feature = "chacha20poly1305", feature = "xchacha20poly1305") + any( + feature = "blake3", + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + feature = "argon2" + ) ), feature(portable_simd) )] -#![cfg_attr( - all( - target_arch = "riscv32", - any(feature = "sha2", feature = "aes-gcm", feature = "aes-gcm-siv", feature = "aegis256") - ), - feature(riscv_ext_intrinsics) -)] +#![cfg_attr(all(target_arch = "riscv32", feature = "sha2"), feature(riscv_ext_intrinsics))] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] @@ -340,20 +355,12 @@ mod secret; ))] pub mod hashes; -#[cfg_attr( - not(any(feature = "kmac", feature = "ascon-hash", feature = "sha3")), - allow(dead_code) -)] #[inline] #[track_caller] +#[cfg(any(test, feature = "kmac", feature = "sha3", feature = "ascon-hash"))] pub(crate) fn bytes_to_bits(len: usize) -> u64 { - let Ok(bytes) = u64::try_from(len) else { - panic!("byte length exceeds u64"); - }; - let Some(bits) = bytes.checked_mul(8) else { - panic!("byte length bit count exceeds u64"); - }; - bits + let bytes = u64::try_from(len).expect("byte length exceeds u64"); + bytes.checked_mul(8).expect("byte length bit count exceeds u64") } // Checksum re-exports. @@ -417,10 +424,10 @@ pub use auth::{Pbkdf2Error, Pbkdf2Params, Pbkdf2Sha256, Pbkdf2Sha512, Pbkdf2Veri pub use auth::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; #[cfg(feature = "rsa")] pub use auth::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, RsaKeyGenerationError, - RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, RsaPrivateScratch, - RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, RsaPublicKey, - RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, + RsaBlindingPair, RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationContract, + RsaKeyGenerationError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, + RsaPrivateScratch, RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, + RsaPublicKey, RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaSignatureSigner, RsaSignatureVerifier, RsaTlsSignatureSchemes, RsaX509PublicKey, RsaX509PublicKeyAlgorithm, }; #[cfg(feature = "scrypt")] @@ -471,8 +478,6 @@ pub use hashes::fast::{Xxh3_128Hasher, Xxh3BuildHasher, Xxh3Hasher}; feature = "xchacha20poly1305", feature = "aegis256", feature = "ascon-aead", - feature = "ecdsa-p256", - feature = "ecdsa-p384", feature = "ed25519", feature = "ml-kem", feature = "x25519" @@ -898,12 +903,14 @@ let third = HmacSha256Tag::from_bytes([0u8; HmacSha256Tag::LENGTH]); let _ = first.ct_eq(&second) == second.ct_eq(&third); ``` -```compile_fail +`Debug` output is deliberately opaque and must not reveal the decision. + +```rust use rscrypto::HmacSha256Tag; let left = HmacSha256Tag::from_bytes([0u8; HmacSha256Tag::LENGTH]); let right = HmacSha256Tag::from_bytes([0u8; HmacSha256Tag::LENGTH]); -let _ = format!("{:?}", left.ct_eq(&right)); +assert_eq!(format!("{:?}", left.ct_eq(&right)), "CtDecision(..)"); ``` ```rust @@ -1192,8 +1199,10 @@ mod direct_io_write_tests { #[test] fn digest_state_accepts_direct_io_write() { let mut digest = Sha256::new(); - digest.write_all(b"hello ").unwrap(); - digest.write_all(b"world").unwrap(); + digest + .write_all(b"hello ") + .expect("in-memory digest write must succeed"); + digest.write_all(b"world").expect("in-memory digest write must succeed"); assert_eq!(digest.finalize(), Sha256::digest(b"hello world")); } @@ -1201,8 +1210,12 @@ mod direct_io_write_tests { #[test] fn checksum_state_accepts_direct_io_write() { let mut checksum = Crc32C::new(); - checksum.write_all(b"hello ").unwrap(); - checksum.write_all(b"world").unwrap(); + checksum + .write_all(b"hello ") + .expect("in-memory checksum write must succeed"); + checksum + .write_all(b"world") + .expect("in-memory checksum write must succeed"); assert_eq!(checksum.finalize(), Crc32C::checksum(b"hello world")); } @@ -1212,7 +1225,9 @@ mod direct_io_write_tests { let mut digest = Sha256::new(); let bufs = [IoSlice::new(b"hello "), IoSlice::new(b"world")]; - let written = digest.write_vectored(&bufs).unwrap(); + let written = digest + .write_vectored(&bufs) + .expect("in-memory vectored digest write must succeed"); assert_eq!(written, b"hello world".len()); assert_eq!(digest.finalize(), Sha256::digest(b"hello world")); @@ -1221,7 +1236,6 @@ mod direct_io_write_tests { #[cfg(test)] mod send_sync_assertions { - #![allow(unused_imports)] use super::*; fn assert_send_sync() {} @@ -1343,14 +1357,14 @@ mod send_sync_assertions { #[cfg(all(feature = "checksums", feature = "std"))] fn io_adapter_types_are_send_and_sync() { // ChecksumReader/Writer are Send+Sync when their inner types are - assert_send_sync::>, Crc32C>>(); + assert_send_sync::>(); assert_send_sync::, Crc32C>>(); } #[test] #[cfg(all(feature = "hashes", feature = "std"))] fn digest_io_adapter_types_are_send_and_sync() { - assert_send_sync::>, Sha256>>(); + assert_send_sync::>(); assert_send_sync::, Sha256>>(); } @@ -1500,14 +1514,14 @@ mod send_sync_assertions { #[test] #[cfg(all(feature = "checksums", feature = "std"))] fn io_adapter_types_are_debug() { - assert_debug::>, Crc32C>>(); + assert_debug::>(); assert_debug::, Crc32C>>(); } #[test] #[cfg(all(feature = "hashes", feature = "std"))] fn digest_io_adapter_types_are_debug() { - assert_debug::>, Sha256>>(); + assert_debug::>(); assert_debug::, Sha256>>(); } } diff --git a/src/macros.rs b/src/macros.rs index a6851b4f..96d785f7 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -9,6 +9,7 @@ macro_rules! define_unit_error { $vis struct $name; impl $name { + /// Construct this error value. #[inline] #[must_use] pub const fn new() -> Self { @@ -37,7 +38,6 @@ macro_rules! define_unit_error { macro_rules! impl_std_io_write_for_checksum { ($type:ty) => { #[cfg(feature = "std")] - #[allow(clippy::std_instead_of_core)] impl std::io::Write for $type { #[inline] fn write(&mut self, buf: &[u8]) -> std::io::Result { @@ -54,9 +54,7 @@ macro_rules! impl_std_io_write_for_checksum { fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { let mut written = 0usize; for buf in bufs { - written = written - .checked_add(buf.len()) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "vectored write length overflow"))?; + written = written.saturating_add(buf.len()); } for buf in bufs { ::update(self, buf.as_ref()); @@ -78,7 +76,6 @@ macro_rules! impl_std_io_write_for_checksum { macro_rules! impl_std_io_write_for_digest { ($type:ty) => { #[cfg(feature = "std")] - #[allow(clippy::std_instead_of_core)] impl std::io::Write for $type { #[inline] fn write(&mut self, buf: &[u8]) -> std::io::Result { @@ -95,9 +92,7 @@ macro_rules! impl_std_io_write_for_digest { fn write_vectored(&mut self, bufs: &[std::io::IoSlice<'_>]) -> std::io::Result { let mut written = 0usize; for buf in bufs { - written = written - .checked_add(buf.len()) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "vectored write length overflow"))?; + written = written.saturating_add(buf.len()); } for buf in bufs { ::update(self, buf.as_ref()); @@ -109,7 +104,7 @@ macro_rules! impl_std_io_write_for_digest { } #[cfg(all( - any(target_arch = "aarch64", target_arch = "riscv64", target_arch = "x86_64"), + any(target_arch = "aarch64", target_arch = "x86_64"), any(feature = "chacha20poly1305", feature = "xchacha20poly1305") ))] macro_rules! define_target_feature_forwarder { @@ -128,6 +123,7 @@ macro_rules! define_target_feature_forwarder { } #[target_feature(enable = $feature)] + #[doc = concat!("Forward through the `", $feature, "` target-feature boundary.\n\n# Safety\n\nThe current CPU must support the named target feature. The safe outer wrapper may call this function only after its dispatch invariant has established that capability.")] unsafe fn __target_feature_forwarder_impl($($arg : $arg_ty),*) $(-> $ret)? { // SAFETY: $inner_safety unsafe { $call } @@ -158,8 +154,7 @@ macro_rules! define_sha_family_dispatch { #[derive(Clone, Copy)] struct Entry { compress_blocks: $compress_fn_ty, - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] name: &'static str, } @@ -200,22 +195,22 @@ macro_rules! define_sha_family_dispatch { boundaries: table.boundaries, xs: Entry { compress_blocks: $compress_fn(xs_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: xs_id.as_str(), }, s: Entry { compress_blocks: $compress_fn(s_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: s_id.as_str(), }, m: Entry { compress_blocks: $compress_fn(m_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: m_id.as_str(), }, l: Entry { compress_blocks: $compress_fn(l_id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: l_id.as_str(), }, } @@ -237,11 +232,10 @@ macro_rules! define_sha_family_dispatch { } } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] #[inline] #[must_use] - pub fn kernel_name_for_len(len: usize) -> &'static str { + pub(crate) fn kernel_name_for_len(len: usize) -> &'static str { if $compile_time_hw { return $compile_time_name; } @@ -251,7 +245,7 @@ macro_rules! define_sha_family_dispatch { #[inline] #[must_use] - pub fn digest(data: &[u8]) -> [u8; $output_len] { + pub(crate) fn digest(data: &[u8]) -> [u8; $output_len] { if $compile_time_hw { return digest_oneshot(data, $compile_time_best); } @@ -332,8 +326,7 @@ macro_rules! define_blake2_dispatch { struct Resolved { compress: $compress_fn_ty, compress_blocks: $compress_blocks_fn_ty, - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] name: &'static str, } @@ -350,7 +343,7 @@ macro_rules! define_blake2_dispatch { return Resolved { compress: $compress_fn(id), compress_blocks: $compress_blocks_fn(id), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: id.as_str(), }; } @@ -359,7 +352,7 @@ macro_rules! define_blake2_dispatch { Resolved { compress: $compress_fn($portable_kernel), compress_blocks: $compress_blocks_fn($portable_kernel), - #[cfg(any(test, feature = "diag"))] + #[cfg(feature = "diag")] name: $portable_kernel.as_str(), } } @@ -392,11 +385,10 @@ macro_rules! define_blake2_dispatch { ACTIVE.get_or_init(resolve).compress_blocks } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] #[inline] #[must_use] - pub fn kernel_name_for_len(_len: usize) -> &'static str { + pub(super) fn kernel_name_for_len(_len: usize) -> &'static str { if super::kernels::COMPILE_TIME_HW { return compile_time_name(); } @@ -408,8 +400,7 @@ macro_rules! define_blake2_dispatch { ACTIVE.get_or_init(resolve).name } - #[cfg(any(test, feature = "diag"))] - #[allow(dead_code)] + #[cfg(feature = "diag")] const fn compile_time_name() -> &'static str { if cfg!(all( target_arch = "x86_64", diff --git a/src/platform/caps.rs b/src/platform/caps.rs index c96bcb84..04ce30d8 100644 --- a/src/platform/caps.rs +++ b/src/platform/caps.rs @@ -152,7 +152,11 @@ impl Caps { #[inline] #[must_use] pub const fn count(self) -> u32 { - self.0[0].count_ones() + self.0[1].count_ones() + self.0[2].count_ones() + self.0[3].count_ones() + self.0[0] + .count_ones() + .strict_add(self.0[1].count_ones()) + .strict_add(self.0[2].count_ones()) + .strict_add(self.0[3].count_ones()) } /// Create a capability set with a single bit set. @@ -222,18 +226,28 @@ impl core::ops::BitOrAssign for Caps { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Arch { + /// 64-bit x86 (`target_arch = "x86_64"`). X86_64, + /// 32-bit x86 (`target_arch = "x86"`). X86, + /// 64-bit Arm (`target_arch = "aarch64"`). Aarch64, + /// 32-bit Arm (`target_arch = "arm"`). Arm, + /// 64-bit RISC-V (`target_arch = "riscv64"`). Riscv64, + /// 32-bit RISC-V (`target_arch = "riscv32"`). Riscv32, - /// IBM POWER (Rust target_arch = "powerpc64"). + /// 64-bit IBM POWER (`target_arch = "powerpc64"`). Power, + /// IBM Z (`target_arch = "s390x"`). S390x, + /// 32-bit WebAssembly (`target_arch = "wasm32"`). Wasm32, + /// 64-bit WebAssembly (`target_arch = "wasm64"`). Wasm64, #[default] + /// Any target architecture not represented by another variant. Other, } @@ -334,74 +348,123 @@ pub mod x86 { use super::Caps; // ─── SSE Family ─── + /// Streaming SIMD Extensions 2. pub const SSE2: Caps = Caps::bit(0); + /// Streaming SIMD Extensions 3. pub const SSE3: Caps = Caps::bit(1); + /// Supplemental Streaming SIMD Extensions 3. pub const SSSE3: Caps = Caps::bit(2); + /// Streaming SIMD Extensions 4.1. pub const SSE41: Caps = Caps::bit(3); + /// Streaming SIMD Extensions 4.2. pub const SSE42: Caps = Caps::bit(4); - pub const SSE4A: Caps = Caps::bit(5); // AMD only + /// AMD Streaming SIMD Extensions 4a. + pub const SSE4A: Caps = Caps::bit(5); // ─── AVX Family ─── + /// Advanced Vector Extensions. pub const AVX: Caps = Caps::bit(6); + /// Advanced Vector Extensions 2. pub const AVX2: Caps = Caps::bit(7); + /// Fused multiply-add instructions. pub const FMA: Caps = Caps::bit(8); + /// Half-precision conversion instructions. pub const F16C: Caps = Caps::bit(9); // ─── Crypto Extensions ─── + /// AES New Instructions. pub const AESNI: Caps = Caps::bit(10); + /// Carry-less multiplication of 64-bit operands. pub const PCLMULQDQ: Caps = Caps::bit(11); + /// SHA-1 and SHA-256 instruction extensions. pub const SHA: Caps = Caps::bit(12); + /// SHA-512 instruction extensions. pub const SHA512: Caps = Caps::bit(13); // ─── AVX-512 Foundation ─── + /// AVX-512 Foundation instructions. pub const AVX512F: Caps = Caps::bit(14); + /// AVX-512 Vector Length extensions for 128- and 256-bit operands. pub const AVX512VL: Caps = Caps::bit(15); + /// AVX-512 byte and word instructions. pub const AVX512BW: Caps = Caps::bit(16); + /// AVX-512 doubleword and quadword instructions. pub const AVX512DQ: Caps = Caps::bit(17); + /// AVX-512 conflict-detection instructions. pub const AVX512CD: Caps = Caps::bit(18); // ─── AVX-512 Crypto/Advanced ─── + /// Vector carry-less multiplication of 64-bit operands. pub const VPCLMULQDQ: Caps = Caps::bit(19); + /// Vector AES instructions. pub const VAES: Caps = Caps::bit(20); + /// Galois Field New Instructions. pub const GFNI: Caps = Caps::bit(21); // ─── AVX-512 Extended ─── + /// AVX-512 integer fused multiply-add instructions. pub const AVX512IFMA: Caps = Caps::bit(22); + /// AVX-512 Vector Byte Manipulation Instructions. pub const AVX512VBMI: Caps = Caps::bit(23); + /// AVX-512 Vector Byte Manipulation Instructions 2. pub const AVX512VBMI2: Caps = Caps::bit(24); + /// AVX-512 Vector Neural Network Instructions. pub const AVX512VNNI: Caps = Caps::bit(25); + /// AVX-512 bit-algorithm instructions. pub const AVX512BITALG: Caps = Caps::bit(26); + /// AVX-512 population-count instructions for doubleword and quadword lanes. pub const AVX512VPOPCNTDQ: Caps = Caps::bit(27); + /// AVX-512 pairwise intersection instructions. pub const AVX512VP2INTERSECT: Caps = Caps::bit(28); + /// AVX-512 half-precision floating-point instructions. pub const AVX512FP16: Caps = Caps::bit(29); + /// AVX-512 BFloat16 instructions. pub const AVX512BF16: Caps = Caps::bit(30); // ─── Bit Manipulation ─── + /// Bit Manipulation Instruction Set 1. pub const BMI1: Caps = Caps::bit(31); + /// Bit Manipulation Instruction Set 2. pub const BMI2: Caps = Caps::bit(32); + /// Population-count instruction. pub const POPCNT: Caps = Caps::bit(33); + /// Leading-zero count instruction. pub const LZCNT: Caps = Caps::bit(34); + /// Multi-precision add-carry instruction extensions. pub const ADX: Caps = Caps::bit(35); // ─── AVX10 (unified AVX-512 replacement) ─── + /// Advanced Vector Extensions 10.1. pub const AVX10_1: Caps = Caps::bit(36); + /// Advanced Vector Extensions 10.2. pub const AVX10_2: Caps = Caps::bit(37); // ─── AMX (Advanced Matrix Extensions) ─── + /// Advanced Matrix Extensions tile configuration and movement instructions. pub const AMX_TILE: Caps = Caps::bit(38); + /// Advanced Matrix Extensions BFloat16 operations. pub const AMX_BF16: Caps = Caps::bit(39); + /// Advanced Matrix Extensions 8-bit integer operations. pub const AMX_INT8: Caps = Caps::bit(40); + /// Advanced Matrix Extensions half-precision floating-point operations. pub const AMX_FP16: Caps = Caps::bit(41); + /// Advanced Matrix Extensions complex-number operations. pub const AMX_COMPLEX: Caps = Caps::bit(42); // ─── Miscellaneous ─── + /// Direct-store instruction for register values. pub const MOVDIRI: Caps = Caps::bit(43); + /// Direct-store instruction for 64-byte memory operands. pub const MOVDIR64B: Caps = Caps::bit(44); + /// Instruction-execution serialization. pub const SERIALIZE: Caps = Caps::bit(45); + /// Hardware random-number generation. pub const RDRAND: Caps = Caps::bit(46); + /// Hardware entropy-seed generation. pub const RDSEED: Caps = Caps::bit(47); // ─── APX (Advanced Performance Extensions) ─── + /// Advanced Performance Extensions foundation. pub const APX: Caps = Caps::bit(48); // ─── Vendor Identification ─── @@ -465,73 +528,120 @@ pub mod aarch64 { use super::Caps; // ─── Basic SIMD ─── - pub const NEON: Caps = Caps::bit(64); // Baseline on AArch64 + /// Advanced SIMD (Neon), which is part of the AArch64 baseline. + pub const NEON: Caps = Caps::bit(64); // ─── Crypto Extensions ─── + /// Arm AES instructions. pub const AES: Caps = Caps::bit(65); - pub const PMULL: Caps = Caps::bit(66); // Often bundled with AES + /// Polynomial multiply long instructions. + pub const PMULL: Caps = Caps::bit(66); + /// Arm SHA-1 and SHA-256 instructions. pub const SHA2: Caps = Caps::bit(67); - pub const SHA3: Caps = Caps::bit(68); // Includes EOR3 + /// Arm SHA-3 instructions, including EOR3. + pub const SHA3: Caps = Caps::bit(68); + /// Arm SHA-512 instructions. pub const SHA512: Caps = Caps::bit(69); + /// Arm SM3 instructions. pub const SM3: Caps = Caps::bit(70); + /// Arm SM4 instructions. pub const SM4: Caps = Caps::bit(71); // ─── CRC Extension ─── + /// Arm CRC32 and CRC32C instructions. pub const CRC: Caps = Caps::bit(72); // ─── Additional SIMD ─── + /// Advanced SIMD integer dot-product instructions. pub const DOTPROD: Caps = Caps::bit(73); + /// Advanced SIMD 8-bit integer matrix-multiply instructions. pub const I8MM: Caps = Caps::bit(74); + /// BFloat16 arithmetic instructions. pub const BF16: Caps = Caps::bit(75); + /// Half-precision floating-point arithmetic instructions. pub const FP16: Caps = Caps::bit(76); + /// Floating-point round-to-integer instructions provided by FEAT_FRINTTS. pub const FRINTTS: Caps = Caps::bit(77); // ─── SVE Family ─── + /// Scalable Vector Extension. pub const SVE: Caps = Caps::bit(78); + /// Scalable Vector Extension version 2. pub const SVE2: Caps = Caps::bit(79); + /// SVE2 AES instructions. pub const SVE2_AES: Caps = Caps::bit(80); + /// SVE2 SHA-3 instructions. pub const SVE2_SHA3: Caps = Caps::bit(81); + /// SVE2 SM4 instructions. pub const SVE2_SM4: Caps = Caps::bit(82); + /// SVE2 bit-permutation instructions. pub const SVE2_BITPERM: Caps = Caps::bit(83); - pub const SVE2_PMULL: Caps = Caps::bit(97); // SVE2 PMULL - pub const SVE2_I8MM: Caps = Caps::bit(98); // SVE2 Int8 matmul - pub const SVE2_F32MM: Caps = Caps::bit(99); // SVE2 FP32 matmul - pub const SVE2_F64MM: Caps = Caps::bit(100); // SVE2 FP64 matmul - pub const SVE2_BF16: Caps = Caps::bit(101); // SVE2 BFloat16 - pub const SVE2_EBF16: Caps = Caps::bit(102); // SVE2 Extended BFloat16 + /// SVE2 polynomial multiply long instructions. + pub const SVE2_PMULL: Caps = Caps::bit(97); + /// SVE2 8-bit integer matrix-multiply instructions. + pub const SVE2_I8MM: Caps = Caps::bit(98); + /// SVE single-precision floating-point matrix-multiply instructions. + pub const SVE2_F32MM: Caps = Caps::bit(99); + /// SVE double-precision floating-point matrix-multiply instructions. + pub const SVE2_F64MM: Caps = Caps::bit(100); + /// SVE BFloat16 instructions. + pub const SVE2_BF16: Caps = Caps::bit(101); + /// SVE extended BFloat16 instructions. + pub const SVE2_EBF16: Caps = Caps::bit(102); // ─── Atomics (ARMv8.1+) ─── - pub const LSE: Caps = Caps::bit(84); // Large System Extensions - pub const LSE2: Caps = Caps::bit(85); // ARMv8.4 atomics + /// Large System Extensions atomic instructions. + pub const LSE: Caps = Caps::bit(84); + /// Large System Extensions version 2 atomic instructions. + pub const LSE2: Caps = Caps::bit(85); // ─── Memory Operations ─── - pub const MOPS: Caps = Caps::bit(86); // FEAT_MOPS memcpy acceleration + /// Memory Copy and Memory Set instructions. + pub const MOPS: Caps = Caps::bit(86); // ─── Scalable Matrix Extension ─── + /// Scalable Matrix Extension. pub const SME: Caps = Caps::bit(87); + /// Scalable Matrix Extension version 2. pub const SME2: Caps = Caps::bit(88); // ─── SME2p1 and extended SME features ─── - pub const SME2P1: Caps = Caps::bit(89); // SME version 2.1 - pub const SME_I16I64: Caps = Caps::bit(90); // SME Int16xInt64 - pub const SME_F64F64: Caps = Caps::bit(91); // SME Float64xFloat64 - pub const SME_B16B16: Caps = Caps::bit(92); // SME BFloat16xBFloat16 (Apple M5) - pub const SME_F16F16: Caps = Caps::bit(93); // SME Float16xFloat16 (Apple M5) - pub const SME_I8I32: Caps = Caps::bit(103); // SME Int8xInt32 - pub const SME_F16F32: Caps = Caps::bit(104); // SME Float16xFloat32 - pub const SME_B16F32: Caps = Caps::bit(105); // SME BFloat16xFloat32 - pub const SME_F32F32: Caps = Caps::bit(106); // SME Float32xFloat32 - pub const SME_FA64: Caps = Caps::bit(107); // SME Full A64 - pub const SME_I16I32: Caps = Caps::bit(108); // SME Int16xInt32 - pub const SME_BI32I32: Caps = Caps::bit(109); // SME BrainInt32xInt32 - pub const EBF16: Caps = Caps::bit(110); // Extended BFloat16 (NEON) + /// Scalable Matrix Extension version 2.1. + pub const SME2P1: Caps = Caps::bit(89); + /// SME 16-bit integer outer products accumulated into 64-bit elements. + pub const SME_I16I64: Caps = Caps::bit(90); + /// SME double-precision floating-point outer products. + pub const SME_F64F64: Caps = Caps::bit(91); + /// SME BFloat16 outer products accumulated into BFloat16 elements. + pub const SME_B16B16: Caps = Caps::bit(92); + /// SME half-precision outer products accumulated into half-precision elements. + pub const SME_F16F16: Caps = Caps::bit(93); + /// SME 8-bit integer outer products accumulated into 32-bit elements. + pub const SME_I8I32: Caps = Caps::bit(103); + /// SME half-precision outer products accumulated into single-precision elements. + pub const SME_F16F32: Caps = Caps::bit(104); + /// SME BFloat16 outer products accumulated into single-precision elements. + pub const SME_B16F32: Caps = Caps::bit(105); + /// SME single-precision floating-point outer products. + pub const SME_F32F32: Caps = Caps::bit(106); + /// Full A64 instruction availability in SME streaming mode. + pub const SME_FA64: Caps = Caps::bit(107); + /// SME 16-bit integer outer products accumulated into 32-bit elements. + pub const SME_I16I32: Caps = Caps::bit(108); + /// SME matrix operations selected by the architectural BI32I32 field. + pub const SME_BI32I32: Caps = Caps::bit(109); + /// Advanced SIMD extended BFloat16 instructions. + pub const EBF16: Caps = Caps::bit(110); // ─── SVE2.1 features ─── - pub const SVE2P1: Caps = Caps::bit(94); // SVE version 2.1 - pub const SVE_B16B16: Caps = Caps::bit(95); // SVE BFloat16xBFloat16 + /// Scalable Vector Extension version 2.1. + pub const SVE2P1: Caps = Caps::bit(94); + /// SVE BFloat16 multiply-accumulate instructions with BFloat16 results. + pub const SVE_B16B16: Caps = Caps::bit(95); // ─── Hardware RNG ─── - pub const RNG: Caps = Caps::bit(96); // RNDR/RNDRRS + /// Random-number instructions RNDR and RNDRRS. + pub const RNG: Caps = Caps::bit(96); // ─── Combined Capability Masks ─── @@ -563,45 +673,78 @@ pub mod riscv { use super::Caps; // ─── Vector Extension ─── + /// Standard vector extension. pub const V: Caps = Caps::bit(128); // ─── Bit Manipulation ─── + /// Basic bit-manipulation instructions. pub const ZBB: Caps = Caps::bit(129); + /// Single-bit manipulation instructions. pub const ZBS: Caps = Caps::bit(130); + /// Address-generation instructions. pub const ZBA: Caps = Caps::bit(131); - pub const ZBC: Caps = Caps::bit(132); // Carryless multiply + /// Carry-less multiplication instructions. + pub const ZBC: Caps = Caps::bit(132); // ─── Scalar Crypto ─── + /// Bit-manipulation instructions for cryptography. pub const ZBKB: Caps = Caps::bit(133); + /// Carry-less multiplication instructions for cryptography. pub const ZBKC: Caps = Caps::bit(134); + /// Crossbar-permutation instructions for cryptography. pub const ZBKX: Caps = Caps::bit(135); - pub const ZKND: Caps = Caps::bit(136); // AES decrypt - pub const ZKNE: Caps = Caps::bit(137); // AES encrypt - pub const ZKNH: Caps = Caps::bit(138); // SHA2 - pub const ZKSED: Caps = Caps::bit(139); // SM4 - pub const ZKSH: Caps = Caps::bit(140); // SM3 - pub const ZKN: Caps = Caps::bit(150); // NIST crypto bundle - pub const ZKS: Caps = Caps::bit(151); // ShangMi crypto bundle - pub const ZK: Caps = Caps::bit(152); // Standard scalar crypto bundle - pub const ZKT: Caps = Caps::bit(153); // Data-independent execution latency + /// NIST-suite AES decryption and key-schedule instructions. + pub const ZKND: Caps = Caps::bit(136); + /// NIST-suite AES encryption and key-schedule instructions. + pub const ZKNE: Caps = Caps::bit(137); + /// NIST-suite SHA-2 hash instructions. + pub const ZKNH: Caps = Caps::bit(138); + /// ShangMi-suite SM4 block-cipher instructions. + pub const ZKSED: Caps = Caps::bit(139); + /// ShangMi-suite SM3 hash instructions. + pub const ZKSH: Caps = Caps::bit(140); + /// Scalar NIST cryptography suite. + pub const ZKN: Caps = Caps::bit(150); + /// Scalar ShangMi cryptography suite. + pub const ZKS: Caps = Caps::bit(151); + /// Standard scalar cryptography suite. + pub const ZK: Caps = Caps::bit(152); + /// Scalar data-independent execution-latency guarantee. + pub const ZKT: Caps = Caps::bit(153); // ─── Vector Crypto ─── + /// Vector basic bit-manipulation instructions. pub const ZVBB: Caps = Caps::bit(141); + /// Vector carry-less multiplication instructions. pub const ZVBC: Caps = Caps::bit(142); + /// Vector bit-manipulation instructions for cryptography. pub const ZVKB: Caps = Caps::bit(143); + /// Vector GCM and GMAC instructions. pub const ZVKG: Caps = Caps::bit(144); + /// Vector AES block-cipher instructions. pub const ZVKNED: Caps = Caps::bit(145); + /// Vector SHA-256 hash instructions. pub const ZVKNHA: Caps = Caps::bit(146); + /// Vector SHA-256 and SHA-512 hash instructions. pub const ZVKNHB: Caps = Caps::bit(147); + /// Vector SM4 block-cipher instructions. pub const ZVKSED: Caps = Caps::bit(148); + /// Vector SM3 hash instructions. pub const ZVKSH: Caps = Caps::bit(149); - pub const ZVKT: Caps = Caps::bit(154); // Vector data-independent execution latency - pub const ZVKN: Caps = Caps::bit(155); // Vector NIST crypto bundle - pub const ZVKNC: Caps = Caps::bit(156); // Vector NIST crypto bundle + CLMUL - pub const ZVKNG: Caps = Caps::bit(157); // Vector NIST crypto bundle + GCM - pub const ZVKS: Caps = Caps::bit(158); // Vector ShangMi crypto bundle - pub const ZVKSC: Caps = Caps::bit(159); // Vector ShangMi crypto bundle + CLMUL - pub const ZVKSG: Caps = Caps::bit(160); // Vector ShangMi crypto bundle + GCM + /// Vector data-independent execution-latency guarantee. + pub const ZVKT: Caps = Caps::bit(154); + /// Vector NIST cryptography suite. + pub const ZVKN: Caps = Caps::bit(155); + /// Vector NIST cryptography suite with carry-less multiplication. + pub const ZVKNC: Caps = Caps::bit(156); + /// Vector NIST cryptography suite with GCM and GMAC instructions. + pub const ZVKNG: Caps = Caps::bit(157); + /// Vector ShangMi cryptography suite. + pub const ZVKS: Caps = Caps::bit(158); + /// Vector ShangMi cryptography suite with carry-less multiplication. + pub const ZVKSC: Caps = Caps::bit(159); + /// Vector ShangMi cryptography suite with GCM and GMAC instructions. + pub const ZVKSG: Caps = Caps::bit(160); } // WebAssembly Features (bits 192-207) @@ -610,7 +753,9 @@ pub mod riscv { pub mod wasm { use super::Caps; + /// WebAssembly 128-bit SIMD instructions. pub const SIMD128: Caps = Caps::bit(192); + /// WebAssembly relaxed SIMD instructions. pub const RELAXED_SIMD: Caps = Caps::bit(193); } diff --git a/src/platform/detect.rs b/src/platform/detect.rs index 278c3dec..232e84ed 100644 --- a/src/platform/detect.rs +++ b/src/platform/detect.rs @@ -1,23 +1,3 @@ -//! CPU feature detection. -//! -//! This module provides runtime CPU capability detection with caching. -//! It handles compile-time and runtime detection, caching, and user overrides. -//! -//! # Detection Tiers -//! -//! 1. **Compile-time**: `cfg!(target_feature)` - zero cost, dead code elimination -//! 2. **Runtime (std)**: `is_x86_feature_detected!` + `OnceLock` caching -//! 3. **Runtime (no_std)**: Atomic-based caching for embedded targets -//! -//! # Override Support -//! -//! ``` -//! use rscrypto::platform::{Detected, expert}; -//! expert::try_set_override(Some(Detected::portable()))?; -//! expert::try_set_override(None)?; -//! # Ok::<(), rscrypto::platform::expert::OverrideError>(()) -//! ``` - use crate::platform::caps::{Arch, Caps}; /// Errors when configuring runtime detection overrides. @@ -120,15 +100,9 @@ fn validate_override(value: Option) -> Result, Overri /// /// Results are cached after first call. /// -/// # Examples -/// -/// ``` -/// let det = rscrypto::platform::get(); -/// assert_eq!(det.arch, rscrypto::platform::Arch::current()); -/// ``` #[inline] #[must_use] -pub fn get() -> Detected { +pub(super) fn get() -> Detected { // Miri cannot interpret SIMD intrinsics #[cfg(miri)] { @@ -137,8 +111,7 @@ pub fn get() -> Detected { #[cfg(not(miri))] { - #[allow(unused_mut)] - let mut det = { + let det = { #[cfg(feature = "std")] { *STD_CACHE.get_or_init(detect_with_override) @@ -172,7 +145,7 @@ pub fn get() -> Detected { /// `Cargo.toml` for deployment context (FIPS / DO-178C / ISO 26262). #[inline] #[must_use] -pub fn caps() -> Caps { +pub(super) fn caps() -> Caps { #[cfg(feature = "portable-only")] { Caps::NONE @@ -186,7 +159,7 @@ pub fn caps() -> Caps { /// Get the detected architecture. #[inline] #[must_use] -pub fn arch() -> Arch { +pub(super) fn arch() -> Arch { get().arch } diff --git a/src/platform/detect/arch/aarch64.rs b/src/platform/detect/arch/aarch64.rs index c3011446..7bd534bb 100644 --- a/src/platform/detect/arch/aarch64.rs +++ b/src/platform/detect/arch/aarch64.rs @@ -96,7 +96,8 @@ fn hwcap_batch_aarch64() -> Caps { // Parse as array of (u64, u64) pairs let entries = buf.get(..n)?; - for chunk in entries.chunks_exact(16) { + let (entries, _) = entries.as_chunks::<16>(); + for chunk in entries { let a_type = u64::from_ne_bytes(chunk.get(0..8)?.try_into().ok()?); let a_val = u64::from_ne_bytes(chunk.get(8..16)?.try_into().ok()?); @@ -380,6 +381,7 @@ fn runtime_aarch64() -> Caps { #[cfg(all( target_arch = "aarch64", feature = "std", + any(test, feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64"), any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -403,7 +405,6 @@ enum AppleSiliconGen { feature = "std", any(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64") ))] -#[cfg_attr(miri, allow(dead_code))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Aarch64TuneFamily { #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos"))] @@ -427,6 +428,7 @@ pub(crate) enum Aarch64TuneFamily { #[cfg(all( target_arch = "aarch64", feature = "std", + any(test, feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64"), any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] fn detect_apple_silicon_gen() -> Option { @@ -459,9 +461,7 @@ fn detect_apple_silicon_gen() -> Option { // Direct extern "C" linkage to libSystem's sysctlbyname // (libSystem is always linked on Apple platforms) - // SAFETY: This extern block declares a C function from libSystem. // The function signature matches Apple's sysctlbyname(3). - #[allow(unsafe_code)] unsafe extern "C" { fn sysctlbyname( name: *const u8, @@ -478,7 +478,6 @@ fn detect_apple_silicon_gen() -> Option { // SAFETY: sysctlbyname is safe to call with valid pointers. // "hw.cpufamily" is a valid null-terminated string. // The output buffer is properly sized for u32. - #[allow(unsafe_code)] let ret = unsafe { sysctlbyname( c"hw.cpufamily".as_ptr().cast(), @@ -500,10 +499,10 @@ fn detect_apple_silicon_gen() -> Option { CPUFAMILY_ARM_DONAN | CPUFAMILY_ARM_BRAVA => Some(AppleSiliconGen::M4), CPUFAMILY_ARM_HIDRA | CPUFAMILY_ARM_SOTRA => Some(AppleSiliconGen::M5), // A-series chips - treat as M-series equivalent for tuning - CPUFAMILY_ARM_COLL => Some(AppleSiliconGen::M2), // A17 Pro ≈ M2 architecture + CPUFAMILY_ARM_COLL => Some(AppleSiliconGen::M2), // A17 Pro ≈ M2 architecture CPUFAMILY_ARM_TAHITI | CPUFAMILY_ARM_TUPAI => Some(AppleSiliconGen::M4), // A18 ≈ M4 architecture - CPUFAMILY_ARM_TILOS | CPUFAMILY_ARM_THERA => Some(AppleSiliconGen::M5), // A19 ≈ M5 architecture - _ => None, // Unknown future chip - will fall back to feature-based detection + CPUFAMILY_ARM_TILOS | CPUFAMILY_ARM_THERA => Some(AppleSiliconGen::M5), // A19 ≈ M5 architecture + _ => None, // Unknown future chip - will fall back to feature-based detection } } @@ -514,7 +513,6 @@ fn detect_apple_silicon_gen() -> Option { any(feature = "crc16", feature = "crc24", feature = "crc32", feature = "crc64") ))] #[must_use] -#[cfg_attr(miri, allow(dead_code))] pub(crate) fn detect_aarch64_tune_family() -> Option { #[cfg(all( feature = "std", @@ -552,8 +550,6 @@ fn detect_apple_sme_features() -> Caps { // Helper to read a u32 sysctl value (returns 0 on error or false, 1 on true) fn sysctl_u32(name: &[u8]) -> u32 { // Direct extern "C" linkage to libSystem's sysctlbyname - // SAFETY: This extern block declares a C function from libSystem. - #[allow(unsafe_code)] unsafe extern "C" { fn sysctlbyname( name: *const u8, @@ -570,7 +566,6 @@ fn detect_apple_sme_features() -> Caps { // SAFETY: sysctlbyname is safe to call with valid pointers. // name is a valid null-terminated C string. // The output buffer is properly sized for u32. - #[allow(unsafe_code)] let ret = unsafe { sysctlbyname( name.as_ptr(), @@ -614,171 +609,6 @@ fn detect_apple_sme_features() -> Caps { caps } -// TODO(T7): Feed Apple SVL into production SME dispatch only after an -// SVL-sensitive kernel wins representative target-native benchmarks. -/// Detect SME streaming vector length in bytes on Apple platforms. -/// -/// Returns the maximum SVL in bytes, or 0 if SME is not supported or detection failed. -/// -/// On Apple Silicon: -/// - M4: SME with 128-bit tiles (SVL = 16 bytes) -/// - M5: SME2p1 with 128-bit tiles (SVL = 16 bytes) -/// -/// Note: Apple's implementation uses fixed 128-bit SVL, unlike server ARM chips -/// which may support 128-512 bit configurable SVL. -#[cfg(all( - target_arch = "aarch64", - feature = "std", - any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") -))] -#[allow(dead_code)] -fn detect_apple_sme_tile_size() -> u16 { - fn sysctl_u32(name: &[u8]) -> u32 { - #[allow(unsafe_code)] - unsafe extern "C" { - fn sysctlbyname( - name: *const u8, - oldp: *mut core::ffi::c_void, - oldlenp: *mut usize, - newp: *const core::ffi::c_void, - newlen: usize, - ) -> i32; - } - - let mut value: u32 = 0; - let mut size = core::mem::size_of::(); - - #[allow(unsafe_code)] - // SAFETY: `sysctlbyname` expects `name` to be a valid NUL-terminated C string (caller provides this), - // `oldp`/`oldlenp` point to writable locals, and `newp` is null with `newlen = 0` (no write). - let ret = unsafe { - sysctlbyname( - name.as_ptr(), - core::ptr::addr_of_mut!(value).cast(), - core::ptr::addr_of_mut!(size), - core::ptr::null(), - 0, - ) - }; - - if ret == 0 { value } else { 0 } - } - - let svl_bytes = sysctl_u32(c"hw.optional.arm.sme_max_svl_b".to_bytes_with_nul()); - if svl_bytes > 0 { - return svl_bytes as u16; - } - - if let Some(AppleSiliconGen::M4 | AppleSiliconGen::M5) = detect_apple_silicon_gen() { - return 16; - } - - 0 -} - -// TODO(T7): Feed SVE VL into production dispatch only after a length-sensitive -// SVE kernel wins representative target-native benchmarks. -/// Detect SVE vector length in bits via prctl(PR_SVE_GET_VL). -/// -/// Uses raw syscall to avoid libc dependency. Returns 0 if SVE is not supported. -#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "std"))] -#[allow(dead_code)] -fn detect_sve_vlen() -> u16 { - const SYS_PRCTL: u64 = 167; - const PR_SVE_GET_VL: u64 = 51; - const PR_SVE_VL_LEN_MASK: u64 = 0xFFFF; - - let result: i64; - - // SAFETY: prctl(PR_SVE_GET_VL) is always safe to call. - // Returns the vector length in bytes on success, or -EINVAL if SVE unsupported. - #[allow(unsafe_code)] - unsafe { - core::arch::asm!( - "svc #0", - in("x8") SYS_PRCTL, - in("x0") PR_SVE_GET_VL, - in("x1") 0u64, - in("x2") 0u64, - in("x3") 0u64, - in("x4") 0u64, - lateout("x0") result, - options(nostack) - ); - } - - if result < 0 { - return 0; - } - - let vl_bytes = (result as u64) & PR_SVE_VL_LEN_MASK; - let vl_bits = vl_bytes.strict_mul(8); - if vl_bits > u16::MAX as u64 { - u16::MAX - } else { - vl_bits as u16 - } -} - -/// Fallback SVE vector length detection for non-Linux platforms. -#[cfg(all(target_arch = "aarch64", not(all(target_os = "linux", feature = "std"))))] -#[allow(dead_code)] -fn detect_sve_vlen() -> u16 { - 0 -} - -// TODO(T7): Feed SME VL into production dispatch only after a length-sensitive -// SME kernel wins representative target-native benchmarks. -/// Detect SME streaming vector length in bits via prctl(PR_SME_GET_VL). -/// -/// Uses raw syscall to avoid libc dependency. Returns 0 if SME is not supported. -/// The SME vector length determines the tile size (SVL × SVL bits). -#[cfg(all(target_arch = "aarch64", target_os = "linux", feature = "std"))] -#[allow(dead_code)] -fn detect_sme_vlen() -> u16 { - const SYS_PRCTL: u64 = 167; - const PR_SME_GET_VL: u64 = 63; - const PR_SME_VL_LEN_MASK: u64 = 0xFFFF; - - let result: i64; - - // SAFETY: prctl(PR_SME_GET_VL) is always safe to call. - // Returns the streaming vector length in bytes on success, or -EINVAL if SME unsupported. - #[allow(unsafe_code)] - unsafe { - core::arch::asm!( - "svc #0", - in("x8") SYS_PRCTL, - in("x0") PR_SME_GET_VL, - in("x1") 0u64, - in("x2") 0u64, - in("x3") 0u64, - in("x4") 0u64, - lateout("x0") result, - options(nostack) - ); - } - - if result < 0 { - return 0; - } - - let vl_bytes = (result as u64) & PR_SME_VL_LEN_MASK; - let vl_bits = vl_bytes.strict_mul(8); - if vl_bits > u16::MAX as u64 { - u16::MAX - } else { - vl_bits as u16 - } -} - -/// Fallback SME vector length detection for non-Linux platforms. -#[cfg(all(target_arch = "aarch64", not(all(target_os = "linux", feature = "std"))))] -#[allow(dead_code)] -fn detect_sme_vlen() -> u16 { - 0 -} - // MIDR_EL1 Detection (Linux aarch64) #[cfg(all( @@ -859,7 +689,10 @@ fn parse_u32_auto_radix(value: &str) -> Option { return u32::from_str_radix(hex, 16).ok(); } - value.parse::().ok().or_else(|| u32::from_str_radix(value, 16).ok()) + value + .parse::() + .ok() + .or_else(|| u32::from_str_radix(value, 16).ok()) } #[cfg(all( diff --git a/src/platform/detect/arch/power.rs b/src/platform/detect/arch/power.rs index 4e8c685f..3cf05471 100644 --- a/src/platform/detect/arch/power.rs +++ b/src/platform/detect/arch/power.rs @@ -40,9 +40,10 @@ fn runtime_power() -> Caps { let mut hwcap = 0u64; let mut hwcap2 = 0u64; - for chunk in buf.get(..n)?.chunks_exact(16) { - let a_type = u64::from_ne_bytes(chunk.get(0..8)?.try_into().ok()?); - let a_val = u64::from_ne_bytes(chunk.get(8..16)?.try_into().ok()?); + for chunk in buf.get(..n)?.as_chunks::<16>().0 { + let (types, values) = chunk.as_chunks::<8>().0.split_at(1); + let a_type = u64::from_ne_bytes(types[0]); + let a_val = u64::from_ne_bytes(values[0]); if a_type == AT_HWCAP { hwcap = a_val; diff --git a/src/platform/detect/arch/riscv.rs b/src/platform/detect/arch/riscv.rs index e5f9541a..7b421512 100644 --- a/src/platform/detect/arch/riscv.rs +++ b/src/platform/detect/arch/riscv.rs @@ -72,6 +72,12 @@ fn runtime_riscv() -> Caps { const RISCV_HWPROBE_EXT_ZVKT: u64 = 1 << 26; #[inline] + /// Invoke the Linux RISC-V hardware-probe syscall. + /// + /// # Safety + /// + /// `pairs` must be valid for writes to `pair_count` consecutive `RiscvHwprobe` + /// values and must remain valid until the syscall returns. unsafe fn syscall_riscv_hwprobe(pairs: *mut RiscvHwprobe, pair_count: usize) -> isize { let ret: usize; // SAFETY: We follow the Linux RISC-V syscall ABI directly: @@ -89,7 +95,7 @@ fn runtime_riscv() -> Caps { options(nostack, preserves_flags), ); } - ret as isize + isize::from_ne_bytes(ret.to_ne_bytes()) } let mut probes = [ diff --git a/src/platform/detect/arch/s390x.rs b/src/platform/detect/arch/s390x.rs index e575385a..85cf8dd1 100644 --- a/src/platform/detect/arch/s390x.rs +++ b/src/platform/detect/arch/s390x.rs @@ -47,9 +47,10 @@ fn runtime_s390x_linux() -> Caps { let mut buf = [0u8; 4096]; let n = file.read(&mut buf).ok()?; - for chunk in buf.get(..n)?.chunks_exact(16) { - let a_type = u64::from_ne_bytes(chunk.get(0..8)?.try_into().ok()?); - let a_val = u64::from_ne_bytes(chunk.get(8..16)?.try_into().ok()?); + for chunk in buf.get(..n)?.as_chunks::<16>().0 { + let (types, values) = chunk.as_chunks::<8>().0.split_at(1); + let a_type = u64::from_ne_bytes(types[0]); + let a_val = u64::from_ne_bytes(values[0]); if a_type == AT_HWCAP { return Some(a_val); } @@ -64,12 +65,12 @@ fn runtime_s390x_linux() -> Caps { let stfle = stfle_facilities(); #[inline(always)] - fn has_facility(words: &[u64; 4], bit: usize) -> bool { - // SAFETY: Facility bits are 0-255; words[bit/64] accesses indices 0-3 in a 4-element array. - // Using get() satisfies clippy::indexing_slicing while maintaining performance. + fn has_facility(words: &[u64; 4], bit: u8) -> bool { + let word_index = usize::from(bit >> 6); + let shift = 63u32.strict_sub(u32::from(bit & 63)); words - .get(bit / 64) - .map(|w| w & (1u64 << (63 - (bit % 64))) != 0) + .get(word_index) + .map(|word| word & 1u64.strict_shl(shift) != 0) .unwrap_or(false) } @@ -152,7 +153,7 @@ fn stfle_facilities() -> [u64; 4] { core::arch::asm!( "stfle 0({ptr})", ptr = in(reg) facilities.as_mut_ptr(), - inout("r0") facilities.len() as u64 - 1 => _, + inout("r0") 3u64 => _, options(nostack) ); } diff --git a/src/platform/detect/arch/x86.rs b/src/platform/detect/arch/x86.rs index 17835c4e..10b49387 100644 --- a/src/platform/detect/arch/x86.rs +++ b/src/platform/detect/arch/x86.rs @@ -20,8 +20,16 @@ fn detect_x86_64() -> Detected { #[cfg(feature = "std")] let mut caps = caps_static.union(runtime_caps); - #[cfg(not(feature = "std"))] + #[cfg(all( + not(feature = "std"), + any(target_os = "linux", target_os = "android") + ))] let mut caps = caps_static; + #[cfg(all( + not(feature = "std"), + not(any(target_os = "linux", target_os = "android")) + ))] + let caps = caps_static; #[cfg(feature = "std")] { @@ -36,7 +44,7 @@ fn detect_x86_64() -> Detected { caps = gate_x86_amx_permission(caps, false); } - // Hybrid Intel AVX-512 Safety: Clear AVX-512 caps on hybrid CPUs + // Clear AVX-512 capabilities on hybrid Intel CPUs. // On hybrid Intel CPUs (Alder Lake, Raptor Lake, etc.), the P-cores have // AVX-512 but E-cores don't. If a thread migrates to an E-core while // executing AVX-512 code, it will SIGILL. The only safe approach is to @@ -74,14 +82,20 @@ fn detect_x86_64() -> Detected { } } -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "std", target_os = "linux", target_os = "android") +))] const X86_ALL_AMX: Caps = crate::platform::caps::x86::AMX_TILE .union(crate::platform::caps::x86::AMX_BF16) .union(crate::platform::caps::x86::AMX_INT8) .union(crate::platform::caps::x86::AMX_FP16) .union(crate::platform::caps::x86::AMX_COMPLEX); -#[cfg(target_arch = "x86_64")] +#[cfg(all( + target_arch = "x86_64", + any(feature = "std", target_os = "linux", target_os = "android") +))] #[inline] const fn gate_x86_amx_permission(caps: Caps, permitted: bool) -> Caps { if permitted { @@ -115,32 +129,41 @@ fn detect_x86() -> Detected { #[cfg(all(target_arch = "x86_64", feature = "std"))] #[inline] -// MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. -#[allow(unsafe_code, unused_unsafe)] -fn cpuid_leaf(leaf: u32) -> core::arch::x86_64::CpuidResult { - // SAFETY: CPUID leaf read is safe here because: - // 1. This function is compiled only for x86_64 targets. - // 2. CPUID is a non-privileged CPU-identification instruction. - // 3. The intrinsic returns register values and does not access Rust memory. - unsafe { core::arch::x86_64::__cpuid(leaf) } +fn cpuid_leaf(leaf: u32) -> CpuidRegisters { + cpuid_leaf_count(leaf, 0) } #[cfg(all(target_arch = "x86_64", feature = "std"))] #[inline] -// MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. -#[allow(unsafe_code, unused_unsafe)] -fn cpuid_leaf_count(leaf: u32, subleaf: u32) -> core::arch::x86_64::CpuidResult { - // SAFETY: CPUID leaf/subleaf read is safe here because: - // 1. This function is compiled only for x86_64 targets. - // 2. CPUID is a non-privileged CPU-identification instruction. - // 3. The intrinsic returns register values and does not access Rust memory. - unsafe { core::arch::x86_64::__cpuid_count(leaf, subleaf) } +fn cpuid_leaf_count(leaf: u32, subleaf: u32) -> CpuidRegisters { + let mut eax = leaf; + let mut ecx = subleaf; + let ebx: u32; + let edx: u32; + + // SAFETY: CPUID is a non-privileged identification instruction on x86_64. + // The sequence preserves RBX, reports every modified register, restores RSP, + // and neither reads nor writes Rust memory. + unsafe { + core::arch::asm!( + "push rbx", + "cpuid", + "mov {ebx:e}, ebx", + "pop rbx", + ebx = lateout(reg) ebx, + inout("eax") eax, + inout("ecx") ecx, + lateout("edx") edx, + options(preserves_flags), + ); + } + + CpuidRegisters { eax, ebx, ecx, edx } } #[cfg(all(target_arch = "x86", feature = "std"))] #[inline] // MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. -#[allow(unsafe_code, unused_unsafe)] fn cpuid_leaf(leaf: u32) -> core::arch::x86::CpuidResult { // SAFETY: CPUID leaf read is safe here because: // 1. This function is compiled only for x86 targets. @@ -174,18 +197,6 @@ struct CpuidRegisters { edx: u32, } -#[cfg(all(target_arch = "x86_64", feature = "std"))] -impl From for CpuidRegisters { - fn from(result: core::arch::x86_64::CpuidResult) -> Self { - Self { - eax: result.eax, - ebx: result.ebx, - ecx: result.ecx, - edx: result.edx, - } - } -} - #[cfg(all(target_arch = "x86_64", feature = "std"))] #[derive(Clone, Copy, Default)] struct CpuidSnapshot { @@ -214,37 +225,35 @@ struct CpuidSnapshot { /// OSXSAVE and XGETBV(XCR0) to ensure the OS will save/restore extended registers. /// Without this check, using AVX/AVX-512 instructions could cause SIGILL. /// -/// # Safety -/// Uses XGETBV, which requires `unsafe` and is only called when OSXSAVE is set. +/// XGETBV is read only after CPUID reports OSXSAVE support. #[cfg(all(target_arch = "x86_64", feature = "std"))] -#[allow(unsafe_code)] fn cpuid_batch_x86_64() -> CpuidBatch { use core::arch::x86_64::_xgetbv; - let leaf0 = CpuidRegisters::from(cpuid_leaf(0)); + let leaf0 = cpuid_leaf(0); let leaf1 = if leaf0.eax >= 1 { - CpuidRegisters::from(cpuid_leaf(1)) + cpuid_leaf(1) } else { CpuidRegisters::default() }; let leaf7_0 = if leaf0.eax >= 7 { - CpuidRegisters::from(cpuid_leaf_count(7, 0)) + cpuid_leaf_count(7, 0) } else { CpuidRegisters::default() }; let leaf7_1 = if leaf0.eax >= 7 && leaf7_0.eax >= 1 { - CpuidRegisters::from(cpuid_leaf_count(7, 1)) + cpuid_leaf_count(7, 1) } else { CpuidRegisters::default() }; let leaf24_0 = if leaf0.eax >= 0x24 && leaf7_1.edx & (1 << 19) != 0 { - CpuidRegisters::from(cpuid_leaf_count(0x24, 0)) + cpuid_leaf_count(0x24, 0) } else { CpuidRegisters::default() }; - let extended_leaf0 = CpuidRegisters::from(cpuid_leaf(0x8000_0000)); + let extended_leaf0 = cpuid_leaf(0x8000_0000); let extended_leaf1 = if extended_leaf0.eax >= 0x8000_0001 { - CpuidRegisters::from(cpuid_leaf(0x8000_0001)) + cpuid_leaf(0x8000_0001) } else { CpuidRegisters::default() }; @@ -274,7 +283,6 @@ fn cpuid_batch_x86_64() -> CpuidBatch { feature = "std", any(target_os = "linux", target_os = "android") ))] -#[allow(unsafe_code)] fn amx_xstate_permission_x86_64() -> bool { const SYS_ARCH_PRCTL: isize = 158; const ARCH_GET_XCOMP_PERM: usize = 0x1022; @@ -294,7 +302,7 @@ fn amx_xstate_permission_x86_64() -> bool { "syscall", inlateout("rax") result, in("rdi") ARCH_GET_XCOMP_PERM, - in("rsi") &mut permissions, + in("rsi") &raw mut permissions, lateout("rcx") _, lateout("r11") _, options(nostack), @@ -358,13 +366,17 @@ fn decode_cpuid_x86_64(snapshot: CpuidSnapshot) -> CpuidBatch { // Extract extended family (bits 27:20) + base family (bits 11:8) let base_family = (cpuid1.eax >> 8) & 0xF; let ext_family = (cpuid1.eax >> 20) & 0xFF; - let family = base_family + ext_family; + let family = if base_family == 0xF { + base_family.strict_add(ext_family) + } else { + base_family + }; // Extract model (bits 7:4 + extended model bits 19:16 for family 6/15) let base_model = (cpuid1.eax >> 4) & 0xF; let ext_model = (cpuid1.eax >> 16) & 0xF; let model = if base_family == 6 || base_family == 15 { - base_model + (ext_model << 4) + base_model | ext_model.strict_shl(4) } else { base_model }; @@ -596,7 +608,6 @@ fn decode_cpuid_x86_64(snapshot: CpuidSnapshot) -> CpuidBatch { /// # Safety /// Uses CPUID instruction which requires unsafe, but is always safe to call on x86. #[cfg(all(target_arch = "x86", feature = "std"))] -#[allow(unsafe_code)] fn runtime_x86_32() -> Caps { use crate::platform::caps::x86; diff --git a/src/platform/detect/cache_override.rs b/src/platform/detect/cache_override.rs index ec083fc1..4f62bb30 100644 --- a/src/platform/detect/cache_override.rs +++ b/src/platform/detect/cache_override.rs @@ -140,7 +140,7 @@ mod atomic_cache { } #[cfg(all(not(feature = "std"), target_has_atomic = "64"))] - pub fn get_or_init(f: fn() -> Detected) -> Detected { + pub(super) fn get_or_init(f: fn() -> Detected) -> Detected { if STATE.load(Ordering::Acquire) == STATE_READY { return load_cached(); } @@ -176,7 +176,7 @@ mod atomic_cache { } } - pub fn try_set_override(value: Option, already_initialized: bool) -> Result<(), OverrideError> { + pub(super) fn try_set_override(value: Option, already_initialized: bool) -> Result<(), OverrideError> { let _guard = lock_override(); if OVERRIDE_SEALED.load(Ordering::Relaxed) { @@ -204,12 +204,12 @@ mod atomic_cache { Ok(()) } - pub fn has_override() -> bool { + pub(super) fn has_override() -> bool { OVERRIDE_SET.load(Ordering::Acquire) } #[cfg(not(miri))] - pub fn seal_and_get_override() -> Option { + pub(super) fn seal_and_get_override() -> Option { let _guard = lock_override(); OVERRIDE_SEALED.store(true, Ordering::Relaxed); diff --git a/src/platform/detect/compile_time.rs b/src/platform/detect/compile_time.rs index ce430373..cb70f02b 100644 --- a/src/platform/detect/compile_time.rs +++ b/src/platform/detect/compile_time.rs @@ -5,73 +5,62 @@ /// Detects features enabled via `-C target-feature=...` or `-C target-cpu=native`. /// Returns a `const` value—the compiler eliminates all runtime checks. /// -/// # When to Use -/// -/// - Building specialized binaries for known hardware -/// - Maximum performance when target features are guaranteed -/// - Embedded/bare-metal where runtime detection isn't available -/// -/// For generic binaries that run on multiple CPUs, use [`get()`] instead. -/// -/// # Examples -/// -/// ``` -/// use rscrypto::platform::caps_static; -/// -/// // Evaluates at compile time—no runtime cost -/// const CAPS: rscrypto::platform::Caps = caps_static(); -/// -/// // On x86_64, SSE2 is always present -/// #[cfg(target_arch = "x86_64")] -/// { -/// use rscrypto::platform::caps::x86; -/// assert!(CAPS.has(x86::SSE2)); -/// } -/// -/// // On aarch64, NEON is always present -/// #[cfg(target_arch = "aarch64")] -/// { -/// use rscrypto::platform::caps::aarch64; -/// assert!(CAPS.has(aarch64::NEON)); -/// } -/// ``` -/// -/// When compiled with `-C target-cpu=znver4`: -/// -/// ``` -/// # #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -/// # fn example() { -/// use rscrypto::platform::{caps::x86, caps_static}; -/// -/// const CAPS: rscrypto::platform::Caps = caps_static(); -/// // AVX-512 features are detected at compile time -/// assert!(CAPS.has(x86::AVX512F)); -/// # } -/// ``` -/// /// # Implementation /// /// Uses `cfg!()` macro inside `const fn`. The compiler evaluates `cfg!()` at /// compile time and eliminates dead branches via constant propagation. #[inline(always)] #[must_use] -pub const fn caps_static() -> Caps { +pub(super) const fn caps_static() -> Caps { // Note: imports are architecture-conditional to avoid warnings use crate::platform::caps::Caps; // Declarative macro for compile-time feature detection. // Uses cfg!() which returns a const bool, enabling use in const fn. // The compiler eliminates dead branches entirely. - #[allow(unused_macros)] // Only used on x86/x86_64/aarch64 + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + ))] macro_rules! detect { ($caps:ident; $($feature:literal => $cap:expr),+ $(,)?) => { $(if cfg!(target_feature = $feature) { $caps = $caps.union($cap); })+ }; } - #[allow(unused_mut)] + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + ))] let mut result = Caps::NONE; + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + )))] + return Caps::NONE; + // x86/x86_64 #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { @@ -305,5 +294,16 @@ pub const fn caps_static() -> Caps { ); } + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "riscv32", + target_arch = "wasm32", + target_arch = "wasm64", + target_arch = "s390x", + target_arch = "powerpc64" + ))] result } diff --git a/src/platform/detect/tests.rs b/src/platform/detect/tests.rs index 4659d670..948bfb8b 100644 --- a/src/platform/detect/tests.rs +++ b/src/platform/detect/tests.rs @@ -258,21 +258,6 @@ mod tests { } } - // SVE Vector Length Detection Tests - - #[test] - #[cfg(all(target_arch = "aarch64", target_os = "linux", not(miri)))] - fn test_sve_vlen_detection_runs() { - // Just verify detection doesn't crash - let vlen = detect_sve_vlen(); - // VL should be 0 (no SVE) or a valid power-of-2 in [128, 2048] - if vlen > 0 { - assert!(vlen >= 128, "SVE VL too small: {vlen}"); - assert!(vlen <= 2048, "SVE VL too large: {vlen}"); - assert!(vlen.is_power_of_two(), "SVE VL not power of 2: {vlen}"); - } - } - // Hybrid Intel Detection Tests #[test] @@ -442,27 +427,31 @@ mod tests { snapshot.leaf7_0.ebx |= 1 << 5; }); + enum Leaf7Register { + Ebx, + Ecx, + Edx, + } let avx512_cases = [ - (x86::AVX512DQ, "ebx", 17), - (x86::AVX512IFMA, "ebx", 21), - (x86::AVX512CD, "ebx", 28), - (x86::AVX512BW, "ebx", 30), - (x86::AVX512VL, "ebx", 31), - (x86::AVX512VBMI, "ecx", 1), - (x86::AVX512VBMI2, "ecx", 6), - (x86::AVX512VNNI, "ecx", 11), - (x86::AVX512BITALG, "ecx", 12), - (x86::AVX512VPOPCNTDQ, "ecx", 14), - (x86::AVX512VP2INTERSECT, "edx", 8), + (x86::AVX512DQ, Leaf7Register::Ebx, 17), + (x86::AVX512IFMA, Leaf7Register::Ebx, 21), + (x86::AVX512CD, Leaf7Register::Ebx, 28), + (x86::AVX512BW, Leaf7Register::Ebx, 30), + (x86::AVX512VL, Leaf7Register::Ebx, 31), + (x86::AVX512VBMI, Leaf7Register::Ecx, 1), + (x86::AVX512VBMI2, Leaf7Register::Ecx, 6), + (x86::AVX512VNNI, Leaf7Register::Ecx, 11), + (x86::AVX512BITALG, Leaf7Register::Ecx, 12), + (x86::AVX512VPOPCNTDQ, Leaf7Register::Ecx, 14), + (x86::AVX512VP2INTERSECT, Leaf7Register::Edx, 8), ]; for (feature, register, bit) in avx512_cases { assert_feature(avx512_caps() | feature, |snapshot| { enable_avx512(snapshot); match register { - "ebx" => snapshot.leaf7_0.ebx |= 1 << bit, - "ecx" => snapshot.leaf7_0.ecx |= 1 << bit, - "edx" => snapshot.leaf7_0.edx |= 1 << bit, - _ => unreachable!(), + Leaf7Register::Ebx => snapshot.leaf7_0.ebx |= 1 << bit, + Leaf7Register::Ecx => snapshot.leaf7_0.ecx |= 1 << bit, + Leaf7Register::Edx => snapshot.leaf7_0.edx |= 1 << bit, } }); } @@ -616,18 +605,9 @@ mod tests { any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] fn test_macos_extended_features() { - // Test that new feature detection works on macOS use crate::platform::caps::aarch64; let det = get(); - // Verify extended features are detected on capable hardware - // On M1+, we should detect these features: - std::eprintln!("Detected features: {}", det.caps.count()); - std::eprintln!(" I8MM: {}", det.caps.has(aarch64::I8MM)); - std::eprintln!(" BF16: {}", det.caps.has(aarch64::BF16)); - std::eprintln!(" FRINTTS: {}", det.caps.has(aarch64::FRINTTS)); - std::eprintln!(" LSE2: {}", det.caps.has(aarch64::LSE2)); - // FRINTTS is detectable via std::arch on macOS; LSE2 is not exposed by // Apple's sysctl and therefore cannot be asserted here. assert!(det.caps.has(aarch64::FRINTTS), "FRINTTS should be detected on M1+"); @@ -640,54 +620,18 @@ mod tests { not(miri), any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") ))] - fn test_detect_apple_sme_features_exists() { - // Verify the SME detection function exists and returns valid caps - let sme_caps = detect_apple_sme_features(); - // The function should always return valid Caps (may be empty on M1-M3) - // On M4+, SME should be detected - std::eprintln!("SME caps detected: {}", sme_caps.count()); - std::eprintln!(" SME: {}", sme_caps.has(crate::platform::caps::aarch64::SME)); - std::eprintln!(" SME2: {}", sme_caps.has(crate::platform::caps::aarch64::SME2)); - } - - #[test] - #[cfg(all( - target_arch = "aarch64", - feature = "std", - not(miri), - any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos") - ))] - fn test_detect_apple_silicon_gen_exists() { - // Verify chip generation detection works - if let Some(chip_gen) = detect_apple_silicon_gen() { - std::eprintln!("Detected Apple Silicon generation: {:?}", chip_gen); - // Basic sanity checks - match chip_gen { - AppleSiliconGen::M1 | AppleSiliconGen::M2 | AppleSiliconGen::M3 => { - // M1-M3 should not have SME - std::eprintln!("M1-M3 chip detected (no SME expected)"); - } - AppleSiliconGen::M4 => { - // M4 should have SME - std::eprintln!("M4 chip detected (SME expected)"); - } - AppleSiliconGen::M5 => { - // M5 should have SME2 - std::eprintln!("M5 chip detected (SME2 expected)"); - } - } - } else { - std::eprintln!("Unknown or A-series chip detected"); - } - } - - // Override Mechanism Tests + fn test_detect_apple_sme_features_are_scoped() { + use crate::platform::caps::aarch64; - #[test] - fn test_has_override_exists() { - // Verify the override API exists and returns a bool. - // Note: Due to global state from other tests, we can't assert a specific value. - let _ = has_override(); + let sme_caps = detect_apple_sme_features(); + let owned_caps = aarch64::SME + .union(aarch64::SME2) + .union(aarch64::SME2P1) + .union(aarch64::SME_I16I64) + .union(aarch64::SME_F64F64) + .union(aarch64::SME_B16B16) + .union(aarch64::SME_F16F16); + assert!(sme_caps.difference(owned_caps).is_empty()); } #[test] diff --git a/src/platform/mod.rs b/src/platform/mod.rs index bdef3c2d..1b0aa646 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -98,6 +98,14 @@ pub fn arch() -> Arch { /// /// Normal callers should use [`get()`], [`caps()`], or [`arch()`]. Overrides /// must be configured before the first cached detection. +/// +/// ``` +/// use rscrypto::platform::{Detected, expert}; +/// +/// expert::try_set_override(Some(Detected::portable()))?; +/// expert::try_set_override(None)?; +/// # Ok::<(), rscrypto::platform::expert::OverrideError>(()) +/// ``` pub mod expert { pub use super::detect::{OverrideError, detect_uncached, has_override, try_set_override}; } @@ -108,6 +116,27 @@ pub mod expert { /// or `-C target-cpu=native`. Use this for zero-overhead dispatch. /// /// This is a compile-time constant and performs no runtime detection. +/// For generic binaries that run on multiple CPUs, use [`caps()`] instead. +/// +/// # Examples +/// +/// ``` +/// use rscrypto::platform::caps_static; +/// +/// const CAPS: rscrypto::platform::Caps = caps_static(); +/// +/// #[cfg(target_arch = "x86_64")] +/// { +/// use rscrypto::platform::caps::x86; +/// assert!(CAPS.has(x86::SSE2)); +/// } +/// +/// #[cfg(target_arch = "aarch64")] +/// { +/// use rscrypto::platform::caps::aarch64; +/// assert!(CAPS.has(aarch64::NEON)); +/// } +/// ``` #[inline(always)] #[must_use] pub const fn caps_static() -> Caps { diff --git a/src/platform/target_matrix.rs b/src/platform/target_matrix.rs index 687c8901..07bbbfae 100644 --- a/src/platform/target_matrix.rs +++ b/src/platform/target_matrix.rs @@ -30,7 +30,7 @@ fn matches_target_arch(target: &str, arch: Arch) -> bool { /// an arch prefix like `x86_64-`, so false positives are impossible. #[inline] #[must_use] -pub fn manifest_has_arch(arch: Arch) -> bool { +pub(super) fn manifest_has_arch(arch: Arch) -> bool { let Some(_) = manifest_prefix_for_arch(arch) else { return true; }; @@ -41,8 +41,10 @@ pub fn manifest_has_arch(arch: Arch) -> bool { for (i, b) in TARGET_MATRIX_MANIFEST.bytes().enumerate() { if b == b'"' { if in_quote { - let value = &TARGET_MATRIX_MANIFEST[start..i]; - if matches_target_arch(value, arch) { + if TARGET_MATRIX_MANIFEST + .get(start..i) + .is_some_and(|value| matches_target_arch(value, arch)) + { return true; } } else { diff --git a/src/secret.rs b/src/secret.rs index 2d201f9c..7156e2a3 100644 --- a/src/secret.rs +++ b/src/secret.rs @@ -237,7 +237,7 @@ impl Drop for ZeroizingBytes { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_fixed_stack(input: [u8; 32]) -> u8 { +pub(crate) fn diag_zeroize_fixed_stack(input: [u8; 32]) -> u8 { let secret = SecretBytes::new(input); core::hint::black_box(secret.as_bytes()[0]) } @@ -246,7 +246,7 @@ pub fn diag_zeroize_fixed_stack(input: [u8; 32]) -> u8 { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_fixed_move(input: [u8; 32]) -> u8 { +pub(crate) fn diag_zeroize_fixed_move(input: [u8; 32]) -> u8 { let secret = SecretBytes::new(input); let mut exposed = secret.expose(); let output = core::hint::black_box(exposed[0]); @@ -258,7 +258,7 @@ pub fn diag_zeroize_fixed_move(input: [u8; 32]) -> u8 { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_early_return(input: [u8; 32], stop: bool) -> u8 { +pub(crate) fn diag_zeroize_early_return(input: [u8; 32], stop: bool) -> u8 { let secret = SecretBytes::new(input); if core::hint::black_box(stop) { return 0; @@ -270,7 +270,7 @@ pub fn diag_zeroize_early_return(input: [u8; 32], stop: bool) -> u8 { #[doc(hidden)] #[unsafe(no_mangle)] #[inline(never)] -pub fn diag_zeroize_variable_heap(input: Vec) -> usize { +pub(crate) fn diag_zeroize_variable_heap(input: Vec) -> usize { let secret = SecretVec::new(input); core::hint::black_box(secret.len()) } diff --git a/src/traits/aead.rs b/src/traits/aead.rs index b81723fa..e4968e9a 100644 --- a/src/traits/aead.rs +++ b/src/traits/aead.rs @@ -15,6 +15,7 @@ use crate::aead::{AeadBufferError, AeadNonce, OpenError, SealError}; /// Zero-sized capability proving that a call came through a crate-owned nonce /// issuer or the explicit expert extension. #[doc(hidden)] +#[derive(Debug)] pub struct SealToken(()); impl SealToken { diff --git a/src/traits/ct.rs b/src/traits/ct.rs index 71bbf665..cce1cc8c 100644 --- a/src/traits/ct.rs +++ b/src/traits/ct.rs @@ -15,9 +15,37 @@ pub struct CtDecision { mask: u8, } +impl core::fmt::Debug for CtDecision { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("CtDecision(..)") + } +} + impl CtDecision { const TRUE_MASK: u8 = u8::MAX; + #[cfg(any( + test, + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "argon2", + feature = "ascon-aead", + feature = "blake3", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "hmac", + feature = "hmac-sha3", + feature = "kmac", + feature = "ml-kem", + feature = "poly1305", + feature = "rsa", + feature = "scrypt", + feature = "x25519", + feature = "xchacha20poly1305" + ))] #[inline(always)] const fn from_difference(difference: u64) -> Self { let nonzero = ((difference | difference.wrapping_neg()) >> 63) as u8; @@ -37,14 +65,14 @@ impl CtDecision { self.mask == Self::TRUE_MASK } + #[cfg(any(feature = "ed25519", feature = "kmac", feature = "rsa"))] #[inline(always)] - #[allow(dead_code)] pub(crate) const fn into_u8(self) -> u8 { self.mask & 1 } + #[cfg(feature = "ml-kem")] #[inline(always)] - #[allow(dead_code)] pub(crate) const fn into_mask(self) -> u8 { self.mask } @@ -83,22 +111,40 @@ impl core::ops::Not for CtDecision { } } +#[cfg(any( + test, + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "argon2", + feature = "ascon-aead", + feature = "blake3", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "hmac", + feature = "hmac-sha3", + feature = "kmac", + feature = "ml-kem", + feature = "poly1305", + feature = "rsa", + feature = "scrypt", + feature = "x25519", + feature = "xchacha20poly1305" +))] #[inline(always)] fn byte_difference(left: &[u8], right: &[u8]) -> u64 { let mut difference = 0u64; - let mut left_chunks = left.chunks_exact(8); - let mut right_chunks = right.chunks_exact(8); - - for (left_chunk, right_chunk) in left_chunks.by_ref().zip(right_chunks.by_ref()) { - let (Ok(left_bytes), Ok(right_bytes)) = (<&[u8; 8]>::try_from(left_chunk), <&[u8; 8]>::try_from(right_chunk)) - else { - return u64::MAX; - }; - difference |= u64::from_ne_bytes(*left_bytes) ^ u64::from_ne_bytes(*right_bytes); + let (left_chunks, left_remainder) = left.as_chunks::<8>(); + let (right_chunks, right_remainder) = right.as_chunks::<8>(); + + for (left_chunk, right_chunk) in left_chunks.iter().zip(right_chunks) { + difference |= u64::from_ne_bytes(*left_chunk) ^ u64::from_ne_bytes(*right_chunk); } let mut remainder = 0u8; - for (left_byte, right_byte) in left_chunks.remainder().iter().zip(right_chunks.remainder()) { + for (left_byte, right_byte) in left_remainder.iter().zip(right_remainder) { remainder |= left_byte ^ right_byte; } difference | u64::from(remainder) @@ -110,8 +156,27 @@ fn byte_difference(left: &[u8], right: &[u8]) -> u64 { /// concrete cryptographic type that owns the bytes. Optimized machine-code /// evidence is tracked separately in `ct.toml`; source structure is not a /// universal constant-time guarantee. +#[cfg(any( + test, + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + all(feature = "argon2", feature = "phc-strings"), + feature = "ascon-aead", + feature = "blake3", + feature = "chacha20poly1305", + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + feature = "hmac", + feature = "hmac-sha3", + feature = "ml-kem", + feature = "poly1305", + all(feature = "scrypt", feature = "phc-strings"), + feature = "x25519", + feature = "xchacha20poly1305" +))] #[inline(always)] -#[allow(dead_code)] pub(crate) fn fixed_eq(left: &[u8; N], right: &[u8; N]) -> CtDecision { // SECURITY: Keep the accumulated word opaque before declassification. LLVM can otherwise fold // equality into target-specific vector reductions; exact binary evidence still owns the @@ -124,8 +189,15 @@ pub(crate) fn fixed_eq(left: &[u8; N], right: &[u8; N]) -> CtDec /// Length mismatch is intentionally observable. Equal-length contents are /// traversed without content-dependent exits. Keep callers individually /// classified in `ct.toml`; fixed-shape owner types must use [`fixed_eq`]. +#[cfg(any( + test, + feature = "argon2", + feature = "kmac", + feature = "ml-kem", + feature = "rsa", + feature = "scrypt" +))] #[inline] -#[allow(dead_code)] pub(crate) fn public_len_eq(left: &[u8], right: &[u8]) -> CtDecision { if left.len() != right.len() { return CtDecision::from_difference(1); @@ -175,6 +247,18 @@ pub fn zeroize(buf: &mut [u8]) { core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); } +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake2b", + feature = "blake2s", + feature = "blake3", + feature = "chacha20poly1305", + feature = "poly1305", + feature = "sha2", + feature = "sha3", + feature = "xchacha20poly1305" +))] mod word_zero_sealed { /// Marker for primitive integer types whose zero representation is /// `0` and whose `write_volatile` of zero is a sound clear. @@ -182,8 +266,7 @@ mod word_zero_sealed { /// Sealed: only the integer types listed here are accepted as scratch /// types for [`zeroize_words_no_fence`] / [`zeroize_words`]. New /// implementors must be reviewed for soundness (no padding, no Drop). - #[allow(dead_code)] - pub trait WordZero: Copy { + pub(crate) trait WordZero: Copy { const ZERO: Self; } @@ -207,6 +290,18 @@ mod word_zero_sealed { } } +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake2b", + feature = "blake2s", + feature = "blake3", + feature = "chacha20poly1305", + feature = "poly1305", + feature = "sha2", + feature = "sha3", + feature = "xchacha20poly1305" +))] pub(crate) use word_zero_sealed::WordZero; /// Volatile-zero a slice of `WordZero` integers without a compiler fence. @@ -216,8 +311,19 @@ pub(crate) use word_zero_sealed::WordZero; /// loops over `core::ptr::write_volatile` patterns. Caller is responsible /// for emitting a single `compiler_fence(SeqCst)` after all related /// zeroizations. +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake2b", + feature = "blake2s", + feature = "blake3", + feature = "chacha20poly1305", + feature = "poly1305", + feature = "sha2", + feature = "sha3", + feature = "xchacha20poly1305" +))] #[inline(always)] -#[allow(dead_code)] pub(crate) fn zeroize_words_no_fence(words: &mut [T]) { for word in words { // SAFETY: `word` is a valid, aligned, dereferenceable pointer to `T`. @@ -228,8 +334,14 @@ pub(crate) fn zeroize_words_no_fence(words: &mut [T]) { } /// Volatile-zero a slice of `WordZero` integers and emit a compiler fence. +#[cfg(any( + feature = "aes-gcm", + feature = "ascon-aead", + feature = "blake3", + feature = "sha2", + feature = "sha3" +))] #[inline(always)] -#[allow(dead_code)] pub(crate) fn zeroize_words(words: &mut [T]) { zeroize_words_no_fence(words); core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst); diff --git a/src/traits/error.rs b/src/traits/error.rs index 96045ac3..f6fb9b71 100644 --- a/src/traits/error.rs +++ b/src/traits/error.rs @@ -78,9 +78,12 @@ mod tests { #[test] fn is_clone() { + fn clone_via_trait_bound(value: &T) -> T { + value.clone() + } + let e = VerificationError::new(); - #[allow(clippy::clone_on_copy)] - let cloned = e.clone(); + let cloned = clone_via_trait_bound(&e); assert_eq!(e, cloned); } @@ -110,7 +113,7 @@ mod tests { fn verify_match() -> Result<(), VerificationError> { Ok(()) } - assert!(verify_match().is_ok()); + assert_eq!(verify_match(), Ok(())); } #[test] @@ -127,7 +130,7 @@ mod tests { fn returns_err() -> Result<(), VerificationError> { Err(VerificationError::new()) } - let err = returns_err().unwrap_err(); + let err = returns_err().expect_err("returns_err must return VerificationError"); assert_eq!(err.to_string(), "verification failed"); } diff --git a/src/traits/io.rs b/src/traits/io.rs index 70f50960..ddc3d670 100644 --- a/src/traits/io.rs +++ b/src/traits/io.rs @@ -179,6 +179,7 @@ impl core::fmt::Debu #[cfg(feature = "std")] impl ChecksumReader { + /// Wrap `inner` with a checksum initialized to its algorithm-defined default. #[inline] #[must_use] pub fn new(inner: R) -> Self { @@ -188,6 +189,7 @@ impl ChecksumReader { } } + /// Wrap `inner` with a checksum initialized from `initial`. #[inline] #[must_use] pub fn with_initial(inner: R, initial: C::Output) -> Self { @@ -197,32 +199,42 @@ impl ChecksumReader { } } + /// Return the checksum of the bytes read through this adapter so far. #[inline] #[must_use] pub fn checksum(&self) -> C::Output { self.hasher.finalize() } + /// Borrow the checksum state mutably. + /// + /// Direct updates affect the returned checksum but do not read from the inner reader. #[inline] pub fn hasher_mut(&mut self) -> &mut C { &mut self.hasher } + /// Consume the adapter and return the inner reader and current checksum. #[inline] pub fn into_parts(self) -> (R, C::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner reader, discarding the checksum state. #[inline] pub fn into_inner(self) -> R { self.inner } + /// Borrow the inner reader. #[inline] pub fn inner(&self) -> &R { &self.inner } + /// Borrow the inner reader mutably. + /// + /// Bytes read directly from this value are not included in the checksum. #[inline] pub fn inner_mut(&mut self) -> &mut R { &mut self.inner @@ -303,6 +315,7 @@ impl core::fmt::Debu #[cfg(feature = "std")] impl ChecksumWriter { + /// Wrap `inner` with a checksum initialized to its algorithm-defined default. #[inline] #[must_use] pub fn new(inner: W) -> Self { @@ -312,6 +325,7 @@ impl ChecksumWriter { } } + /// Wrap `inner` with a checksum initialized from `initial`. #[inline] #[must_use] pub fn with_initial(inner: W, initial: C::Output) -> Self { @@ -321,32 +335,42 @@ impl ChecksumWriter { } } + /// Return the checksum of the bytes written through this adapter so far. #[inline] #[must_use] pub fn checksum(&self) -> C::Output { self.hasher.finalize() } + /// Borrow the checksum state mutably. + /// + /// Direct updates affect the returned checksum but do not write to the inner writer. #[inline] pub fn hasher_mut(&mut self) -> &mut C { &mut self.hasher } + /// Consume the adapter and return the inner writer and current checksum. #[inline] pub fn into_parts(self) -> (W, C::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner writer, discarding the checksum state. #[inline] pub fn into_inner(self) -> W { self.inner } + /// Borrow the inner writer. #[inline] pub fn inner(&self) -> &W { &self.inner } + /// Borrow the inner writer mutably. + /// + /// Bytes written directly to this value are not included in the checksum. #[inline] pub fn inner_mut(&mut self) -> &mut W { &mut self.inner @@ -424,6 +448,7 @@ impl core::fmt #[cfg(feature = "std")] impl DigestReader { + /// Wrap `inner` with a new digest state. #[inline] #[must_use] pub fn new(inner: R) -> Self { @@ -433,32 +458,42 @@ impl DigestReader { } } + /// Return the digest of the bytes read through this adapter so far. #[inline] #[must_use] pub fn digest(&self) -> D::Output { self.hasher.finalize() } + /// Borrow the digest state mutably. + /// + /// Direct updates affect the returned digest but do not read from the inner reader. #[inline] pub fn hasher_mut(&mut self) -> &mut D { &mut self.hasher } + /// Consume the adapter and return the inner reader and current digest. #[inline] pub fn into_parts(self) -> (R, D::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner reader, discarding the digest state. #[inline] pub fn into_inner(self) -> R { self.inner } + /// Borrow the inner reader. #[inline] pub fn inner(&self) -> &R { &self.inner } + /// Borrow the inner reader mutably. + /// + /// Bytes read directly from this value are not included in the digest. #[inline] pub fn inner_mut(&mut self) -> &mut R { &mut self.inner @@ -538,6 +573,7 @@ impl core::fmt #[cfg(feature = "std")] impl DigestWriter { + /// Wrap `inner` with a new digest state. #[inline] #[must_use] pub fn new(inner: W) -> Self { @@ -547,32 +583,42 @@ impl DigestWriter { } } + /// Return the digest of the bytes written through this adapter so far. #[inline] #[must_use] pub fn digest(&self) -> D::Output { self.hasher.finalize() } + /// Borrow the digest state mutably. + /// + /// Direct updates affect the returned digest but do not write to the inner writer. #[inline] pub fn hasher_mut(&mut self) -> &mut D { &mut self.hasher } + /// Consume the adapter and return the inner writer and current digest. #[inline] pub fn into_parts(self) -> (W, D::Output) { (self.inner, self.hasher.finalize()) } + /// Consume the adapter and return the inner writer, discarding the digest state. #[inline] pub fn into_inner(self) -> W { self.inner } + /// Borrow the inner writer. #[inline] pub fn inner(&self) -> &W { &self.inner } + /// Borrow the inner writer mutably. + /// + /// Bytes written directly to this value are not included in the digest. #[inline] pub fn inner_mut(&mut self) -> &mut W { &mut self.inner @@ -717,7 +763,9 @@ mod tests { #[test] fn checksum_writer_hashes_only_written_prefix() { let mut writer = ChecksumWriter::<_, Sum>::new(PartialWriter::new(4)); - let written = writer.write(b"abcdef").unwrap(); + let written = writer + .write(b"abcdef") + .expect("in-memory checksum writer must accept input"); assert_eq!(written, 4); assert_eq!(writer.checksum(), checksum_sum(b"abcd")); @@ -730,7 +778,9 @@ mod tests { fn checksum_writer_vectored_hashes_only_written_prefix() { let mut writer = ChecksumWriter::<_, Sum>::new(PartialWriter::new(5)); let bufs = [IoSlice::new(b"ab"), IoSlice::new(b"cdef"), IoSlice::new(b"gh")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory checksum writer must accept vectored input"); assert_eq!(written, 5); assert_eq!(writer.checksum(), checksum_sum(b"abcde")); @@ -742,7 +792,9 @@ mod tests { #[test] fn digest_writer_hashes_only_written_prefix() { let mut writer = DigestWriter::<_, SumDigest>::new(PartialWriter::new(3)); - let written = writer.write(b"abcdef").unwrap(); + let written = writer + .write(b"abcdef") + .expect("in-memory digest writer must accept input"); assert_eq!(written, 3); assert_eq!(writer.digest(), digest_sum(b"abc")); @@ -755,7 +807,9 @@ mod tests { fn digest_writer_vectored_hashes_only_written_prefix() { let mut writer = DigestWriter::<_, SumDigest>::new(PartialWriter::new(6)); let bufs = [IoSlice::new(b"ab"), IoSlice::new(b"cdef"), IoSlice::new(b"ghij")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory digest writer must accept vectored input"); assert_eq!(written, 6); assert_eq!(writer.digest(), digest_sum(b"abcdef")); @@ -776,7 +830,9 @@ mod tests { IoSliceMut::new(&mut third), ]; - let read = reader.read_vectored(&mut bufs).unwrap(); + let read = reader + .read_vectored(&mut bufs) + .expect("in-memory checksum reader must provide vectored input"); assert_eq!(read, 5); assert_eq!(second, *b"ab"); assert_eq!(third, *b"cde"); @@ -795,7 +851,9 @@ mod tests { IoSliceMut::new(&mut third), ]; - let read = reader.read_vectored(&mut bufs).unwrap(); + let read = reader + .read_vectored(&mut bufs) + .expect("in-memory digest reader must provide vectored input"); assert_eq!(read, 5); assert_eq!(second, *b"ab"); assert_eq!(third, *b"cde"); @@ -806,7 +864,9 @@ mod tests { fn checksum_writer_vectored_skips_empty_buffers() { let mut writer = ChecksumWriter::<_, Sum>::new(PartialWriter::new(5)); let bufs = [IoSlice::new(b""), IoSlice::new(b"ab"), IoSlice::new(b"cde")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory checksum writer must accept nonempty vectored input"); assert_eq!(written, 5); assert_eq!(writer.checksum(), checksum_sum(b"abcde")); @@ -820,7 +880,9 @@ mod tests { fn digest_writer_vectored_skips_empty_buffers() { let mut writer = DigestWriter::<_, SumDigest>::new(PartialWriter::new(5)); let bufs = [IoSlice::new(b""), IoSlice::new(b"ab"), IoSlice::new(b"cde")]; - let written = writer.write_vectored(&bufs).unwrap(); + let written = writer + .write_vectored(&bufs) + .expect("in-memory digest writer must accept nonempty vectored input"); assert_eq!(written, 5); assert_eq!(writer.digest(), digest_sum(b"abcde")); diff --git a/tests/aead_foundations.rs b/tests/aead_foundations.rs index 7e583af7..e5b016ae 100644 --- a/tests/aead_foundations.rs +++ b/tests/aead_foundations.rs @@ -71,10 +71,14 @@ fn aead_encrypt_and_decrypt_helpers_round_trip() { let aead = fixture_cipher(); let mut sealed = [0u8; 12 + ChaCha20Poly1305::TAG_SIZE]; - aead.encrypt(&nonce, aad, &plaintext, &mut sealed).unwrap(); + aead + .encrypt(&nonce, aad, &plaintext, &mut sealed) + .expect("combined AEAD seal buffer must fit plaintext and tag"); let mut opened = [0u8; 12]; - aead.decrypt(&nonce, aad, &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, aad, &sealed, &mut opened) + .expect("freshly sealed combined AEAD ciphertext must authenticate"); assert_eq!(opened, plaintext); } @@ -86,15 +90,21 @@ fn aead_seal_random_issues_fresh_nonces_and_round_trips() { let aead = fixture_cipher(); let mut sealed_a = [0u8; 12 + ChaCha20Poly1305::TAG_SIZE]; - let nonce_a = aead.seal_random(aad, b"hello world!", &mut sealed_a).unwrap(); + let nonce_a = aead + .seal_random(aad, b"hello world!", &mut sealed_a) + .expect("first random-nonce AEAD seal must succeed"); let mut sealed_b = [0u8; 12 + ChaCha20Poly1305::TAG_SIZE]; - let nonce_b = aead.seal_random(aad, b"hello world!", &mut sealed_b).unwrap(); + let nonce_b = aead + .seal_random(aad, b"hello world!", &mut sealed_b) + .expect("second random-nonce AEAD seal must succeed"); assert_ne!(nonce_a, nonce_b, "successive random seals must not reuse a nonce"); let mut opened = [0u8; 12]; - aead.decrypt(&nonce_a, aad, &sealed_a, &mut opened).unwrap(); + aead + .decrypt(&nonce_a, aad, &sealed_a, &mut opened) + .expect("fresh random-nonce AEAD ciphertext must authenticate"); assert_eq!(&opened, b"hello world!"); } @@ -105,10 +115,14 @@ fn aead_seal_random_in_place_returns_nonce_and_tag_for_open() { let aead = fixture_cipher(); let mut buffer = *b"detached"; - let (nonce, tag) = aead.seal_random_in_place(aad, &mut buffer).unwrap(); + let (nonce, tag) = aead + .seal_random_in_place(aad, &mut buffer) + .expect("random-nonce detached AEAD seal must succeed"); assert_ne!(&buffer, b"detached"); - aead.decrypt_in_place(&nonce, aad, &mut buffer, &tag).unwrap(); + aead + .decrypt_in_place(&nonce, aad, &mut buffer, &tag) + .expect("fresh random-nonce detached ciphertext must authenticate"); assert_eq!(&buffer, b"detached"); } @@ -118,7 +132,9 @@ fn aead_seal_random_maps_buffer_errors_without_mutating_output() { let aead = fixture_cipher(); let mut out = [0xA5; 3]; - let err = aead.seal_random(b"", b"data", &mut out).unwrap_err(); + let err = aead + .seal_random(b"", b"data", &mut out) + .expect_err("undersized random-nonce AEAD output must be rejected"); assert_eq!(err, RandomSealError::seal(SealError::buffer())); assert_eq!( @@ -137,15 +153,19 @@ fn detached_aliases_match_core_behavior() { let mut left = *b"detached"; let mut right = left; - let tag_left = aead.encrypt_in_place(&nonce, aad, &mut left).unwrap(); - let tag_right = aead.encrypt_in_place_detached(&nonce, aad, &mut right).unwrap(); + let tag_left = aead + .encrypt_in_place(&nonce, aad, &mut left) + .expect("core detached AEAD seal must succeed"); + let tag_right = aead + .encrypt_in_place_detached(&nonce, aad, &mut right) + .expect("detached AEAD alias must seal the same input"); assert_eq!(left, right); assert!(tag_left.ct_eq(&tag_right).declassify()); aead .decrypt_in_place_detached(&nonce, aad, &mut right, &tag_right) - .unwrap(); + .expect("detached AEAD alias must open its fresh ciphertext"); assert_eq!(right, *b"detached"); } @@ -156,7 +176,9 @@ fn aead_open_reports_buffer_and_verification_failures() { let aead = fixture_cipher(); let mut sealed = [0u8; 4 + ChaCha20Poly1305::TAG_SIZE]; - aead.encrypt(&nonce, b"aad", b"data", &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"aad", b"data", &mut sealed) + .expect("AEAD error fixture seal buffer must fit plaintext and tag"); let mut short_out = [0u8; 3]; assert_eq!( @@ -184,8 +206,12 @@ fn aead_combined_open_accepts_empty_plaintext() { let mut sealed = [0u8; ChaCha20Poly1305::TAG_SIZE]; let mut opened = [0u8; 0]; - aead.encrypt(&nonce, b"aad", b"", &mut sealed).unwrap(); - aead.decrypt(&nonce, b"aad", &sealed, &mut opened).unwrap(); + aead + .encrypt(&nonce, b"aad", b"", &mut sealed) + .expect("AEAD must seal an empty plaintext"); + aead + .decrypt(&nonce, b"aad", &sealed, &mut opened) + .expect("AEAD must open its empty-plaintext ciphertext"); } #[test] @@ -210,7 +236,9 @@ fn all_aeads_clear_in_place_buffer_on_verification_failure() { let cipher = $cipher; let nonce = $nonce; let mut buffer = *b"failed-open-clear"; - let tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect(concat!($name, " failed-open fixture must seal")); let mut bad_tag = tag.to_bytes(); bad_tag[0] ^= 0x80; let bad_tag = <$tag>::from_bytes(bad_tag); @@ -282,7 +310,13 @@ fn all_aeads_clear_in_place_buffer_on_verification_failure() { #[test] #[cfg(feature = "aead")] fn aead_length_helpers_reject_invalid_sizes() { - assert_eq!(ChaCha20Poly1305::ciphertext_len(5).unwrap(), 21); - assert_eq!(ChaCha20Poly1305::plaintext_len(21).unwrap(), 5); + assert_eq!( + ChaCha20Poly1305::ciphertext_len(5).expect("five-byte plaintext length must fit ChaCha20-Poly1305"), + 21 + ); + assert_eq!( + ChaCha20Poly1305::plaintext_len(21).expect("21-byte ciphertext length must include a full tag"), + 5 + ); assert_eq!(ChaCha20Poly1305::plaintext_len(15), Err(AeadBufferError::new())); } diff --git a/tests/aead_kernel_equivalence.rs b/tests/aead_kernel_equivalence.rs index 3693fabc..f5ec44dc 100644 --- a/tests/aead_kernel_equivalence.rs +++ b/tests/aead_kernel_equivalence.rs @@ -21,7 +21,7 @@ use rscrypto::{ }; /// Function pointer type for ChaCha20 XOR-keystream kernels. -type XorKeystreamFn = fn(&[u8; 32], u32, &[u8; 12], &mut [u8]); +type XorKeystreamFn = unsafe fn(&[u8; 32], u32, &[u8; 12], &mut [u8]); #[derive(Clone, Copy)] struct Backend { @@ -119,7 +119,8 @@ fn all_chacha20_backends_match_portable_at_counter_zero() { for backend in runnable_backends() { let mut actual = plain.clone(); - (backend.xor_keystream)(&key, 0, &nonce, &mut actual); + // SAFETY: `runnable_backends` proves the exact capability set; `TEST_SIZES` uses at most 65 blocks from zero. + unsafe { (backend.xor_keystream)(&key, 0, &nonce, &mut actual) }; assert_eq!( actual, expected, "ChaCha20 backend {} diverged from portable at len={len}, counter=0", @@ -152,7 +153,9 @@ fn all_chacha20_backends_match_portable_at_arbitrary_counters() { for backend in runnable_backends() { let mut actual = plain.clone(); - (backend.xor_keystream)(&key, counter, &nonce, &mut actual); + // SAFETY: `runnable_backends` proves the exact capability set; at most 16 blocks from the largest tested + // counter, 0x1000_0000, stays within the `u32` counter range. + unsafe { (backend.xor_keystream)(&key, counter, &nonce, &mut actual) }; assert_eq!( actual, expected, "ChaCha20 backend {} diverged from portable at len={len}, counter={counter}", @@ -176,8 +179,11 @@ fn all_chacha20_backends_self_inverse() { for backend in runnable_backends() { let mut buffer = original.clone(); - (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); - (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); + // SAFETY: `runnable_backends` proves the exact capability set; `TEST_SIZES` uses at most 65 blocks from zero. + unsafe { + (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); + (backend.xor_keystream)(&key, 0, &nonce, &mut buffer); + } assert_eq!( buffer, original, "ChaCha20 backend {} not self-inverse at len={len}", diff --git a/tests/aead_wycheproof.rs b/tests/aead_wycheproof.rs index 16db55de..0be63f26 100644 --- a/tests/aead_wycheproof.rs +++ b/tests/aead_wycheproof.rs @@ -37,9 +37,7 @@ struct AeadCase { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value[name].as_str().expect(name) } fn groups(suite: &Value) -> &[Value] { @@ -53,6 +51,11 @@ fn tests(group: &Value) -> &[Value] { } fn case_from_json(test: &Value) -> AeadCase { + let result = field(test, "result"); + assert!( + matches!(result, "valid" | "invalid"), + "unsupported Wycheproof AEAD result `{result}`" + ); AeadCase { tc_id: test["tcId"].as_u64().expect("tcId must be numeric"), comment: field(test, "comment").to_owned(), @@ -62,11 +65,7 @@ fn case_from_json(test: &Value) -> AeadCase { msg: decode_hex_vec(field(test, "msg")), ct: decode_hex_vec(field(test, "ct")), tag: decode_hex_vec(field(test, "tag")), - valid: match field(test, "result") { - "valid" => true, - "invalid" => false, - other => panic!("unsupported Wycheproof AEAD result `{other}`"), - }, + valid: result == "valid", } } @@ -80,14 +79,34 @@ fn assert_open_failure_clears_buffer(case: &AeadCase, buffer: &[u8]) { } fn assert_chacha20poly1305_case(case: &AeadCase) { - let key = ChaCha20Poly1305Key::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = ChaCha20Poly1305Tag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = ChaCha20Poly1305Key::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof ChaCha20-Poly1305 key must be 32 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof ChaCha20-Poly1305 nonce must be 12 bytes"), + ); + let tag = ChaCha20Poly1305Tag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof ChaCha20-Poly1305 tag must be 16 bytes"), + ); let cipher = ChaCha20Poly1305::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof ChaCha20-Poly1305 vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof ChaCha20-Poly1305 tcId {} opened incorrectly", @@ -95,7 +114,9 @@ fn assert_chacha20poly1305_case(case: &AeadCase) { ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof ChaCha20-Poly1305 vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof ChaCha20-Poly1305 tcId {} encrypted incorrectly", @@ -114,14 +135,34 @@ fn assert_chacha20poly1305_case(case: &AeadCase) { } fn assert_xchacha20poly1305_case(case: &AeadCase) { - let key = XChaCha20Poly1305Key::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce192::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = XChaCha20Poly1305Tag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = XChaCha20Poly1305Key::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof XChaCha20-Poly1305 key must be 32 bytes"), + ); + let nonce = Nonce192::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof XChaCha20-Poly1305 nonce must be 24 bytes"), + ); + let tag = XChaCha20Poly1305Tag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof XChaCha20-Poly1305 tag must be 16 bytes"), + ); let cipher = XChaCha20Poly1305::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof XChaCha20-Poly1305 vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof XChaCha20-Poly1305 tcId {} opened incorrectly", @@ -129,7 +170,9 @@ fn assert_xchacha20poly1305_case(case: &AeadCase) { ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof XChaCha20-Poly1305 vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof XChaCha20-Poly1305 tcId {} encrypted incorrectly", @@ -148,21 +191,43 @@ fn assert_xchacha20poly1305_case(case: &AeadCase) { } fn assert_aes128gcm_case(case: &AeadCase) { - let key = Aes128GcmKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes128GcmTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes128GcmKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM key must be 16 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM nonce must be 12 bytes"), + ); + let tag = Aes128GcmTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM tag must be 16 bytes"), + ); let cipher = Aes128Gcm::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-128-GCM vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-128-GCM tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-128-GCM vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-128-GCM tcId {} encrypted incorrectly", @@ -181,21 +246,43 @@ fn assert_aes128gcm_case(case: &AeadCase) { } fn assert_aes256gcm_case(case: &AeadCase) { - let key = Aes256GcmKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes256GcmTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes256GcmKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM key must be 32 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM nonce must be 12 bytes"), + ); + let tag = Aes256GcmTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM tag must be 16 bytes"), + ); let cipher = Aes256Gcm::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-256-GCM vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-256-GCM tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-256-GCM vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-256-GCM tcId {} encrypted incorrectly", @@ -214,21 +301,43 @@ fn assert_aes256gcm_case(case: &AeadCase) { } fn assert_aes128gcmsiv_case(case: &AeadCase) { - let key = Aes128GcmSivKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes128GcmSivTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes128GcmSivKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM-SIV key must be 16 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM-SIV nonce must be 12 bytes"), + ); + let tag = Aes128GcmSivTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-128-GCM-SIV tag must be 16 bytes"), + ); let cipher = Aes128GcmSiv::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-128-GCM-SIV vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-128-GCM-SIV tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-128-GCM-SIV vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-128-GCM-SIV tcId {} encrypted incorrectly", @@ -247,21 +356,43 @@ fn assert_aes128gcmsiv_case(case: &AeadCase) { } fn assert_aes256gcmsiv_case(case: &AeadCase) { - let key = Aes256GcmSivKey::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce96::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aes256GcmSivTag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aes256GcmSivKey::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM-SIV key must be 32 bytes"), + ); + let nonce = Nonce96::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AES-GCM-SIV nonce must be 12 bytes"), + ); + let tag = Aes256GcmSivTag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AES-256-GCM-SIV tag must be 16 bytes"), + ); let cipher = Aes256GcmSiv::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AES-256-GCM-SIV vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AES-256-GCM-SIV tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AES-256-GCM-SIV vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AES-256-GCM-SIV tcId {} encrypted incorrectly", @@ -280,21 +411,43 @@ fn assert_aes256gcmsiv_case(case: &AeadCase) { } fn assert_aegis256_case(case: &AeadCase) { - let key = Aegis256Key::from_bytes(case.key.as_slice().try_into().unwrap()); - let nonce = Nonce256::from_bytes(case.nonce.as_slice().try_into().unwrap()); - let tag = Aegis256Tag::from_bytes(case.tag.as_slice().try_into().unwrap()); + let key = Aegis256Key::from_bytes( + case + .key + .as_slice() + .try_into() + .expect("Wycheproof AEGIS-256 key must be 32 bytes"), + ); + let nonce = Nonce256::from_bytes( + case + .nonce + .as_slice() + .try_into() + .expect("Wycheproof AEGIS-256 nonce must be 32 bytes"), + ); + let tag = Aegis256Tag::from_bytes( + case + .tag + .as_slice() + .try_into() + .expect("Wycheproof AEGIS-256 tag must be 16 bytes"), + ); let cipher = Aegis256::new(&key); let mut buffer = case.ct.clone(); if case.valid { - cipher.decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, &case.aad, &mut buffer, &tag) + .expect("valid Wycheproof AEGIS-256 vector must authenticate"); assert_eq!( buffer, case.msg, "Wycheproof AEGIS-256 tcId {} opened incorrectly", case.tc_id ); let mut sealed = case.msg.clone(); - let actual_tag = cipher.encrypt_in_place(&nonce, &case.aad, &mut sealed).unwrap(); + let actual_tag = cipher + .encrypt_in_place(&nonce, &case.aad, &mut sealed) + .expect("valid Wycheproof AEGIS-256 vector must seal"); assert_eq!( sealed, case.ct, "Wycheproof AEGIS-256 tcId {} encrypted incorrectly", @@ -352,10 +505,17 @@ fn wycheproof_aes_gcm_open_vectors_match_expected_results() { && matches!(group["keySize"].as_u64(), Some(128 | 256)) && group["tagSize"].as_u64() == Some(128) }, - |case| match case.key.len() { - 16 => assert_aes128gcm_case(case), - 32 => assert_aes256gcm_case(case), - len => panic!("unsupported AES-GCM key length {len}"), + |case| { + assert!( + matches!(case.key.len(), 16 | 32), + "unsupported AES-GCM key length {}", + case.key.len() + ); + if case.key.len() == 16 { + assert_aes128gcm_case(case); + } else { + assert_aes256gcm_case(case); + } }, ); } @@ -374,10 +534,17 @@ fn wycheproof_aes_gcm_siv_open_vectors_match_expected_results() { && matches!(group["keySize"].as_u64(), Some(128 | 256)) && group["tagSize"].as_u64() == Some(128) }, - |case| match case.key.len() { - 16 => assert_aes128gcmsiv_case(case), - 32 => assert_aes256gcmsiv_case(case), - len => panic!("unsupported AES-GCM-SIV key length {len}"), + |case| { + assert!( + matches!(case.key.len(), 16 | 32), + "unsupported AES-GCM-SIV key length {}", + case.key.len() + ); + if case.key.len() == 16 { + assert_aes128gcmsiv_case(case); + } else { + assert_aes256gcmsiv_case(case); + } }, ); } diff --git a/tests/aegis256_oracle.rs b/tests/aegis256_oracle.rs index afbd757a..3056239b 100644 --- a/tests/aegis256_oracle.rs +++ b/tests/aegis256_oracle.rs @@ -26,7 +26,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 32], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AEGIS-256 oracle input"); // Encrypt with oracle (consumes self, returns (Vec, [u8; 16])). let (oracle_ct, oracle_tag) = Oracle::<16>::new(key_bytes, nonce_bytes).encrypt(plaintext, aad); @@ -35,13 +37,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 32], aad: &[u8 assert_eq!(tag.as_bytes(), &oracle_tag, "tag mismatch (len={})", plaintext.len()); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AEGIS-256 ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); // Cross-decrypt: oracle decrypts rscrypto's ciphertext. let oracle_pt = Oracle::<16>::new(key_bytes, nonce_bytes) .decrypt(&oracle_ct, &oracle_tag, aad) - .unwrap(); + .expect("AEGIS-256 oracle must authenticate rscrypto ciphertext"); assert_eq!( oracle_pt, plaintext, @@ -94,7 +98,7 @@ fn aegis256_oracle_varied_sizes() { fn aegis256_oracle_large_input() { let key = [0x77u8; 32]; let nonce = [0x88u8; 32]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -107,14 +111,15 @@ fn aegis256_rejects_modified_tag() { let cipher = Aegis256::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AEGIS-256 tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aegis256Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aegis256Tag::from_bytes(tag)) + .expect_err("AEGIS-256 must reject a modified tag"); } #[test] @@ -124,10 +129,14 @@ fn aegis256_rejects_modified_ciphertext() { let cipher = Aegis256::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AEGIS-256 ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AEGIS-256 must reject modified ciphertext"); } #[test] @@ -137,7 +146,11 @@ fn aegis256_rejects_wrong_aad() { let cipher = Aegis256::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AEGIS-256 AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AEGIS-256 must reject incorrect associated data"); } diff --git a/tests/aes128gcm_oracle.rs b/tests/aes128gcm_oracle.rs index e8b5378d..07050174 100644 --- a/tests/aes128gcm_oracle.rs +++ b/tests/aes128gcm_oracle.rs @@ -24,10 +24,10 @@ use rscrypto::{ fn deterministic_bytes(seed: u8, len: usize) -> Vec { let mut out = Vec::with_capacity(len); - let mut x = seed as u32; + let mut x = u32::from(seed); for _ in 0..len { x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - out.push((x >> 24) as u8); + out.push(x.to_be_bytes()[0]); } out } @@ -42,13 +42,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-128-GCM oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto must seal valid AES-128-GCM oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -59,7 +61,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-128-GCM ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -108,7 +112,7 @@ fn aes128gcm_oracle_large_input() { let key = [0x77u8; 16]; let nonce = [0x88u8; 12]; // 8 KiB — exercises multi-block GHASH and CTR paths. - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -120,8 +124,8 @@ fn aes128gcm_oracle_aad_size_sweep() { // Sweeps cover the wide-GHASH 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -132,13 +136,15 @@ fn aes128gcm_oracle_all_short_lengths() { for size in 0..=255usize { let mut nonce = [0x18u8; 12]; - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); + let size_word = u32::try_from(size).expect("short-length case must fit in a 32-bit nonce field"); + let size_byte = size.to_le_bytes()[0]; + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); let aad = if size % 3 == 0 { Vec::new() } else { - deterministic_bytes(0x42 ^ size as u8, size % 97) + deterministic_bytes(0x42 ^ size_byte, size % 97) }; - let plaintext = deterministic_bytes(0x81 ^ size as u8, size); + let plaintext = deterministic_bytes(0x81 ^ size_byte, size); assert_matches_oracle(&key, &nonce, &aad, &plaintext); } } @@ -150,9 +156,11 @@ fn aes128gcm_oracle_large_aligned_and_unaligned_lengths() { for (case_idx, &size) in [4096usize, 4097, 8191, 8192, 8193, 16_384, 16_385].iter().enumerate() { let mut nonce = [0x71u8; 12]; - nonce[4..8].copy_from_slice(&(case_idx as u32).to_be_bytes()); - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0x33 ^ case_idx as u8, size); + let case_word = u32::try_from(case_idx).expect("GCM oracle case index must fit in a 32-bit nonce field"); + let size_word = u32::try_from(size).expect("GCM oracle size must fit in a 32-bit nonce field"); + nonce[4..8].copy_from_slice(&case_word.to_be_bytes()); + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); + let plaintext = deterministic_bytes(0x33 ^ case_idx.to_le_bytes()[0], size); for aad in &aad_cases { assert_matches_oracle(&key, &nonce, aad, &plaintext); } @@ -168,14 +176,15 @@ fn aes128gcm_rejects_modified_tag() { let cipher = Aes128Gcm::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-128-GCM tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmTag::from_bytes(tag)) + .expect_err("AES-128-GCM must reject a modified tag"); } #[test] @@ -185,10 +194,14 @@ fn aes128gcm_rejects_modified_ciphertext() { let cipher = Aes128Gcm::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-128-GCM ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-128-GCM must reject modified ciphertext"); } #[test] @@ -198,7 +211,11 @@ fn aes128gcm_rejects_wrong_aad() { let cipher = Aes128Gcm::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-128-GCM AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-128-GCM must reject incorrect associated data"); } diff --git a/tests/aes128gcmsiv_oracle.rs b/tests/aes128gcmsiv_oracle.rs index c84db9f5..19d7443e 100644 --- a/tests/aes128gcmsiv_oracle.rs +++ b/tests/aes128gcmsiv_oracle.rs @@ -14,10 +14,7 @@ #![cfg(feature = "aead")] -use aes_gcm_siv::{ - Aes128GcmSiv as Oracle, KeyInit, - aead::{AeadInPlace, generic_array::GenericArray}, -}; +use aes_gcm_siv::{Aes128GcmSiv as Oracle, KeyInit, Nonce as OracleNonce, aead::AeadInOut}; use rscrypto::{ Aes128GcmSiv, Aes128GcmSivKey, Aes128GcmSivTag, aead::{Nonce96, expert::AeadWithNonce}, @@ -28,18 +25,20 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 let nonce = Nonce96::from_bytes(*nonce_bytes); let cipher = Aes128GcmSiv::new(&key); - let oracle = Oracle::new(GenericArray::from_slice(key_bytes)); - let oracle_nonce = GenericArray::from_slice(nonce_bytes); + let oracle = Oracle::new_from_slice(key_bytes).expect("AES-128-GCM-SIV oracle key length must be valid"); + let oracle_nonce = OracleNonce::from(*nonce_bytes); // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-128-GCM-SIV oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle - .encrypt_in_place_detached(oracle_nonce, aad, &mut oracle_buf) - .unwrap(); + .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) + .expect("RustCrypto must seal valid AES-128-GCM-SIV oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -50,7 +49,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-128-GCM-SIV ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -97,7 +98,7 @@ fn aes128gcmsiv_oracle_block_boundary_sizes() { fn aes128gcmsiv_oracle_large_input() { let key = [0x77u8; 16]; let nonce = [0x88u8; 12]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -109,8 +110,8 @@ fn aes128gcmsiv_oracle_aad_size_sweep() { // Sweeps cover the wide-POLYVAL 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -124,10 +125,14 @@ fn aes128gcmsiv_is_deterministic_under_nonce_reuse() { let plaintext = b"same nonce, same plaintext, same tag"; let mut first = plaintext.to_vec(); - let first_tag = cipher.encrypt_in_place(&nonce, aad, &mut first).unwrap(); + let first_tag = cipher + .encrypt_in_place(&nonce, aad, &mut first) + .expect("first deterministic AES-128-GCM-SIV seal must succeed"); let mut second = plaintext.to_vec(); - let second_tag = cipher.encrypt_in_place(&nonce, aad, &mut second).unwrap(); + let second_tag = cipher + .encrypt_in_place(&nonce, aad, &mut second) + .expect("second deterministic AES-128-GCM-SIV seal must succeed"); assert_eq!(first, second, "ciphertext changed across identical AES-GCM-SIV inputs"); assert_eq!( @@ -146,14 +151,15 @@ fn aes128gcmsiv_rejects_modified_tag() { let cipher = Aes128GcmSiv::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-128-GCM-SIV tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmSivTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes128GcmSivTag::from_bytes(tag)) + .expect_err("AES-128-GCM-SIV must reject a modified tag"); } #[test] @@ -163,10 +169,14 @@ fn aes128gcmsiv_rejects_modified_ciphertext() { let cipher = Aes128GcmSiv::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-128-GCM-SIV ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-128-GCM-SIV must reject modified ciphertext"); } #[test] @@ -176,7 +186,11 @@ fn aes128gcmsiv_rejects_wrong_aad() { let cipher = Aes128GcmSiv::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-128-GCM-SIV AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-128-GCM-SIV must reject incorrect associated data"); } diff --git a/tests/aes256gcm_oracle.rs b/tests/aes256gcm_oracle.rs index dedd58bf..c9bc834d 100644 --- a/tests/aes256gcm_oracle.rs +++ b/tests/aes256gcm_oracle.rs @@ -24,10 +24,10 @@ use rscrypto::{ fn deterministic_bytes(seed: u8, len: usize) -> Vec { let mut out = Vec::with_capacity(len); - let mut x = seed as u32; + let mut x = u32::from(seed); for _ in 0..len { x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - out.push((x >> 24) as u8); + out.push(x.to_be_bytes()[0]); } out } @@ -42,13 +42,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-256-GCM oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto must seal valid AES-256-GCM oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -59,7 +61,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-256-GCM ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -108,7 +112,7 @@ fn aes256gcm_oracle_large_input() { let key = [0x77u8; 32]; let nonce = [0x88u8; 12]; // 8 KiB — exercises multi-block GHASH and CTR paths. - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -120,8 +124,8 @@ fn aes256gcm_oracle_aad_size_sweep() { // Sweeps cover the wide-GHASH 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -132,13 +136,15 @@ fn aes256gcm_oracle_all_short_lengths() { for size in 0..=255usize { let mut nonce = [0x28u8; 12]; - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); + let size_word = u32::try_from(size).expect("short-length case must fit in a 32-bit nonce field"); + let size_byte = size.to_le_bytes()[0]; + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); let aad = if size % 3 == 0 { Vec::new() } else { - deterministic_bytes(0x52 ^ size as u8, size % 97) + deterministic_bytes(0x52 ^ size_byte, size % 97) }; - let plaintext = deterministic_bytes(0x91 ^ size as u8, size); + let plaintext = deterministic_bytes(0x91 ^ size_byte, size); assert_matches_oracle(&key, &nonce, &aad, &plaintext); } } @@ -150,9 +156,11 @@ fn aes256gcm_oracle_large_aligned_and_unaligned_lengths() { for (case_idx, &size) in [4096usize, 4097, 8191, 8192, 8193, 16_384, 16_385].iter().enumerate() { let mut nonce = [0x81u8; 12]; - nonce[4..8].copy_from_slice(&(case_idx as u32).to_be_bytes()); - nonce[8..12].copy_from_slice(&(size as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0x43 ^ case_idx as u8, size); + let case_word = u32::try_from(case_idx).expect("GCM oracle case index must fit in a 32-bit nonce field"); + let size_word = u32::try_from(size).expect("GCM oracle size must fit in a 32-bit nonce field"); + nonce[4..8].copy_from_slice(&case_word.to_be_bytes()); + nonce[8..12].copy_from_slice(&size_word.to_be_bytes()); + let plaintext = deterministic_bytes(0x43 ^ case_idx.to_le_bytes()[0], size); for aad in &aad_cases { assert_matches_oracle(&key, &nonce, aad, &plaintext); } @@ -168,14 +176,15 @@ fn aes256gcm_rejects_modified_tag() { let cipher = Aes256Gcm::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-256-GCM tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmTag::from_bytes(tag)) + .expect_err("AES-256-GCM must reject a modified tag"); } #[test] @@ -185,10 +194,14 @@ fn aes256gcm_rejects_modified_ciphertext() { let cipher = Aes256Gcm::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-256-GCM ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-256-GCM must reject modified ciphertext"); } #[test] @@ -198,7 +211,11 @@ fn aes256gcm_rejects_wrong_aad() { let cipher = Aes256Gcm::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-256-GCM AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-256-GCM must reject incorrect associated data"); } diff --git a/tests/aes256gcmsiv_oracle.rs b/tests/aes256gcmsiv_oracle.rs index cdf3bb03..ee084554 100644 --- a/tests/aes256gcmsiv_oracle.rs +++ b/tests/aes256gcmsiv_oracle.rs @@ -13,10 +13,7 @@ #![cfg(feature = "aead")] -use aes_gcm_siv::{ - Aes256GcmSiv as Oracle, KeyInit, - aead::{AeadInPlace, generic_array::GenericArray}, -}; +use aes_gcm_siv::{Aes256GcmSiv as Oracle, KeyInit, Nonce as OracleNonce, aead::AeadInOut}; use rscrypto::{ Aes256GcmSiv, Aes256GcmSivKey, Aes256GcmSivTag, aead::{Nonce96, expert::AeadWithNonce}, @@ -27,18 +24,20 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 let nonce = Nonce96::from_bytes(*nonce_bytes); let cipher = Aes256GcmSiv::new(&key); - let oracle = Oracle::new(GenericArray::from_slice(key_bytes)); - let oracle_nonce = GenericArray::from_slice(nonce_bytes); + let oracle = Oracle::new_from_slice(key_bytes).expect("AES-256-GCM-SIV oracle key length must be valid"); + let oracle_nonce = OracleNonce::from(*nonce_bytes); // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid AES-256-GCM-SIV oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle - .encrypt_in_place_detached(oracle_nonce, aad, &mut oracle_buf) - .unwrap(); + .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) + .expect("RustCrypto must seal valid AES-256-GCM-SIV oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -49,7 +48,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 32], nonce_bytes: &[u8; 12], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-256-GCM-SIV ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -96,7 +97,7 @@ fn aes256gcmsiv_oracle_block_boundary_sizes() { fn aes256gcmsiv_oracle_large_input() { let key = [0x77u8; 32]; let nonce = [0x88u8; 12]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -108,8 +109,8 @@ fn aes256gcmsiv_oracle_aad_size_sweep() { // Sweeps cover the wide-POLYVAL 4-block boundary (64-byte chunks) and the // partial-tail seam at +/-1 around 16-byte block boundaries. - for aad_len in [0, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { - let aad: Vec = (0..aad_len).map(|i| (i & 0xFF) as u8).collect(); + for aad_len in [0usize, 1, 15, 16, 17, 32, 33, 47, 48, 49, 64, 65, 80, 81, 128, 1024] { + let aad: Vec = (0..aad_len).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, &aad, plaintext); } } @@ -123,10 +124,14 @@ fn aes256gcmsiv_is_deterministic_under_nonce_reuse() { let plaintext = b"same nonce, same plaintext, same tag"; let mut first = plaintext.to_vec(); - let first_tag = cipher.encrypt_in_place(&nonce, aad, &mut first).unwrap(); + let first_tag = cipher + .encrypt_in_place(&nonce, aad, &mut first) + .expect("first deterministic AES-256-GCM-SIV seal must succeed"); let mut second = plaintext.to_vec(); - let second_tag = cipher.encrypt_in_place(&nonce, aad, &mut second).unwrap(); + let second_tag = cipher + .encrypt_in_place(&nonce, aad, &mut second) + .expect("second deterministic AES-256-GCM-SIV seal must succeed"); assert_eq!(first, second, "ciphertext changed across identical AES-GCM-SIV inputs"); assert_eq!( @@ -145,14 +150,15 @@ fn aes256gcmsiv_rejects_modified_tag() { let cipher = Aes256GcmSiv::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("AES-256-GCM-SIV tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmSivTag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &Aes256GcmSivTag::from_bytes(tag)) + .expect_err("AES-256-GCM-SIV must reject a modified tag"); } #[test] @@ -162,10 +168,14 @@ fn aes256gcmsiv_rejects_modified_ciphertext() { let cipher = Aes256GcmSiv::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("AES-256-GCM-SIV ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("AES-256-GCM-SIV must reject modified ciphertext"); } #[test] @@ -175,7 +185,11 @@ fn aes256gcmsiv_rejects_wrong_aad() { let cipher = Aes256GcmSiv::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("AES-256-GCM-SIV AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("AES-256-GCM-SIV must reject incorrect associated data"); } diff --git a/tests/aes_gcm_aarch64_asm_oracle.rs b/tests/aes_gcm_aarch64_asm_oracle.rs index be3c0c86..ba431e80 100644 --- a/tests/aes_gcm_aarch64_asm_oracle.rs +++ b/tests/aes_gcm_aarch64_asm_oracle.rs @@ -27,20 +27,24 @@ fn deterministic_bytes(seed: u8, len: usize) -> Vec { fn assert_aes128_matches_oracle(len: usize, aad: &[u8]) { let key_bytes = [0x31u8; 16]; let mut nonce_bytes = [0x42u8; 12]; - nonce_bytes[4..8].copy_from_slice(&(aad.len() as u32).to_be_bytes()); - nonce_bytes[8..12].copy_from_slice(&(len as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0x80 ^ len as u8, len); + let aad_len = u32::try_from(aad.len()).expect("oracle AAD length must fit the 32-bit nonce field"); + let text_len = u32::try_from(len).expect("oracle text length must fit the 32-bit nonce field"); + nonce_bytes[4..8].copy_from_slice(&aad_len.to_be_bytes()); + nonce_bytes[8..12].copy_from_slice(&text_len.to_be_bytes()); + let plaintext = deterministic_bytes(0x80 ^ len.to_le_bytes()[0], len); let cipher = Aes128Gcm::new(&Aes128GcmKey::from_bytes(key_bytes)); let nonce = Nonce96::from_bytes(nonce_bytes); let mut ours = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("bounded AES-128-GCM oracle input must seal"); let oracle = Aes128Oracle::new(&Array(key_bytes)); let mut expected = plaintext.clone(); let expected_tag = oracle .encrypt_inout_detached(&Array(nonce_bytes), aad, expected.as_mut_slice().into()) - .unwrap(); + .expect("bounded AES-128-GCM oracle input must seal"); assert_eq!(ours, expected, "AES-128-GCM ciphertext mismatch at len {len}"); assert_eq!( @@ -49,27 +53,33 @@ fn assert_aes128_matches_oracle(len: usize, aad: &[u8]) { "AES-128-GCM tag mismatch at len {len}" ); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-128-GCM ciphertext and tag must open"); assert_eq!(ours, plaintext, "AES-128-GCM open mismatch at len {len}"); } fn assert_aes256_matches_oracle(len: usize, aad: &[u8]) { let key_bytes = [0x53u8; 32]; let mut nonce_bytes = [0x64u8; 12]; - nonce_bytes[4..8].copy_from_slice(&(aad.len() as u32).to_be_bytes()); - nonce_bytes[8..12].copy_from_slice(&(len as u32).to_be_bytes()); - let plaintext = deterministic_bytes(0xA0 ^ len as u8, len); + let aad_len = u32::try_from(aad.len()).expect("oracle AAD length must fit the 32-bit nonce field"); + let text_len = u32::try_from(len).expect("oracle text length must fit the 32-bit nonce field"); + nonce_bytes[4..8].copy_from_slice(&aad_len.to_be_bytes()); + nonce_bytes[8..12].copy_from_slice(&text_len.to_be_bytes()); + let plaintext = deterministic_bytes(0xA0 ^ len.to_le_bytes()[0], len); let cipher = Aes256Gcm::new(&Aes256GcmKey::from_bytes(key_bytes)); let nonce = Nonce96::from_bytes(nonce_bytes); let mut ours = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("bounded AES-256-GCM oracle input must seal"); let oracle = Aes256Oracle::new(&Array(key_bytes)); let mut expected = plaintext.clone(); let expected_tag = oracle .encrypt_inout_detached(&Array(nonce_bytes), aad, expected.as_mut_slice().into()) - .unwrap(); + .expect("bounded AES-256-GCM oracle input must seal"); assert_eq!(ours, expected, "AES-256-GCM ciphertext mismatch at len {len}"); assert_eq!( @@ -78,7 +88,9 @@ fn assert_aes256_matches_oracle(len: usize, aad: &[u8]) { "AES-256-GCM tag mismatch at len {len}" ); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh AES-256-GCM ciphertext and tag must open"); assert_eq!(ours, plaintext, "AES-256-GCM open mismatch at len {len}"); } diff --git a/tests/api_consistency.rs b/tests/api_consistency.rs index 28c78d41..15aed348 100644 --- a/tests/api_consistency.rs +++ b/tests/api_consistency.rs @@ -1,12 +1,23 @@ -#![allow(unused_imports)] - #[cfg(all(feature = "checksums", feature = "std"))] -use std::io::{Cursor, Read, Write}; +use std::io::Write; #[cfg(any(feature = "ecdsa-p256", feature = "ecdsa-p384"))] use rscrypto::EcdsaKeyGenerationError; +use rscrypto::Kem; #[cfg(any(feature = "hmac", feature = "hmac-sha3"))] use rscrypto::Mac; +#[cfg(any( + feature = "ecdsa-p256", + feature = "ecdsa-p384", + feature = "ed25519", + all(feature = "rsa", feature = "getrandom") +))] +use rscrypto::TrySigner; +#[cfg(all(feature = "rsa", feature = "getrandom"))] +use rscrypto::TrySignerInto; +use rscrypto::VerificationError; +#[cfg(any(feature = "ecdsa-p256", feature = "ecdsa-p384", feature = "ed25519", feature = "rsa"))] +use rscrypto::Verifier; #[cfg(feature = "aead")] use rscrypto::aead::expert::AeadWithNonce; #[cfg(feature = "aead")] @@ -24,7 +35,7 @@ use rscrypto::{ #[cfg(feature = "ecdsa-p256")] use rscrypto::{EcdsaP256Keypair, EcdsaP256PublicKey, EcdsaP256SecretKey}; #[cfg(feature = "ecdsa-p384")] -use rscrypto::{EcdsaP384Keypair, EcdsaP384PublicKey, EcdsaP384SecretKey}; +use rscrypto::{EcdsaP384Keypair, EcdsaP384PublicKey}; #[cfg(feature = "ed25519")] use rscrypto::{Ed25519Keypair, Ed25519PublicKey, Ed25519SecretKey}; #[cfg(feature = "hkdf")] @@ -33,7 +44,6 @@ use rscrypto::{HkdfSha256, HkdfSha384, HkdfSha512, auth::HkdfOutputLengthError}; use rscrypto::{HmacSha3_224, HmacSha3_256, HmacSha3_384, HmacSha3_512}; #[cfg(feature = "hmac")] use rscrypto::{HmacSha256, HmacSha384, HmacSha512}; -use rscrypto::{Kem, TrySigner, TrySignerInto, VerificationError, Verifier}; #[cfg(feature = "kmac")] use rscrypto::{Kmac128, Kmac256}; #[cfg(feature = "poly1305")] @@ -123,7 +133,9 @@ fn assert_mac_api() { let expected = mac.finalize(); mac.reset(); mac.update(b"abc"); - assert!(mac.verify(&expected).is_ok()); + mac + .verify(&expected) + .expect("MAC must verify its freshly finalized tag"); } #[cfg(any(feature = "hmac", feature = "hmac-sha3"))] @@ -174,27 +186,41 @@ where let plaintext = b"abc"; let mut sealed = [0u8; 19]; - aead.encrypt(&nonce, b"aad", plaintext, &mut sealed).unwrap(); + aead + .encrypt(&nonce, b"aad", plaintext, &mut sealed) + .expect("fixed-size AEAD seal buffer must fit plaintext and tag"); let mut opened = [0u8; 3]; - aead.decrypt(&nonce, b"aad", &sealed, &mut opened).unwrap(); + aead + .decrypt(&nonce, b"aad", &sealed, &mut opened) + .expect("freshly sealed AEAD ciphertext must authenticate"); assert_eq!(&opened, plaintext); #[cfg(feature = "alloc")] { - let sealed_vec = aead.encrypt_to_vec(&nonce, b"aad", plaintext).unwrap(); + let sealed_vec = aead + .encrypt_to_vec(&nonce, b"aad", plaintext) + .expect("AEAD vector allocation must accept a three-byte plaintext"); assert_eq!(sealed_vec.as_slice(), sealed); - let opened_vec = aead.decrypt_to_vec(&nonce, b"aad", &sealed_vec).unwrap(); + let opened_vec = aead + .decrypt_to_vec(&nonce, b"aad", &sealed_vec) + .expect("freshly sealed AEAD vector must authenticate"); assert_eq!(opened_vec.as_slice(), plaintext); let mut tampered = sealed_vec; tampered[0] ^= 1; - assert!(aead.decrypt_to_vec(&nonce, b"aad", &tampered).is_err()); + aead + .decrypt_to_vec(&nonce, b"aad", &tampered) + .expect_err("tampered AEAD ciphertext must fail authentication"); } let mut detached = *b"abc"; - let tag = aead.encrypt_in_place_detached(&nonce, b"aad", &mut detached).unwrap(); - aead.decrypt_in_place(&nonce, b"aad", &mut detached, &tag).unwrap(); + let tag = aead + .encrypt_in_place_detached(&nonce, b"aad", &mut detached) + .expect("detached AEAD seal must accept a three-byte plaintext"); + aead + .decrypt_in_place(&nonce, b"aad", &mut detached, &tag) + .expect("freshly sealed detached AEAD ciphertext must authenticate"); assert_eq!(&detached, plaintext); } @@ -248,7 +274,7 @@ fn all_xofs_follow_new_update_finalize_xof_and_xof() { cshake.update(data); assert_eq!(streaming, squeeze_32(cshake.finalize_xof())); - let mut cxof = AsconCxof128::new(b"ctx=v1").unwrap(); + let mut cxof = AsconCxof128::new(b"ctx=v1").expect("short Ascon-CXOF customization must be valid"); cxof.update(data); let streaming = squeeze_32(cxof.finalize_xof()); cxof.reset(); @@ -292,7 +318,9 @@ fn kmac_follows_new_update_finalize_into_reset_and_verify() { kmac128.update(b"abc"); kmac128.finalize_into(&mut actual128); assert_eq!(actual128, expected128); - assert!(kmac128.verify(&expected128).is_ok()); + kmac128 + .verify(&expected128) + .expect("KMAC128 must verify its freshly finalized tag"); let mut kmac = Kmac256::new(b"api-consistency-key", b"ctx=v1"); kmac.update(b"abc"); @@ -304,7 +332,9 @@ fn kmac_follows_new_update_finalize_into_reset_and_verify() { kmac.update(b"abc"); kmac.finalize_into(&mut actual); assert_eq!(actual, expected); - assert!(kmac.verify(&expected).is_ok()); + kmac + .verify(&expected) + .expect("KMAC256 must verify its freshly finalized tag"); } #[test] @@ -314,32 +344,38 @@ fn hkdfs_follow_new_expand_and_derive_array_conventions() { let hkdf384 = HkdfSha384::new(b"salt", b"ikm"); let hkdf512 = HkdfSha512::new(b"salt", b"ikm"); - let okm256 = hkdf256.expand_array::<32>(b"info").unwrap(); - let okm384 = hkdf384.expand_array::<48>(b"info").unwrap(); - let okm512 = hkdf512.expand_array::<64>(b"info").unwrap(); + let okm256 = hkdf256 + .expand_array::<32>(b"info") + .expect("32-byte HKDF-SHA-256 output must be valid"); + let okm384 = hkdf384 + .expand_array::<48>(b"info") + .expect("48-byte HKDF-SHA-384 output must be valid"); + let okm512 = hkdf512 + .expand_array::<64>(b"info") + .expect("64-byte HKDF-SHA-512 output must be valid"); assert_eq!( okm256, - HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").unwrap() + HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").expect("32-byte HKDF-SHA-256 derivation must be valid") ); assert_eq!( okm384, - HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").unwrap() + HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").expect("48-byte HKDF-SHA-384 derivation must be valid") ); assert_eq!( okm512, - HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").unwrap() + HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").expect("64-byte HKDF-SHA-512 derivation must be valid") ); assert_eq!( - HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").unwrap(), + HkdfSha256::derive_array::<32>(b"salt", b"ikm", b"info").expect("32-byte HKDF-SHA-256 derivation must be valid"), okm256 ); assert_eq!( - HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").unwrap(), + HkdfSha384::derive_array::<48>(b"salt", b"ikm", b"info").expect("48-byte HKDF-SHA-384 derivation must be valid"), okm384 ); assert_eq!( - HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").unwrap(), + HkdfSha512::derive_array::<64>(b"salt", b"ikm", b"info").expect("64-byte HKDF-SHA-512 derivation must be valid"), okm512 ); assert_eq!(hkdf256.prk().len(), 32); @@ -361,7 +397,8 @@ fn poly1305_consumes_one_time_key_and_verifies() { assert_eq!(tag.as_bytes().len(), Poly1305Tag::LENGTH); let key = Poly1305OneTimeKey::from_bytes([0x42; Poly1305OneTimeKey::LENGTH]); - assert!(Poly1305::verify_once(key, b"api-consistency-poly1305", &tag).is_ok()); + Poly1305::verify_once(key, b"api-consistency-poly1305", &tag) + .expect("Poly1305 must verify its freshly generated tag"); } #[test] @@ -371,17 +408,21 @@ fn ed25519_types_follow_byte_roundtrip_and_verify_conventions() { out.fill(0x24); Ok::<(), ()>(()) }) - .unwrap(); + .expect("deterministic Ed25519 entropy callback must generate a key"); let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); let public = Ed25519PublicKey::from_bytes(keypair.public_key().to_bytes()); - let signature = TrySigner::try_sign(&keypair, b"api-consistency-ed25519").unwrap(); + let signature = TrySigner::try_sign(&keypair, b"api-consistency-ed25519") + .expect("Ed25519 signing with a valid generated key must succeed"); assert_eq!(*secret.expose_secret().as_bytes(), *secret.as_bytes()); assert_eq!(secret.duplicate_secret().as_bytes(), secret.as_bytes()); assert_eq!(public.to_bytes(), *public.as_bytes()); assert_eq!(signature.to_bytes(), *signature.as_bytes()); - assert!(public.verify(b"api-consistency-ed25519", &signature).is_ok()); - assert!(Verifier::verify(&public, b"api-consistency-ed25519", &signature).is_ok()); + public + .verify(b"api-consistency-ed25519", &signature) + .expect("Ed25519 public-key method must verify the fresh signature"); + Verifier::verify(&public, b"api-consistency-ed25519", &signature) + .expect("Ed25519 Verifier trait must verify the fresh signature"); } #[test] @@ -391,9 +432,11 @@ fn x25519_types_follow_byte_roundtrip_conventions() { out.fill(0x42); Ok::<(), ()>(()) }) - .unwrap(); + .expect("deterministic X25519 entropy callback must generate a key"); let public = X25519PublicKey::from_bytes(secret.public_key().to_bytes()); - let shared = secret.diffie_hellman(&public).unwrap(); + let shared = secret + .diffie_hellman(&public) + .expect("X25519 with a valid public key must produce a nonzero shared secret"); assert_eq!(*secret.expose_secret().as_bytes(), *secret.as_bytes()); assert_eq!(public.to_bytes(), *public.as_bytes()); @@ -407,20 +450,24 @@ fn ecdsa_p256_keygen_and_native_signature_traits_are_consistent() { out.fill(1); Ok::<(), ()>(()) }) - .unwrap(); - let public = EcdsaP256PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()).unwrap(); - let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p256").unwrap(); + .expect("deterministic P-256 entropy callback must generate a key"); + let public = EcdsaP256PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()) + .expect("generated P-256 public key must round-trip through SEC1"); + let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p256") + .expect("P-256 signing with a valid generated key must succeed"); - assert!(Verifier::verify(&public, b"api-consistency-ecdsa-p256", &signature).is_ok()); + Verifier::verify(&public, b"api-consistency-ecdsa-p256", &signature) + .expect("P-256 Verifier trait must verify the fresh signature"); let rejected = EcdsaP256SecretKey::try_generate_with(|out| { out.fill(0); Ok::<(), ()>(()) }) - .unwrap_err(); + .expect_err("an all-zero P-256 secret key must be rejected"); assert_eq!(rejected, EcdsaKeyGenerationError::InvalidSecretKey); - let rng_error = EcdsaP256SecretKey::try_generate_with(|_| Err("rng")).unwrap_err(); + let rng_error = EcdsaP256SecretKey::try_generate_with(|_| Err("rng")) + .expect_err("P-256 key generation must preserve entropy-source failure"); assert_eq!(rng_error, EcdsaKeyGenerationError::Random("rng")); } @@ -431,26 +478,35 @@ fn ecdsa_p384_keygen_and_native_signature_traits_are_consistent() { out.fill(1); Ok::<(), ()>(()) }) - .unwrap(); - let public = EcdsaP384PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()).unwrap(); - let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p384").unwrap(); - - assert!(Verifier::verify(&public, b"api-consistency-ecdsa-p384", &signature).is_ok()); + .expect("deterministic P-384 entropy callback must generate a key"); + let public = EcdsaP384PublicKey::from_sec1_bytes(&keypair.public_key().to_sec1_bytes()) + .expect("generated P-384 public key must round-trip through SEC1"); + let signature = TrySigner::try_sign(&keypair, b"api-consistency-ecdsa-p384") + .expect("P-384 signing with a valid generated key must succeed"); + + Verifier::verify(&public, b"api-consistency-ecdsa-p384", &signature) + .expect("P-384 Verifier trait must verify the fresh signature"); } #[test] #[cfg(feature = "rsa")] fn rsa_signature_verifier_requires_a_bound_profile() { - let key = RsaPublicKey::from_spki_der(include_bytes!("../benches/rsa_fixtures/rsa3072_spki.der")).unwrap(); + let key = RsaPublicKey::from_spki_der(include_bytes!("../benches/rsa_fixtures/rsa3072_spki.der")) + .expect("embedded RSA-3072 SPKI fixture must parse"); let message = b"rscrypto RSA-PSS verification fixture"; let pss_signature = include_bytes!("../benches/rsa_fixtures/rsa3072_pss_sha256.sig"); let pss_verifier = key.verifier(RsaSignatureProfile::pss(RsaPssProfile::Sha256)); - assert!(pss_verifier.verify(message, pss_signature).is_ok()); - assert!(Verifier::verify(&pss_verifier, message, pss_signature.as_slice()).is_ok()); + pss_verifier + .verify(message, pss_signature) + .expect("profile-bound RSA verifier must accept the matching PSS fixture"); + Verifier::verify(&pss_verifier, message, pss_signature.as_slice()) + .expect("RSA Verifier trait must accept the matching PSS fixture"); let wrong_profile = key.verifier(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256)); - assert!(wrong_profile.verify(message, pss_signature).is_err()); + wrong_profile + .verify(message, pss_signature) + .expect_err("RSA verifier must reject a PSS signature under the PKCS1-v1_5 profile"); } #[test] @@ -483,7 +539,7 @@ fn kem_trait_uses_typed_key_ciphertext_and_secret_outputs() { out.copy_from_slice(&[1, 2, 3, 4]); Ok(()) }) - .unwrap(); + .expect("infallible ToyKem key generation must succeed"); assert_eq!(encapsulation_key.as_ref(), &[1, 2, 3]); assert_eq!(decapsulation_key.as_ref(), &[1, 2, 3, 4]); @@ -491,11 +547,12 @@ fn kem_trait_uses_typed_key_ciphertext_and_secret_outputs() { out.copy_from_slice(&[8, 9]); Ok(()) }) - .unwrap(); + .expect("infallible ToyKem encapsulation must succeed"); assert_eq!(ciphertext.as_ref(), &[1, 2, 3, 8, 9]); assert_eq!(encapsulated_secret.as_ref(), &[8, 9]); - let decapsulated_secret = ToyKem::decapsulate(&decapsulation_key, &ciphertext).unwrap(); + let decapsulated_secret = + ToyKem::decapsulate(&decapsulation_key, &ciphertext).expect("infallible ToyKem decapsulation must succeed"); assert_eq!(decapsulated_secret.as_ref(), &[4, 9]); } @@ -576,24 +633,30 @@ fn all_aeads_follow_new_encrypt_decrypt_and_detached_aliases() { #[cfg(all(feature = "aead", feature = "getrandom", feature = "alloc"))] fn aead_random_to_vec_seals_and_opens() { let cipher = ChaCha20Poly1305::new(&ChaCha20Poly1305Key::from_bytes([0x33; ChaCha20Poly1305::KEY_SIZE])); - let (nonce, sealed) = cipher.seal_random_to_vec(b"aad", b"plaintext").unwrap(); - let opened = cipher.decrypt_to_vec(&nonce, b"aad", &sealed).unwrap(); + let (nonce, sealed) = cipher + .seal_random_to_vec(b"aad", b"plaintext") + .expect("random-nonce AEAD sealing must succeed"); + let opened = cipher + .decrypt_to_vec(&nonce, b"aad", &sealed) + .expect("freshly sealed random-nonce ciphertext must authenticate"); assert_eq!(opened, b"plaintext"); } #[test] #[cfg(all(feature = "checksums", feature = "std"))] -fn checksum_adapters_use_checksum() -> std::io::Result<()> { +fn checksum_adapters_use_checksum() { use rscrypto::{Checksum as _, Crc32C}; - let mut reader = Crc32C::reader(Cursor::new(b"abc".to_vec())); - std::io::copy(&mut reader, &mut std::io::sink())?; + let mut reader = Crc32C::reader(&b"abc"[..]); + let mut copied = Vec::new(); + std::io::copy(&mut reader, &mut copied).expect("checksum reader must stream into a Vec"); + assert_eq!(copied, b"abc"); assert_eq!(reader.checksum(), Crc32C::checksum(b"abc")); let mut writer = Crc32C::writer(Vec::new()); - writer.write_all(b"abc")?; + writer + .write_all(b"abc") + .expect("Vec-backed checksum writer must accept three bytes"); assert_eq!(writer.checksum(), Crc32C::checksum(b"abc")); - - Ok(()) } diff --git a/tests/argon2_differential.rs b/tests/argon2_differential.rs index 9c63baa9..85801505 100644 --- a/tests/argon2_differential.rs +++ b/tests/argon2_differential.rs @@ -34,31 +34,34 @@ fn oracle_hash( p: u32, out_len: usize, ) -> Vec { - let params = argon2::Params::new(m_kib, t, p, Some(out_len)).unwrap(); + let params = argon2::Params::new(m_kib, t, p, Some(out_len)) + .expect("RustCrypto must accept generated Argon2 differential parameters"); let ctx = argon2::Argon2::new(algo, argon2::Version::V0x13, params); let mut out = vec![0u8; out_len]; - ctx.hash_password_into(password, salt, &mut out).unwrap(); + ctx + .hash_password_into(password, salt, &mut out) + .expect("RustCrypto Argon2 differential derivation must succeed"); out } fn rs_hash_id(password: &[u8], salt: &[u8], m_kib: u32, t: u32, p: u32, out_len: usize) -> Vec { - let params = Argon2Params::new(m_kib, t, p).unwrap(); + let params = Argon2Params::new(m_kib, t, p).expect("generated Argon2id parameters must be valid"); let mut out = vec![0u8; out_len]; - Argon2id::derive(¶ms, password, salt, &mut out).unwrap(); + Argon2id::derive(¶ms, password, salt, &mut out).expect("Argon2id differential derivation must succeed"); out } fn rs_hash_d(password: &[u8], salt: &[u8], m_kib: u32, t: u32, p: u32, out_len: usize) -> Vec { - let params = Argon2Params::new(m_kib, t, p).unwrap(); + let params = Argon2Params::new(m_kib, t, p).expect("generated Argon2d parameters must be valid"); let mut out = vec![0u8; out_len]; - Argon2d::derive(¶ms, password, salt, &mut out).unwrap(); + Argon2d::derive(¶ms, password, salt, &mut out).expect("Argon2d differential derivation must succeed"); out } fn rs_hash_i(password: &[u8], salt: &[u8], m_kib: u32, t: u32, p: u32, out_len: usize) -> Vec { - let params = Argon2Params::new(m_kib, t, p).unwrap(); + let params = Argon2Params::new(m_kib, t, p).expect("generated Argon2i parameters must be valid"); let mut out = vec![0u8; out_len]; - Argon2i::derive(¶ms, password, salt, &mut out).unwrap(); + Argon2i::derive(¶ms, password, salt, &mut out).expect("Argon2i differential derivation must succeed"); out } @@ -74,7 +77,7 @@ proptest! { p in 1u32..=2, out_len in proptest::sample::select(vec![16usize, 32, 48]), ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let actual = rs_hash_id(&password, &salt, m, t, p, out_len); let expected = oracle_hash(argon2::Algorithm::Argon2id, &password, &salt, m, t, p, out_len); prop_assert_eq!(actual, expected); @@ -89,7 +92,7 @@ proptest! { p in 1u32..=2, out_len in proptest::sample::select(vec![16usize, 32, 48]), ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let actual = rs_hash_d(&password, &salt, m, t, p, out_len); let expected = oracle_hash(argon2::Algorithm::Argon2d, &password, &salt, m, t, p, out_len); prop_assert_eq!(actual, expected); @@ -104,7 +107,7 @@ proptest! { p in 1u32..=2, out_len in proptest::sample::select(vec![16usize, 32, 48]), ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let actual = rs_hash_i(&password, &salt, m, t, p, out_len); let expected = oracle_hash(argon2::Algorithm::Argon2i, &password, &salt, m, t, p, out_len); prop_assert_eq!(actual, expected); @@ -118,12 +121,14 @@ proptest! { t in 1u32..=2, p in 1u32..=2, ) { - prop_assume!(m >= 8 * p); + prop_assume!(m >= p.strict_mul(8)); let params = Argon2Params::new(m, t, p) - .unwrap(); + .expect("generated Argon2 verification parameters must be valid"); let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, &password, &salt, &mut hash).unwrap(); - prop_assert!(Argon2id::verify(¶ms, &password, &salt, &hash).is_ok()); + Argon2id::derive(¶ms, &password, &salt, &mut hash) + .expect("Argon2id verification fixture must derive"); + Argon2id::verify(¶ms, &password, &salt, &hash) + .expect("fresh Argon2id property-test hash must verify"); } } @@ -173,45 +178,43 @@ fn argon2id_p8_matches_oracle() { /// — see the doc comment on `verify` in `src/auth/argon2/mod.rs`. #[test] fn argon2id_verify_rejects_length_mismatch() { - let params = Argon2Params::new(8, 1, 1).unwrap(); + let params = Argon2Params::new(8, 1, 1).expect("length-mismatch test parameters must be valid"); let password = b"pw"; let salt = b"saltsalt"; let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, password, salt, &mut hash).unwrap(); + Argon2id::derive(¶ms, password, salt, &mut hash).expect("length-mismatch fixture must derive"); // Sanity: correct length verifies. - assert!(Argon2id::verify(¶ms, password, salt, &hash).is_ok()); + Argon2id::verify(¶ms, password, salt, &hash).expect("fresh Argon2id hash must verify"); // Various wrong lengths: shorter, longer, empty. let too_short: &[u8] = &hash[..16]; let too_long: &[u8] = &[hash.as_ref(), &[0u8; 8]].concat()[..40]; let empty: &[u8] = &[]; - assert!(Argon2id::verify(¶ms, password, salt, too_short).is_err()); - assert!(Argon2id::verify(¶ms, password, salt, too_long).is_err()); - assert!(Argon2id::verify(¶ms, password, salt, empty).is_err()); + Argon2id::verify(¶ms, password, salt, too_short).expect_err("Argon2id must reject a short hash"); + Argon2id::verify(¶ms, password, salt, too_long).expect_err("Argon2id must reject a long hash"); + Argon2id::verify(¶ms, password, salt, empty).expect_err("Argon2id must reject an empty hash"); // And a length one byte off in either direction — the boundary cases. let off_minus_one: &[u8] = &hash[..31]; let off_plus_one: &[u8] = &[hash.as_ref(), &[0u8; 1]].concat(); - assert!(Argon2id::verify(¶ms, password, salt, off_minus_one).is_err()); - assert!(Argon2id::verify(¶ms, password, salt, off_plus_one).is_err()); + Argon2id::verify(¶ms, password, salt, off_minus_one).expect_err("Argon2id must reject a hash one byte too short"); + Argon2id::verify(¶ms, password, salt, off_plus_one).expect_err("Argon2id must reject a hash one byte too long"); } #[test] fn argon2id_verify_rejects_byte_flip_at_every_position() { - let params = Argon2Params::new(32, 2, 1).unwrap(); + let params = Argon2Params::new(32, 2, 1).expect("byte-flip test parameters must be valid"); let password = b"correct horse battery staple"; let salt = b"random-salt-1234"; let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, password, salt, &mut hash).unwrap(); + Argon2id::derive(¶ms, password, salt, &mut hash).expect("byte-flip fixture must derive"); for pos in 0..hash.len() { let mut tampered = hash; tampered[pos] ^= 0x01; - assert!( - Argon2id::verify(¶ms, password, salt, &tampered).is_err(), - "verify must reject flip at byte {pos}" - ); + Argon2id::verify(¶ms, password, salt, &tampered) + .expect_err("Argon2id must reject a hash with any flipped byte"); } } diff --git a/tests/argon2_kernels.rs b/tests/argon2_kernels.rs index 99d8dd1b..92c291b1 100644 --- a/tests/argon2_kernels.rs +++ b/tests/argon2_kernels.rs @@ -13,15 +13,14 @@ //! full hash with a non-trivial cost matrix surfaces it. #![cfg(all(feature = "argon2", feature = "diag"))] -#![allow(clippy::unwrap_used)] use rscrypto::{ Argon2Params, auth::{argon2, argon2::Argon2Variant}, }; -fn params(m_kib: u32, t: u32, p: u32, _out_len: u32) -> Argon2Params { - Argon2Params::new(m_kib, t, p).unwrap() +fn params(m_kib: u32, t: u32, p: u32) -> Argon2Params { + Argon2Params::new(m_kib, t, p).expect("Argon2 kernel-test parameters must be valid") } const PASSWORD: &[u8] = b"correct horse battery staple"; @@ -30,12 +29,14 @@ const SALT: &[u8] = b"rscrypto-salt-16b!!!"; /// Every kernel must agree at a range of cost points. fn all_kernels_agree(variant: Argon2Variant) { for &(m, t, p) in &[(16u32, 1u32, 1u32), (32, 2, 1), (64, 1, 2), (32, 3, 1)] { - let params = params(m, t, p, 32); + let params = params(m, t, p); let mut expected = [0u8; 32]; - argon2::diag_hash_portable(¶ms, PASSWORD, SALT, variant, &mut expected).unwrap(); + argon2::diag_hash_portable(¶ms, PASSWORD, SALT, variant, &mut expected) + .expect("portable Argon2 kernel-test hash must succeed"); let mut active_out = [0u8; 32]; - argon2::diag_hash_active(¶ms, PASSWORD, SALT, variant, &mut active_out).unwrap(); + argon2::diag_hash_active(¶ms, PASSWORD, SALT, variant, &mut active_out) + .expect("active Argon2 kernel-test hash must succeed"); assert_eq!( active_out, expected, "active kernel diverged on m={m} t={t} p={p} variant={variant:?}" @@ -44,7 +45,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "aarch64")] { let mut neon_out = [0u8; 32]; - argon2::diag_hash_aarch64_neon(¶ms, PASSWORD, SALT, variant, &mut neon_out).unwrap(); + argon2::diag_hash_aarch64_neon(¶ms, PASSWORD, SALT, variant, &mut neon_out) + .expect("AArch64 NEON Argon2 kernel-test hash must succeed"); assert_eq!( neon_out, expected, "aarch64-neon diverged on m={m} t={t} p={p} variant={variant:?}" @@ -57,7 +59,8 @@ fn all_kernels_agree(variant: Argon2Variant) { if host.has(rscrypto::auth::argon2::required_caps(argon2::KernelId::X86Avx2)) { let mut out = [0u8; 32]; - argon2::diag_hash_x86_avx2(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_x86_avx2(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("x86 AVX2 Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "x86-avx2 diverged on m={m} t={t} p={p} variant={variant:?}" @@ -66,7 +69,8 @@ fn all_kernels_agree(variant: Argon2Variant) { if host.has(rscrypto::auth::argon2::required_caps(argon2::KernelId::X86Avx512)) { let mut out = [0u8; 32]; - argon2::diag_hash_x86_avx512(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_x86_avx512(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("x86 AVX-512 Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "x86-avx512 diverged on m={m} t={t} p={p} variant={variant:?}" @@ -77,7 +81,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "powerpc64")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::PowerVsx)) { let mut out = [0u8; 32]; - argon2::diag_hash_power_vsx(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_power_vsx(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("POWER VSX Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "power-vsx diverged on m={m} t={t} p={p} variant={variant:?}" @@ -87,7 +92,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "s390x")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::S390xVector)) { let mut out = [0u8; 32]; - argon2::diag_hash_s390x_vector(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_s390x_vector(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("s390x vector Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "s390x-vector diverged on m={m} t={t} p={p} variant={variant:?}" @@ -97,7 +103,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "riscv64")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::Riscv64V)) { let mut out = [0u8; 32]; - argon2::diag_hash_riscv64_v(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_riscv64_v(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("RISC-V vector Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "riscv64-v diverged on m={m} t={t} p={p} variant={variant:?}" @@ -107,7 +114,8 @@ fn all_kernels_agree(variant: Argon2Variant) { #[cfg(target_arch = "wasm32")] if rscrypto::platform::caps().has(rscrypto::auth::argon2::required_caps(argon2::KernelId::WasmSimd128)) { let mut out = [0u8; 32]; - argon2::diag_hash_wasm_simd128(¶ms, PASSWORD, SALT, variant, &mut out).unwrap(); + argon2::diag_hash_wasm_simd128(¶ms, PASSWORD, SALT, variant, &mut out) + .expect("WASM SIMD128 Argon2 kernel-test hash must succeed"); assert_eq!( out, expected, "wasm-simd128 diverged on m={m} t={t} p={p} variant={variant:?}" @@ -136,9 +144,16 @@ fn argon2id_all_kernels_agree() { #[test] fn single_block_compress_matches_across_kernels() { use argon2::DIAG_BLOCK_WORDS; - let x: [u64; DIAG_BLOCK_WORDS] = core::array::from_fn(|i| (i as u64).wrapping_mul(0x0f0f_0f0f_0f0f_0f0f)); - let y: [u64; DIAG_BLOCK_WORDS] = - core::array::from_fn(|i| (i as u64).wrapping_mul(0xa5a5_a5a5_a5a5_a5a5).rotate_left(i as u32)); + let x: [u64; DIAG_BLOCK_WORDS] = core::array::from_fn(|index| { + u64::try_from(index) + .expect("Argon2 block-word index must fit in u64") + .wrapping_mul(0x0f0f_0f0f_0f0f_0f0f) + }); + let y: [u64; DIAG_BLOCK_WORDS] = core::array::from_fn(|index| { + let index_word = u64::try_from(index).expect("Argon2 block-word index must fit in u64"); + let rotation = u32::try_from(index).expect("Argon2 block-word index must fit in u32"); + index_word.wrapping_mul(0xa5a5_a5a5_a5a5_a5a5).rotate_left(rotation) + }); for xor_into in [false, true] { let mut expected = if xor_into { diff --git a/tests/argon2_miri.rs b/tests/argon2_miri.rs index 8460ee76..fd9111a6 100644 --- a/tests/argon2_miri.rs +++ b/tests/argon2_miri.rs @@ -62,8 +62,10 @@ fn argon2id_minimal_verify_no_ub() { let params = Argon2Params::new(8, 1, 1).expect("minimal params are valid"); let mut hash = [0u8; 4]; - Argon2id::derive(¶ms, b"correct", b"saltsalt", &mut hash).unwrap(); + Argon2id::derive(¶ms, b"correct", b"saltsalt", &mut hash) + .expect("Argon2id Miri verification fixture must derive"); - assert!(Argon2id::verify(¶ms, b"correct", b"saltsalt", &hash).is_ok()); - assert!(Argon2id::verify(¶ms, b"wrong", b"saltsalt", &hash).is_err()); + Argon2id::verify(¶ms, b"correct", b"saltsalt", &hash).expect("fresh Argon2id Miri hash must verify"); + Argon2id::verify(¶ms, b"wrong", b"saltsalt", &hash) + .expect_err("Argon2id Miri verification must reject the wrong password"); } diff --git a/tests/argon2_parallel.rs b/tests/argon2_parallel.rs index 99eb5f2a..b477567d 100644 --- a/tests/argon2_parallel.rs +++ b/tests/argon2_parallel.rs @@ -12,8 +12,8 @@ use rscrypto::{Argon2Params, Argon2d, Argon2i, Argon2id}; -fn rs_params(m_kib: u32, t: u32, p: u32, _out_len: u32) -> Argon2Params { - Argon2Params::new(m_kib, t, p).unwrap() +fn rs_params(m_kib: u32, t: u32, p: u32) -> Argon2Params { + Argon2Params::new(m_kib, t, p).expect("Argon2 parallel-test parameters must be valid") } fn oracle_hash( @@ -25,10 +25,13 @@ fn oracle_hash( p: u32, out_len: usize, ) -> Vec { - let params = argon2::Params::new(m_kib, t, p, Some(out_len)).unwrap(); + let params = argon2::Params::new(m_kib, t, p, Some(out_len)) + .expect("RustCrypto must accept valid Argon2 parallel-test parameters"); let ctx = argon2::Argon2::new(algo, argon2::Version::V0x13, params); let mut out = vec![0u8; out_len]; - ctx.hash_password_into(password, salt, &mut out).unwrap(); + ctx + .hash_password_into(password, salt, &mut out) + .expect("RustCrypto Argon2 parallel-test derivation must succeed"); out } @@ -46,9 +49,9 @@ fn argon2id_p8_matches_oracle() { let p = 8u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2id p=8 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2id, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2id p=8 mismatch vs RustCrypto oracle"); @@ -61,9 +64,9 @@ fn argon2d_p8_matches_oracle() { let p = 8u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2d::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2d::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2d p=8 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2d, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2d p=8 mismatch vs RustCrypto oracle"); @@ -76,9 +79,9 @@ fn argon2i_p8_matches_oracle() { let p = 8u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2i::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2i::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2i p=8 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2i, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2i p=8 mismatch vs RustCrypto oracle"); @@ -91,9 +94,9 @@ fn argon2id_p16_matches_oracle() { let p = 16u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2id p=16 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2id, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2id p=16 mismatch vs RustCrypto oracle"); @@ -105,28 +108,29 @@ fn argon2id_p16_matches_oracle() { /// produce the same tag. Any non-determinism would indicate a data race. #[test] fn argon2id_parallel_is_deterministic() { - let params = rs_params(1024, 2, 8, 32); + let params = rs_params(1024, 2, 8); let mut reference = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut reference).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut reference) + .expect("reference Argon2id parallel derivation must succeed"); for trial in 0..16 { let mut out = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut out).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut out).expect("repeated Argon2id derivation must succeed"); assert_eq!(out, reference, "non-deterministic output on trial {trial}"); } } #[test] fn argon2d_parallel_is_deterministic() { - let params = rs_params(1024, 2, 8, 32); + let params = rs_params(1024, 2, 8); let mut reference = [0u8; 32]; - Argon2d::derive(¶ms, PASSWORD, SALT, &mut reference).unwrap(); + Argon2d::derive(¶ms, PASSWORD, SALT, &mut reference).expect("reference Argon2d parallel derivation must succeed"); for trial in 0..16 { let mut out = [0u8; 32]; - Argon2d::derive(¶ms, PASSWORD, SALT, &mut out).unwrap(); + Argon2d::derive(¶ms, PASSWORD, SALT, &mut out).expect("repeated Argon2d derivation must succeed"); assert_eq!(out, reference, "non-deterministic output on trial {trial}"); } } @@ -142,9 +146,9 @@ fn argon2id_p1_fast_path_matches_oracle() { let p = 1u32; let out_len = 32usize; - let params = rs_params(m, t, p, out_len as u32); + let params = rs_params(m, t, p); let mut actual = vec![0u8; out_len]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut actual).expect("Argon2id p=1 derivation must succeed"); let expected = oracle_hash(argon2::Algorithm::Argon2id, PASSWORD, SALT, m, t, p, out_len); assert_eq!(actual, expected, "argon2id p=1 fast-path mismatch vs oracle"); @@ -152,10 +156,11 @@ fn argon2id_p1_fast_path_matches_oracle() { #[test] fn argon2id_parallel_verify_round_trip() { - let params = rs_params(256, 2, 8, 32); + let params = rs_params(256, 2, 8); let mut hash = [0u8; 32]; - Argon2id::derive(¶ms, PASSWORD, SALT, &mut hash).unwrap(); + Argon2id::derive(¶ms, PASSWORD, SALT, &mut hash).expect("Argon2id verification fixture must derive"); - assert!(Argon2id::verify(¶ms, PASSWORD, SALT, &hash).is_ok()); - assert!(Argon2id::verify(¶ms, b"wrong-password-zzzzzzzzzzzzzzzzzz", SALT, &hash).is_err()); + Argon2id::verify(¶ms, PASSWORD, SALT, &hash).expect("fresh Argon2id hash must verify"); + Argon2id::verify(¶ms, b"wrong-password-zzzzzzzzzzzzzzzzzz", SALT, &hash) + .expect_err("Argon2id must reject the wrong password"); } diff --git a/tests/argon2_vectors.rs b/tests/argon2_vectors.rs index e7fafc3f..ad88bfe2 100644 --- a/tests/argon2_vectors.rs +++ b/tests/argon2_vectors.rs @@ -33,7 +33,7 @@ fn rfc9106_appendix_a1_argon2d() { RFC_SALT, &mut out, ) - .unwrap(); + .expect("RFC 9106 Argon2d vector derivation must succeed"); assert_eq!(out, expected); } @@ -51,7 +51,7 @@ fn rfc9106_appendix_a2_argon2i() { RFC_SALT, &mut out, ) - .unwrap(); + .expect("RFC 9106 Argon2i vector derivation must succeed"); assert_eq!(out, expected); } @@ -69,7 +69,7 @@ fn rfc9106_appendix_a3_argon2id() { RFC_SALT, &mut out, ) - .unwrap(); + .expect("RFC 9106 Argon2id vector derivation must succeed"); assert_eq!(out, expected); } @@ -82,9 +82,12 @@ fn all_three_variants_produce_distinct_output() { let mut d = [0u8; 32]; let mut i = [0u8; 32]; let mut id = [0u8; 32]; - Argon2d::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut d).unwrap(); - Argon2i::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut i).unwrap(); - Argon2id::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut id).unwrap(); + Argon2d::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut d) + .expect("Argon2d distinct-output fixture must derive"); + Argon2i::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut i) + .expect("Argon2i distinct-output fixture must derive"); + Argon2id::derive_with_context(¶ms, canonical_context(), RFC_PASSWORD, RFC_SALT, &mut id) + .expect("Argon2id distinct-output fixture must derive"); assert_ne!(d, i); assert_ne!(d, id); assert_ne!(i, id); diff --git a/tests/ascon_aead_oracle.rs b/tests/ascon_aead_oracle.rs index ad8454a5..800bd718 100644 --- a/tests/ascon_aead_oracle.rs +++ b/tests/ascon_aead_oracle.rs @@ -36,13 +36,15 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 16], aad: &[u8 // Encrypt with rscrypto. let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto must seal valid Ascon-AEAD128 oracle input"); // Encrypt with oracle. let mut oracle_buf = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buf.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto must seal valid Ascon-AEAD128 oracle input"); assert_eq!(ours, oracle_buf, "ciphertext mismatch (len={})", plaintext.len()); assert_eq!( @@ -53,7 +55,9 @@ fn assert_matches_oracle(key_bytes: &[u8; 16], nonce_bytes: &[u8; 16], aad: &[u8 ); // Decrypt with rscrypto. - cipher.decrypt_in_place(&nonce, aad, &mut ours, &tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &tag) + .expect("fresh Ascon-AEAD128 ciphertext must authenticate"); assert_eq!(ours, plaintext, "decrypt round-trip failed (len={})", plaintext.len()); } @@ -101,7 +105,7 @@ fn ascon_aead128_oracle_rate_boundary_sizes() { fn ascon_aead128_oracle_large_input() { let key = [0x77u8; 16]; let nonce = [0x88u8; 16]; - let plaintext: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let plaintext: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_matches_oracle(&key, &nonce, b"large", &plaintext); } @@ -114,14 +118,15 @@ fn ascon_aead128_rejects_modified_tag() { let cipher = AsconAead128::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("Ascon-AEAD128 tag-forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &AsconAead128Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &AsconAead128Tag::from_bytes(tag)) + .expect_err("Ascon-AEAD128 must reject a modified tag"); } #[test] @@ -131,10 +136,14 @@ fn ascon_aead128_rejects_modified_ciphertext() { let cipher = AsconAead128::new(&key); let mut buffer = *b"tamper-detect"; - let tag = cipher.encrypt_in_place(&nonce, b"", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"", &mut buffer) + .expect("Ascon-AEAD128 ciphertext-tampering fixture must seal"); buffer[0] ^= 1; - assert!(cipher.decrypt_in_place(&nonce, b"", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"", &mut buffer, &tag) + .expect_err("Ascon-AEAD128 must reject modified ciphertext"); } #[test] @@ -144,7 +153,11 @@ fn ascon_aead128_rejects_wrong_aad() { let cipher = AsconAead128::new(&key); let mut buffer = *b"aad-mismatch"; - let tag = cipher.encrypt_in_place(&nonce, b"correct", &mut buffer).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, b"correct", &mut buffer) + .expect("Ascon-AEAD128 AAD-mismatch fixture must seal"); - assert!(cipher.decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag).is_err()); + cipher + .decrypt_in_place(&nonce, b"wrong", &mut buffer, &tag) + .expect_err("Ascon-AEAD128 must reject incorrect associated data"); } diff --git a/tests/ascon_cxof_vectors.rs b/tests/ascon_cxof_vectors.rs index 09e837c7..701d2cdc 100644 --- a/tests/ascon_cxof_vectors.rs +++ b/tests/ascon_cxof_vectors.rs @@ -19,7 +19,8 @@ fn ascon_cxof128_matches_ascon_c_kat_vector() { "4F50159EF70BB3DAD8807E034EAEBD44C4FA2CBBC8CF1F05511AB66CDCC529905CA12083FC186AD899B270B1473DC5F7EC88D1052082DCDFE69FB75D269E7B74", ); - let actual = squeeze_all(AsconCxof128::xof(&customization, &msg).unwrap(), expected.len()); + let reader = AsconCxof128::xof(&customization, &msg).expect("empty Ascon-CXOF128 customization must be valid"); + let actual = squeeze_all(reader, expected.len()); assert_eq!(actual, expected); } @@ -28,7 +29,7 @@ fn ascon_cxof128_reset_restores_customized_state() { let customization = b"ctx=v1"; let data = b"abc"; - let mut hasher = AsconCxof128::new(customization).unwrap(); + let mut hasher = AsconCxof128::new(customization).expect("short Ascon-CXOF128 customization must be valid"); hasher.update(data); let expected = squeeze_all(hasher.finalize_xof(), 64); @@ -40,6 +41,6 @@ fn ascon_cxof128_reset_restores_customized_state() { #[test] fn ascon_cxof128_rejects_long_customization() { - let err = AsconCxof128::new(&[0u8; 257]).unwrap_err(); + let err = AsconCxof128::new(&[0u8; 257]).expect_err("257-byte Ascon-CXOF128 customization must be rejected"); assert_eq!(err.to_string(), "Ascon-CXOF128 customization exceeds 256 bytes"); } diff --git a/tests/ascon_differential.rs b/tests/ascon_differential.rs index 94fd2407..5040c374 100644 --- a/tests/ascon_differential.rs +++ b/tests/ascon_differential.rs @@ -10,8 +10,8 @@ fn ascon_hash256_streaming(data: &[u8]) -> [u8; 32] { let mut hasher = AsconHash256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = usize::from(data[i]).strict_rem(97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); hasher.update(&data[i..end]); i = end; } @@ -57,8 +57,8 @@ proptest! { split_data in any::(), split_out in any::(), ) { - let split_data = split_data % (data.len() + 1); - let split_out = split_out % (out_len + 1); + let split_data = split_data.strict_rem(data.len().strict_add(1)); + let split_out = split_out.strict_rem(out_len.strict_add(1)); let mut expected = vec![0u8; out_len]; { diff --git a/tests/ascon_final_kats.rs b/tests/ascon_final_kats.rs index 857a1ad3..11fd8d6c 100644 --- a/tests/ascon_final_kats.rs +++ b/tests/ascon_final_kats.rs @@ -3,8 +3,11 @@ mod common; #[cfg(feature = "ascon-aead")] -use common::decode_hex_array; +#[path = "common/array.rs"] +mod hex_array; use common::decode_hex_vec; +#[cfg(feature = "ascon-aead")] +use hex_array::decode_hex_array; #[cfg(feature = "ascon-aead")] const AEAD_VECTORS: &str = include_str!("../testdata/ascon/asconaead128.txt"); @@ -20,11 +23,13 @@ fn field<'a>(case: &'a str, name: &str) -> &'a str { .lines() .find_map(|line| line.split_once(" = ").filter(|(key, _)| *key == name)) .map(|(_, value)| value) - .unwrap_or_else(|| panic!("missing Ascon KAT field `{name}`")) + .expect("Ascon KAT field must be present") } fn case_number(case: &str) -> usize { - field(case, "Count").parse().expect("invalid Ascon KAT case number") + field(case, "Count") + .parse() + .expect("Ascon KAT case number must be valid") } #[cfg(feature = "ascon-aead")] @@ -35,7 +40,7 @@ fn ascon_aead128_matches_final_reference_kats() { aead::{Nonce128, expert::AeadWithNonce as _}, }; - let mut rows = 0; + let mut rows = 0usize; for case in cases(AEAD_VECTORS) { let count = case_number(case); let key = AsconAead128Key::from_bytes(decode_hex_array(field(case, "Key"))); @@ -58,7 +63,7 @@ fn ascon_aead128_matches_final_reference_kats() { let mut ciphertext = plaintext.clone(); let tag = cipher .encrypt_in_place(&nonce, &aad, &mut ciphertext) - .unwrap_or_else(|err| panic!("Ascon-AEAD128 KAT {count} encryption failed: {err}")); + .expect("Ascon-AEAD128 KAT encryption must succeed"); assert_eq!( ciphertext, expected_ciphertext, "Ascon-AEAD128 KAT {count} ciphertext mismatch" @@ -72,10 +77,10 @@ fn ascon_aead128_matches_final_reference_kats() { let mut decrypted = expected_ciphertext.to_vec(); cipher .decrypt_in_place(&nonce, &aad, &mut decrypted, &expected_tag) - .unwrap_or_else(|err| panic!("Ascon-AEAD128 KAT {count} decryption failed: {err}")); + .expect("Ascon-AEAD128 KAT decryption must succeed"); assert_eq!(decrypted, plaintext, "Ascon-AEAD128 KAT {count} plaintext mismatch"); - rows += 1; + rows = rows.strict_add(1); } assert_eq!(rows, 1089, "incomplete Ascon-AEAD128 KAT corpus"); } @@ -85,7 +90,7 @@ fn ascon_aead128_matches_final_reference_kats() { fn ascon_cxof128_matches_final_reference_kats() { use rscrypto::{AsconCxof128, traits::Xof as _}; - let mut rows = 0; + let mut rows = 0usize; for case in cases(CXOF_VECTORS) { let count = case_number(case); let message = decode_hex_vec(field(case, "Msg")); @@ -94,11 +99,11 @@ fn ascon_cxof128_matches_final_reference_kats() { let mut actual = vec![0u8; expected.len()]; AsconCxof128::xof(&customization, &message) - .unwrap_or_else(|err| panic!("Ascon-CXOF128 KAT {count} setup failed: {err}")) + .expect("Ascon-CXOF128 KAT setup must succeed") .squeeze(&mut actual); assert_eq!(actual, expected, "Ascon-CXOF128 KAT {count} output mismatch"); - rows += 1; + rows = rows.strict_add(1); } assert_eq!(rows, 1089, "incomplete Ascon-CXOF128 KAT corpus"); } diff --git a/tests/ascon_hash_oracle.rs b/tests/ascon_hash_oracle.rs index 04dc0333..7c88cc91 100644 --- a/tests/ascon_hash_oracle.rs +++ b/tests/ascon_hash_oracle.rs @@ -56,7 +56,7 @@ fn ascon_hash256_oracle_rate_boundaries() { #[test] fn ascon_hash256_oracle_large() { - let input: Vec = (0..8192).map(|i| (i & 0xFF) as u8).collect(); + let input: Vec = (0usize..8192).map(|i| i.to_le_bytes()[0]).collect(); assert_hash_matches_oracle(&input); } @@ -133,7 +133,7 @@ fn ascon_xof128_oracle_large_squeeze() { #[test] fn ascon_xof128_oracle_large_input() { - let input: Vec = (0..4096).map(|i| (i & 0xFF) as u8).collect(); + let input: Vec = (0usize..4096).map(|i| i.to_le_bytes()[0]).collect(); assert_xof_matches_oracle(&input, 64); } diff --git a/tests/ascon_official_vectors.rs b/tests/ascon_official_vectors.rs index 1b6186cd..d4a0757d 100644 --- a/tests/ascon_official_vectors.rs +++ b/tests/ascon_official_vectors.rs @@ -6,17 +6,16 @@ use rscrypto::{ hashes::crypto::{AsconHash256, AsconXof}, traits::{Digest as _, Xof as _}, }; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; #[test] fn ascon_hash256_official_vectors() { let data = include_bytes!("../testdata/ascon/asconhash.blb"); - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("ascon hash vector corpus must parse") .enumerate() { - let [input, output] = - row.unwrap_or_else(|err| panic!("ascon-hash256 vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("Ascon-Hash256 vector row must decode"); let actual = AsconHash256::digest(input); assert_eq!( &actual[..], @@ -30,12 +29,11 @@ fn ascon_hash256_official_vectors() { #[test] fn ascon_xof_official_vectors() { let data = include_bytes!("../testdata/ascon/asconxof.blb"); - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("ascon xof vector corpus must parse") .enumerate() { - let [input, output] = - row.unwrap_or_else(|err| panic!("ascon-xof128 vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("Ascon-XOF128 vector row must decode"); let mut actual = vec![0u8; output.len()]; AsconXof::xof(input).squeeze(&mut actual); assert_eq!( diff --git a/tests/blake2_differential.rs b/tests/blake2_differential.rs index 547a02d3..b32f32cb 100644 --- a/tests/blake2_differential.rs +++ b/tests/blake2_differential.rs @@ -1,11 +1,14 @@ #![cfg(feature = "hashes")] use blake2::{ - Blake2b256 as OracleBlake2b256, Blake2b512 as OracleBlake2b512, Blake2bMac, Blake2s128 as OracleBlake2s128, - Blake2s256 as OracleBlake2s256, Blake2sMac, Digest as _, + Blake2b as OracleBlake2b, Blake2b512 as OracleBlake2b512, Blake2bMac, Blake2bVarCore, Blake2s as OracleBlake2s, + Blake2s256 as OracleBlake2s256, Blake2sMac, Blake2sVarCore, + digest::{ + Digest as _, Mac as _, Output, + consts::{U16, U32, U64}, + core_api::{Buffer, UpdateCore, VariableOutputCore}, + }, }; -use digest::typenum::{U16, U32, U64}; -use hmac::{Mac as _, digest::KeyInit}; use proptest::{prelude::*, test_runner::Config as ProptestConfig}; use rscrypto::{ Blake2b256, Blake2b512, Blake2bKey, Blake2bParams, Blake2s128, Blake2s256, Blake2sKey, Blake2sParams, Digest, @@ -15,18 +18,47 @@ type OracleBlake2bMac256 = Blake2bMac; type OracleBlake2bMac512 = Blake2bMac; type OracleBlake2sMac128 = Blake2sMac; type OracleBlake2sMac256 = Blake2sMac; +type OracleBlake2b256 = OracleBlake2b; +type OracleBlake2s128 = OracleBlake2s; fn split_at_ratio(data: &[u8], ratio: u8) -> (&[u8], &[u8]) { let idx = if data.is_empty() { 0 } else { - data.len().strict_mul(ratio as usize) / 255 + data.len().strict_mul(usize::from(ratio)) / 255 }; data.split_at(idx.min(data.len())) } fn patterned_input(seed: u8, len: usize) -> Vec { - (0..len).map(|i| seed.wrapping_add((i % 251) as u8)).collect() + (0..len) + .map(|i| { + let offset = u8::try_from(i % 251).expect("remainder modulo 251 must fit in one byte"); + seed.wrapping_add(offset) + }) + .collect() +} + +fn oracle_blake2b_unkeyed(data: &[u8], salt: &[u8], personal: &[u8]) -> [u8; N] { + let mut core = Blake2bVarCore::new_with_params(salt, personal, 0, N); + let mut buffer = Buffer::::default(); + buffer.digest_blocks(data, |blocks| core.update_blocks(blocks)); + let mut full = Output::::default(); + core.finalize_variable_core(&mut buffer, &mut full); + let mut out = [0u8; N]; + out.copy_from_slice(&full[..N]); + out +} + +fn oracle_blake2s_unkeyed(data: &[u8], salt: &[u8], personal: &[u8]) -> [u8; N] { + let mut core = Blake2sVarCore::new_with_params(salt, personal, 0, N); + let mut buffer = Buffer::::default(); + buffer.digest_blocks(data, |blocks| core.update_blocks(blocks)); + let mut full = Output::::default(); + core.finalize_variable_core(&mut buffer, &mut full); + let mut out = [0u8; N]; + out.copy_from_slice(&full[..N]); + out } proptest! { @@ -41,7 +73,7 @@ proptest! { ) { let (left, right) = split_at_ratio(&data, split); let key = &patterned_input(0x42, key_len); - let typed_key = Blake2bKey::new(key).unwrap(); + let typed_key = Blake2bKey::new(key).expect("generated BLAKE2b key length must be valid"); let tail = patterned_input(0xA5, tail_len); let expected_256 = OracleBlake2b256::digest(&data); @@ -59,7 +91,8 @@ proptest! { streaming_512.update(right); prop_assert_eq!(&streaming_512.finalize()[..], expected_512.as_slice()); - let mut oracle_keyed_256 = OracleBlake2bMac256::new_from_slice(key).unwrap(); + let mut oracle_keyed_256 = OracleBlake2bMac256::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2b-256 key"); oracle_keyed_256.update(&data); let expected_keyed_256 = oracle_keyed_256.finalize().into_bytes(); prop_assert_eq!( @@ -67,7 +100,8 @@ proptest! { &expected_keyed_256[..] ); - let mut oracle_keyed_512 = OracleBlake2bMac512::new_from_slice(key).unwrap(); + let mut oracle_keyed_512 = OracleBlake2bMac512::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2b-512 key"); oracle_keyed_512.update(&data); let expected_keyed_512 = oracle_keyed_512.finalize().into_bytes(); prop_assert_eq!( @@ -77,14 +111,14 @@ proptest! { let mut reset_256 = Blake2b256::new(); reset_256.update(&data); - let _ = reset_256.finalize(); + let _first_digest = reset_256.finalize(); reset_256.reset(); reset_256.update(&tail); prop_assert_eq!(reset_256.finalize(), Blake2b256::digest(&tail)); let mut reset_512 = Blake2b512::new(); reset_512.update(&data); - let _ = reset_512.finalize(); + let _first_digest = reset_512.finalize(); reset_512.reset(); reset_512.update(&tail); prop_assert_eq!(reset_512.finalize(), Blake2b512::digest(&tail)); @@ -99,7 +133,7 @@ proptest! { ) { let (left, right) = split_at_ratio(&data, split); let key = &patterned_input(0x24, key_len); - let typed_key = Blake2sKey::new(key).unwrap(); + let typed_key = Blake2sKey::new(key).expect("generated BLAKE2s key length must be valid"); let tail = patterned_input(0x5A, tail_len); let expected_128 = OracleBlake2s128::digest(&data); @@ -117,7 +151,8 @@ proptest! { streaming_256.update(right); prop_assert_eq!(&streaming_256.finalize()[..], expected_256.as_slice()); - let mut oracle_keyed_128 = OracleBlake2sMac128::new_from_slice(key).unwrap(); + let mut oracle_keyed_128 = OracleBlake2sMac128::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2s-128 key"); oracle_keyed_128.update(&data); let expected_keyed_128 = oracle_keyed_128.finalize().into_bytes(); prop_assert_eq!( @@ -125,7 +160,8 @@ proptest! { &expected_keyed_128[..] ); - let mut oracle_keyed_256 = OracleBlake2sMac256::new_from_slice(key).unwrap(); + let mut oracle_keyed_256 = OracleBlake2sMac256::new_from_slice(key) + .expect("RustCrypto must accept the generated BLAKE2s-256 key"); oracle_keyed_256.update(&data); let expected_keyed_256 = oracle_keyed_256.finalize().into_bytes(); prop_assert_eq!( @@ -135,14 +171,14 @@ proptest! { let mut reset_128 = Blake2s128::new(); reset_128.update(&data); - let _ = reset_128.finalize(); + let _first_digest = reset_128.finalize(); reset_128.reset(); reset_128.update(&tail); prop_assert_eq!(reset_128.finalize(), Blake2s128::digest(&tail)); let mut reset_256 = Blake2s256::new(); reset_256.update(&data); - let _ = reset_256.finalize(); + let _first_digest = reset_256.finalize(); reset_256.reset(); reset_256.update(&tail); prop_assert_eq!(reset_256.finalize(), Blake2s256::digest(&tail)); @@ -163,17 +199,18 @@ proptest! { let mut personal_field = [0u8; 16]; personal_field[..personal.len()].copy_from_slice(&personal); - let key_opt: Option<&[u8]> = if key.is_empty() { None } else { Some(&key) }; - - // Oracle: Blake2bMac::new_with_salt_and_personal(key, salt, personal). - // When key=None the MAC reduces to a plain keyless hash with params. - let mut oracle_256 = OracleBlake2bMac256::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_256.update(&data); - let expected_256 = oracle_256.finalize().into_bytes(); + let expected_256: [u8; 32] = if key.is_empty() { + oracle_blake2b_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2bMac256::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2b-256 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let mut params = Blake2bParams::new().salt(salt_field).personal(personal_field); if !key.is_empty() { - params = params.key(Blake2bKey::new(&key).unwrap()); + params = params.key(Blake2bKey::new(&key).expect("generated BLAKE2b parameter key must be valid")); } let ours_oneshot_256 = params.hash_256(&data); prop_assert_eq!(&ours_oneshot_256[..], &expected_256[..]); @@ -183,9 +220,14 @@ proptest! { ours_stream_256.update(&data); prop_assert_eq!(&ours_stream_256.finalize()[..], &expected_256[..]); - let mut oracle_512 = OracleBlake2bMac512::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_512.update(&data); - let expected_512 = oracle_512.finalize().into_bytes(); + let expected_512: [u8; 64] = if key.is_empty() { + oracle_blake2b_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2bMac512::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2b-512 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let ours_oneshot_512 = params.hash_512(&data); prop_assert_eq!(&ours_oneshot_512[..], &expected_512[..]); @@ -206,15 +248,18 @@ proptest! { let mut personal_field = [0u8; 8]; personal_field[..personal.len()].copy_from_slice(&personal); - let key_opt: Option<&[u8]> = if key.is_empty() { None } else { Some(&key) }; - - let mut oracle_256 = OracleBlake2sMac256::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_256.update(&data); - let expected_256 = oracle_256.finalize().into_bytes(); + let expected_256: [u8; 32] = if key.is_empty() { + oracle_blake2s_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2sMac256::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2s-256 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let mut params = Blake2sParams::new().salt(salt_field).personal(personal_field); if !key.is_empty() { - params = params.key(Blake2sKey::new(&key).unwrap()); + params = params.key(Blake2sKey::new(&key).expect("generated BLAKE2s parameter key must be valid")); } let ours_oneshot_256 = params.hash_256(&data); prop_assert_eq!(&ours_oneshot_256[..], &expected_256[..]); @@ -223,9 +268,14 @@ proptest! { ours_stream_256.update(&data); prop_assert_eq!(&ours_stream_256.finalize()[..], &expected_256[..]); - let mut oracle_128 = OracleBlake2sMac128::new_with_salt_and_personal(key_opt, &salt, &personal).unwrap(); - oracle_128.update(&data); - let expected_128 = oracle_128.finalize().into_bytes(); + let expected_128: [u8; 16] = if key.is_empty() { + oracle_blake2s_unkeyed(&data, &salt, &personal) + } else { + let mut oracle = OracleBlake2sMac128::new_with_salt_and_personal(&key, &salt, &personal) + .expect("RustCrypto must accept generated BLAKE2s-128 parameters"); + oracle.update(&data); + oracle.finalize().into_bytes().into() + }; let ours_oneshot_128 = params.hash_128(&data); prop_assert_eq!(&ours_oneshot_128[..], &expected_128[..]); diff --git a/tests/blake2_official_vectors.rs b/tests/blake2_official_vectors.rs index 8dcf140f..025f211b 100644 --- a/tests/blake2_official_vectors.rs +++ b/tests/blake2_official_vectors.rs @@ -3,7 +3,7 @@ mod support; use rscrypto::{Blake2b512, Blake2bKey, Blake2s256, Blake2sKey, Digest}; -use support::blobby_compat::Blob3Iterator; +use support::blobby_compat::BlobIterator; fn run_blake2_vectors( data: &'static [u8], @@ -11,12 +11,11 @@ fn run_blake2_vectors( mut one_shot: impl FnMut(&[u8], &[u8]) -> [u8; OUT], mut streaming: impl FnMut(&[u8], &[u8]) -> [u8; OUT], ) { - for (index, row) in Blob3Iterator::new(data) + for (index, row) in BlobIterator::<3>::new(data) .expect("blake2 vector corpus must parse") .enumerate() { - let [input, key, output] = - row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {index}: {err:?}")); + let [input, key, output] = row.expect("BLAKE2 vector row must decode"); let actual = one_shot(input, key); assert_eq!( @@ -48,14 +47,16 @@ fn blake2s_official_vectors() { if key.is_empty() { Blake2s256::digest(input) } else { - Blake2s256::keyed_digest(Blake2sKey::new(key).unwrap(), input) + let key = Blake2sKey::new(key).expect("official BLAKE2s key length must be valid"); + Blake2s256::keyed_digest(key, input) } }, |input, key| { let mut hasher = if key.is_empty() { Blake2s256::new() } else { - Blake2s256::new_keyed(Blake2sKey::new(key).unwrap()) + let key = Blake2sKey::new(key).expect("official BLAKE2s key length must be valid"); + Blake2s256::new_keyed(key) }; for chunk in input.chunks(11) { hasher.update(chunk); @@ -75,14 +76,16 @@ fn blake2b_official_vectors() { if key.is_empty() { Blake2b512::digest(input) } else { - Blake2b512::keyed_digest(Blake2bKey::new(key).unwrap(), input) + let key = Blake2bKey::new(key).expect("official BLAKE2b key length must be valid"); + Blake2b512::keyed_digest(key, input) } }, |input, key| { let mut hasher = if key.is_empty() { Blake2b512::new() } else { - Blake2b512::new_keyed(Blake2bKey::new(key).unwrap()) + let key = Blake2bKey::new(key).expect("official BLAKE2b key length must be valid"); + Blake2b512::new_keyed(key) }; for chunk in input.chunks(17) { hasher.update(chunk); diff --git a/tests/blake3_differential.rs b/tests/blake3_differential.rs index 6d834803..c9a3ff4e 100644 --- a/tests/blake3_differential.rs +++ b/tests/blake3_differential.rs @@ -37,7 +37,9 @@ fn blake3_ref_derive_xof(context: &str, data: &[u8], out: &mut [u8]) { } fn patterned_bytes(len: usize) -> Vec { - (0..len).map(|i| (i % 251) as u8).collect() + (0..len) + .map(|i| u8::try_from(i % 251).expect("remainder modulo 251 must fit in one byte")) + .collect() } proptest! { @@ -53,8 +55,8 @@ proptest! { let mut h = Blake3::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 251) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 251).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/blake3_official_vectors.rs b/tests/blake3_official_vectors.rs index 18ee7228..b80b05bd 100644 --- a/tests/blake3_official_vectors.rs +++ b/tests/blake3_official_vectors.rs @@ -3,7 +3,7 @@ mod support; use rscrypto::{Digest, hashes::crypto::Blake3, traits::Xof as _}; -use support::blobby_compat::Blob6Iterator; +use support::blobby_compat::BlobIterator; fn update_input_pattern(hasher: &mut Blake3, len: usize) { let mut remaining = len; @@ -12,11 +12,12 @@ fn update_input_pattern(hasher: &mut Blake3, len: usize) { while remaining != 0 { let take = core::cmp::min(remaining, buf.len()); for (i, b) in buf[..take].iter_mut().enumerate() { - *b = ((offset + i) % 251) as u8; + let value = offset.strict_add(i).strict_rem(251); + *b = u8::try_from(value).expect("BLAKE3 fixture byte must fit in u8"); } hasher.update(&buf[..take]); - offset += take; - remaining -= take; + offset = offset.strict_add(take); + remaining = remaining.strict_sub(take); } } @@ -28,7 +29,7 @@ fn decode_u64_le(bytes: &[u8]) -> u64 { #[test] fn blake3_official_test_vectors() { let blb = include_bytes!("../testdata/blake3/test_vectors.blb"); - for (i, row) in Blob6Iterator::new(blb) + for (i, row) in BlobIterator::<6>::new(blb) .expect("blake3 vector corpus must parse") .enumerate() { @@ -39,14 +40,15 @@ fn blake3_official_test_vectors() { hash_xof, keyed_hash_xof, derive_key_xof, - ] = row.unwrap_or_else(|err| panic!("blake3 vector row decode failed at case {i}: {err:?}")); + ] = row.expect("BLAKE3 vector row must decode"); assert_eq!(key_bytes.len(), 32, "blake3 key length mismatch at case {i}"); let mut key = [0u8; 32]; key.copy_from_slice(key_bytes); let context = core::str::from_utf8(context_bytes).expect("blake3 context_string is valid UTF-8"); - let input_len = decode_u64_le(input_len_bytes) as usize; + let input_len = + usize::try_from(decode_u64_le(input_len_bytes)).expect("BLAKE3 vector input length must fit in usize"); // Hash mode { diff --git a/tests/chacha20poly1305.rs b/tests/chacha20poly1305.rs index 834d24e9..cedec351 100644 --- a/tests/chacha20poly1305.rs +++ b/tests/chacha20poly1305.rs @@ -15,9 +15,10 @@ use common::decode_hex_vec as decode_hex; fn pattern_bytes(len: usize, seed: u8) -> Vec { let mut out = vec![0u8; len]; for (index, byte) in out.iter_mut().enumerate() { + let index = index.to_le_bytes()[0]; *byte = seed - .wrapping_add((index as u8).wrapping_mul(17)) - .wrapping_add((index as u8).rotate_left(1)); + .wrapping_add(index.wrapping_mul(17)) + .wrapping_add(index.rotate_left(1)); } out } @@ -35,18 +36,22 @@ fn chacha20poly1305_matches_rfc_8439_vector() { ); let expected_tag = decode_hex("1ae10b594f09e26a7e902ecbd0600691"); - let key = ChaCha20Poly1305Key::from_bytes(key.try_into().unwrap()); - let nonce = Nonce96::from_bytes(nonce.try_into().unwrap()); + let key = ChaCha20Poly1305Key::from_bytes(key.try_into().expect("RFC 8439 key must be 32 bytes")); + let nonce = Nonce96::from_bytes(nonce.try_into().expect("RFC 8439 nonce must be 12 bytes")); let cipher = ChaCha20Poly1305::new(&key); let mut sealed = vec![0u8; plaintext.len() + ChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut sealed).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut sealed) + .expect("RFC 8439 seal buffer must fit plaintext and tag"); assert_eq!(&sealed[..plaintext.len()], expected_ciphertext.as_slice()); assert_eq!(&sealed[plaintext.len()..], expected_tag.as_slice()); let mut opened = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &sealed, &mut opened) + .expect("RFC 8439 ciphertext and tag must authenticate"); assert_eq!(opened, plaintext); } @@ -65,18 +70,22 @@ fn chacha20poly1305_matches_rustcrypto_oracle() { let oracle_nonce = Array(nonce_bytes); let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto ChaCha20-Poly1305 oracle input must seal"); let mut oracle_buffer = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buffer.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto ChaCha20-Poly1305 oracle input must seal"); assert_eq!(ours, oracle_buffer); assert_eq!(tag.as_bytes(), oracle_tag.as_slice()); let typed_tag = ChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &typed_tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &typed_tag) + .expect("fresh rscrypto ChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!(ours, plaintext); } @@ -87,14 +96,15 @@ fn chacha20poly1305_rejects_modified_tag() { let cipher = ChaCha20Poly1305::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("ChaCha20-Poly1305 forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &ChaCha20Poly1305Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &ChaCha20Poly1305Tag::from_bytes(tag)) + .expect_err("modified ChaCha20-Poly1305 tag must fail authentication"); } #[cfg(feature = "diag")] @@ -110,9 +120,12 @@ fn chacha20poly1305_diag_owned_decrypt_large_inputs_match_normal_path() { let plaintext = pattern_bytes(plaintext_len, 0x51); let aad = pattern_bytes(257, 0xa7); let mut ciphertext = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut ciphertext).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut ciphertext) + .expect("diagnostic ChaCha20-Poly1305 fixture must seal"); - rscrypto::aead::diag_chacha20poly1305_decrypt_in_place_owned(&cipher, &nonce, &aad, &mut ciphertext, &tag).unwrap(); + rscrypto::aead::diag_chacha20poly1305_decrypt_in_place_owned(&cipher, &nonce, &aad, &mut ciphertext, &tag) + .expect("owned diagnostic ChaCha20-Poly1305 path must open its fixture"); assert_eq!( ciphertext, plaintext, @@ -130,10 +143,10 @@ fn chacha20poly1305_diag_owned_decrypt_zeroes_large_buffer_on_bad_tag() { let plaintext = pattern_bytes(4097, 0x6d); let aad = pattern_bytes(33, 0x95); - let mut ciphertext = plaintext.clone(); + let mut ciphertext = plaintext; let mut tag = cipher .encrypt_in_place(&nonce, &aad, &mut ciphertext) - .unwrap() + .expect("diagnostic ChaCha20-Poly1305 bad-tag fixture must seal") .to_bytes(); tag[7] ^= 0x80; @@ -145,7 +158,7 @@ fn chacha20poly1305_diag_owned_decrypt_zeroes_large_buffer_on_bad_tag() { &ChaCha20Poly1305Tag::from_bytes(tag), ); - assert!(result.is_err()); + result.expect_err("owned diagnostic ChaCha20-Poly1305 path must reject a modified tag"); assert!( ciphertext.iter().all(|&byte| byte == 0), "owned diagnostic decrypt must zero caller buffer on verification failure" @@ -189,10 +202,11 @@ fn chacha20poly1305_diag_x86_asm_encrypt_matches_owned_path() { let mut expected = plaintext.clone(); let expected_tag = - rscrypto::aead::diag_chacha20poly1305_encrypt_in_place_owned(&cipher, &nonce, &aad, &mut expected).unwrap(); + rscrypto::aead::diag_chacha20poly1305_encrypt_in_place_owned(&cipher, &nonce, &aad, &mut expected) + .expect("owned diagnostic ChaCha20-Poly1305 path must seal"); let actual_tag = actual_tag .expect("x86 asm path must apply when AVX2+BMI2 are present") - .unwrap(); + .expect("x86 assembly ChaCha20-Poly1305 path must seal"); assert_eq!( actual, expected, @@ -208,10 +222,10 @@ fn chacha20poly1305_diag_x86_asm_encrypt_matches_owned_path() { #[test] fn chacha20poly1305_rejects_wrong_tag_length() { - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 0]).is_err()); - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 15]).is_err()); - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 17]).is_err()); - assert!(ChaCha20Poly1305::tag_from_slice(&[0u8; 16]).is_ok()); + ChaCha20Poly1305::tag_from_slice(&[0u8; 0]).expect_err("empty ChaCha20-Poly1305 tag must be rejected"); + ChaCha20Poly1305::tag_from_slice(&[0u8; 15]).expect_err("short ChaCha20-Poly1305 tag must be rejected"); + ChaCha20Poly1305::tag_from_slice(&[0u8; 17]).expect_err("long ChaCha20-Poly1305 tag must be rejected"); + let _tag = ChaCha20Poly1305::tag_from_slice(&[0u8; 16]).expect("16-byte ChaCha20-Poly1305 tag must be accepted"); } #[test] @@ -237,7 +251,9 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { let aad = pattern_bytes(aad_len, 0x9b); let mut combined = vec![0u8; plaintext_len + ChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut combined).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut combined) + .expect("rscrypto combined ChaCha20-Poly1305 oracle input must seal"); let oracle_combined = oracle .encrypt( @@ -247,14 +263,16 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("RustCrypto combined ChaCha20-Poly1305 oracle input must seal"); assert_eq!( combined, oracle_combined, "combined ciphertext mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut opened = vec![0u8; plaintext_len]; - cipher.decrypt(&nonce, &aad, &oracle_combined, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &oracle_combined, &mut opened) + .expect("RustCrypto ChaCha20-Poly1305 ciphertext must open in rscrypto"); assert_eq!( opened, plaintext, "combined decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" @@ -268,14 +286,16 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("rscrypto ChaCha20-Poly1305 ciphertext must open in RustCrypto"); assert_eq!( oracle_opened, plaintext, "oracle decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut detached = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut detached).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut detached) + .expect("rscrypto detached ChaCha20-Poly1305 oracle input must seal"); assert_eq!( detached, oracle_combined[..plaintext_len], @@ -290,7 +310,7 @@ fn chacha20poly1305_boundary_and_large_inputs_match_oracle() { let typed_tag = ChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); cipher .decrypt_in_place(&nonce, &aad, &mut detached, &typed_tag) - .unwrap(); + .expect("fresh detached ChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!( detached, plaintext, "detached decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" diff --git a/tests/common/array.rs b/tests/common/array.rs new file mode 100644 index 00000000..c647a0a0 --- /dev/null +++ b/tests/common/array.rs @@ -0,0 +1,10 @@ +//! Fixed-size hexadecimal test-vector decoding. + +/// Decode a hex string into a fixed-size byte array. +/// +/// Panics if the decoded length does not equal `N`. +pub(crate) fn decode_hex_array(hex: &str) -> [u8; N] { + crate::common::decode_hex_vec(hex) + .try_into() + .expect("hex length must match the destination array") +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 005b5b10..00950740 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -3,30 +3,24 @@ /// Decode a hex string into a `Vec`. /// /// Panics on odd-length input or invalid hex characters. -#[allow(dead_code)] -pub fn decode_hex_vec(hex: &str) -> Vec { +pub(crate) fn decode_hex_vec(hex: &str) -> Vec { assert_eq!(hex.len() % 2, 0, "hex length must be even"); hex .as_bytes() .chunks_exact(2) - .map(|pair| (nibble(pair[0]) << 4) | nibble(pair[1])) + .map(|pair| { + let high = nibble(pair[0]).expect("hex input must contain only hexadecimal digits"); + let low = nibble(pair[1]).expect("hex input must contain only hexadecimal digits"); + (high << 4) | low + }) .collect() } -/// Decode a hex string into a fixed-size byte array. -/// -/// Panics if the decoded length does not equal `N`. -#[allow(dead_code)] -pub fn decode_hex_array(hex: &str) -> [u8; N] { - let v = decode_hex_vec(hex); - v.try_into().expect("hex length does not match array size") -} - -fn nibble(b: u8) -> u8 { +fn nibble(b: u8) -> Option { match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => b - b'a' + 10, - b'A'..=b'F' => b - b'A' + 10, - _ => panic!("invalid hex byte: {b:#04x}"), + b'0'..=b'9' => Some(b.strict_sub(b'0')), + b'a'..=b'f' => Some(b.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(b.strict_sub(b'A').strict_add(10)), + _ => None, } } diff --git a/tests/common_properties.rs b/tests/common_properties.rs index a5fa9b82..1f971712 100644 --- a/tests/common_properties.rs +++ b/tests/common_properties.rs @@ -25,6 +25,18 @@ use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm, Crc24OpenPgp, Cr const REF_CRC24_OPENPGP: RefCrc = RefCrc::::new(&crc::CRC_24_OPENPGP); +fn reference_u16(value: u64) -> u16 { + u16::try_from(value).expect("CRC-16 reference output must fit in 16 bits") +} + +fn reference_u32(value: u64) -> u32 { + u32::try_from(value).expect("CRC-32 reference output must fit in 32 bits") +} + +fn low_byte(value: usize) -> u8 { + value.to_le_bytes()[0] +} + // Combine Correctness Tests // // These tests prove the fundamental combine property: @@ -52,7 +64,7 @@ proptest! { let combined = Crc16Ccitt::combine(crc_a, crc_b, b.len()); // Reference: independent crc-fast implementation over full data - let expected = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data) as u16; + let expected = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -71,7 +83,7 @@ proptest! { let crc_b = Crc16Ibm::checksum(b); let combined = Crc16Ibm::combine(crc_a, crc_b, b.len()); - let expected = crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data) as u16; + let expected = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -113,7 +125,7 @@ proptest! { let crc_b = Crc32::checksum(b); let combined = Crc32::combine(crc_a, crc_b, b.len()); - let expected = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data) as u32; + let expected = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -132,7 +144,7 @@ proptest! { let crc_b = Crc32C::checksum(b); let combined = Crc32C::combine(crc_a, crc_b, b.len()); - let expected = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data) as u32; + let expected = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data)); prop_assert_eq!(combined, expected, "combine(crc(A), crc(B), len(B)) != crc(A||B) at split {}/{}", @@ -210,7 +222,7 @@ fn apply_chunking(data: &[u8], chunk_pattern: &[usize]) -> C::Outpu let end = (offset.strict_add(chunk_size)).min(data.len()); hasher.update(&data[offset..end]); offset = end; - pattern_idx = pattern_idx.strict_add(1) % chunk_pattern.len(); + pattern_idx = pattern_idx.strict_add(1).strict_rem(chunk_pattern.len()); } hasher.finalize() @@ -234,7 +246,7 @@ fn apply_chunking_vectored(data: &[u8], chunk_pattern: &[usize]) -> let end = (offset.strict_add(chunk_size)).min(data.len()); chunks.push(&data[offset..end]); offset = end; - pattern_idx = pattern_idx.strict_add(1) % chunk_pattern.len(); + pattern_idx = pattern_idx.strict_add(1).strict_rem(chunk_pattern.len()); } hasher.update_vectored(&chunks); @@ -352,7 +364,7 @@ macro_rules! test_combine_all_splits { // Test various small sizes including edge cases for size in [0, 1, 2, 3, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256] { - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(17)).collect(); + let data: Vec = (0..size).map(|i| low_byte(i).wrapping_mul(17)).collect(); for split in 0..=data.len() { let (a, b) = data.split_at(split); @@ -379,19 +391,19 @@ macro_rules! test_combine_all_splits { } test_combine_all_splits!(crc16_ccitt_combine_all_splits, Crc16Ccitt, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, data) as u16 + reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, data)) }); test_combine_all_splits!(crc16_ibm_combine_all_splits, Crc16Ibm, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc16Arc, data) as u16 + reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16Arc, data)) }); test_combine_all_splits!(crc24_openpgp_combine_all_splits, Crc24OpenPgp, |data: &[u8]| { REF_CRC24_OPENPGP.checksum(data) & 0x00FF_FFFF }); test_combine_all_splits!(crc32_ieee_combine_all_splits, Crc32, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, data) as u32 + reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, data)) }); test_combine_all_splits!(crc32c_combine_all_splits, Crc32C, |data: &[u8]| { - crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, data) as u32 + reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, data)) }); test_combine_all_splits!(crc64_xz_combine_all_splits, Crc64, |data: &[u8]| { crc_fast::checksum(CrcAlgorithm::Crc64Xz, data) @@ -409,7 +421,7 @@ macro_rules! test_chunking_edge_cases { for size in [ 0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 512, 1024, ] { - let data: Vec = (0..size).map(|i| (i as u8).wrapping_mul(23)).collect(); + let data: Vec = (0..size).map(|i| low_byte(i).wrapping_mul(23)).collect(); let expected = <$crc_type>::checksum(&data); // Byte-at-a-time diff --git a/tests/crc16_properties.rs b/tests/crc16_properties.rs index 8a869309..16f3afe7 100644 --- a/tests/crc16_properties.rs +++ b/tests/crc16_properties.rs @@ -11,20 +11,24 @@ use crc_fast::CrcAlgorithm; use proptest::prelude::*; use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm}; +fn reference_u16(value: u64) -> u16 { + u16::try_from(value).expect("CRC-16 reference output must fit in 16 bits") +} + proptest! { // Cross-validation against crc-fast-rust #[test] fn crc16_ccitt_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc16Ccitt::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data) as u16; + let reference = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, &data)); prop_assert_eq!(ours, reference); } #[test] fn crc16_ibm_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc16Ibm::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data) as u16; + let reference = reference_u16(crc_fast::checksum(CrcAlgorithm::Crc16Arc, &data)); prop_assert_eq!(ours, reference); } @@ -41,7 +45,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u16); + prop_assert_eq!(ours.finalize(), reference_u16(reference.finalize())); } #[test] @@ -57,7 +61,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u16); + prop_assert_eq!(ours.finalize(), reference_u16(reference.finalize())); } #[test] @@ -74,8 +78,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc16IbmSdlc, b); - let ref_combined = - crc_fast::checksum_combine(CrcAlgorithm::Crc16IbmSdlc, ref_crc_a, ref_crc_b, b.len() as u64) as u16; + let ref_combined = reference_u16(crc_fast::checksum_combine( + CrcAlgorithm::Crc16IbmSdlc, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } @@ -94,8 +102,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc16Arc, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc16Arc, b); - let ref_combined = - crc_fast::checksum_combine(CrcAlgorithm::Crc16Arc, ref_crc_a, ref_crc_b, b.len() as u64) as u16; + let ref_combined = reference_u16(crc_fast::checksum_combine( + CrcAlgorithm::Crc16Arc, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } diff --git a/tests/crc32_properties.rs b/tests/crc32_properties.rs index 864dd25f..4ba575b2 100644 --- a/tests/crc32_properties.rs +++ b/tests/crc32_properties.rs @@ -11,20 +11,24 @@ use crc_fast::CrcAlgorithm; use proptest::prelude::*; use rscrypto::{Checksum, ChecksumCombine, Crc32, Crc32C}; +fn reference_u32(value: u64) -> u32 { + u32::try_from(value).expect("CRC-32 reference output must fit in 32 bits") +} + proptest! { // Cross-validation against crc-fast-rust #[test] fn crc32_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc32::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data) as u32; + let reference = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, &data)); prop_assert_eq!(ours, reference); } #[test] fn crc32c_matches_crc_fast_rust(data in proptest::collection::vec(any::(), 0..=4096)) { let ours = Crc32C::checksum(&data); - let reference = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data) as u32; + let reference = reference_u32(crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, &data)); prop_assert_eq!(ours, reference); } @@ -38,7 +42,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u32); + prop_assert_eq!(ours.finalize(), reference_u32(reference.finalize())); } #[test] @@ -51,7 +55,7 @@ proptest! { reference.update(part); } - prop_assert_eq!(ours.finalize(), reference.finalize() as u32); + prop_assert_eq!(ours.finalize(), reference_u32(reference.finalize())); } #[test] @@ -65,7 +69,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc32IsoHdlc, b); - let ref_combined = crc_fast::checksum_combine(CrcAlgorithm::Crc32IsoHdlc, ref_crc_a, ref_crc_b, b.len() as u64) as u32; + let ref_combined = reference_u32(crc_fast::checksum_combine( + CrcAlgorithm::Crc32IsoHdlc, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } @@ -81,7 +90,12 @@ proptest! { let ref_crc_a = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, a); let ref_crc_b = crc_fast::checksum(CrcAlgorithm::Crc32Iscsi, b); - let ref_combined = crc_fast::checksum_combine(CrcAlgorithm::Crc32Iscsi, ref_crc_a, ref_crc_b, b.len() as u64) as u32; + let ref_combined = reference_u32(crc_fast::checksum_combine( + CrcAlgorithm::Crc32Iscsi, + ref_crc_a, + ref_crc_b, + u64::try_from(b.len()).expect("test input length must fit in u64"), + )); prop_assert_eq!(combined, ref_combined); } diff --git a/tests/cshake256_differential.rs b/tests/cshake256_differential.rs index 02cd04f6..98e26668 100644 --- a/tests/cshake256_differential.rs +++ b/tests/cshake256_differential.rs @@ -22,9 +22,19 @@ fn cshake256_ref(function_name: &[u8], customization: &[u8], data: &[u8], out: & fn decode_hex_64(value: &str) -> [u8; 64] { assert_eq!(value.len(), 128); let mut out = [0u8; 64]; - for (index, byte) in out.iter_mut().enumerate() { - let offset = index * 2; - *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + let mut digits = value.chars(); + for byte in &mut out { + let high = digits + .next() + .expect("cSHAKE vector must contain 128 hexadecimal digits"); + let low = digits.next().expect("cSHAKE vector must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("cSHAKE vector must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("cSHAKE vector must contain only hexadecimal digits"); + *byte = u8::try_from(high.strict_mul(16).strict_add(low)).expect("two hexadecimal digits must fit in one byte"); } out } @@ -59,8 +69,8 @@ proptest! { let mut hasher = Cshake128::new(&function_name, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); hasher.update(&data[i..end]); i = end; } @@ -99,8 +109,8 @@ proptest! { let mut hasher = Cshake256::new(&function_name, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); hasher.update(&data[i..end]); i = end; } diff --git a/tests/cshake256_nist_vectors.rs b/tests/cshake256_nist_vectors.rs index c9e603d4..9e8f66e8 100644 --- a/tests/cshake256_nist_vectors.rs +++ b/tests/cshake256_nist_vectors.rs @@ -6,7 +6,9 @@ mod common; use common::decode_hex_vec as decode_hex; fn ascending_bytes(len: usize) -> Vec { - (0..len).map(|byte| byte as u8).collect() + (0..len) + .map(|byte| u8::try_from(byte).expect("cSHAKE fixture byte must fit in u8")) + .collect() } #[test] diff --git a/tests/ecdsa_oracle.rs b/tests/ecdsa_oracle.rs index 4c6c14b5..46571f7d 100644 --- a/tests/ecdsa_oracle.rs +++ b/tests/ecdsa_oracle.rs @@ -23,19 +23,24 @@ fn array_from_slice(slice: &[u8]) -> [u8; N] { fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long DER length must contain a nonzero byte"); let body = &bytes[first..]; - let mut out = Vec::with_capacity(body.len() + 1); - out.push(0x80 | (body.len() as u8)); + let mut out = Vec::with_capacity(body.len().strict_add(1)); + let body_len = u8::try_from(body.len()).expect("DER length-of-length must fit in one byte"); + out.push(0x80 | body_len); out.extend_from_slice(body); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + value.len() + 4); + let capacity = value.len().strict_add(5); + let mut out = Vec::with_capacity(capacity); out.push(tag); out.extend_from_slice(&der_len(value.len())); out.extend_from_slice(value); @@ -47,7 +52,7 @@ fn spki_der(curve_oid: &[u8], sec1: &[u8]) -> Vec { algorithm.extend_from_slice(&tlv(0x06, ID_EC_PUBLIC_KEY_OID)); algorithm.extend_from_slice(&tlv(0x06, curve_oid)); - let mut bit_string = Vec::with_capacity(sec1.len() + 1); + let mut bit_string = Vec::with_capacity(sec1.len().strict_add(1)); bit_string.push(0); bit_string.extend_from_slice(sec1); @@ -75,8 +80,12 @@ fn p256_verify_accepts_rustcrypto_raw_and_der_signatures() { .expect("P-256 raw signature must parse"); let der = EcdsaP256Signature::from_der(oracle_signature.to_der().as_bytes()).expect("P-256 DER signature must parse"); - assert!(public.verify(message, &raw).is_ok()); - assert!(public.verify(message, &der).is_ok()); + public + .verify(message, &raw) + .expect("rscrypto must verify the RustCrypto P-256 raw signature"); + public + .verify(message, &der) + .expect("rscrypto must verify the RustCrypto P-256 DER signature"); } #[cfg(feature = "ecdsa-p256")] @@ -97,15 +106,14 @@ fn p256_verify_rejects_tampered_rustcrypto_signature() { bytes[17] ^= 0x40; let tampered = EcdsaP256Signature::from_bytes(bytes).expect("tampered P-256 signature scalar shape must parse"); - assert!(public.verify(message, &tampered).is_err()); - assert!( - public - .verify( - b"wrong message", - &EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap() - ) - .is_err() - ); + public + .verify(message, &tampered) + .expect_err("rscrypto must reject a tampered P-256 signature"); + let original = EcdsaP256Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("original P-256 signature must parse"); + public + .verify(b"wrong message", &original) + .expect_err("rscrypto must reject a P-256 signature for the wrong message"); } #[cfg(feature = "ecdsa-p256")] @@ -123,7 +131,9 @@ fn p256_sign_derives_oracle_public_key_and_signature_verifies_with_rustcrypto() let oracle_signature = P256OracleSignature::from_slice(rs_signature.as_bytes()).expect("P-256 oracle signature must parse"); - assert!(rs_public.verify(message, &rs_signature).is_ok()); + rs_public + .verify(message, &rs_signature) + .expect("rscrypto must verify its P-256 signature"); p256::ecdsa::signature::Verifier::verify(verifying_key, message, &oracle_signature) .expect("RustCrypto must verify rscrypto P-256 signature"); } @@ -150,7 +160,9 @@ fn p256_blinded_sign_matches_deterministic_signature_and_rustcrypto_oracle() { P256OracleSignature::from_slice(blinded.as_bytes()).expect("P-256 oracle signature must parse"); assert_eq!(deterministic, blinded); - assert!(rs_public.verify(message, &blinded).is_ok()); + rs_public + .verify(message, &blinded) + .expect("rscrypto must verify its blinded P-256 signature"); p256::ecdsa::signature::Verifier::verify(&oracle_public, message, &oracle_signature) .expect("RustCrypto must verify rscrypto blinded P-256 signature"); } @@ -173,8 +185,12 @@ fn p384_verify_accepts_rustcrypto_raw_and_der_signatures() { .expect("P-384 raw signature must parse"); let der = EcdsaP384Signature::from_der(oracle_signature.to_der().as_bytes()).expect("P-384 DER signature must parse"); - assert!(public.verify(message, &raw).is_ok()); - assert!(public.verify(message, &der).is_ok()); + public + .verify(message, &raw) + .expect("rscrypto must verify the RustCrypto P-384 raw signature"); + public + .verify(message, &der) + .expect("rscrypto must verify the RustCrypto P-384 DER signature"); } #[cfg(feature = "ecdsa-p384")] @@ -195,15 +211,14 @@ fn p384_verify_rejects_tampered_rustcrypto_signature() { bytes[29] ^= 0x10; let tampered = EcdsaP384Signature::from_bytes(bytes).expect("tampered P-384 signature scalar shape must parse"); - assert!(public.verify(message, &tampered).is_err()); - assert!( - public - .verify( - b"wrong message", - &EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())).unwrap() - ) - .is_err() - ); + public + .verify(message, &tampered) + .expect_err("rscrypto must reject a tampered P-384 signature"); + let original = EcdsaP384Signature::from_bytes(array_from_slice(oracle_signature.to_bytes().as_ref())) + .expect("original P-384 signature must parse"); + public + .verify(b"wrong message", &original) + .expect_err("rscrypto must reject a P-384 signature for the wrong message"); } #[cfg(feature = "ecdsa-p384")] @@ -221,7 +236,9 @@ fn p384_sign_derives_oracle_public_key_and_signature_verifies_with_rustcrypto() let oracle_signature = P384OracleSignature::from_slice(rs_signature.as_bytes()).expect("P-384 oracle signature must parse"); - assert!(rs_public.verify(message, &rs_signature).is_ok()); + rs_public + .verify(message, &rs_signature) + .expect("rscrypto must verify its P-384 signature"); p384::ecdsa::signature::Verifier::verify(verifying_key, message, &oracle_signature) .expect("RustCrypto must verify rscrypto P-384 signature"); } @@ -248,7 +265,9 @@ fn p384_blinded_sign_matches_deterministic_signature_and_rustcrypto_oracle() { P384OracleSignature::from_slice(blinded.as_bytes()).expect("P-384 oracle signature must parse"); assert_eq!(deterministic, blinded); - assert!(rs_public.verify(message, &blinded).is_ok()); + rs_public + .verify(message, &blinded) + .expect("rscrypto must verify its blinded P-384 signature"); p384::ecdsa::signature::Verifier::verify(&oracle_public, message, &oracle_signature) .expect("RustCrypto must verify rscrypto blinded P-384 signature"); } diff --git a/tests/ecdsa_wycheproof.rs b/tests/ecdsa_wycheproof.rs index 9de609cc..acfa75f4 100644 --- a/tests/ecdsa_wycheproof.rs +++ b/tests/ecdsa_wycheproof.rs @@ -29,9 +29,11 @@ struct Suite<'a> { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] + value + .get(name) + .expect("Wycheproof field must be present") .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof field must be a string") } fn assert_der_vectors( @@ -47,7 +49,10 @@ fn assert_der_vectors( assert_eq!(suite["schema"].as_str(), Some("ecdsa_verify_schema_v1.json")); assert_eq!( suite["numberOfTests"].as_u64(), - Some((spec.expected.valid + spec.expected.invalid) as u64) + Some( + u64::try_from(spec.expected.valid.strict_add(spec.expected.invalid)) + .expect("Wycheproof test count must fit in u64") + ) ); let groups = suite["testGroups"] @@ -70,7 +75,12 @@ fn assert_der_vectors( let signature = decode_hex_vec(field(test, "sig")); let verified = verify_der(&public, &message, &signature); - match field(test, "result") { + let disposition = field(test, "result"); + assert!( + matches!(disposition, "valid" | "invalid"), + "unsupported Wycheproof ECDSA result `{disposition}`" + ); + match disposition { "valid" => { counts.valid = counts.valid.strict_add(1); assert!( @@ -89,7 +99,7 @@ fn assert_der_vectors( field(test, "comment") ); } - other => panic!("unsupported Wycheproof ECDSA result `{other}`"), + _ => {} } } } diff --git a/tests/ed25519_oracle.rs b/tests/ed25519_oracle.rs index 4e89f2bb..238d15ab 100644 --- a/tests/ed25519_oracle.rs +++ b/tests/ed25519_oracle.rs @@ -5,7 +5,7 @@ use rscrypto::{Ed25519Keypair, Ed25519PublicKey, Ed25519SecretKey, Ed25519Signat fn patterned_message(len: usize, mul: u8, add: u8) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(mul).wrapping_add(add)) + .map(|i| i.to_le_bytes()[0].wrapping_mul(mul).wrapping_add(add)) .collect() } @@ -32,12 +32,18 @@ fn ed25519_matches_dalek_for_deterministic_cases() { assert_eq!(public.to_bytes(), verifying_key.to_bytes()); assert_eq!(ours.to_bytes(), oracle.to_bytes()); - assert!(public.verify(&message, &ours).is_ok()); + public + .verify(&message, &ours) + .expect("rscrypto signature must verify with its public key"); let oracle_public = Ed25519PublicKey::from_bytes(verifying_key.to_bytes()); let oracle_signature = Ed25519Signature::from_bytes(oracle.to_bytes()); - assert!(oracle_public.verify(&message, &oracle_signature).is_ok()); - assert!(verifying_key.verify_strict(&message, &oracle).is_ok()); + oracle_public + .verify(&message, &oracle_signature) + .expect("Dalek signature must verify through rscrypto"); + verifying_key + .verify_strict(&message, &oracle) + .expect("Dalek signature must verify through Dalek"); } } @@ -56,6 +62,10 @@ fn ed25519_and_dalek_agree_on_rejection_for_tampered_signature() { let ours = Ed25519Signature::from_bytes(signature); let oracle = DalekSignature::from_bytes(&signature); - assert!(public.verify(&message, &ours).is_err()); - assert!(verifying_key.verify_strict(&message, &oracle).is_err()); + public + .verify(&message, &ours) + .expect_err("rscrypto must reject a tampered signature"); + verifying_key + .verify_strict(&message, &oracle) + .expect_err("Dalek must reject a tampered signature"); } diff --git a/tests/ed25519_rfc8032_vectors.rs b/tests/ed25519_rfc8032_vectors.rs index 36a7d1ca..a50b66ff 100644 --- a/tests/ed25519_rfc8032_vectors.rs +++ b/tests/ed25519_rfc8032_vectors.rs @@ -12,7 +12,10 @@ struct Vector<'a> { } mod common; -use common::{decode_hex_array as decode_hex, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array as decode_hex; #[test] fn ed25519_rfc8032_vectors() { diff --git a/tests/ed25519_wycheproof.rs b/tests/ed25519_wycheproof.rs index 29c66bc7..7b395a4a 100644 --- a/tests/ed25519_wycheproof.rs +++ b/tests/ed25519_wycheproof.rs @@ -4,7 +4,10 @@ use rscrypto::{Ed25519PublicKey, Ed25519Signature}; use serde_json::Value; mod common; -use common::{decode_hex_array, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array; const ED25519: &str = include_str!("../testdata/auth/wycheproof/ed25519_test.json"); @@ -15,9 +18,11 @@ struct Counts { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] + value + .get(name) + .expect("Wycheproof field must be present") .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof field must be a string") } fn groups(suite: &Value) -> &[Value] { @@ -45,7 +50,12 @@ fn wycheproof_ed25519_verify_vectors_match_expected_results() { let message = decode_hex_vec(field(test, "msg")); let signature = decode_hex_vec(field(test, "sig")); - match field(test, "result") { + let disposition = field(test, "result"); + assert!( + matches!(disposition, "valid" | "invalid"), + "unsupported Wycheproof Ed25519 result `{disposition}`" + ); + match disposition { "valid" => { counts.valid = counts.valid.strict_add(1); let signature = Ed25519Signature::from_bytes( @@ -53,25 +63,24 @@ fn wycheproof_ed25519_verify_vectors_match_expected_results() { .try_into() .expect("valid Wycheproof Ed25519 signatures must be 64 bytes"), ); - assert!( - public.verify(&message, &signature).is_ok(), - "Wycheproof Ed25519 tcId {} rejected a valid signature", - test["tcId"] - ); + public + .verify(&message, &signature) + .expect("Wycheproof valid Ed25519 signature must verify"); } "invalid" => { counts.invalid = counts.invalid.strict_add(1); if signature.len() == Ed25519Signature::LENGTH { - let signature = Ed25519Signature::from_bytes(signature.try_into().unwrap()); - assert!( - public.verify(&message, &signature).is_err(), - "Wycheproof Ed25519 tcId {} accepted an invalid signature: {}", - test["tcId"], - field(test, "comment") + let signature = Ed25519Signature::from_bytes( + signature + .try_into() + .expect("length-checked Wycheproof Ed25519 signature must fit an array"), ); + public + .verify(&message, &signature) + .expect_err("Wycheproof invalid Ed25519 signature must be rejected"); } } - other => panic!("unsupported Wycheproof Ed25519 result `{other}`"), + _ => {} } } } diff --git a/tests/fast_hash_allocations.rs b/tests/fast_hash_allocations.rs index d36eccef..fc2cdb37 100644 --- a/tests/fast_hash_allocations.rs +++ b/tests/fast_hash_allocations.rs @@ -18,14 +18,16 @@ std::thread_local! { } fn record_allocation() { - let _ = ALLOCATIONS.try_with(|allocations| { + discard_measurement_result(ALLOCATIONS.try_with(|allocations| { let count = allocations.get(); if count != MEASUREMENT_DISABLED { allocations.set(count.strict_add(1)); } - }); + })); } +fn discard_measurement_result(_result: Result<(), std::thread::AccessError>) {} + // SAFETY: Delegating allocation to `System` because: // 1. Every operation forwards the original pointer and layout unchanged. // 2. `System` defines and upholds the global allocator contract. @@ -66,7 +68,7 @@ struct AllocationMeasurement; impl Drop for AllocationMeasurement { fn drop(&mut self) { - let _ = ALLOCATIONS.try_with(|allocations| allocations.set(MEASUREMENT_DISABLED)); + discard_measurement_result(ALLOCATIONS.try_with(|allocations| allocations.set(MEASUREMENT_DISABLED))); } } @@ -99,7 +101,8 @@ fn fast_hashers_and_preallocated_maps_hash_without_allocating() { for chunk in long_input.chunks(127) { xxh3.write(chunk); } - let _ = xxh3.finish(); + let hash = xxh3.finish(); + assert_eq!(xxh3.finish(), hash); }); let xxh3_128_direct = measure_allocations(|| { @@ -107,13 +110,15 @@ fn fast_hashers_and_preallocated_maps_hash_without_allocating() { for chunk in long_input.chunks(127) { xxh3_128.write(chunk); } - let _ = xxh3_128.finish(); + let hash = xxh3_128.finish(); + assert_eq!(xxh3_128.finish(), hash); }); let rapid_direct = measure_allocations(|| { let mut rapid = rapid_builder.build_hasher(); rapid.write(&long_input); - let _ = rapid.finish(); + let hash = rapid.finish(); + assert_eq!(rapid.finish(), hash); }); let rapid_stream_direct = measure_allocations(|| { @@ -121,17 +126,18 @@ fn fast_hashers_and_preallocated_maps_hash_without_allocating() { for chunk in long_input.chunks(127) { rapid_stream.write(chunk); } - let _ = rapid_stream.finish(); + let hash = rapid_stream.finish(); + assert_eq!(rapid_stream.finish(), hash); }); let xxh3_map_ops = measure_allocations(|| { xxh3_map.insert("allocation-free", 1); - let _ = xxh3_map.get("allocation-free"); + assert_eq!(xxh3_map.get("allocation-free"), Some(&1)); }); let rapid_map_ops = measure_allocations(|| { rapid_map.insert("allocation-free", 1); - let _ = rapid_map.get("allocation-free"); + assert_eq!(rapid_map.get("allocation-free"), Some(&1)); }); assert_eq!(xxh3_direct, 0, "XXH3 Hasher must not allocate"); diff --git a/tests/getrandom_smoke.rs b/tests/getrandom_smoke.rs index 870c9147..471dc0e7 100644 --- a/tests/getrandom_smoke.rs +++ b/tests/getrandom_smoke.rs @@ -5,13 +5,23 @@ #![cfg(feature = "getrandom")] -#[allow(unused_macros)] +#[cfg(any( + feature = "chacha20poly1305", + feature = "xchacha20poly1305", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "ascon-aead", + feature = "aegis256", + feature = "aead", + feature = "ed25519", + feature = "x25519" +))] macro_rules! random_smoke { ($name:ident, $ty:ty) => { #[test] fn $name() { - let a = <$ty>::try_random().unwrap(); - let b = <$ty>::try_random().unwrap(); + let a = <$ty>::try_random().expect("OS entropy must produce the first value"); + let b = <$ty>::try_random().expect("OS entropy must produce the second value"); // Must not be all-zero (overwhelmingly unlikely from a CSPRNG). assert!(a.as_bytes().iter().any(|&b| b != 0), "try_random() returned all zeros"); @@ -89,7 +99,7 @@ mod ed25519_rng { #[test] fn keypair_try_generate() { - let keypair = Ed25519Keypair::try_generate().unwrap(); + let keypair = Ed25519Keypair::try_generate().expect("OS entropy must generate an Ed25519 keypair"); assert!(keypair.secret_key().as_bytes().iter().any(|&b| b != 0)); } } @@ -101,7 +111,7 @@ mod x25519_rng { #[test] fn secret_key_try_generate() { - let secret = X25519SecretKey::try_generate().unwrap(); + let secret = X25519SecretKey::try_generate().expect("OS entropy must generate an X25519 secret key"); assert!(secret.as_bytes().iter().any(|&b| b != 0)); } } @@ -114,8 +124,8 @@ mod rapidhash_rng { #[test] fn random_state_uses_platform_entropy() { - let first = RapidRandomState::try_new().unwrap(); - let second = RapidRandomState::try_new().unwrap(); + let first = RapidRandomState::try_new().expect("OS entropy must initialize the first random state"); + let second = RapidRandomState::try_new().expect("OS entropy must initialize the second random state"); assert_ne!( first.hash_one(b"collection key"), second.hash_one(b"collection key"), @@ -130,7 +140,7 @@ mod ecdsa_p256_rng { #[test] fn secret_key_try_generate() { - let secret = EcdsaP256SecretKey::try_generate().unwrap(); + let secret = EcdsaP256SecretKey::try_generate().expect("OS entropy must generate a P-256 secret key"); assert!(secret.as_bytes().iter().any(|&b| b != 0)); } } @@ -141,7 +151,7 @@ mod ecdsa_p384_rng { #[test] fn secret_key_try_generate() { - let secret = EcdsaP384SecretKey::try_generate().unwrap(); + let secret = EcdsaP384SecretKey::try_generate().expect("OS entropy must generate a P-384 secret key"); assert!(secret.as_bytes().iter().any(|&b| b != 0)); } } @@ -152,9 +162,9 @@ mod mlkem_rng { #[test] fn keypair_try_generate() { - let (ek512, dk512) = MlKem512::try_generate_keypair().unwrap(); - let (ek768, dk768) = MlKem768::try_generate_keypair().unwrap(); - let (ek1024, dk1024) = MlKem1024::try_generate_keypair().unwrap(); + let (ek512, dk512) = MlKem512::try_generate_keypair().expect("OS entropy must generate an ML-KEM-512 keypair"); + let (ek768, dk768) = MlKem768::try_generate_keypair().expect("OS entropy must generate an ML-KEM-768 keypair"); + let (ek1024, dk1024) = MlKem1024::try_generate_keypair().expect("OS entropy must generate an ML-KEM-1024 keypair"); assert!(ek512.as_bytes().iter().any(|&b| b != 0)); assert!(dk512.as_bytes().iter().any(|&b| b != 0)); diff --git a/tests/hkdf_sha256_vectors.rs b/tests/hkdf_sha256_vectors.rs index d19d755c..a1af3ac8 100644 --- a/tests/hkdf_sha256_vectors.rs +++ b/tests/hkdf_sha256_vectors.rs @@ -3,10 +3,12 @@ use rscrypto::{HkdfSha256, auth::HkdfOutputLengthError}; mod common; +#[path = "common/array.rs"] +mod hex_array; use common::decode_hex_vec; fn decode_hex_array(hex: &str) -> [u8; N] { - common::decode_hex_array(&hex.replace('\n', "")) + hex_array::decode_hex_array(&hex.replace('\n', "")) } #[test] @@ -21,7 +23,9 @@ fn hkdf_sha256_rfc5869_case_1() { &decode_hex_array::<32>("077709362c2e32df0ddc3f0dc47bba6390b6c73bb50f9c3122ec844ad7c2b3e5") ); - let okm = hkdf.expand_array::<42>(&info).unwrap(); + let okm = hkdf + .expand_array::<42>(&info) + .expect("RFC 5869 HKDF-SHA-256 case 1 expansion must succeed"); assert_eq!( okm, decode_hex_array::<42>( @@ -33,7 +37,8 @@ fn hkdf_sha256_rfc5869_case_1() { #[test] fn hkdf_sha256_rfc5869_case_3() { - let okm = HkdfSha256::derive_array::<42>(b"", &[0x0b; 22], b"").unwrap(); + let okm = HkdfSha256::derive_array::<42>(b"", &[0x0b; 22], b"") + .expect("RFC 5869 HKDF-SHA-256 case 3 derivation must succeed"); assert_eq!( okm, decode_hex_array::<42>( @@ -45,7 +50,8 @@ fn hkdf_sha256_rfc5869_case_3() { #[test] fn hkdf_sha256_rejects_oversized_output() { - let mut out = vec![0u8; HkdfSha256::MAX_OUTPUT_SIZE + 1]; - let err = HkdfSha256::derive(b"salt", b"ikm", b"info", &mut out).unwrap_err(); + let mut out = vec![0u8; HkdfSha256::MAX_OUTPUT_SIZE.strict_add(1)]; + let err = + HkdfSha256::derive(b"salt", b"ikm", b"info", &mut out).expect_err("HKDF-SHA-256 must reject oversized output"); assert_eq!(err, HkdfOutputLengthError::new()); } diff --git a/tests/hkdf_sha384_vectors.rs b/tests/hkdf_sha384_vectors.rs index 53068622..0ad18327 100644 --- a/tests/hkdf_sha384_vectors.rs +++ b/tests/hkdf_sha384_vectors.rs @@ -4,7 +4,10 @@ use hkdf::Hkdf as RustCryptoHkdf; use rscrypto::{HkdfSha384, auth::HkdfOutputLengthError}; mod common; -use common::{decode_hex_array, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array; #[test] fn hkdf_sha384_case_1() { @@ -20,7 +23,9 @@ fn hkdf_sha384_case_1() { ) ); - let okm = hkdf.expand_array::<42>(&info).unwrap(); + let okm = hkdf + .expand_array::<42>(&info) + .expect("HKDF-SHA-384 case 1 expansion must succeed"); assert_eq!( okm, decode_hex_array::<42>("9b5097a86038b805309076a44b3a9f38063e25b516dcbf369f394cfab43685f748b6457763e4f0204fc5",) @@ -53,7 +58,9 @@ fn hkdf_sha384_case_2() { ) ); - let okm = hkdf.expand_array::<82>(&info).unwrap(); + let okm = hkdf + .expand_array::<82>(&info) + .expect("HKDF-SHA-384 case 2 expansion must succeed"); assert_eq!( okm, decode_hex_array::<82>( @@ -65,7 +72,7 @@ fn hkdf_sha384_case_2() { #[test] fn hkdf_sha384_case_3() { - let okm = HkdfSha384::derive_array::<42>(b"", &[0x0b; 22], b"").unwrap(); + let okm = HkdfSha384::derive_array::<42>(b"", &[0x0b; 22], b"").expect("HKDF-SHA-384 case 3 derivation must succeed"); assert_eq!( okm, decode_hex_array::<42>("c8c96e710f89b0d7990bca68bcdec8cf854062e54c73a7abc743fade9b242daacc1cea5670415b52849c",) @@ -84,8 +91,12 @@ fn hkdf_sha384_matches_rustcrypto() { let mut ours = vec![0u8; len]; let mut theirs = vec![0u8; len]; - hkdf.expand(&info, &mut ours).unwrap(); - rustcrypto.expand(&info, &mut theirs).unwrap(); + hkdf + .expand(&info, &mut ours) + .expect("rscrypto HKDF-SHA-384 differential expansion must succeed"); + rustcrypto + .expand(&info, &mut theirs) + .expect("RustCrypto HKDF-SHA-384 differential expansion must succeed"); assert_eq!(ours, theirs, "HKDF-SHA384 mismatch at output len {len}"); } @@ -98,13 +109,17 @@ fn hkdf_sha384_derive_matches_extract_then_expand() { let info = b"context"; let extracted = HkdfSha384::new(salt, ikm); - let derived = HkdfSha384::derive_array::<96>(salt, ikm, info).unwrap(); - assert_eq!(derived, extracted.expand_array::<96>(info).unwrap()); + let derived = HkdfSha384::derive_array::<96>(salt, ikm, info).expect("HKDF-SHA-384 one-shot derivation must succeed"); + let expanded = extracted + .expand_array::<96>(info) + .expect("HKDF-SHA-384 extracted-state expansion must succeed"); + assert_eq!(derived, expanded); } #[test] fn hkdf_sha384_rejects_oversized_output() { - let mut out = vec![0u8; HkdfSha384::MAX_OUTPUT_SIZE + 1]; - let err = HkdfSha384::derive(b"salt", b"ikm", b"info", &mut out).unwrap_err(); + let mut out = vec![0u8; HkdfSha384::MAX_OUTPUT_SIZE.strict_add(1)]; + let err = + HkdfSha384::derive(b"salt", b"ikm", b"info", &mut out).expect_err("HKDF-SHA-384 must reject oversized output"); assert_eq!(err, HkdfOutputLengthError::new()); } diff --git a/tests/hkdf_sha512_vectors.rs b/tests/hkdf_sha512_vectors.rs index 0ae5c5da..da71b8b6 100644 --- a/tests/hkdf_sha512_vectors.rs +++ b/tests/hkdf_sha512_vectors.rs @@ -4,7 +4,10 @@ use hkdf::Hkdf as RustCryptoHkdf; use rscrypto::{HkdfSha512, auth::HkdfOutputLengthError}; mod common; -use common::{decode_hex_array, decode_hex_vec}; +#[path = "common/array.rs"] +mod hex_array; +use common::decode_hex_vec; +use hex_array::decode_hex_array; #[test] fn hkdf_sha512_rfc5869_case_1() { @@ -21,7 +24,9 @@ fn hkdf_sha512_rfc5869_case_1() { ) ); - let okm = hkdf.expand_array::<42>(&info).unwrap(); + let okm = hkdf + .expand_array::<42>(&info) + .expect("HKDF-SHA-512 vector expansion must succeed"); assert_eq!( okm, decode_hex_array::<42>("832390086cda71fb47625bb5ceB168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb",) @@ -40,8 +45,12 @@ fn hkdf_sha512_matches_rustcrypto() { let mut ours = vec![0u8; len]; let mut theirs = vec![0u8; len]; - hkdf.expand(&info, &mut ours).unwrap(); - rustcrypto.expand(&info, &mut theirs).unwrap(); + hkdf + .expand(&info, &mut ours) + .expect("rscrypto HKDF-SHA-512 differential expansion must succeed"); + rustcrypto + .expand(&info, &mut theirs) + .expect("RustCrypto HKDF-SHA-512 differential expansion must succeed"); assert_eq!(ours, theirs, "HKDF-SHA512 mismatch at output len {len}"); } @@ -54,13 +63,18 @@ fn hkdf_sha512_derive_matches_extract_then_expand() { let info = b"context"; let extracted = HkdfSha512::new(salt, ikm); - let derived = HkdfSha512::derive_array::<128>(salt, ikm, info).unwrap(); - assert_eq!(derived, extracted.expand_array::<128>(info).unwrap()); + let derived = + HkdfSha512::derive_array::<128>(salt, ikm, info).expect("HKDF-SHA-512 one-shot derivation must succeed"); + let expanded = extracted + .expand_array::<128>(info) + .expect("HKDF-SHA-512 extracted-state expansion must succeed"); + assert_eq!(derived, expanded); } #[test] fn hkdf_sha512_rejects_oversized_output() { - let mut out = vec![0u8; HkdfSha512::MAX_OUTPUT_SIZE + 1]; - let err = HkdfSha512::derive(b"salt", b"ikm", b"info", &mut out).unwrap_err(); + let mut out = vec![0u8; HkdfSha512::MAX_OUTPUT_SIZE.strict_add(1)]; + let err = + HkdfSha512::derive(b"salt", b"ikm", b"info", &mut out).expect_err("HKDF-SHA-512 must reject oversized output"); assert_eq!(err, HkdfOutputLengthError::new()); } diff --git a/tests/hkdf_wycheproof.rs b/tests/hkdf_wycheproof.rs index da17f963..126ca489 100644 --- a/tests/hkdf_wycheproof.rs +++ b/tests/hkdf_wycheproof.rs @@ -9,7 +9,7 @@ use common::decode_hex_vec; const HKDF_SHA256: &str = include_str!("../testdata/auth/wycheproof/hkdf_sha256_test.json"); const HKDF_SHA384: &str = include_str!("../testdata/auth/wycheproof/hkdf_sha384_test.json"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] struct Counts { valid: usize, invalid: usize, @@ -18,9 +18,24 @@ struct Counts { type HkdfDerive = fn(&[u8], &[u8], &[u8], &mut [u8]) -> Result<(), HkdfOutputLengthError>; fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") +} + +enum VectorResult { + Valid, + Invalid, +} + +fn vector_result(test: &Value) -> VectorResult { + let result = match field(test, "result") { + "valid" => Some(VectorResult::Valid), + "invalid" => Some(VectorResult::Invalid), + _ => None, + }; + result.expect("HKDF Wycheproof result must be valid or invalid") } fn groups(suite: &Value) -> &[Value] { @@ -43,25 +58,22 @@ fn run_hkdf_suite(suite_json: &str, algorithm: &str, derive: HkdfDerive, expecte let ikm = decode_hex_vec(field(test, "ikm")); let salt = decode_hex_vec(field(test, "salt")); let info = decode_hex_vec(field(test, "info")); - let size = test["size"].as_u64().expect("size must be numeric") as usize; + let size = usize::try_from(test["size"].as_u64().expect("size must be numeric")) + .expect("HKDF Wycheproof size must fit in usize"); let mut okm = vec![0u8; size]; - match field(test, "result") { - "valid" => { - counts.valid += 1; + match vector_result(test) { + VectorResult::Valid => { + counts.valid = counts.valid.strict_add(1); let expected_okm = decode_hex_vec(field(test, "okm")); assert_eq!(expected_okm.len(), size, "{algorithm} tcId {tc_id} okm length mismatch"); - derive(&salt, &ikm, &info, &mut okm).unwrap_or_else(|err| panic!("{algorithm} tcId {tc_id} failed: {err}")); + derive(&salt, &ikm, &info, &mut okm).expect("known-valid HKDF Wycheproof vector must derive"); assert_eq!(okm, expected_okm, "{algorithm} tcId {tc_id} OKM mismatch"); } - "invalid" => { - counts.invalid += 1; - assert!( - derive(&salt, &ikm, &info, &mut okm).is_err(), - "{algorithm} tcId {tc_id} accepted invalid size" - ); + VectorResult::Invalid => { + counts.invalid = counts.invalid.strict_add(1); + derive(&salt, &ikm, &info, &mut okm).expect_err("HKDF must reject an invalid Wycheproof output size"); } - other => panic!("{algorithm} tcId {tc_id} has unsupported result `{other}`"), } } } diff --git a/tests/hmac_sha256_proptest.rs b/tests/hmac_sha256_proptest.rs index e747e7e6..0088c73a 100644 --- a/tests/hmac_sha256_proptest.rs +++ b/tests/hmac_sha256_proptest.rs @@ -14,7 +14,8 @@ proptest! { let ours = HmacSha256::mac(&key, &data); - let mut oracle = Hmac::::new_from_slice(&key).unwrap(); + let mut oracle = Hmac::::new_from_slice(&key) + .expect("RustCrypto HMAC-SHA-256 must accept every generated key length"); oracle.update(&data); let expected = oracle.finalize().into_bytes(); diff --git a/tests/hmac_sha256_vectors.rs b/tests/hmac_sha256_vectors.rs index 1cf2ed8d..5217c179 100644 --- a/tests/hmac_sha256_vectors.rs +++ b/tests/hmac_sha256_vectors.rs @@ -3,7 +3,9 @@ use rscrypto::{HmacSha256, HmacSha256Tag}; mod common; -use common::decode_hex_array as decode_hex; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array as decode_hex; #[test] fn hmac_sha256_rfc4231_vectors() { @@ -37,7 +39,7 @@ fn hmac_sha256_rfc4231_vectors() { actual.ct_eq(&expected).declassify(), "HMAC-SHA256 RFC 4231 vector {i} mismatch" ); - assert!(HmacSha256::verify_tag(key, data, &expected).is_ok()); + HmacSha256::verify_tag(key, data, &expected).expect("HMAC-SHA-256 must verify an RFC 4231 tag"); } } @@ -51,10 +53,7 @@ fn hmac_sha256_verify_rejects_corrupted_tag() { let mut corrupted = tag.to_bytes(); corrupted[index] ^= 0x80; let corrupted = HmacSha256Tag::from_bytes(corrupted); - assert!( - HmacSha256::verify_tag(key, data, &corrupted).is_err(), - "HMAC-SHA256 accepted a tag corrupted at byte {index}" - ); + HmacSha256::verify_tag(key, data, &corrupted).expect_err("HMAC-SHA-256 must reject a corrupted full-length tag"); } } @@ -64,14 +63,13 @@ fn hmac_sha256_verifies_rfc4231_tag_truncated_to_64_bits() { let data = b"Hi There"; let expected = decode_hex::<8>("b0344c61d8db3853"); - assert!(HmacSha256::verify_truncated_tag_64(&key, data, &expected).is_ok()); + HmacSha256::verify_truncated_tag_64(&key, data, &expected) + .expect("HMAC-SHA-256 must verify the RFC 4231 truncated tag"); for index in 0..expected.len() { let mut corrupted = expected; corrupted[index] ^= 0x80; - assert!( - HmacSha256::verify_truncated_tag_64(&key, data, &corrupted).is_err(), - "HMAC-SHA256 accepted a 64-bit tag corrupted at byte {index}" - ); + HmacSha256::verify_truncated_tag_64(&key, data, &corrupted) + .expect_err("HMAC-SHA-256 must reject a corrupted 64-bit tag"); } } diff --git a/tests/hmac_sha2_family_vectors.rs b/tests/hmac_sha2_family_vectors.rs index 4abe584b..78429ec9 100644 --- a/tests/hmac_sha2_family_vectors.rs +++ b/tests/hmac_sha2_family_vectors.rs @@ -3,7 +3,9 @@ use rscrypto::{HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, Mac}; mod common; -use common::decode_hex_array as decode_hex; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array as decode_hex; #[test] fn hmac_sha384_rfc4231_vectors() { @@ -50,7 +52,7 @@ fn hmac_sha384_rfc4231_vectors() { actual.ct_eq(&expected).declassify(), "HMAC-SHA384 RFC 4231 vector {i} mismatch" ); - assert!(HmacSha384::verify_tag(key, data, &expected).is_ok()); + HmacSha384::verify_tag(key, data, &expected).expect("HMAC-SHA-384 must verify an RFC 4231 tag"); } let truncated = HmacSha384::mac(&[0x0c; 20], b"Test With Truncation"); @@ -105,7 +107,7 @@ fn hmac_sha512_rfc4231_vectors() { actual.ct_eq(&expected).declassify(), "HMAC-SHA512 RFC 4231 vector {i} mismatch" ); - assert!(HmacSha512::verify_tag(key, data, &expected).is_ok()); + HmacSha512::verify_tag(key, data, &expected).expect("HMAC-SHA-512 must verify an RFC 4231 tag"); } let truncated = HmacSha512::mac(&[0x0c; 20], b"Test With Truncation"); @@ -125,10 +127,7 @@ fn hmac_sha384_verify_rejects_corrupted_tag_positions() { let mut corrupted = tag.to_bytes(); corrupted[index] ^= 0x80; let corrupted = HmacSha384Tag::from_bytes(corrupted); - assert!( - HmacSha384::verify_tag(key, data, &corrupted).is_err(), - "HMAC-SHA384 accepted a tag corrupted at byte {index}" - ); + HmacSha384::verify_tag(key, data, &corrupted).expect_err("HMAC-SHA-384 must reject a corrupted tag"); } } @@ -142,10 +141,7 @@ fn hmac_sha512_verify_rejects_corrupted_tag_positions() { let mut corrupted = tag.to_bytes(); corrupted[index] ^= 0x80; let corrupted = HmacSha512Tag::from_bytes(corrupted); - assert!( - HmacSha512::verify_tag(key, data, &corrupted).is_err(), - "HMAC-SHA512 accepted a tag corrupted at byte {index}" - ); + HmacSha512::verify_tag(key, data, &corrupted).expect_err("HMAC-SHA-512 must reject a corrupted tag"); } } diff --git a/tests/hmac_sha384_proptest.rs b/tests/hmac_sha384_proptest.rs index ae121f29..661cb699 100644 --- a/tests/hmac_sha384_proptest.rs +++ b/tests/hmac_sha384_proptest.rs @@ -14,7 +14,8 @@ proptest! { let ours = HmacSha384::mac(&key, &data); - let mut oracle = Hmac::::new_from_slice(&key).unwrap(); + let mut oracle = Hmac::::new_from_slice(&key) + .expect("RustCrypto HMAC-SHA-384 must accept every generated key length"); oracle.update(&data); let expected = oracle.finalize().into_bytes(); diff --git a/tests/hmac_sha3_vectors.rs b/tests/hmac_sha3_vectors.rs index 49b083df..1b570c2b 100644 --- a/tests/hmac_sha3_vectors.rs +++ b/tests/hmac_sha3_vectors.rs @@ -9,9 +9,9 @@ use sha3::Digest as _; fn pattern(len: usize, mul: u8, add: u8) -> Vec { (0..len) .map(|i| { - (i as u8) + i.to_le_bytes()[0] .wrapping_mul(mul) - .wrapping_add(((i >> 3) as u8).wrapping_add(add)) + .wrapping_add((i >> 3).to_le_bytes()[0].wrapping_add(add)) }) .collect() } @@ -71,7 +71,7 @@ macro_rules! assert_hmac_sha3 { key_len, data_len ); - assert!(<$ours>::verify_tag(&key, &data, &expected).is_ok()); + <$ours>::verify_tag(&key, &data, &expected).expect("HMAC-SHA3 must verify the independent-oracle tag"); let mut streaming = <$ours>::new(&key); for chunk in data.chunks(chunk_len) { @@ -98,7 +98,8 @@ macro_rules! assert_hmac_sha3 { if !corrupted.is_empty() { corrupted[corrupted.len() / 2] ^= 0x80; } - assert!(<$ours>::verify_tag(&key, &data, &<$tag>::from_bytes(corrupted)).is_err()); + <$ours>::verify_tag(&key, &data, &<$tag>::from_bytes(corrupted)) + .expect_err("HMAC-SHA3 must reject a corrupted tag"); } }}; } diff --git a/tests/hmac_sha512_proptest.rs b/tests/hmac_sha512_proptest.rs index 0352f94a..edad7773 100644 --- a/tests/hmac_sha512_proptest.rs +++ b/tests/hmac_sha512_proptest.rs @@ -14,7 +14,8 @@ proptest! { let ours = HmacSha512::mac(&key, &data); - let mut oracle = Hmac::::new_from_slice(&key).unwrap(); + let mut oracle = Hmac::::new_from_slice(&key) + .expect("RustCrypto HMAC-SHA-512 must accept every generated key length"); oracle.update(&data); let expected = oracle.finalize().into_bytes(); diff --git a/tests/hmac_wycheproof.rs b/tests/hmac_wycheproof.rs index 748d5575..4761854f 100644 --- a/tests/hmac_wycheproof.rs +++ b/tests/hmac_wycheproof.rs @@ -10,16 +10,31 @@ const HMAC_SHA256: &str = include_str!("../testdata/auth/wycheproof/hmac_sha256_ const HMAC_SHA384: &str = include_str!("../testdata/auth/wycheproof/hmac_sha384_test.json"); const HMAC_SHA512: &str = include_str!("../testdata/auth/wycheproof/hmac_sha512_test.json"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] struct Counts { valid: usize, invalid: usize, } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") +} + +enum VectorResult { + Valid, + Invalid, +} + +fn vector_result(test: &Value) -> VectorResult { + let result = match field(test, "result") { + "valid" => Some(VectorResult::Valid), + "invalid" => Some(VectorResult::Invalid), + _ => None, + }; + result.expect("HMAC Wycheproof result must be valid or invalid") } fn groups(suite: &Value) -> &[Value] { @@ -55,34 +70,26 @@ fn run_hmac_suite( let key = decode_hex_vec(field(test, "key")); let msg = decode_hex_vec(field(test, "msg")); let tag = decode_hex_vec(field(test, "tag")); - let tag_bytes: [u8; TAG_SIZE] = match tag.try_into() { - Ok(tag) => tag, - Err(tag) => panic!("{algorithm} tcId {tc_id} tag has wrong length: {}", tag.len()), - }; + let tag_bytes: [u8; TAG_SIZE] = tag + .try_into() + .expect("full-tag HMAC Wycheproof vector must have the algorithm tag length"); let tag = M::Tag::from(tag_bytes); - match field(test, "result") { - "valid" => { - counts.valid += 1; + match vector_result(test) { + VectorResult::Valid => { + counts.valid = counts.valid.strict_add(1); let actual = M::mac(&key, &msg); assert_eq!( >::as_ref(&actual), >::as_ref(&tag), "{algorithm} tcId {tc_id} MAC mismatch" ); - assert!( - M::verify_tag(&key, &msg, &tag).is_ok(), - "{algorithm} tcId {tc_id} verify failed" - ); + M::verify_tag(&key, &msg, &tag).expect("HMAC must verify a known-valid Wycheproof tag"); } - "invalid" => { - counts.invalid += 1; - assert!( - M::verify_tag(&key, &msg, &tag).is_err(), - "{algorithm} tcId {tc_id} accepted an invalid tag" - ); + VectorResult::Invalid => { + counts.invalid = counts.invalid.strict_add(1); + M::verify_tag(&key, &msg, &tag).expect_err("HMAC must reject a known-invalid Wycheproof tag"); } - other => panic!("{algorithm} tcId {tc_id} has unsupported result `{other}`"), } } } diff --git a/tests/kmac128_differential.rs b/tests/kmac128_differential.rs index 4a6e7750..a4014240 100644 --- a/tests/kmac128_differential.rs +++ b/tests/kmac128_differential.rs @@ -12,21 +12,37 @@ fn kmac128_ref(key: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { } fn encoded_string_len(len: usize) -> usize { - let bits = len * 8; - let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); - 1 + width + len + let bits = len.strict_mul(8); + let width_bits = usize::BITS.strict_sub(bits.leading_zeros()); + let width = usize::try_from(width_bits) + .expect("encoded-string width must fit usize") + .div_ceil(8) + .max(1); + 1usize.strict_add(width).strict_add(len) } fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { - (2 + segments.iter().map(|&len| encoded_string_len(len)).sum::()).is_multiple_of(rate) + 2usize + .strict_add(segments.iter().map(|&len| encoded_string_len(len)).sum::()) + .is_multiple_of(rate) } fn decode_hex_32(value: &str) -> [u8; 32] { assert_eq!(value.len(), 64); let mut out = [0u8; 32]; - for (index, byte) in out.iter_mut().enumerate() { - let offset = index * 2; - *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + let mut digits = value.chars(); + for byte in &mut out { + let high = digits + .next() + .expect("KMAC-128 vector must contain 64 hexadecimal digits"); + let low = digits.next().expect("KMAC-128 vector must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("KMAC-128 vector must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("KMAC-128 vector must contain only hexadecimal digits"); + *byte = u8::try_from(high.strict_mul(16).strict_add(low)).expect("two hexadecimal digits must fit in one byte"); } out } @@ -47,7 +63,7 @@ proptest! { let mut actual = vec![0u8; out_len]; Kmac128::mac_into(&key, &customization, &data, &mut actual); - prop_assert_eq!(actual, expected.clone()); + prop_assert_eq!(actual.as_slice(), expected.as_slice()); if expected.is_empty() { prop_assert!(Kmac128::verify_tag(&key, &customization, &data, &expected).is_err()); } else { @@ -77,8 +93,8 @@ proptest! { let mut kmac = Kmac128::new(&key, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); kmac.update(&data[i..end]); i = end; } diff --git a/tests/kmac128_nist_vectors.rs b/tests/kmac128_nist_vectors.rs index a9d9e2ec..7a8545e5 100644 --- a/tests/kmac128_nist_vectors.rs +++ b/tests/kmac128_nist_vectors.rs @@ -6,7 +6,9 @@ mod common; use common::decode_hex_vec as decode_hex; fn ascending_bytes(len: usize) -> Vec { - (0..len).map(|byte| byte as u8).collect() + (0..len) + .map(|byte| u8::try_from(byte).expect("NIST KMAC-128 sample index must fit in one byte")) + .collect() } fn key_bytes() -> Vec { @@ -22,7 +24,7 @@ fn kmac128_nist_sample_1_matches() { let mut actual = vec![0u8; expected.len()]; Kmac128::mac_into(&key, b"", &data, &mut actual); assert_eq!(actual, expected, "kmac128 sample 1 one-shot mismatch"); - assert!(Kmac128::verify_tag(&key, b"", &data, &expected).is_ok()); + Kmac128::verify_tag(&key, b"", &data, &expected).expect("KMAC-128 must verify NIST sample 1"); } #[test] @@ -34,7 +36,7 @@ fn kmac128_nist_sample_2_matches() { let mut actual = vec![0u8; expected.len()]; Kmac128::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac128 sample 2 one-shot mismatch"); - assert!(Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-128 must verify NIST sample 2"); } #[test] @@ -46,7 +48,7 @@ fn kmac128_nist_sample_3_matches() { let mut actual = vec![0u8; expected.len()]; Kmac128::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac128 sample 3 one-shot mismatch"); - assert!(Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-128 must verify NIST sample 3"); let mut streaming = Kmac128::new(&key, b"My Tagged Application"); streaming.update(&data[..168]); @@ -62,17 +64,13 @@ fn kmac128_verify_rejects_empty_and_corrupted_tags() { let data = ascending_bytes(200); let expected = Kmac128::mac_array::<32>(&key, b"My Tagged Application", &data); - assert!( - Kmac128::verify_tag(&key, b"My Tagged Application", &data, &[]).is_err(), - "KMAC128 must reject an empty expected tag" - ); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &[]) + .expect_err("KMAC-128 must reject an empty expected tag"); for index in [0, expected.len() / 2, expected.len() - 1] { let mut corrupted = expected; corrupted[index] ^= 0x80; - assert!( - Kmac128::verify_tag(&key, b"My Tagged Application", &data, &corrupted).is_err(), - "KMAC128 accepted a tag corrupted at byte {index}" - ); + Kmac128::verify_tag(&key, b"My Tagged Application", &data, &corrupted) + .expect_err("KMAC-128 must reject a corrupted tag"); } } diff --git a/tests/kmac256_differential.rs b/tests/kmac256_differential.rs index 707d14b3..d21b457b 100644 --- a/tests/kmac256_differential.rs +++ b/tests/kmac256_differential.rs @@ -12,21 +12,37 @@ fn kmac256_ref(key: &[u8], customization: &[u8], data: &[u8], out: &mut [u8]) { } fn encoded_string_len(len: usize) -> usize { - let bits = len * 8; - let width = ((usize::BITS - bits.leading_zeros()) as usize).div_ceil(8).max(1); - 1 + width + len + let bits = len.strict_mul(8); + let width_bits = usize::BITS.strict_sub(bits.leading_zeros()); + let width = usize::try_from(width_bits) + .expect("encoded-string width must fit usize") + .div_ceil(8) + .max(1); + 1usize.strict_add(width).strict_add(len) } fn bytepad_is_aligned(rate: usize, segments: &[usize]) -> bool { - (2 + segments.iter().map(|&len| encoded_string_len(len)).sum::()).is_multiple_of(rate) + 2usize + .strict_add(segments.iter().map(|&len| encoded_string_len(len)).sum::()) + .is_multiple_of(rate) } fn decode_hex_64(value: &str) -> [u8; 64] { assert_eq!(value.len(), 128); let mut out = [0u8; 64]; - for (index, byte) in out.iter_mut().enumerate() { - let offset = index * 2; - *byte = u8::from_str_radix(&value[offset..offset + 2], 16).unwrap(); + let mut digits = value.chars(); + for byte in &mut out { + let high = digits + .next() + .expect("KMAC-256 vector must contain 128 hexadecimal digits"); + let low = digits.next().expect("KMAC-256 vector must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("KMAC-256 vector must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("KMAC-256 vector must contain only hexadecimal digits"); + *byte = u8::try_from(high.strict_mul(16).strict_add(low)).expect("two hexadecimal digits must fit in one byte"); } out } @@ -47,7 +63,7 @@ proptest! { let mut actual = vec![0u8; out_len]; Kmac256::mac_into(&key, &customization, &data, &mut actual); - prop_assert_eq!(actual, expected.clone()); + prop_assert_eq!(actual.as_slice(), expected.as_slice()); if expected.is_empty() { prop_assert!(Kmac256::verify_tag(&key, &customization, &data, &expected).is_err()); } else { @@ -77,8 +93,8 @@ proptest! { let mut kmac = Kmac256::new(&key, &customization); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); kmac.update(&data[i..end]); i = end; } diff --git a/tests/kmac256_nist_vectors.rs b/tests/kmac256_nist_vectors.rs index 35e1d95b..87b24585 100644 --- a/tests/kmac256_nist_vectors.rs +++ b/tests/kmac256_nist_vectors.rs @@ -6,7 +6,9 @@ mod common; use common::decode_hex_vec as decode_hex; fn ascending_bytes(len: usize) -> Vec { - (0..len).map(|byte| byte as u8).collect() + (0..len) + .map(|byte| u8::try_from(byte).expect("NIST KMAC-256 sample index must fit in one byte")) + .collect() } fn key_bytes() -> Vec { @@ -25,7 +27,7 @@ fn kmac256_nist_sample_4_matches() { let mut actual = vec![0u8; expected.len()]; Kmac256::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac256 sample 4 one-shot mismatch"); - assert!(Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-256 must verify NIST sample 4"); let mut streaming = Kmac256::new(&key, b"My Tagged Application"); streaming.update(&data[..1]); @@ -47,7 +49,7 @@ fn kmac256_nist_sample_5_matches() { let mut actual = vec![0u8; expected.len()]; Kmac256::mac_into(&key, b"", &data, &mut actual); assert_eq!(actual, expected, "kmac256 sample 5 one-shot mismatch"); - assert!(Kmac256::verify_tag(&key, b"", &data, &expected).is_ok()); + Kmac256::verify_tag(&key, b"", &data, &expected).expect("KMAC-256 must verify NIST sample 5"); let mut streaming = Kmac256::new(&key, b""); streaming.update(&data[..136]); @@ -69,7 +71,7 @@ fn kmac256_nist_sample_6_matches() { let mut actual = vec![0u8; expected.len()]; Kmac256::mac_into(&key, b"My Tagged Application", &data, &mut actual); assert_eq!(actual, expected, "kmac256 sample 6 one-shot mismatch"); - assert!(Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).is_ok()); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &expected).expect("KMAC-256 must verify NIST sample 6"); let mut streaming = Kmac256::new(&key, b"My Tagged Application"); streaming.update(&data[..136]); @@ -85,23 +87,17 @@ fn kmac256_verify_rejects_empty_and_corrupted_tags() { let data = ascending_bytes(200); let expected = Kmac256::mac_array::<64>(&key, b"My Tagged Application", &data); - assert!( - Kmac256::verify_tag(&key, b"My Tagged Application", &data, &[]).is_err(), - "KMAC256 must reject an empty expected tag" - ); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &[]) + .expect_err("KMAC-256 must reject an empty expected tag"); let one_byte = [expected[0] ^ 0x80]; - assert!( - Kmac256::verify_tag(&key, b"My Tagged Application", &data, &one_byte).is_err(), - "KMAC256 accepted a corrupted one-byte tag" - ); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &one_byte) + .expect_err("KMAC-256 must reject a corrupted one-byte tag"); for index in [0, expected.len() / 2, expected.len() - 1] { let mut corrupted = expected; corrupted[index] ^= 0x80; - assert!( - Kmac256::verify_tag(&key, b"My Tagged Application", &data, &corrupted).is_err(), - "KMAC256 accepted a tag corrupted at byte {index}" - ); + Kmac256::verify_tag(&key, b"My Tagged Application", &data, &corrupted) + .expect_err("KMAC-256 must reject a corrupted full-length tag"); } } diff --git a/tests/kmac_wycheproof.rs b/tests/kmac_wycheproof.rs index 2b43d65b..210d4720 100644 --- a/tests/kmac_wycheproof.rs +++ b/tests/kmac_wycheproof.rs @@ -8,16 +8,31 @@ use common::decode_hex_vec; const KMAC256_NO_CUSTOMIZATION: &str = include_str!("../testdata/auth/wycheproof/kmac256_no_customization_test.json"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] struct Counts { valid: usize, invalid: usize, } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") +} + +enum VectorResult { + Valid, + Invalid, +} + +fn vector_result(test: &Value) -> VectorResult { + let result = match field(test, "result") { + "valid" => Some(VectorResult::Valid), + "invalid" => Some(VectorResult::Invalid), + _ => None, + }; + result.expect("KMAC Wycheproof result must be valid or invalid") } fn groups(suite: &Value) -> &[Value] { @@ -42,25 +57,20 @@ fn kmac256_no_customization_wycheproof_vectors() { let msg = decode_hex_vec(field(test, "msg")); let tag = decode_hex_vec(field(test, "tag")); - match field(test, "result") { - "valid" => { - counts.valid += 1; + match vector_result(test) { + VectorResult::Valid => { + counts.valid = counts.valid.strict_add(1); let mut actual = vec![0u8; tag.len()]; Kmac256::mac_into(&key, b"", &msg, &mut actual); assert_eq!(actual, tag, "KMAC256 tcId {tc_id} tag mismatch"); - assert!( - Kmac256::verify_tag_primitive(&key, b"", &msg, &tag).is_ok(), - "KMAC256 tcId {tc_id} verify failed" - ); + Kmac256::verify_tag_primitive(&key, b"", &msg, &tag) + .expect("KMAC256 must verify a known-valid Wycheproof tag"); } - "invalid" => { - counts.invalid += 1; - assert!( - Kmac256::verify_tag_primitive(&key, b"", &msg, &tag).is_err(), - "KMAC256 tcId {tc_id} accepted an invalid tag" - ); + VectorResult::Invalid => { + counts.invalid = counts.invalid.strict_add(1); + Kmac256::verify_tag_primitive(&key, b"", &msg, &tag) + .expect_err("KMAC256 must reject a known-invalid Wycheproof tag"); } - other => panic!("KMAC256 tcId {tc_id} has unsupported result `{other}`"), } } } @@ -77,5 +87,5 @@ fn kmac256_no_customization_wycheproof_vectors() { #[test] fn kmac256_rejects_empty_verification_tag() { - assert!(Kmac256::verify_tag(b"key", b"", b"message", b"").is_err()); + Kmac256::verify_tag(b"key", b"", b"message", b"").expect_err("KMAC256 must reject an empty verification tag"); } diff --git a/tests/migration_aws_lc_rs.rs b/tests/migration_aws_lc_rs.rs index 6c5d5254..ec6efb39 100644 --- a/tests/migration_aws_lc_rs.rs +++ b/tests/migration_aws_lc_rs.rs @@ -62,15 +62,17 @@ fn test_aws_lc_rs_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equiva aws_hkdf::Salt::new(aws_hkdf::HKDF_SHA256, salt) .extract(ikm) .expand(&[info], AwsHkdfLen(aws_okm.len())) - .unwrap() + .expect("42-byte AWS-LC HKDF expansion must be valid") .fill(&mut aws_okm) - .unwrap(); + .expect("AWS-LC HKDF output buffer must match the requested length"); let mut ours_okm = [0u8; 42]; - HkdfSha256::new(salt, ikm).expand(info, &mut ours_okm).unwrap(); + HkdfSha256::new(salt, ikm) + .expand(info, &mut ours_okm) + .expect("42-byte rscrypto HKDF expansion must be valid"); assert_eq!(ours_okm, aws_okm); - let iterations = NonZeroU32::new(600_000).unwrap(); + let iterations = NonZeroU32::new(600_000).expect("PBKDF2 iteration count must be nonzero"); let mut aws_pbkdf2 = [0u8; 32]; aws_pbkdf2::derive( aws_pbkdf2::PBKDF2_HMAC_SHA256, @@ -81,7 +83,8 @@ fn test_aws_lc_rs_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equiva ); let mut ours_pbkdf2 = [0u8; 32]; - Pbkdf2Sha256::derive_key(b"migration password", salt, iterations.get(), &mut ours_pbkdf2).unwrap(); + Pbkdf2Sha256::derive_key(b"migration password", salt, iterations.get(), &mut ours_pbkdf2) + .expect("rscrypto PBKDF2 parameters must be valid"); assert_eq!(ours_pbkdf2, aws_pbkdf2); } @@ -91,21 +94,29 @@ fn test_aws_lc_rs_aead_migration_examples_are_byte_equivalent() { let aes = Aes256Gcm::new(&Aes256GcmKey::from_bytes(KEY_32)); let nonce = Nonce96::from_bytes(NONCE_12); let mut ours_aes = vec![0u8; DATA.len() + 16]; - aes.encrypt(&nonce, AAD, DATA, &mut ours_aes).unwrap(); + aes + .encrypt(&nonce, AAD, DATA, &mut ours_aes) + .expect("rscrypto AES-256-GCM seal buffer must fit plaintext and tag"); assert_eq!(ours_aes, aws_aes); let mut opened = vec![0u8; DATA.len()]; - aes.decrypt(&nonce, AAD, &ours_aes, &mut opened).unwrap(); + aes + .decrypt(&nonce, AAD, &ours_aes, &mut opened) + .expect("freshly sealed rscrypto AES-256-GCM ciphertext must authenticate"); assert_eq!(opened, DATA); let aws_chacha = aws_aead_seal(&aws_aead::CHACHA20_POLY1305, &KEY_32, DATA); let chacha = ChaCha20Poly1305::new(&ChaCha20Poly1305Key::from_bytes(KEY_32)); let mut ours_chacha = vec![0u8; DATA.len() + 16]; - chacha.encrypt(&nonce, AAD, DATA, &mut ours_chacha).unwrap(); + chacha + .encrypt(&nonce, AAD, DATA, &mut ours_chacha) + .expect("rscrypto ChaCha20-Poly1305 seal buffer must fit plaintext and tag"); assert_eq!(ours_chacha, aws_chacha); let mut opened = vec![0u8; DATA.len()]; - chacha.decrypt(&nonce, AAD, &ours_chacha, &mut opened).unwrap(); + chacha + .decrypt(&nonce, AAD, &ours_chacha, &mut opened) + .expect("freshly sealed rscrypto ChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!(opened, DATA); } @@ -114,7 +125,8 @@ fn test_aws_lc_rs_ed25519_and_x25519_migration_examples_are_byte_equivalent() { use aws_lc_rs::signature::KeyPair as _; let seed = [0x13; 32]; - let aws_ed25519 = aws_signature::Ed25519KeyPair::from_seed_unchecked(&seed).unwrap(); + let aws_ed25519 = + aws_signature::Ed25519KeyPair::from_seed_unchecked(&seed).expect("32-byte AWS-LC Ed25519 seed must be valid"); let ours_ed25519 = Ed25519SecretKey::from_bytes(seed); let ours_public = ours_ed25519.public_key(); let ours_signature = ours_ed25519.sign(DATA); @@ -124,25 +136,36 @@ fn test_aws_lc_rs_ed25519_and_x25519_migration_examples_are_byte_equivalent() { aws_signature::UnparsedPublicKey::new(&aws_signature::ED25519, ours_public.as_bytes()) .verify(DATA, ours_signature.as_bytes()) - .unwrap(); - ours_public.verify(DATA, &ours_signature).unwrap(); + .expect("AWS-LC must verify the rscrypto Ed25519 signature"); + ours_public + .verify(DATA, &ours_signature) + .expect("rscrypto must verify its Ed25519 signature"); let alice_bytes = [0x18; 32]; let bob_bytes = [0x34; 32]; let ours_alice = X25519SecretKey::from_bytes(alice_bytes); let ours_bob_public = X25519SecretKey::from_bytes(bob_bytes).public_key(); - let ours_shared = ours_alice.diffie_hellman(&ours_bob_public).unwrap(); - - let aws_alice = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &alice_bytes).unwrap(); - let aws_bob = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &bob_bytes).unwrap(); - let aws_bob_public = aws_bob.compute_public_key().unwrap(); + let ours_shared = ours_alice + .diffie_hellman(&ours_bob_public) + .expect("fixed X25519 keys must produce a nonzero shared secret"); + + let aws_alice = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &alice_bytes) + .expect("32-byte AWS-LC Alice X25519 key must be valid"); + let aws_bob = aws_agreement::PrivateKey::from_private_key(&aws_agreement::X25519, &bob_bytes) + .expect("32-byte AWS-LC Bob X25519 key must be valid"); + let aws_bob_public = aws_bob + .compute_public_key() + .expect("AWS-LC must derive Bob's X25519 public key"); let mut aws_bob_public_bytes = [0u8; 32]; aws_bob_public_bytes.copy_from_slice(aws_bob_public.as_ref()); assert_eq!(ours_bob_public.as_bytes(), &aws_bob_public_bytes); let peer = X25519PublicKey::from_bytes(aws_bob_public_bytes); assert_eq!( - ours_alice.diffie_hellman(&peer).unwrap().as_bytes(), + ours_alice + .diffie_hellman(&peer) + .expect("AWS-LC X25519 public key must produce a nonzero rscrypto shared secret") + .as_bytes(), ours_shared.as_bytes() ); @@ -152,30 +175,32 @@ fn test_aws_lc_rs_ed25519_and_x25519_migration_examples_are_byte_equivalent() { out.copy_from_slice(bytes); Ok::<[u8; 32], ()>(out) }) - .unwrap(); + .expect("AWS-LC X25519 agreement must accept the fixed key pair"); assert_eq!(ours_shared.as_bytes(), &aws_shared); } #[test] fn test_aws_lc_rs_rsa_verify_migration_examples_accept_the_same_fixtures() { - let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("embedded RSA-3072 SPKI fixture must parse"); ours .verify_pss(RsaPssProfile::Sha256, MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PSS fixture"); ours .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PKCS1-v1_5 fixture"); aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA256, RSA3072_SPKI) .verify(MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("AWS-LC must verify the RSA-PSS fixture"); aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA256, RSA3072_SPKI) .verify(MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("AWS-LC must verify the RSA-PKCS1-v1_5 fixture"); } fn aws_aead_seal(algorithm: &'static aws_aead::Algorithm, key_bytes: &[u8], plaintext: &[u8]) -> Vec { - let key = aws_aead::LessSafeKey::new(aws_aead::UnboundKey::new(algorithm, key_bytes).unwrap()); + let key = aws_aead::LessSafeKey::new( + aws_aead::UnboundKey::new(algorithm, key_bytes).expect("AWS-LC AEAD key length must match the algorithm"), + ); let mut out = plaintext.to_vec(); key .seal_in_place_append_tag( @@ -183,6 +208,6 @@ fn aws_aead_seal(algorithm: &'static aws_aead::Algorithm, key_bytes: &[u8], plai aws_aead::Aad::from(AAD), &mut out, ) - .unwrap(); + .expect("AWS-LC AEAD sealing must accept the fixed nonce and buffer"); out } diff --git a/tests/migration_docs.rs b/tests/migration_docs.rs index ca5fd13b..cb360982 100644 --- a/tests/migration_docs.rs +++ b/tests/migration_docs.rs @@ -59,7 +59,7 @@ fn migration_docs_do_not_delegate_accuracy_to_a_validation_index() { #[test] fn migration_docs_do_not_reference_retired_rscrypto_versions() { for path in migration_markdown_files(&migration_root()) { - let text = fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + let text = fs::read_to_string(&path).expect("migration markdown file must be readable"); assert!( !text.contains("rscrypto = { version = \"0.1\""), "{} still contains an rscrypto 0.1 dependency example", @@ -81,7 +81,7 @@ fn migration_docs_do_not_reference_retired_rscrypto_versions() { #[test] fn migration_docs_local_markdown_links_resolve() { for path in migration_markdown_files(&migration_root()) { - let text = fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + let text = fs::read_to_string(&path).expect("migration markdown file must be readable"); for link in local_markdown_links(&text) { let target = link.split_once('#').map_or(link.as_str(), |(base, _)| base); if target.is_empty() { @@ -300,8 +300,8 @@ fn migration_markdown_files(root: &Path) -> Vec { } fn collect_markdown_files(dir: &Path, out: &mut Vec) { - for entry in fs::read_dir(dir).unwrap_or_else(|err| panic!("read dir {}: {err}", dir.display())) { - let entry = entry.unwrap_or_else(|err| panic!("read dir entry in {}: {err}", dir.display())); + for entry in fs::read_dir(dir).expect("migration directory must be readable") { + let entry = entry.expect("migration directory entry must be readable"); let path = entry.path(); if path.is_dir() { collect_markdown_files(&path, out); @@ -315,14 +315,11 @@ fn local_markdown_links(text: &str) -> Vec { let mut links = Vec::new(); let mut rest = text; - while let Some(start) = rest.find("](") { - rest = &rest[start + 2..]; - let Some(end) = rest.find(')') else { + while let Some((_, after_open)) = rest.split_once("](") { + let Some((target, after_close)) = after_open.split_once(')') else { break; }; - - let target = &rest[..end]; - rest = &rest[end + 1..]; + rest = after_close; if target.starts_with("http://") || target.starts_with("https://") diff --git a/tests/migration_dryoc.rs b/tests/migration_dryoc.rs index 02332061..aa3558ea 100644 --- a/tests/migration_dryoc.rs +++ b/tests/migration_dryoc.rs @@ -14,24 +14,32 @@ const KEY_64: [u8; 64] = [0x24; 64]; #[test] fn test_dryoc_blake2b_migration_examples_are_byte_equivalent() { let mut dryoc_b256 = [0u8; 32]; - crypto_generichash(&mut dryoc_b256, DATA, None).unwrap(); + crypto_generichash(&mut dryoc_b256, DATA, None).expect("dryoc BLAKE2b-256 migration hash must succeed"); assert_eq!(Blake2b256::digest(DATA), dryoc_b256); let mut dryoc_b512 = [0u8; 64]; - crypto_generichash(&mut dryoc_b512, DATA, None).unwrap(); + crypto_generichash(&mut dryoc_b512, DATA, None).expect("dryoc BLAKE2b-512 migration hash must succeed"); assert_eq!(Blake2b512::digest(DATA), dryoc_b512); let mut dryoc_keyed_b256 = [0u8; 32]; - crypto_generichash(&mut dryoc_keyed_b256, DATA, Some(&KEY_32)).unwrap(); + crypto_generichash(&mut dryoc_keyed_b256, DATA, Some(&KEY_32)) + .expect("dryoc keyed BLAKE2b-256 migration hash must succeed"); assert_eq!( - Blake2b256::keyed_digest(Blake2bKey::new(&KEY_32).unwrap(), DATA), + Blake2b256::keyed_digest( + Blake2bKey::new(&KEY_32).expect("32-byte BLAKE2b migration key must be valid"), + DATA, + ), dryoc_keyed_b256 ); let mut dryoc_keyed_b512 = [0u8; 64]; - crypto_generichash(&mut dryoc_keyed_b512, DATA, Some(&KEY_64)).unwrap(); + crypto_generichash(&mut dryoc_keyed_b512, DATA, Some(&KEY_64)) + .expect("dryoc keyed BLAKE2b-512 migration hash must succeed"); assert_eq!( - Blake2b512::keyed_digest(Blake2bKey::new(&KEY_64).unwrap(), DATA), + Blake2b512::keyed_digest( + Blake2bKey::new(&KEY_64).expect("64-byte BLAKE2b migration key must be valid"), + DATA, + ), dryoc_keyed_b512 ); } @@ -41,7 +49,8 @@ fn test_dryoc_ed25519_migration_examples_are_byte_equivalent() { let seed = [0x13; 32]; let (dryoc_public, dryoc_secret) = crypto_sign_seed_keypair(&seed); let mut dryoc_signature = [0u8; 64]; - crypto_sign_detached(&mut dryoc_signature, DATA, &dryoc_secret).unwrap(); + crypto_sign_detached(&mut dryoc_signature, DATA, &dryoc_secret) + .expect("dryoc Ed25519 migration signing must succeed"); let ours = Ed25519SecretKey::from_bytes(seed); let ours_public = ours.public_key(); @@ -50,8 +59,11 @@ fn test_dryoc_ed25519_migration_examples_are_byte_equivalent() { assert_eq!(ours_public.as_bytes(), &dryoc_public); assert_eq!(ours_signature.as_bytes(), &dryoc_signature); - crypto_sign_verify_detached(ours_signature.as_bytes(), DATA, &dryoc_public).unwrap(); - ours_public.verify(DATA, &ours_signature).unwrap(); + crypto_sign_verify_detached(ours_signature.as_bytes(), DATA, &dryoc_public) + .expect("dryoc must verify the rscrypto Ed25519 migration signature"); + ours_public + .verify(DATA, &ours_signature) + .expect("rscrypto must verify its Ed25519 migration signature"); } #[test] @@ -61,13 +73,16 @@ fn test_dryoc_x25519_migration_examples_are_byte_equivalent() { let ours_alice = X25519SecretKey::from_bytes(alice_bytes); let ours_bob_public = X25519SecretKey::from_bytes(bob_bytes).public_key(); - let ours_shared = ours_alice.diffie_hellman(&ours_bob_public).unwrap(); + let ours_shared = ours_alice + .diffie_hellman(&ours_bob_public) + .expect("rscrypto X25519 migration exchange must produce a nonzero secret"); let mut dryoc_bob_public = [0u8; 32]; crypto_scalarmult_base(&mut dryoc_bob_public, &bob_bytes); assert_eq!(ours_bob_public.as_bytes(), &dryoc_bob_public); let mut dryoc_shared = [0u8; 32]; - crypto_scalarmult(&mut dryoc_shared, &alice_bytes, &dryoc_bob_public); + crypto_scalarmult(&mut dryoc_shared, &alice_bytes, &dryoc_bob_public) + .expect("dryoc X25519 migration exchange must produce a nonzero secret"); assert_eq!(ours_shared.as_bytes(), &dryoc_shared); } diff --git a/tests/migration_ring.rs b/tests/migration_ring.rs index a2b51d44..70edbb04 100644 --- a/tests/migration_ring.rs +++ b/tests/migration_ring.rs @@ -53,15 +53,17 @@ fn test_ring_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equivalent( ring_hkdf::Salt::new(ring_hkdf::HKDF_SHA256, salt) .extract(ikm) .expand(&[info], RingHkdfLen(ring_okm.len())) - .unwrap() + .expect("ring HKDF must accept the migration output length") .fill(&mut ring_okm) - .unwrap(); + .expect("ring HKDF migration expansion must succeed"); let mut ours_okm = [0u8; 42]; - HkdfSha256::new(salt, ikm).expand(info, &mut ours_okm).unwrap(); + HkdfSha256::new(salt, ikm) + .expand(info, &mut ours_okm) + .expect("rscrypto HKDF migration expansion must succeed"); assert_eq!(ours_okm, ring_okm); - let iterations = NonZeroU32::new(600_000).unwrap(); + let iterations = NonZeroU32::new(600_000).expect("migration iteration count must be nonzero"); let mut ring_pbkdf2 = [0u8; 32]; ring_pbkdf2::derive( ring_pbkdf2::PBKDF2_HMAC_SHA256, @@ -72,7 +74,8 @@ fn test_ring_digest_hmac_hkdf_and_pbkdf2_migration_examples_are_byte_equivalent( ); let mut ours_pbkdf2 = [0u8; 32]; - Pbkdf2Sha256::derive_key(b"ring migration password", salt, iterations.get(), &mut ours_pbkdf2).unwrap(); + Pbkdf2Sha256::derive_key(b"ring migration password", salt, iterations.get(), &mut ours_pbkdf2) + .expect("rscrypto PBKDF2 migration derivation must succeed"); assert_eq!(ours_pbkdf2, ring_pbkdf2); } @@ -82,13 +85,17 @@ fn test_ring_aead_migration_examples_are_byte_equivalent() { let aes = Aes256Gcm::new(&Aes256GcmKey::from_bytes(KEY_32)); let nonce = Nonce96::from_bytes(NONCE_12); let mut ours_aes = vec![0u8; DATA.len() + 16]; - aes.encrypt(&nonce, AAD, DATA, &mut ours_aes).unwrap(); + aes + .encrypt(&nonce, AAD, DATA, &mut ours_aes) + .expect("rscrypto AES-GCM migration encryption must succeed"); assert_eq!(ours_aes, ring_aes); let ring_chacha = ring_aead_seal(&ring_aead::CHACHA20_POLY1305, &KEY_32, DATA); let chacha = ChaCha20Poly1305::new(&ChaCha20Poly1305Key::from_bytes(KEY_32)); let mut ours_chacha = vec![0u8; DATA.len() + 16]; - chacha.encrypt(&nonce, AAD, DATA, &mut ours_chacha).unwrap(); + chacha + .encrypt(&nonce, AAD, DATA, &mut ours_chacha) + .expect("rscrypto ChaCha20-Poly1305 migration encryption must succeed"); assert_eq!(ours_chacha, ring_chacha); } @@ -97,7 +104,8 @@ fn test_ring_ed25519_and_rsa_verify_migration_examples_are_compatible() { use ring::signature::KeyPair as _; let seed = [0x13; 32]; - let ring_ed25519 = ring::signature::Ed25519KeyPair::from_seed_unchecked(&seed).unwrap(); + let ring_ed25519 = + ring::signature::Ed25519KeyPair::from_seed_unchecked(&seed).expect("ring must accept the Ed25519 migration seed"); let ours_ed25519 = Ed25519SecretKey::from_bytes(seed); let ours_public = ours_ed25519.public_key(); let ours_signature = ours_ed25519.sign(DATA); @@ -107,28 +115,32 @@ fn test_ring_ed25519_and_rsa_verify_migration_examples_are_compatible() { ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, ours_public.as_bytes()) .verify(DATA, ours_signature.as_bytes()) - .unwrap(); - ours_public.verify(DATA, &ours_signature).unwrap(); + .expect("ring must verify the rscrypto Ed25519 migration signature"); + ours_public + .verify(DATA, &ours_signature) + .expect("rscrypto must verify its Ed25519 migration signature"); - let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let ours = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("RSA migration SPKI fixture must parse"); let pkcs1 = ours.to_pkcs1_der(); ours .verify_pss(RsaPssProfile::Sha256, MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PSS migration fixture"); ours .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("rscrypto must verify the RSA-PKCS1-v1_5 migration fixture"); ring::signature::UnparsedPublicKey::new(&ring::signature::RSA_PSS_2048_8192_SHA256, &pkcs1) .verify(MESSAGE_PSS, RSA3072_PSS_SHA256) - .unwrap(); + .expect("ring must verify the RSA-PSS migration fixture"); ring::signature::UnparsedPublicKey::new(&ring::signature::RSA_PKCS1_2048_8192_SHA256, &pkcs1) .verify(MESSAGE_PKCS1V15, RSA3072_PKCS1V15_SHA256) - .unwrap(); + .expect("ring must verify the RSA-PKCS1-v1_5 migration fixture"); } fn ring_aead_seal(algorithm: &'static ring_aead::Algorithm, key_bytes: &[u8], plaintext: &[u8]) -> Vec { - let key = ring_aead::LessSafeKey::new(ring_aead::UnboundKey::new(algorithm, key_bytes).unwrap()); + let unbound_key = + ring_aead::UnboundKey::new(algorithm, key_bytes).expect("ring must accept the migration AEAD key length"); + let key = ring_aead::LessSafeKey::new(unbound_key); let mut out = plaintext.to_vec(); key .seal_in_place_append_tag( @@ -136,6 +148,6 @@ fn ring_aead_seal(algorithm: &'static ring_aead::Algorithm, key_bytes: &[u8], pl ring_aead::Aad::from(AAD), &mut out, ) - .unwrap(); + .expect("ring migration AEAD encryption must succeed"); out } diff --git a/tests/mlkem_acvp.rs b/tests/mlkem_acvp.rs index 9d7686c2..de0d781e 100644 --- a/tests/mlkem_acvp.rs +++ b/tests/mlkem_acvp.rs @@ -15,28 +15,44 @@ const ACVP_ENCAP_DECAP_PROMPT: &str = include_str!("vectors/mlkem_acvp_encapdeca const ACVP_ENCAP_DECAP_EXPECTED: &str = include_str!("vectors/mlkem_acvp_encapdecap_fips203_expected.json"); fn parse_acvp(json: &str) -> Value { - serde_json::from_str(json).unwrap() + serde_json::from_str(json).expect("embedded ACVP fixture must contain valid JSON") } fn string_field<'a>(value: &'a Value, field: &str) -> &'a str { value .get(field) .and_then(Value::as_str) - .unwrap_or_else(|| panic!("missing ACVP string field `{field}`")) + .expect("ACVP string field must exist and contain a string") } fn bool_field(value: &Value, field: &str) -> bool { value .get(field) .and_then(Value::as_bool) - .unwrap_or_else(|| panic!("missing ACVP bool field `{field}`")) + .expect("ACVP bool field must exist and contain a boolean") } fn u64_field(value: &Value, field: &str) -> u64 { value .get(field) .and_then(Value::as_u64) - .unwrap_or_else(|| panic!("missing ACVP integer field `{field}`")) + .expect("ACVP integer field must exist and contain an unsigned integer") +} + +enum ParameterSet { + MlKem512, + MlKem768, + MlKem1024, +} + +fn parameter_set(group: &Value) -> ParameterSet { + let parameter_set = match string_field(group, "parameterSet") { + "ML-KEM-512" => Some(ParameterSet::MlKem512), + "ML-KEM-768" => Some(ParameterSet::MlKem768), + "ML-KEM-1024" => Some(ParameterSet::MlKem1024), + _ => None, + }; + parameter_set.expect("ACVP parameterSet must name a supported ML-KEM profile") } fn test_groups(vectors: &Value) -> &[Value] { @@ -59,22 +75,22 @@ fn group_by_id(vectors: &Value, tg_id: u64) -> &Value { test_groups(vectors) .iter() .find(|group| u64_field(group, "tgId") == tg_id) - .unwrap_or_else(|| panic!("missing ACVP expected group tgId {tg_id}")) + .expect("ACVP expected group must contain every prompt tgId") } fn case_by_id(group: &Value, tc_id: u64) -> &Value { test_cases(group) .iter() .find(|test_case| u64_field(test_case, "tcId") == tc_id) - .unwrap_or_else(|| panic!("missing ACVP expected tcId {tc_id}")) + .expect("ACVP expected group must contain every prompt tcId") } -fn hex_nibble(byte: u8) -> u8 { +fn hex_nibble(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex byte {byte:#x}"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -83,7 +99,9 @@ fn decode_hex(hex: &str) -> Vec { let mut out = Vec::with_capacity(hex.len() / 2); for pair in hex.as_bytes().chunks_exact(2) { - out.push((hex_nibble(pair[0]) << 4) | hex_nibble(pair[1])); + let high = hex_nibble(pair[0]).expect("ACVP fixture must contain only hexadecimal digits"); + let low = hex_nibble(pair[1]).expect("ACVP fixture must contain only hexadecimal digits"); + out.push((high << 4) | low); } out } @@ -117,7 +135,7 @@ macro_rules! assert_keygen_case { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("ML-KEM ACVP key generation must succeed"); let expected_encapsulation_key = array_from_hex::<{ <$profile>::ENCAPSULATION_KEY_SIZE }>(string_field($expected, "ek")); @@ -145,14 +163,15 @@ macro_rules! assert_encapsulation_case { ($profile:ty, $encapsulation_key:ty, $parameter_set:literal, $prompt:expr, $expected:expr) => {{ let tc_id = u64_field($prompt, "tcId"); let encapsulation_key_bytes = decode_hex(string_field($prompt, "ek")); - let encapsulation_key = <$encapsulation_key>::try_from_slice(&encapsulation_key_bytes).unwrap(); + let encapsulation_key = <$encapsulation_key>::try_from_slice(&encapsulation_key_bytes) + .expect("ACVP encapsulation key must have a valid encoding"); let random = array_from_hex::<{ <$profile>::ENCAPSULATION_RANDOM_SIZE }>(string_field($prompt, "m")); let (ciphertext, shared_secret) = <$profile>::encapsulate(&encapsulation_key, |out| { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("ML-KEM ACVP encapsulation must succeed"); let expected_ciphertext = array_from_hex::<{ <$profile>::CIPHERTEXT_SIZE }>(string_field($expected, "c")); let expected_shared_secret = array_from_hex::<{ <$profile>::SHARED_SECRET_SIZE }>(string_field($expected, "k")); @@ -173,9 +192,12 @@ macro_rules! assert_decapsulation_case { let tc_id = u64_field($prompt, "tcId"); let decapsulation_key_bytes = decode_hex(string_field($prompt, "dk")); let ciphertext_bytes = decode_hex(string_field($prompt, "c")); - let decapsulation_key = <$decapsulation_key>::try_from_slice(&decapsulation_key_bytes).unwrap(); - let ciphertext = <$ciphertext>::try_from_slice(&ciphertext_bytes).unwrap(); - let shared_secret = <$profile>::decapsulate(&decapsulation_key, &ciphertext).unwrap(); + let decapsulation_key = <$decapsulation_key>::try_from_slice(&decapsulation_key_bytes) + .expect("ACVP decapsulation key must have a valid encoding"); + let ciphertext = + <$ciphertext>::try_from_slice(&ciphertext_bytes).expect("ACVP ciphertext must have a valid encoding"); + let shared_secret = + <$profile>::decapsulate(&decapsulation_key, &ciphertext).expect("ML-KEM ACVP decapsulation must succeed"); let expected_shared_secret = array_from_hex::<{ <$profile>::SHARED_SECRET_SIZE }>(string_field($expected, "k")); assert_bytes_eq( @@ -215,11 +237,10 @@ fn mlkem_matches_acvp_keygen_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => assert_keygen_case!(MlKem512, "ML-KEM-512", prompt_case, expected_case), - "ML-KEM-768" => assert_keygen_case!(MlKem768, "ML-KEM-768", prompt_case, expected_case), - "ML-KEM-1024" => assert_keygen_case!(MlKem1024, "ML-KEM-1024", prompt_case, expected_case), - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => assert_keygen_case!(MlKem512, "ML-KEM-512", prompt_case, expected_case), + ParameterSet::MlKem768 => assert_keygen_case!(MlKem768, "ML-KEM-768", prompt_case, expected_case), + ParameterSet::MlKem1024 => assert_keygen_case!(MlKem1024, "ML-KEM-1024", prompt_case, expected_case), } } } @@ -240,8 +261,8 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => { + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => { assert_encapsulation_case!( MlKem512, MlKem512EncapsulationKey, @@ -250,7 +271,7 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { expected_case ) } - "ML-KEM-768" => { + ParameterSet::MlKem768 => { assert_encapsulation_case!( MlKem768, MlKem768EncapsulationKey, @@ -259,7 +280,7 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { expected_case ) } - "ML-KEM-1024" => { + ParameterSet::MlKem1024 => { assert_encapsulation_case!( MlKem1024, MlKem1024EncapsulationKey, @@ -268,7 +289,6 @@ fn mlkem_matches_acvp_encapsulation_fips203_vectors() { expected_case ) } - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } @@ -289,8 +309,8 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => assert_decapsulation_case!( + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => assert_decapsulation_case!( MlKem512, MlKem512DecapsulationKey, MlKem512Ciphertext, @@ -298,7 +318,7 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { prompt_case, expected_case ), - "ML-KEM-768" => assert_decapsulation_case!( + ParameterSet::MlKem768 => assert_decapsulation_case!( MlKem768, MlKem768DecapsulationKey, MlKem768Ciphertext, @@ -306,7 +326,7 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { prompt_case, expected_case ), - "ML-KEM-1024" => assert_decapsulation_case!( + ParameterSet::MlKem1024 => assert_decapsulation_case!( MlKem1024, MlKem1024DecapsulationKey, MlKem1024Ciphertext, @@ -314,7 +334,6 @@ fn mlkem_matches_acvp_decapsulation_fips203_vectors() { prompt_case, expected_case ), - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } @@ -335,14 +354,14 @@ fn mlkem_matches_acvp_decapsulation_key_check_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => { + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => { assert_key_check_case!(MlKem512DecapsulationKey, "ML-KEM-512", prompt_case, expected_case, "dk") } - "ML-KEM-768" => { + ParameterSet::MlKem768 => { assert_key_check_case!(MlKem768DecapsulationKey, "ML-KEM-768", prompt_case, expected_case, "dk") } - "ML-KEM-1024" => { + ParameterSet::MlKem1024 => { assert_key_check_case!( MlKem1024DecapsulationKey, "ML-KEM-1024", @@ -351,7 +370,6 @@ fn mlkem_matches_acvp_decapsulation_key_check_fips203_vectors() { "dk" ) } - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } @@ -372,14 +390,14 @@ fn mlkem_matches_acvp_encapsulation_key_check_fips203_vectors() { for prompt_case in test_cases(prompt_group) { let expected_case = case_by_id(expected_group, u64_field(prompt_case, "tcId")); - match string_field(prompt_group, "parameterSet") { - "ML-KEM-512" => { + match parameter_set(prompt_group) { + ParameterSet::MlKem512 => { assert_key_check_case!(MlKem512EncapsulationKey, "ML-KEM-512", prompt_case, expected_case, "ek") } - "ML-KEM-768" => { + ParameterSet::MlKem768 => { assert_key_check_case!(MlKem768EncapsulationKey, "ML-KEM-768", prompt_case, expected_case, "ek") } - "ML-KEM-1024" => { + ParameterSet::MlKem1024 => { assert_key_check_case!( MlKem1024EncapsulationKey, "ML-KEM-1024", @@ -388,7 +406,6 @@ fn mlkem_matches_acvp_encapsulation_key_check_fips203_vectors() { "ek" ) } - parameter_set => panic!("unsupported ACVP ML-KEM parameter set `{parameter_set}`"), } } } diff --git a/tests/mlkem_ops.rs b/tests/mlkem_ops.rs index 3b0adcf3..cc70ea6f 100644 --- a/tests/mlkem_ops.rs +++ b/tests/mlkem_ops.rs @@ -11,7 +11,8 @@ use rscrypto::{ fn deterministic_bytes(offset: u8) -> [u8; N] { let mut out = [0u8; N]; for (i, byte) in out.iter_mut().enumerate() { - *byte = offset.wrapping_add(i as u8); + let i = u8::try_from(i).expect("ML-KEM deterministic entropy index must fit in u8"); + *byte = offset.wrapping_add(i); } out } diff --git a/tests/mlkem_properties.rs b/tests/mlkem_properties.rs index 9c3d1f1d..5f0e34c4 100644 --- a/tests/mlkem_properties.rs +++ b/tests/mlkem_properties.rs @@ -44,7 +44,7 @@ fn mlkem512_matches_fips203_for_reduced_feature_ci_seed() { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM-512 key generation must succeed"); let (fips_ek, fips_dk) = fips_mlkem512::KG::keygen_from_seed(d, z); assert_eq!(*ek.as_bytes(), fips_ek.clone().into_bytes()); @@ -54,7 +54,7 @@ fn mlkem512_matches_fips203_for_reduced_feature_ci_seed() { out.copy_from_slice(&m); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM-512 encapsulation must succeed"); let (fips_shared_secret, fips_ciphertext) = fips_ek.encaps_from_seed(&m); assert_eq!(*ciphertext.as_bytes(), fips_ciphertext.clone().into_bytes()); @@ -63,8 +63,10 @@ fn mlkem512_matches_fips203_for_reduced_feature_ci_seed() { fips_shared_secret.into_bytes() ); - let decapsulated = MlKem512::decapsulate(&dk, &ciphertext).unwrap(); - let fips_decapsulated = fips_dk.try_decaps(&fips_ciphertext).unwrap(); + let decapsulated = MlKem512::decapsulate(&dk, &ciphertext).expect("generated ML-KEM-512 ciphertext must decapsulate"); + let fips_decapsulated = fips_dk + .try_decaps(&fips_ciphertext) + .expect("FIPS 203 ML-KEM-512 ciphertext must decapsulate"); assert_eq!(*decapsulated.expose_secret().as_bytes(), fips_decapsulated.into_bytes()); } @@ -94,7 +96,7 @@ macro_rules! mlkem_profile_properties { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM key generation must succeed"); let (fips_ek, fips_dk) = $fips::KG::keygen_from_seed(d, z); prop_assert_eq!(*ek.as_bytes(), fips_ek.clone().into_bytes()); @@ -104,14 +106,17 @@ macro_rules! mlkem_profile_properties { out.copy_from_slice(&m); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM encapsulation must succeed"); let (fips_shared_secret, fips_ciphertext) = fips_ek.encaps_from_seed(&m); prop_assert_eq!(*ciphertext.as_bytes(), fips_ciphertext.clone().into_bytes()); prop_assert_eq!(*shared_secret.expose_secret().as_bytes(), fips_shared_secret.into_bytes()); - let decapsulated = <$profile>::decapsulate(&dk, &ciphertext).unwrap(); - let fips_decapsulated = fips_dk.try_decaps(&fips_ciphertext).unwrap(); + let decapsulated = <$profile>::decapsulate(&dk, &ciphertext) + .expect("generated ML-KEM ciphertext must decapsulate"); + let fips_decapsulated = fips_dk + .try_decaps(&fips_ciphertext) + .expect("FIPS 203 ML-KEM ciphertext must decapsulate"); prop_assert_eq!(*decapsulated.expose_secret().as_bytes(), fips_decapsulated.into_bytes()); } @@ -128,16 +133,17 @@ macro_rules! mlkem_profile_properties { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM key generation must succeed"); let (ciphertext, encapsulated) = <$profile>::encapsulate(&ek, |out| { out.copy_from_slice(&m); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("deterministic ML-KEM encapsulation must succeed"); let mut modified = ciphertext.to_bytes(); modified[byte_idx] ^= 1u8 << bit_idx; - let rejected = <$profile>::decapsulate(&dk, &<$ciphertext>::from_bytes(modified)).unwrap(); + let rejected = <$profile>::decapsulate(&dk, &<$ciphertext>::from_bytes(modified)) + .expect("modified ML-KEM ciphertext must use implicit rejection"); prop_assert!( !encapsulated.ct_eq(&rejected).declassify(), @@ -150,26 +156,68 @@ macro_rules! mlkem_profile_properties { fn $slice_parsers(bytes in prop::collection::vec(any::(), 0..=3400)) { if bytes.len() != <$profile>::ENCAPSULATION_KEY_SIZE { prop_assert_eq!( - <$encapsulation_key>::try_from_slice(&bytes).unwrap_err(), + <$encapsulation_key>::try_from_slice(&bytes) + .expect_err("wrong-length ML-KEM encapsulation key must be rejected"), MlKemError::InvalidEncapsulationKey ); } else { - let _ = <$encapsulation_key>::try_from_slice(&bytes); + let mut raw = [0u8; <$profile>::ENCAPSULATION_KEY_SIZE]; + raw.copy_from_slice(&bytes); + let direct = <$encapsulation_key>::from_bytes(raw); + match direct.validate() { + Ok(()) => { + let parsed = <$encapsulation_key>::try_from_slice(&bytes) + .expect("canonical ML-KEM encapsulation key must parse"); + prop_assert_eq!(parsed.as_ref(), bytes.as_slice()); + } + Err(expected) => { + prop_assert_eq!( + <$encapsulation_key>::try_from_slice(&bytes) + .expect_err("noncanonical ML-KEM encapsulation key must be rejected"), + expected + ); + } + } } if bytes.len() != <$profile>::DECAPSULATION_KEY_SIZE { prop_assert_eq!( - <$decapsulation_key>::try_from_slice(&bytes).unwrap_err(), + <$decapsulation_key>::try_from_slice(&bytes) + .expect_err("wrong-length ML-KEM decapsulation key must be rejected"), MlKemError::InvalidDecapsulationKey ); } else { - let _ = <$decapsulation_key>::try_from_slice(&bytes); + let mut raw = [0u8; <$profile>::DECAPSULATION_KEY_SIZE]; + raw.copy_from_slice(&bytes); + let direct = <$decapsulation_key>::from_bytes(raw); + match direct.validate() { + Ok(()) => { + let parsed = <$decapsulation_key>::try_from_slice(&bytes) + .expect("valid ML-KEM decapsulation key must parse"); + prop_assert_eq!(parsed.as_ref(), bytes.as_slice()); + } + Err(expected) => { + prop_assert_eq!( + <$decapsulation_key>::try_from_slice(&bytes) + .expect_err("invalid ML-KEM decapsulation key must be rejected"), + expected + ); + } + } } if bytes.len() != <$profile>::CIPHERTEXT_SIZE { - prop_assert_eq!(<$ciphertext>::try_from_slice(&bytes).unwrap_err(), MlKemError::InvalidCiphertext); + prop_assert_eq!( + <$ciphertext>::try_from_slice(&bytes) + .expect_err("wrong-length ML-KEM ciphertext must be rejected"), + MlKemError::InvalidCiphertext + ); } else { - <$ciphertext>::try_from_slice(&bytes).unwrap().validate().unwrap(); + let ciphertext = <$ciphertext>::try_from_slice(&bytes) + .expect("correct-length ML-KEM ciphertext must parse"); + ciphertext + .validate() + .expect("all correctly sized ML-KEM ciphertexts are canonical"); } } } diff --git a/tests/owned_equality.rs b/tests/owned_equality.rs index aadfa205..1029d46d 100644 --- a/tests/owned_equality.rs +++ b/tests/owned_equality.rs @@ -1,3 +1,4 @@ +#[cfg(any(feature = "aes-gcm", feature = "hmac", feature = "x25519"))] fn assert_owned_equality(make: impl Fn([u8; N]) -> T, compare: impl Fn(&T, &T) -> bool) { let bytes = [0x5a; N]; assert!(compare(&make(bytes), &make(bytes))); diff --git a/tests/pbkdf2_differential.rs b/tests/pbkdf2_differential.rs index d2dbb3f6..eefc1bac 100644 --- a/tests/pbkdf2_differential.rs +++ b/tests/pbkdf2_differential.rs @@ -32,21 +32,27 @@ proptest! { oracle_sha256(&password, &salt, iterations, &mut expected); let mut oneshot = vec![0u8; out_len]; - Pbkdf2Sha256::derive_key_primitive(&password, &salt, iterations, &mut oneshot).unwrap(); + Pbkdf2Sha256::derive_key_primitive(&password, &salt, iterations, &mut oneshot) + .expect("PBKDF2-SHA-256 one-shot differential derivation must succeed"); let state = Pbkdf2Sha256::new(&password); let mut reused = vec![0u8; out_len]; - state.derive(&salt, iterations, &mut reused).unwrap(); + state.derive(&salt, iterations, &mut reused) + .expect("PBKDF2-SHA-256 state-reuse differential derivation must succeed"); prop_assert_eq!(oneshot, expected.as_slice()); prop_assert_eq!(reused, expected.as_slice()); - prop_assert!(state.verify_primitive(&salt, iterations, &expected).is_ok()); - prop_assert!(Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &expected).is_ok()); + state.verify_primitive(&salt, iterations, &expected) + .expect("PBKDF2-SHA-256 state must verify the oracle output"); + Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &expected) + .expect("PBKDF2-SHA-256 one-shot API must verify the oracle output"); let mut wrong = expected.clone(); wrong[0] ^= 1; - prop_assert!(state.verify_primitive(&salt, iterations, &wrong).is_err()); - prop_assert!(Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &wrong).is_err()); + state.verify_primitive(&salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-256 state must reject a corrupted output"); + Pbkdf2Sha256::verify_password_primitive(&password, &salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-256 one-shot API must reject a corrupted output"); } #[test] @@ -60,21 +66,27 @@ proptest! { oracle_sha512(&password, &salt, iterations, &mut expected); let mut oneshot = vec![0u8; out_len]; - Pbkdf2Sha512::derive_key_primitive(&password, &salt, iterations, &mut oneshot).unwrap(); + Pbkdf2Sha512::derive_key_primitive(&password, &salt, iterations, &mut oneshot) + .expect("PBKDF2-SHA-512 one-shot differential derivation must succeed"); let state = Pbkdf2Sha512::new(&password); let mut reused = vec![0u8; out_len]; - state.derive(&salt, iterations, &mut reused).unwrap(); + state.derive(&salt, iterations, &mut reused) + .expect("PBKDF2-SHA-512 state-reuse differential derivation must succeed"); prop_assert_eq!(oneshot, expected.as_slice()); prop_assert_eq!(reused, expected.as_slice()); - prop_assert!(state.verify_primitive(&salt, iterations, &expected).is_ok()); - prop_assert!(Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &expected).is_ok()); + state.verify_primitive(&salt, iterations, &expected) + .expect("PBKDF2-SHA-512 state must verify the oracle output"); + Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &expected) + .expect("PBKDF2-SHA-512 one-shot API must verify the oracle output"); let mut wrong = expected.clone(); wrong[0] ^= 1; - prop_assert!(state.verify_primitive(&salt, iterations, &wrong).is_err()); - prop_assert!(Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &wrong).is_err()); + state.verify_primitive(&salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-512 state must reject a corrupted output"); + Pbkdf2Sha512::verify_password_primitive(&password, &salt, iterations, &wrong) + .expect_err("PBKDF2-SHA-512 one-shot API must reject a corrupted output"); } } @@ -106,7 +118,8 @@ fn pbkdf2_sha256_rfc7914_vectors_match_rustcrypto() { ]; for case in cases { - let actual = Pbkdf2Sha256::derive_key_array_primitive::<64>(case.password, case.salt, case.iterations).unwrap(); + let actual = Pbkdf2Sha256::derive_key_array_primitive::<64>(case.password, case.salt, case.iterations) + .expect("RFC 7914 PBKDF2-SHA-256 vector derivation must succeed"); assert_eq!(actual, case.expected); let mut oracle = [0u8; 64]; diff --git a/tests/pbkdf2_kat_vectors.rs b/tests/pbkdf2_kat_vectors.rs index cbc4ab1d..5e8062d3 100644 --- a/tests/pbkdf2_kat_vectors.rs +++ b/tests/pbkdf2_kat_vectors.rs @@ -17,12 +17,23 @@ use rscrypto::{Pbkdf2Sha256, Pbkdf2Sha512}; -fn hex_to_bytes(s: &str) -> Vec { - let s: String = s.chars().filter(|c| !c.is_ascii_whitespace()).collect(); - (0..s.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) - .collect() +fn hex_to_bytes(source: &str) -> Vec { + let mut digits = source.chars().filter(|character| !character.is_ascii_whitespace()); + let mut output = Vec::new(); + + while let Some(high) = digits.next() { + let low = digits.next().expect("KAT hex input must contain complete byte pairs"); + let high = high + .to_digit(16) + .expect("KAT input must contain only hexadecimal digits"); + let low = low + .to_digit(16) + .expect("KAT input must contain only hexadecimal digits"); + let byte = high.strict_mul(16).strict_add(low); + output.push(u8::try_from(byte).expect("two hexadecimal digits must fit in one byte")); + } + + output } #[test] @@ -30,7 +41,8 @@ fn pbkdf2_sha256_kat_c1_dk32() { // P="password" S="salt" c=1 dk_len=32 let expected = hex_to_bytes("120fb6cffcf8b32c43e7225256c4f837 a86548c92ccc35480805987cb70be17b"); let mut out = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 1, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 1, &mut out) + .expect("PBKDF2-SHA-256 c=1 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -39,7 +51,8 @@ fn pbkdf2_sha256_kat_c2_dk32() { // P="password" S="salt" c=2 dk_len=32 let expected = hex_to_bytes("ae4d0c95af6b46d32d0adff928f06dd0 2a303f8ef3c251dfd6e2d85a95474c43"); let mut out = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 2, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 2, &mut out) + .expect("PBKDF2-SHA-256 c=2 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -49,7 +62,8 @@ fn pbkdf2_sha256_kat_c4096_dk32() { // P="password" S="salt" c=4096 dk_len=32 let expected = hex_to_bytes("c5e478d59288c841aa530db6845c4c8d 962893a001ce4e11a4963873aa98134a"); let mut out = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 4096, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 4096, &mut out) + .expect("PBKDF2-SHA-256 c=4096 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -65,7 +79,7 @@ fn pbkdf2_sha256_kat_long_inputs_c4096_dk40() { 4096, &mut out, ) - .unwrap(); + .expect("PBKDF2-SHA-256 long-input KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -75,7 +89,8 @@ fn pbkdf2_sha256_kat_embedded_nul_c4096_dk16() { // NUL-termination bugs in the HMAC key/salt path. let expected = hex_to_bytes("89b69d0516f829893c696226650a8687"); let mut out = [0u8; 16]; - Pbkdf2Sha256::derive_key_primitive(b"pass\x00word", b"sa\x00lt", 4096, &mut out).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"pass\x00word", b"sa\x00lt", 4096, &mut out) + .expect("PBKDF2-SHA-256 embedded-NUL KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -87,7 +102,8 @@ fn pbkdf2_sha512_kat_c1_dk64() { 050235d7d68b1da55e63f73b60a57fce", ); let mut out = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 1, &mut out).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 1, &mut out) + .expect("PBKDF2-SHA-512 c=1 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -99,7 +115,8 @@ fn pbkdf2_sha512_kat_c2_dk64() { be67335c77a6068e04112754f27ccf4e", ); let mut out = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 2, &mut out).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 2, &mut out) + .expect("PBKDF2-SHA-512 c=2 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -112,7 +129,8 @@ fn pbkdf2_sha512_kat_c4096_dk64() { 376060ecd532e039b742a239434af2d5", ); let mut out = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 4096, &mut out).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 4096, &mut out) + .expect("PBKDF2-SHA-512 c=4096 KAT derivation must succeed"); assert_eq!(out.as_slice(), expected.as_slice()); } @@ -120,19 +138,25 @@ fn pbkdf2_sha512_kat_c4096_dk64() { fn pbkdf2_sha256_state_reuse_matches_oneshot() { let state = Pbkdf2Sha256::new(b"password"); let mut from_state = [0u8; 32]; - state.derive(b"salt", 100, &mut from_state).unwrap(); + state + .derive(b"salt", 100, &mut from_state) + .expect("PBKDF2-SHA-256 cached-state derivation must succeed"); let mut from_oneshot = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot) + .expect("PBKDF2-SHA-256 one-shot derivation must succeed"); assert_eq!(from_state, from_oneshot); // The cached state must be reusable for a second derivation. let mut from_state_again = [0u8; 32]; - state.derive(b"salt2", 50, &mut from_state_again).unwrap(); + state + .derive(b"salt2", 50, &mut from_state_again) + .expect("second PBKDF2-SHA-256 cached-state derivation must succeed"); let mut from_oneshot_again = [0u8; 32]; - Pbkdf2Sha256::derive_key_primitive(b"password", b"salt2", 50, &mut from_oneshot_again).unwrap(); + Pbkdf2Sha256::derive_key_primitive(b"password", b"salt2", 50, &mut from_oneshot_again) + .expect("second PBKDF2-SHA-256 one-shot derivation must succeed"); assert_eq!(from_state_again, from_oneshot_again); } @@ -141,10 +165,13 @@ fn pbkdf2_sha256_state_reuse_matches_oneshot() { fn pbkdf2_sha512_state_reuse_matches_oneshot() { let state = Pbkdf2Sha512::new(b"password"); let mut from_state = [0u8; 64]; - state.derive(b"salt", 100, &mut from_state).unwrap(); + state + .derive(b"salt", 100, &mut from_state) + .expect("PBKDF2-SHA-512 cached-state derivation must succeed"); let mut from_oneshot = [0u8; 64]; - Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot).unwrap(); + Pbkdf2Sha512::derive_key_primitive(b"password", b"salt", 100, &mut from_oneshot) + .expect("PBKDF2-SHA-512 one-shot derivation must succeed"); assert_eq!(from_state, from_oneshot); } diff --git a/tests/pbkdf2_wycheproof.rs b/tests/pbkdf2_wycheproof.rs index 744f73c5..039dae0e 100644 --- a/tests/pbkdf2_wycheproof.rs +++ b/tests/pbkdf2_wycheproof.rs @@ -13,9 +13,10 @@ type Pbkdf2Derive = fn(&[u8], &[u8], u32, &mut [u8]) -> Result<(), Pbkdf2Error>; type Pbkdf2Verify = fn(&[u8], &[u8], u32, &[u8]) -> Result<(), rscrypto::VerificationError>; fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + value + .get(name) + .and_then(Value::as_str) + .expect("Wycheproof string field must exist and contain a string") } fn groups(suite: &Value) -> &[Value] { @@ -44,24 +45,25 @@ fn run_pbkdf2_suite( let tc_id = test["tcId"].as_u64().expect("tcId must be numeric"); let password = decode_hex_vec(field(test, "password")); let salt = decode_hex_vec(field(test, "salt")); - let iterations = test["iterationCount"].as_u64().expect("iterationCount must be numeric") as u32; - let dk_len = test["dkLen"].as_u64().expect("dkLen must be numeric") as usize; + let iterations = u32::try_from(test["iterationCount"].as_u64().expect("iterationCount must be numeric")) + .expect("Wycheproof iterationCount must fit in u32"); + let dk_len = usize::try_from(test["dkLen"].as_u64().expect("dkLen must be numeric")) + .expect("Wycheproof dkLen must fit in usize"); let expected_dk = decode_hex_vec(field(test, "dk")); assert_eq!(expected_dk.len(), dk_len, "{algorithm} tcId {tc_id} dkLen mismatch"); - match field(test, "result") { - "valid" => { - valid += 1; - let mut actual = vec![0u8; dk_len]; - derive(&password, &salt, iterations, &mut actual) - .unwrap_or_else(|err| panic!("{algorithm} tcId {tc_id} failed: {err}")); - assert_eq!(actual, expected_dk, "{algorithm} tcId {tc_id} derived key mismatch"); - - if first_valid_for_negative_check.is_none() { - first_valid_for_negative_check = Some((password, salt, iterations, expected_dk)); - } - } - other => panic!("{algorithm} tcId {tc_id} has unsupported result `{other}`"), + assert_eq!( + field(test, "result"), + "valid", + "{algorithm} tcId {tc_id} has an unsupported result" + ); + valid = valid.strict_add(1); + let mut actual = vec![0u8; dk_len]; + derive(&password, &salt, iterations, &mut actual).expect("known-valid PBKDF2 Wycheproof vector must derive"); + assert_eq!(actual, expected_dk, "{algorithm} tcId {tc_id} derived key mismatch"); + + if first_valid_for_negative_check.is_none() { + first_valid_for_negative_check = Some((password, salt, iterations, expected_dk)); } } } @@ -70,21 +72,12 @@ fn run_pbkdf2_suite( let (password, salt, iterations, expected_dk) = first_valid_for_negative_check.expect("PBKDF2 suite must contain at least one valid vector"); - assert!( - verify(&password, &salt, iterations, &expected_dk).is_ok(), - "{algorithm} rejected a known-good derived key" - ); + verify(&password, &salt, iterations, &expected_dk).expect("PBKDF2 must verify a known-good Wycheproof derived key"); let mut wrong_dk = expected_dk.clone(); wrong_dk[0] ^= 1; - assert!( - verify(&password, &salt, iterations, &wrong_dk).is_err(), - "{algorithm} accepted a corrupted derived key" - ); - assert!( - verify(b"wrong password", &salt, iterations, &expected_dk).is_err(), - "{algorithm} accepted the wrong password" - ); + verify(&password, &salt, iterations, &wrong_dk).expect_err("PBKDF2 must reject a corrupted Wycheproof derived key"); + verify(b"wrong password", &salt, iterations, &expected_dk).expect_err("PBKDF2 must reject the wrong password"); } #[test] diff --git a/tests/phc_external_entropy.rs b/tests/phc_external_entropy.rs index c76f455f..fed326fa 100644 --- a/tests/phc_external_entropy.rs +++ b/tests/phc_external_entropy.rs @@ -15,7 +15,8 @@ const SALT_B64: &str = "AAECAwQFBgcICQoLDA0ODw"; fn argon2_hashes_with_one_caller_entropy_fill() { use rscrypto::{Argon2Params, Argon2idPassword}; - let passwords = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let params = Argon2Params::new(32, 2, 1).expect("Argon2 entropy-test parameters must be valid"); + let passwords = Argon2idPassword::new(params).expect("Argon2 password service must accept entropy-test parameters"); let fills = Cell::new(0usize); let record = passwords .hash_password_with(PASSWORD, |salt| { @@ -24,7 +25,7 @@ fn argon2_hashes_with_one_caller_entropy_fill() { salt.copy_from_slice(&SALT); Ok::<(), &'static str>(()) }) - .unwrap(); + .expect("Argon2 hashing with caller entropy must succeed"); assert_eq!(fills.get(), 1); assert_eq!(record.split('$').nth(4), Some(SALT_B64)); @@ -39,7 +40,8 @@ fn argon2_hashes_with_one_caller_entropy_fill() { fn scrypt_hashes_with_one_caller_entropy_fill() { use rscrypto::{ScryptParams, ScryptPassword}; - let passwords = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); + let params = ScryptParams::new(4, 1, 1).expect("scrypt entropy-test parameters must be valid"); + let passwords = ScryptPassword::new(params).expect("scrypt password service must accept entropy-test parameters"); let fills = Cell::new(0usize); let record = passwords .hash_password_with(PASSWORD, |salt| { @@ -48,7 +50,7 @@ fn scrypt_hashes_with_one_caller_entropy_fill() { salt.copy_from_slice(&SALT); Ok::<(), &'static str>(()) }) - .unwrap(); + .expect("scrypt hashing with caller entropy must succeed"); assert_eq!(fills.get(), 1); assert_eq!(record.split('$').nth(3), Some(SALT_B64)); @@ -63,10 +65,11 @@ fn scrypt_hashes_with_one_caller_entropy_fill() { fn argon2_preserves_entropy_errors_without_hashing() { use rscrypto::{Argon2Error, Argon2Params, Argon2idPassword}; - let passwords = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let params = Argon2Params::new(32, 2, 1).expect("Argon2 entropy-error parameters must be valid"); + let passwords = Argon2idPassword::new(params).expect("Argon2 password service must accept entropy-error parameters"); let error = passwords .hash_password_with(PASSWORD, |_| Err("entropy unavailable")) - .unwrap_err(); + .expect_err("Argon2 must preserve caller entropy failures"); assert_eq!( error, @@ -79,10 +82,11 @@ fn argon2_preserves_entropy_errors_without_hashing() { fn scrypt_preserves_entropy_errors_without_hashing() { use rscrypto::{ScryptError, ScryptParams, ScryptPassword}; - let passwords = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); + let params = ScryptParams::new(4, 1, 1).expect("scrypt entropy-error parameters must be valid"); + let passwords = ScryptPassword::new(params).expect("scrypt password service must accept entropy-error parameters"); let error = passwords .hash_password_with(PASSWORD, |_| Err("entropy unavailable")) - .unwrap_err(); + .expect_err("scrypt must preserve caller entropy failures"); assert_eq!( error, diff --git a/tests/phc_roundtrip.rs b/tests/phc_roundtrip.rs index 115f1384..af0172e5 100644 --- a/tests/phc_roundtrip.rs +++ b/tests/phc_roundtrip.rs @@ -94,38 +94,59 @@ fn allocations_during(operation: impl FnOnce() -> T) -> (T, usize) { #[test] fn generated_password_records_are_canonical_and_self_verifying() { - let argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); - let argon2_record = argon2.hash_password(b"correct horse battery staple").unwrap(); + let argon2_params = Argon2Params::new(32, 2, 1).expect("Argon2 test parameters must be valid"); + let argon2 = Argon2idPassword::new(argon2_params).expect("Argon2 password service must accept valid parameters"); + let argon2_record = argon2 + .hash_password(b"correct horse battery staple") + .expect("Argon2 password hashing must succeed"); assert!(argon2_record.starts_with("$argon2id$v=19$m=32,t=2,p=1$")); assert_eq!( argon2.verify_password(b"correct horse battery staple", &argon2_record), Ok(PasswordStatus::Current) ); - assert!(argon2.verify_password(b"wrong", &argon2_record).is_err()); - - let scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); - let scrypt_record = scrypt.hash_password(b"correct horse battery staple").unwrap(); + argon2 + .verify_password(b"wrong", &argon2_record) + .expect_err("Argon2 verification must reject the wrong password"); + + let scrypt_params = ScryptParams::new(4, 1, 1).expect("scrypt test parameters must be valid"); + let scrypt = ScryptPassword::new(scrypt_params).expect("scrypt password service must accept valid parameters"); + let scrypt_record = scrypt + .hash_password(b"correct horse battery staple") + .expect("scrypt password hashing must succeed"); assert!(scrypt_record.starts_with("$scrypt$ln=4,r=1,p=1$")); assert_eq!( scrypt.verify_password(b"correct horse battery staple", &scrypt_record), Ok(PasswordStatus::Current) ); - assert!(scrypt.verify_password(b"wrong", &scrypt_record).is_err()); + scrypt + .verify_password(b"wrong", &scrypt_record) + .expect_err("scrypt verification must reject the wrong password"); } #[test] fn verifier_reports_accepted_stale_profiles() { - let old_argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); - let argon2_record = old_argon2.hash_password(b"password").unwrap(); - let current_argon2 = Argon2idPassword::new(Argon2Params::new(40, 2, 1).unwrap()).unwrap(); + let old_argon2_params = Argon2Params::new(32, 2, 1).expect("old Argon2 profile must be valid"); + let old_argon2 = + Argon2idPassword::new(old_argon2_params).expect("old Argon2 password service must accept its profile"); + let argon2_record = old_argon2 + .hash_password(b"password") + .expect("old Argon2 profile must produce a password record"); + let current_argon2_params = Argon2Params::new(40, 2, 1).expect("current Argon2 profile must be valid"); + let current_argon2 = + Argon2idPassword::new(current_argon2_params).expect("current Argon2 password service must accept its profile"); assert_eq!( current_argon2.verify_password(b"password", &argon2_record), Ok(PasswordStatus::NeedsRehash) ); - let old_scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); - let scrypt_record = old_scrypt.hash_password(b"password").unwrap(); - let current_scrypt = ScryptPassword::new(ScryptParams::new(5, 1, 1).unwrap()).unwrap(); + let old_scrypt_params = ScryptParams::new(4, 1, 1).expect("old scrypt profile must be valid"); + let old_scrypt = ScryptPassword::new(old_scrypt_params).expect("old scrypt password service must accept its profile"); + let scrypt_record = old_scrypt + .hash_password(b"password") + .expect("old scrypt profile must produce a password record"); + let current_scrypt_params = ScryptParams::new(5, 1, 1).expect("current scrypt profile must be valid"); + let current_scrypt = + ScryptPassword::new(current_scrypt_params).expect("current scrypt password service must accept its profile"); assert_eq!( current_scrypt.verify_password(b"password", &scrypt_record), Ok(PasswordStatus::NeedsRehash) @@ -134,7 +155,9 @@ fn verifier_reports_accepted_stale_profiles() { #[test] fn every_rejected_phc_class_allocates_nothing() { - let argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let argon2_params = Argon2Params::new(32, 2, 1).expect("Argon2 rejection-test parameters must be valid"); + let argon2 = + Argon2idPassword::new(argon2_params).expect("Argon2 password service must accept rejection-test parameters"); let oversized = "x".repeat(1_025); let argon2_rejections = [ oversized.as_str(), @@ -148,11 +171,13 @@ fn every_rejected_phc_class_allocates_nothing() { ]; for encoded in argon2_rejections { let (result, allocations) = allocations_during(|| argon2.verify_password(b"password", encoded)); - assert!(result.is_err(), "rejected Argon2 PHC: {encoded}"); + result.expect_err("known-invalid Argon2 PHC record must be rejected"); assert_eq!(allocations, 0, "rejected Argon2 PHC allocated: {encoded}"); } - let scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); + let scrypt_params = ScryptParams::new(4, 1, 1).expect("scrypt rejection-test parameters must be valid"); + let scrypt = + ScryptPassword::new(scrypt_params).expect("scrypt password service must accept rejection-test parameters"); let scrypt_rejections = [ oversized.as_str(), "not-a-phc-record", @@ -165,34 +190,42 @@ fn every_rejected_phc_class_allocates_nothing() { ]; for encoded in scrypt_rejections { let (result, allocations) = allocations_during(|| scrypt.verify_password(b"password", encoded)); - assert!(result.is_err(), "rejected scrypt PHC: {encoded}"); + result.expect_err("known-invalid scrypt PHC record must be rejected"); assert_eq!(allocations, 0, "rejected scrypt PHC allocated: {encoded}"); } } #[test] fn public_verifiers_reject_noncanonical_and_cross_algorithm_records() { - let argon2 = Argon2idPassword::new(Argon2Params::new(32, 2, 1).unwrap()).unwrap(); - let scrypt = ScryptPassword::new(ScryptParams::new(4, 1, 1).unwrap()).unwrap(); - let valid_argon2 = argon2.hash_password(b"password").unwrap(); - let valid_scrypt = scrypt.hash_password(b"password").unwrap(); - - assert!(argon2.verify_password(b"password", &valid_scrypt).is_err()); - assert!(scrypt.verify_password(b"password", &valid_argon2).is_err()); - assert!( - argon2 - .verify_password( - b"password", - "$argon2id$v=19$t=2,m=32,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - ) - .is_err() - ); - assert!( - scrypt - .verify_password( - b"password", - "$scrypt$r=1,ln=4,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - ) - .is_err() - ); + let argon2_params = Argon2Params::new(32, 2, 1).expect("Argon2 cross-algorithm parameters must be valid"); + let argon2 = + Argon2idPassword::new(argon2_params).expect("Argon2 password service must accept cross-algorithm parameters"); + let scrypt_params = ScryptParams::new(4, 1, 1).expect("scrypt cross-algorithm parameters must be valid"); + let scrypt = + ScryptPassword::new(scrypt_params).expect("scrypt password service must accept cross-algorithm parameters"); + let valid_argon2 = argon2 + .hash_password(b"password") + .expect("Argon2 must produce a cross-algorithm fixture"); + let valid_scrypt = scrypt + .hash_password(b"password") + .expect("scrypt must produce a cross-algorithm fixture"); + + argon2 + .verify_password(b"password", &valid_scrypt) + .expect_err("Argon2 must reject a scrypt record"); + scrypt + .verify_password(b"password", &valid_argon2) + .expect_err("scrypt must reject an Argon2 record"); + argon2 + .verify_password( + b"password", + "$argon2id$v=19$t=2,m=32,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ) + .expect_err("Argon2 must reject noncanonical parameter order"); + scrypt + .verify_password( + b"password", + "$scrypt$r=1,ln=4,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ) + .expect_err("scrypt must reject noncanonical parameter order"); } diff --git a/tests/platform_amx_permission.rs b/tests/platform_amx_permission.rs index d8837a15..f28d08c1 100644 --- a/tests/platform_amx_permission.rs +++ b/tests/platform_amx_permission.rs @@ -5,7 +5,6 @@ target_os = "linux", not(miri) ))] -#![allow(unsafe_code)] use rscrypto::platform::{self, caps::x86}; @@ -18,17 +17,12 @@ const ARCH_REQ_XCOMP_PERM: usize = 0x1023; const XFEATURE_XTILEDATA: usize = 18; const XCOMP_TILE_MASK: u64 = (1 << 17) | (1 << 18); -#[allow(unused_unsafe)] fn cpu_supports_amx_tile() -> bool { - // MSRV: CPUID is unsafe on Rust 1.91 but safe on the pinned nightly. - // SAFETY: CPUID is a non-privileged x86-64 identification instruction. - let leaf0 = unsafe { core::arch::x86_64::__cpuid(0) }; + let leaf0 = core::arch::x86_64::__cpuid(0); if leaf0.eax < 7 { return false; } - // SAFETY: CPUID leaf 7, subleaf 0 is valid because leaf 0 reports support - // for leaf 7; the intrinsic only returns register values. - let leaf7 = unsafe { core::arch::x86_64::__cpuid_count(7, 0) }; + let leaf7 = core::arch::x86_64::__cpuid_count(7, 0); leaf7.edx & (1 << 24) != 0 } diff --git a/tests/platform_override_race.rs b/tests/platform_override_race.rs index b6f78f72..257b85ce 100644 --- a/tests/platform_override_race.rs +++ b/tests/platform_override_race.rs @@ -10,14 +10,15 @@ fn concurrent_override_writers_are_serialized() { for _ in 0..8 { scope.spawn(|| { for _ in 0..128 { - expert::try_set_override(Some(Detected::portable())).unwrap(); - expert::try_set_override(None).unwrap(); + expert::try_set_override(Some(Detected::portable())) + .expect("portable override must be accepted before detection"); + expert::try_set_override(None).expect("override must be clearable before detection"); } }); } }); - expert::try_set_override(None).unwrap(); + expert::try_set_override(None).expect("override must be clearable after concurrent updates"); } #[test] @@ -39,13 +40,15 @@ fn concurrent_detection_and_override_child() { }); barrier.wait(); - let setter_result = setter.join().unwrap(); - let detected = detector.join().unwrap(); + let setter_result = setter.join().expect("override thread must not panic"); + let detected = detector.join().expect("detection thread must not panic"); - match setter_result { - Ok(()) => assert_eq!(detected, Detected::portable()), - Err(expert::OverrideError::AlreadyInitialized) => {} - Err(error) => panic!("unexpected override result: {error:?}"), + assert!( + matches!(setter_result, Ok(()) | Err(expert::OverrideError::AlreadyInitialized)), + "concurrent override returned an invalid result: {setter_result:?}" + ); + if setter_result == Ok(()) { + assert_eq!(detected, Detected::portable()); } }); } @@ -53,7 +56,7 @@ fn concurrent_detection_and_override_child() { #[test] #[cfg(not(miri))] fn concurrent_detection_and_override_are_linearizable() { - let executable = std::env::current_exe().unwrap(); + let executable = std::env::current_exe().expect("test executable path must be available"); for _ in 0..32 { let status = std::process::Command::new(&executable) @@ -62,7 +65,7 @@ fn concurrent_detection_and_override_are_linearizable() { .arg("--quiet") .env(CHILD_MODE, "1") .status() - .unwrap(); + .expect("race child process must start"); assert!(status.success(), "race child failed with {status}"); } } diff --git a/tests/platform_override_validation.rs b/tests/platform_override_validation.rs index c4f82ab0..e190983f 100644 --- a/tests/platform_override_validation.rs +++ b/tests/platform_override_validation.rs @@ -18,7 +18,7 @@ fn safe_override_rejects_impossible_caps_and_still_allows_portable() { ); let portable = Detected::portable(); - expert::try_set_override(Some(portable)).unwrap(); + expert::try_set_override(Some(portable)).expect("portable override must be accepted before detection"); assert!(expert::has_override()); assert_eq!(platform::get(), portable); @@ -33,7 +33,7 @@ fn safe_override_rejects_impossible_caps_and_still_allows_portable() { #[cfg(miri)] { - expert::try_set_override(None).unwrap(); + expert::try_set_override(None).expect("Miri override must remain mutable"); assert!(!expert::has_override()); } } diff --git a/tests/poly1305_vectors.rs b/tests/poly1305_vectors.rs index 45b3a886..35009dcc 100644 --- a/tests/poly1305_vectors.rs +++ b/tests/poly1305_vectors.rs @@ -3,7 +3,9 @@ use rscrypto::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; mod common; -use common::decode_hex_array; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array; #[test] fn poly1305_matches_rfc_8439_section_2_5_2() { @@ -39,7 +41,7 @@ fn poly1305_verify_rejects_corrupted_tag() { corrupted[7] ^= 0x80; let corrupted = Poly1305Tag::from_bytes(corrupted); - assert!(Poly1305::verify_once(key, message, &corrupted).is_err()); + Poly1305::verify_once(key, message, &corrupted).expect_err("corrupted Poly1305 tag must be rejected"); } #[test] @@ -48,8 +50,8 @@ fn poly1305_try_generate_with_uses_fallible_fill() { out.fill(0x11); Ok::<(), ()>(()) }) - .unwrap(); + .expect("deterministic entropy callback must generate a Poly1305 key"); assert_eq!(key.as_bytes(), &[0x11; Poly1305OneTimeKey::LENGTH]); - assert!(Poly1305OneTimeKey::try_generate_with(|_| Err("rng")).is_err()); + Poly1305OneTimeKey::try_generate_with(|_| Err("rng")).expect_err("entropy-source failure must be returned"); } diff --git a/tests/portable_fallback.rs b/tests/portable_fallback.rs index 1ac59305..b10372f5 100644 --- a/tests/portable_fallback.rs +++ b/tests/portable_fallback.rs @@ -187,7 +187,7 @@ fn crc64_empty_input() { fn crc32_various_sizes() { // Just verify that different sizes don't panic and produce consistent results for size in [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024] { - let data: Vec = (0..size).map(|i| (i & 0xFF) as u8).collect(); + let data: Vec = (0..size).map(|i: usize| i.to_le_bytes()[0]).collect(); let oneshot = Crc32::checksum(&data); @@ -205,7 +205,7 @@ fn crc32_various_sizes() { #[test] fn crc64_various_sizes() { for size in [1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128, 255, 256, 1024] { - let data: Vec = (0..size).map(|i| (i & 0xFF) as u8).collect(); + let data: Vec = (0..size).map(|i: usize| i.to_le_bytes()[0]).collect(); let oneshot = Crc64::checksum(&data); diff --git a/tests/rapidhash_differential.rs b/tests/rapidhash_differential.rs index 3cf0addb..e0cac670 100644 --- a/tests/rapidhash_differential.rs +++ b/tests/rapidhash_differential.rs @@ -23,7 +23,7 @@ fn collection_state_schedules_match_rapidhash() { for seed in [0, 1, u64::MAX, 0x243f_6a88_85a3_08d3] { for len in lengths { let data: Vec = (0..len) - .map(|index| index.wrapping_mul(131).wrapping_add(17) as u8) + .map(|index| index.wrapping_mul(131).wrapping_add(17).to_le_bytes()[0]) .collect(); let deterministic = RapidSeededState::new(seed); @@ -37,7 +37,7 @@ fn collection_state_schedules_match_rapidhash() { out.copy_from_slice(&seed.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); + .expect("deterministic entropy callback must initialize RapidRandomState"); let secrets = rapidhash::v3::RapidSecrets::seed(seed); assert_eq!( hash_bytes(&randomized, &data), diff --git a/tests/root_surface.rs b/tests/root_surface.rs index 6480602c..266339c3 100644 --- a/tests/root_surface.rs +++ b/tests/root_surface.rs @@ -1,9 +1,11 @@ -#![allow(unused_imports)] - #[cfg(feature = "aead")] use rscrypto::Aead; -#[cfg(any(feature = "hmac", feature = "hmac-sha3", feature = "kmac"))] +#[cfg(feature = "hmac")] use rscrypto::Mac; +#[cfg(all(feature = "rsa", feature = "getrandom"))] +use rscrypto::RsaJwtAlgorithm; +#[cfg(all(feature = "hashes", any(feature = "std", feature = "diag")))] +use rscrypto::Sha256; #[cfg(feature = "aead")] use rscrypto::aead::expert::AeadWithNonce; #[cfg(all(feature = "aead", feature = "diag"))] @@ -21,7 +23,7 @@ use rscrypto::aead::{ use rscrypto::auth::HkdfOutputLengthError; #[cfg(all(feature = "checksums", feature = "alloc"))] use rscrypto::checksum::buffered::BufferedCrc32C; -#[cfg(feature = "checksums")] +#[cfg(all(feature = "checksums", feature = "diag"))] use rscrypto::checksum::config::{ Crc16Config, Crc16Force, Crc24Config, Crc24Force, Crc32Config, Crc32Force, Crc64Config, Crc64Force, }; @@ -39,11 +41,12 @@ use rscrypto::hashes::introspect::{ use rscrypto::hashes::{DigestReader, DigestWriter}; #[cfg(feature = "hashes")] use rscrypto::{ - AsconCxof128, AsconCxof128Reader, AsconHash256, AsconXof, AsconXofReader, Blake3, Blake3XofReader, Cshake128, - Cshake128XofReader, Cshake256, Cshake256XofReader, Digest, FastHash, RapidHash64, RapidHasher, RapidRandomState, - RapidSeededState, RapidStreamHasher, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Sha224, Sha256, Sha384, Sha512, - Sha512_256, Shake128, Shake128XofReader, Shake256, Shake256XofReader, Xof, Xxh3, Xxh3_128, + AsconCxof128, AsconCxof128Reader, AsconXof, AsconXofReader, Blake3, Blake3XofReader, Cshake128, Cshake128XofReader, + Cshake256, Cshake256XofReader, Digest, FastHash, RapidHash64, RapidHasher, RapidRandomState, RapidSeededState, + RapidStreamHasher, Shake128, Shake128XofReader, Shake256, Shake256XofReader, Xof, Xxh3, }; +#[cfg(all(feature = "hashes", feature = "diag"))] +use rscrypto::{AsconHash256, Sha3_224, Sha3_256, Sha3_384, Sha3_512, Sha224, Sha384, Sha512, Sha512_256, Xxh3_128}; #[cfg(feature = "checksums")] use rscrypto::{Checksum, ChecksumCombine, Crc16Ccitt, Crc16Ibm, Crc24OpenPgp, Crc32, Crc32C, Crc64, Crc64Nvme}; #[cfg(feature = "ed25519")] @@ -71,7 +74,7 @@ use rscrypto::{ use rscrypto::{Poly1305, Poly1305OneTimeKey, Poly1305Tag}; #[cfg(feature = "rsa")] use rscrypto::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationError, RsaOaepProfile, + RsaBlindingPair, RsaEncryptionError, RsaJwtVerifier, RsaKeyError, RsaKeyGenerationError, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPrivateOpError, RsaPrivateScratch, RsaProtocolAlgorithmError, RsaPssProfile, RsaPublicExponent, RsaPublicExponentPolicy, RsaPublicKey, RsaPublicKeyPolicy, RsaPublicOpError, RsaPublicScratch, RsaSignatureProfile, RsaTlsSignatureSchemes, @@ -81,7 +84,7 @@ use rscrypto::{VerificationError, ct}; #[cfg(feature = "x25519")] use rscrypto::{X25519Error, X25519PublicKey, X25519SecretKey, X25519SharedSecret}; -#[cfg(feature = "rsa")] +#[cfg(all(feature = "rsa", feature = "getrandom"))] fn fill_rsa_random_with(byte: u8) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncryptionError> { move |out| { out.fill(byte); @@ -89,7 +92,7 @@ fn fill_rsa_random_with(byte: u8) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncr } } -#[cfg(feature = "rsa")] +#[cfg(all(feature = "rsa", feature = "getrandom"))] fn fill_rsa_random_from(bytes: &[u8]) -> impl FnMut(&mut [u8]) -> Result<(), RsaEncryptionError> + '_ { let mut offset = 0usize; move |out| { @@ -105,7 +108,7 @@ fn fill_rsa_random_from(bytes: &[u8]) -> impl FnMut(&mut [u8]) -> Result<(), Rsa #[test] fn root_surface_core_exports_compile() { - let _ = VerificationError::new(); + let _error = VerificationError::new(); let mut secret = [0x5a; 8]; ct::zeroize(&mut secret); assert_eq!(secret, [0; 8]); @@ -125,22 +128,22 @@ fn root_surface_aead_exports_compile() { let nonce256 = Nonce256::from_bytes([0x44; Nonce256::LENGTH]); assert_eq!(nonce256.as_bytes().len(), Nonce256::LENGTH); - let _ = AeadBufferError::new(); - let _ = SealError::buffer(); - let _ = SealError::too_large(); - let _ = OpenError::buffer(); - let _ = OpenError::too_large(); - let _ = OpenError::verification(); + let _buffer_error = AeadBufferError::new(); + let _seal_buffer_error = SealError::buffer(); + let _seal_length_error = SealError::too_large(); + let _open_buffer_error = OpenError::buffer(); + let _open_length_error = OpenError::too_large(); + let _verification_error = OpenError::verification(); #[cfg(feature = "diag")] { - let _ = AeadDispatchInfo::current(); - let _ = aes256gcm_backend(); - let _ = aes256gcmsiv_backend(); - let _ = chacha20poly1305_backend(); - let _ = xchacha20poly1305_backend(); - let _ = aegis256_backend(); - let _ = ascon_aead128_backend(); + let _dispatch = AeadDispatchInfo::current(); + let _aes256gcm_backend = aes256gcm_backend(); + let _aes256gcmsiv_backend = aes256gcmsiv_backend(); + let _chacha20poly1305_backend = chacha20poly1305_backend(); + let _xchacha20poly1305_backend = xchacha20poly1305_backend(); + let _aegis256_backend = aegis256_backend(); + let _ascon_aead128_backend = ascon_aead128_backend(); } fn assert_aead_trait() {} @@ -149,20 +152,26 @@ fn root_surface_aead_exports_compile() { let key = XChaCha20Poly1305Key::from_bytes([0x44; XChaCha20Poly1305::KEY_SIZE]); let cipher = XChaCha20Poly1305::new(&key); let mut sealed = [0u8; 20]; - cipher.encrypt(&nonce192, b"aad", b"test", &mut sealed).unwrap(); - let _ = XChaCha20Poly1305Tag::from_bytes([0u8; XChaCha20Poly1305Tag::LENGTH]); + cipher + .encrypt(&nonce192, b"aad", b"test", &mut sealed) + .expect("valid XChaCha20-Poly1305 input must encrypt"); + let _tag = XChaCha20Poly1305Tag::from_bytes([0u8; XChaCha20Poly1305Tag::LENGTH]); let key = ChaCha20Poly1305Key::from_bytes([0x55; ChaCha20Poly1305::KEY_SIZE]); let cipher = ChaCha20Poly1305::new(&key); let mut sealed = [0u8; 20]; - cipher.encrypt(&nonce96, b"aad", b"test", &mut sealed).unwrap(); - let _ = ChaCha20Poly1305Tag::from_bytes([0u8; ChaCha20Poly1305Tag::LENGTH]); + cipher + .encrypt(&nonce96, b"aad", b"test", &mut sealed) + .expect("valid ChaCha20-Poly1305 input must encrypt"); + let _tag = ChaCha20Poly1305Tag::from_bytes([0u8; ChaCha20Poly1305Tag::LENGTH]); let key = Aegis256Key::from_bytes([0x66; Aegis256::KEY_SIZE]); let cipher = Aegis256::new(&key); let mut sealed = [0u8; 20]; - cipher.encrypt(&nonce256, b"aad", b"test", &mut sealed).unwrap(); - let _ = Aegis256Tag::from_bytes([0u8; Aegis256Tag::LENGTH]); + cipher + .encrypt(&nonce256, b"aad", b"test", &mut sealed) + .expect("valid AEGIS-256 input must encrypt"); + let _tag = Aegis256Tag::from_bytes([0u8; Aegis256Tag::LENGTH]); } #[test] @@ -174,26 +183,29 @@ fn root_surface_mac_exports_compile() { let tag = HmacSha256::mac(key, data); let tag384 = HmacSha384::mac(key, data); let tag512 = HmacSha512::mac(key, data); - let _ = HmacSha256Tag::from_bytes(tag.to_bytes()); - let _ = HmacSha384Tag::from_bytes(tag384.to_bytes()); - let _ = HmacSha512Tag::from_bytes(tag512.to_bytes()); + let _tag256 = HmacSha256Tag::from_bytes(tag.to_bytes()); + let _tag384 = HmacSha384Tag::from_bytes(tag384.to_bytes()); + let _tag512 = HmacSha512Tag::from_bytes(tag512.to_bytes()); let mut mac = HmacSha256::new(key); mac.update(data); assert!(tag.ct_eq(&mac.finalize()).declassify()); - assert!(mac.verify(&tag).is_ok()); - let prefix = *tag.as_bytes().first_chunk::<8>().unwrap(); - assert!(HmacSha256::verify_truncated_tag_64(key, data, &prefix).is_ok()); + mac.verify(&tag).expect("matching HMAC-SHA-256 tag must verify"); + let prefix = *tag + .as_bytes() + .first_chunk::<8>() + .expect("an HMAC-SHA-256 tag must contain an eight-byte prefix"); + HmacSha256::verify_truncated_tag_64(key, data, &prefix).expect("matching truncated HMAC-SHA-256 tag must verify"); let mut mac384 = HmacSha384::new(key); mac384.update(data); assert!(tag384.ct_eq(&mac384.finalize()).declassify()); - assert!(mac384.verify(&tag384).is_ok()); + mac384.verify(&tag384).expect("matching HMAC-SHA-384 tag must verify"); let mut mac512 = HmacSha512::new(key); mac512.update(data); assert!(tag512.ct_eq(&mac512.finalize()).declassify()); - assert!(mac512.verify(&tag512).is_ok()); + mac512.verify(&tag512).expect("matching HMAC-SHA-512 tag must verify"); } #[test] @@ -206,15 +218,15 @@ fn root_surface_hmac_sha3_exports_compile() { let tag256 = HmacSha3_256::mac(key, data); let tag384 = HmacSha3_384::mac(key, data); let tag512 = HmacSha3_512::mac(key, data); - let _ = HmacSha3_224Tag::from_bytes(tag224.to_bytes()); - let _ = HmacSha3_256Tag::from_bytes(tag256.to_bytes()); - let _ = HmacSha3_384Tag::from_bytes(tag384.to_bytes()); - let _ = HmacSha3_512Tag::from_bytes(tag512.to_bytes()); - - assert!(HmacSha3_224::verify_tag(key, data, &tag224).is_ok()); - assert!(HmacSha3_256::verify_tag(key, data, &tag256).is_ok()); - assert!(HmacSha3_384::verify_tag(key, data, &tag384).is_ok()); - assert!(HmacSha3_512::verify_tag(key, data, &tag512).is_ok()); + let _tag224 = HmacSha3_224Tag::from_bytes(tag224.to_bytes()); + let _tag256 = HmacSha3_256Tag::from_bytes(tag256.to_bytes()); + let _tag384 = HmacSha3_384Tag::from_bytes(tag384.to_bytes()); + let _tag512 = HmacSha3_512Tag::from_bytes(tag512.to_bytes()); + + HmacSha3_224::verify_tag(key, data, &tag224).expect("matching HMAC-SHA3-224 tag must verify"); + HmacSha3_256::verify_tag(key, data, &tag256).expect("matching HMAC-SHA3-256 tag must verify"); + HmacSha3_384::verify_tag(key, data, &tag384).expect("matching HMAC-SHA3-384 tag must verify"); + HmacSha3_512::verify_tag(key, data, &tag512).expect("matching HMAC-SHA3-512 tag must verify"); } #[test] @@ -224,19 +236,34 @@ fn root_surface_kdf_exports_compile() { let mut out = [0u8; 32]; let hkdf = HkdfSha256::new(b"salt", key); - hkdf.expand(b"info", &mut out).unwrap(); - assert_eq!(out, HkdfSha256::derive_array::<32>(b"salt", key, b"info").unwrap()); + hkdf + .expand(b"info", &mut out) + .expect("32-byte HKDF-SHA-256 output must fit"); + assert_eq!( + out, + HkdfSha256::derive_array::<32>(b"salt", key, b"info").expect("32-byte HKDF-SHA-256 output must fit") + ); let mut out384 = [0u8; 48]; let hkdf384 = HkdfSha384::new(b"salt", key); - hkdf384.expand(b"info", &mut out384).unwrap(); - assert_eq!(out384, HkdfSha384::derive_array::<48>(b"salt", key, b"info").unwrap()); + hkdf384 + .expand(b"info", &mut out384) + .expect("48-byte HKDF-SHA-384 output must fit"); + assert_eq!( + out384, + HkdfSha384::derive_array::<48>(b"salt", key, b"info").expect("48-byte HKDF-SHA-384 output must fit") + ); let mut out512 = [0u8; 64]; let hkdf512 = HkdfSha512::new(b"salt", key); - hkdf512.expand(b"info", &mut out512).unwrap(); - assert_eq!(out512, HkdfSha512::derive_array::<64>(b"salt", key, b"info").unwrap()); - let _ = HkdfOutputLengthError::new(); + hkdf512 + .expand(b"info", &mut out512) + .expect("64-byte HKDF-SHA-512 output must fit"); + assert_eq!( + out512, + HkdfSha512::derive_array::<64>(b"salt", key, b"info").expect("64-byte HKDF-SHA-512 output must fit") + ); + let _length_error = HkdfOutputLengthError::new(); } #[test] @@ -248,13 +275,13 @@ fn root_surface_kmac_exports_compile() { let mut kmac128 = Kmac128::new(key, b"svc=v1"); kmac128.update(data); kmac128.finalize_into(&mut out128); - assert!(Kmac128::verify_tag(key, b"svc=v1", data, &out128).is_ok()); + Kmac128::verify_tag(key, b"svc=v1", data, &out128).expect("matching KMAC128 tag must verify"); let mut out = [0u8; 32]; let mut kmac = Kmac256::new(key, b"svc=v1"); kmac.update(data); kmac.finalize_into(&mut out); - assert!(Kmac256::verify_tag(key, b"svc=v1", data, &out).is_ok()); + Kmac256::verify_tag(key, b"svc=v1", data, &out).expect("matching KMAC256 tag must verify"); } #[test] @@ -262,10 +289,10 @@ fn root_surface_kmac_exports_compile() { fn root_surface_poly1305_exports_compile() { let key = Poly1305OneTimeKey::from_bytes([0x33; Poly1305OneTimeKey::LENGTH]); let tag = Poly1305::authenticate_once(key, b"root-surface-poly1305"); - let _ = Poly1305Tag::from_bytes(tag.to_bytes()); + let _tag = Poly1305Tag::from_bytes(tag.to_bytes()); let key = Poly1305OneTimeKey::from_bytes([0x33; Poly1305OneTimeKey::LENGTH]); - assert!(Poly1305::verify_once(key, b"root-surface-poly1305", &tag).is_ok()); + Poly1305::verify_once(key, b"root-surface-poly1305", &tag).expect("matching Poly1305 tag must verify"); } #[test] @@ -328,13 +355,15 @@ fn root_surface_mlkem_exports_compile() { fn root_surface_signature_exports_compile() { let secret = Ed25519SecretKey::from_bytes([7u8; Ed25519SecretKey::LENGTH]); let keypair = Ed25519Keypair::from_secret_key(secret.duplicate_secret()); - let public = keypair.public_key(); - let signature = keypair.sign(b"root-surface-ed25519"); + let public: Ed25519PublicKey = keypair.public_key(); + let signature: Ed25519Signature = keypair.sign(b"root-surface-ed25519"); assert_eq!(secret.as_bytes().len(), 32); assert_eq!(public.as_bytes().len(), 32); assert_eq!(signature.as_bytes().len(), 64); - assert!(public.verify(b"root-surface-ed25519", &signature).is_ok()); + public + .verify(b"root-surface-ed25519", &signature) + .expect("matching Ed25519 signature must verify"); } #[test] @@ -343,24 +372,25 @@ fn root_surface_rsa_exports_compile() { let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); assert_eq!(policy.min_modulus_bits(), 2048); assert_eq!(policy.max_modulus_bits(), 8192); - let _ = RsaPublicExponentPolicy::Common65537; - let _ = RsaKeyError::InvalidModulus; - let _ = RsaKeyGenerationError::InvalidModulusBits; - let _ = RsaEncryptionError::InvalidLength; - let _ = RsaPrivateOpError::InvalidLength; - let _ = RsaPublicOpError::RepresentativeOutOfRange; - let _ = RsaProtocolAlgorithmError::UnsupportedAlgorithm; - let _ = RsaOaepProfile::Sha256; - let _ = RsaPssProfile::Sha256; - let _ = RsaPkcs1v15Profile::Sha256; - let _: Option = None; - let _: Option = None; - let _: Option = None; - let _: Option> = None; - let _: Option = None; - let _: Option> = None; - let _: Option = None; - let _: Option = None; + let _exponent_policy = RsaPublicExponentPolicy::Common65537; + let _key_error = RsaKeyError::InvalidModulus; + let _generation_error = RsaKeyGenerationError::InvalidModulusBits; + let _encryption_error = RsaEncryptionError::InvalidLength; + let _private_op_error = RsaPrivateOpError::InvalidLength; + let _public_op_error = RsaPublicOpError::RepresentativeOutOfRange; + let _protocol_error = RsaProtocolAlgorithmError::UnsupportedAlgorithm; + let _oaep_profile = RsaOaepProfile::Sha256; + let _pss_profile = RsaPssProfile::Sha256; + let _pkcs1v15_profile = RsaPkcs1v15Profile::Sha256; + let _public_exponent: Option = None; + let _public_key: Option = None; + let _public_scratch: Option = None; + let _jwt_verifier: Option> = None; + let _private_key: Option = None; + let _private_key_parts: Option> = None; + let _blinding_pair = RsaBlindingPair::new(&[], &[]); + let _private_scratch: Option = None; + let _x509_public_key: Option = None; assert_eq!( RsaSignatureProfile::pss(RsaPssProfile::Sha256).pss_parts(), Some((RsaPssProfile::Sha256, 32)) @@ -370,23 +400,20 @@ fn root_surface_rsa_exports_compile() { Some(RsaPkcs1v15Profile::Sha384) ); assert_eq!( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), + RsaSignatureProfile::from_tls13_signature_scheme(0x0804) + .expect("TLS 1.3 rsa_pss_rsae_sha256 must map to an RSA signature profile"), RsaSignatureProfile::pss(RsaPssProfile::Sha256) ); - assert!( - RsaX509PublicKeyAlgorithm::RsaPss - .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) - .is_ok() - ); - assert!( - RsaX509PublicKeyAlgorithm::RsaEncryption - .signature_profile_from_tls13_signature_scheme(0x0804) - .is_ok() - ); + RsaX509PublicKeyAlgorithm::RsaPss + .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) + .expect("an RSA-PSS key must permit the SHA-256 PSS profile"); + RsaX509PublicKeyAlgorithm::RsaEncryption + .signature_profile_from_tls13_signature_scheme(0x0804) + .expect("an rsaEncryption key must accept TLS 1.3 rsa_pss_rsae_sha256"); let advertised = RsaX509PublicKeyAlgorithm::RsaEncryption.advertised_tls13_signature_schemes(); assert_eq!(advertised.len(), 3); assert!(advertised.contains(0x0804)); - let _ = RsaTlsSignatureSchemes::MAX_LEN; + let _maximum_scheme_count = RsaTlsSignatureSchemes::MAX_LEN; } #[test] @@ -403,9 +430,11 @@ fn root_surface_rsa_generated_key_end_to_end() { ]; let policy = RsaPublicKeyPolicy::legacy_verification(); - let key = RsaPrivateKey::generate_with_policy(2048, &policy).unwrap(); + let key = + RsaPrivateKey::generate_with_policy(2048, &policy).expect("the supported 2048-bit policy must generate an RSA key"); let public_key = key.public_key(); - let x509_key = RsaX509PublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy).unwrap(); + let x509_key = RsaX509PublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy) + .expect("the generated public key must round-trip through X.509 SPKI"); let message = b"root-surface-rsa-generated-key"; let mut private_scratch = key.private_scratch(); let mut public_scratch = public_key.public_scratch(); @@ -416,29 +445,31 @@ fn root_surface_rsa_generated_key_end_to_end() { assert_eq!(format!("{pkcs1_der:?}"), "SecretVec(****)"); assert_eq!( RsaPrivateKey::from_pkcs1_der_with_policy(&pkcs1_der, &policy) - .unwrap() + .expect("the generated private key must round-trip through PKCS#1 DER") .public_key(), public_key ); assert_eq!( RsaPrivateKey::from_pkcs8_der_with_policy(&pkcs8_der, &policy) - .unwrap() + .expect("the generated private key must round-trip through PKCS#8 DER") .public_key(), public_key ); assert_eq!( - RsaPublicKey::from_pkcs1_der_with_policy(&public_key.to_pkcs1_der(), &policy).unwrap(), + RsaPublicKey::from_pkcs1_der_with_policy(&public_key.to_pkcs1_der(), &policy) + .expect("the generated public key must round-trip through PKCS#1 DER"), *public_key ); assert_eq!( - RsaPublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy).unwrap(), + RsaPublicKey::from_spki_der_with_policy(&public_key.to_spki_der(), &policy) + .expect("the generated public key must round-trip through SPKI DER"), *public_key ); let mut unprotected_pkcs1 = key.to_pkcs1_der().into_unprotected_vec(); assert_eq!( RsaPrivateKey::from_pkcs1_der_with_policy(&unprotected_pkcs1, &policy) - .unwrap() + .expect("the unprotected PKCS#1 bytes must parse before zeroization") .public_key(), public_key ); @@ -451,54 +482,62 @@ fn root_surface_rsa_generated_key_end_to_end() { (RsaPkcs1v15Profile::Sha512, RsaPssProfile::Sha512), ] { let pkcs1v15_profile = RsaSignatureProfile::pkcs1v15(pkcs1v15_profile); - key.sign_signature(pkcs1v15_profile, message, &mut signature).unwrap(); + key + .sign_signature(pkcs1v15_profile, message, &mut signature) + .expect("the generated key must produce a PKCS#1 v1.5 signature"); public_key .verify_signature(pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("the generated public key must verify its PKCS#1 v1.5 signature"); public_key .verify_signature_with_scratch(pkcs1v15_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 verification must accept the matching signature"); key .sign_signature_with_scratch(pkcs1v15_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 signing must succeed"); public_key .verify_signature(pkcs1v15_profile, message, &signature) - .unwrap(); + .expect("the generated public key must verify the scratch-backed PKCS#1 v1.5 signature"); public_key .verify_signature_with_scratch(pkcs1v15_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed PKCS#1 v1.5 signature"); let pss_profile = RsaSignatureProfile::pss(pss_profile); - key.sign_signature(pss_profile, message, &mut signature).unwrap(); - public_key.verify_signature(pss_profile, message, &signature).unwrap(); + key + .sign_signature(pss_profile, message, &mut signature) + .expect("the generated key must produce an RSA-PSS signature"); + public_key + .verify_signature(pss_profile, message, &signature) + .expect("the generated public key must verify its RSA-PSS signature"); public_key .verify_signature_with_scratch(pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed RSA-PSS verification must accept the matching signature"); key .sign_signature_with_scratch(pss_profile, message, &mut signature, &mut private_scratch) - .unwrap(); - public_key.verify_signature(pss_profile, message, &signature).unwrap(); + .expect("scratch-backed RSA-PSS signing must succeed"); + public_key + .verify_signature(pss_profile, message, &signature) + .expect("the generated public key must verify the scratch-backed RSA-PSS signature"); public_key .verify_signature_with_scratch(pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed RSA-PSS signature"); } let explicit_pss_profile = RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha384, 24); key .sign_signature_with_scratch(explicit_pss_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("RSA-PSS signing with an explicit salt length must succeed"); public_key .verify_signature(explicit_pss_profile, message, &signature) - .unwrap(); + .expect("RSA-PSS verification must honor the explicit salt length"); public_key .verify_signature_with_scratch(explicit_pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed RSA-PSS verification must honor the explicit salt length"); key .sign_x509_signature_algorithm_der(X509_SHA256_WITH_RSA_ENCRYPTION, message, &mut signature) - .unwrap(); + .expect("the PKCS#1 v1.5 X.509 algorithm identifier must be accepted for signing"); x509_key .verify_signature_from_x509_algorithm_der(X509_SHA256_WITH_RSA_ENCRYPTION, message, &signature) - .unwrap(); + .expect("the X.509 key must verify the matching PKCS#1 v1.5 signature"); x509_key .verify_signature_from_x509_algorithm_der_with_scratch( X509_SHA256_WITH_RSA_ENCRYPTION, @@ -506,7 +545,7 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed X.509 verification must accept the matching PKCS#1 v1.5 signature"); key .sign_x509_signature_algorithm_der_with_scratch( X509_PSS_SHA256_ALGORITHM, @@ -514,10 +553,10 @@ fn root_surface_rsa_generated_key_end_to_end() { &mut signature, &mut private_scratch, ) - .unwrap(); + .expect("the RSA-PSS X.509 algorithm identifier must be accepted for scratch-backed signing"); x509_key .verify_signature_from_x509_algorithm_der(X509_PSS_SHA256_ALGORITHM, message, &signature) - .unwrap(); + .expect("the X.509 key must verify the matching RSA-PSS signature"); x509_key .verify_signature_from_x509_algorithm_der_with_scratch( X509_PSS_SHA256_ALGORITHM, @@ -525,17 +564,17 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed X.509 verification must accept the matching RSA-PSS signature"); let pss_sha256 = RsaSignatureProfile::pss(RsaPssProfile::Sha256); let pkcs1v15_sha256 = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); key .sign_tls13_signature_scheme(0x0804, message, &mut signature) - .unwrap(); + .expect("TLS 1.3 rsa_pss_rsae_sha256 signing must succeed"); x509_key .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_sha256, message, &signature) - .unwrap(); + .expect("the matching TLS 1.3 RSA-PSS signature must verify"); x509_key .verify_expected_tls13_signature_scheme_with_scratch( 0x0804, @@ -545,13 +584,13 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed TLS 1.3 RSA-PSS verification must accept the matching signature"); key .sign_tls13_signature_scheme_with_scratch(0x0804, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed TLS 1.3 RSA-PSS signing must succeed"); x509_key .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_sha256, message, &signature) - .unwrap(); + .expect("the scratch-backed TLS 1.3 RSA-PSS signature must verify"); x509_key .verify_expected_tls13_signature_scheme_with_scratch( 0x0804, @@ -561,14 +600,14 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed TLS 1.3 RSA-PSS signature"); key .sign_tls_certificate_signature_scheme(0x0401, message, &mut signature) - .unwrap(); + .expect("TLS certificate rsa_pkcs1_sha256 signing must succeed"); x509_key .verify_expected_tls_certificate_signature_scheme(0x0401, 0x0401, pkcs1v15_sha256, message, &signature) - .unwrap(); + .expect("the matching TLS certificate PKCS#1 v1.5 signature must verify"); x509_key .verify_expected_tls_certificate_signature_scheme_with_scratch( 0x0401, @@ -578,13 +617,13 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed TLS certificate verification must accept the matching signature"); key .sign_tls_certificate_signature_scheme_with_scratch(0x0401, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed TLS certificate PKCS#1 v1.5 signing must succeed"); x509_key .verify_expected_tls_certificate_signature_scheme(0x0401, 0x0401, pkcs1v15_sha256, message, &signature) - .unwrap(); + .expect("the scratch-backed TLS certificate signature must verify"); x509_key .verify_expected_tls_certificate_signature_scheme_with_scratch( 0x0401, @@ -594,17 +633,19 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut x509_scratch, ) - .unwrap(); + .expect("scratch-backed verification must accept the scratch-backed TLS certificate signature"); key .jwt_signer(RsaJwtAlgorithm::Ps256) .try_sign_into(message, &mut signature) - .unwrap(); + .expect("PS256 JWT signing must succeed"); let verifier = public_key.jwt_verifier(RsaJwtAlgorithm::Ps256); - verifier.verify("PS256", message, &signature).unwrap(); + verifier + .verify("PS256", message, &signature) + .expect("the matching PS256 JWT signature must verify"); verifier .verify_with_scratch("PS256", message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed PS256 JWT verification must accept the matching signature"); key .sign_signature_with_scratch( RsaJwtAlgorithm::Rs256.signature_profile(), @@ -612,26 +653,30 @@ fn root_surface_rsa_generated_key_end_to_end() { &mut signature, &mut private_scratch, ) - .unwrap(); + .expect("scratch-backed RS256 signing must succeed"); let verifier = public_key.jwt_verifier(RsaJwtAlgorithm::Rs256); - verifier.verify("RS256", message, &signature).unwrap(); + verifier + .verify("RS256", message, &signature) + .expect("the matching RS256 JWT signature must verify"); verifier .verify_with_scratch("RS256", message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed RS256 JWT verification must accept the matching signature"); - key.sign_cose_algorithm_id(-37, message, &mut signature).unwrap(); + key + .sign_cose_algorithm_id(-37, message, &mut signature) + .expect("COSE PS256 signing must succeed"); public_key .verify_expected_cose_algorithm_id(-37, -37, pss_sha256, message, &signature) - .unwrap(); + .expect("the matching COSE PS256 signature must verify"); public_key .verify_expected_cose_algorithm_id_with_scratch(-37, -37, pss_sha256, message, &signature, &mut public_scratch) - .unwrap(); + .expect("scratch-backed COSE PS256 verification must accept the matching signature"); key .sign_cose_algorithm_id_with_scratch(-257, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("scratch-backed COSE RS256 signing must succeed"); public_key .verify_expected_cose_algorithm_id(-257, -257, pkcs1v15_sha256, message, &signature) - .unwrap(); + .expect("the matching COSE RS256 signature must verify"); public_key .verify_expected_cose_algorithm_id_with_scratch( -257, @@ -641,7 +686,7 @@ fn root_surface_rsa_generated_key_end_to_end() { &signature, &mut public_scratch, ) - .unwrap(); + .expect("scratch-backed COSE RS256 verification must accept the matching signature"); let label = b"root-surface-rsa-label"; let plaintext = b"root-surface-rsa-oaep"; @@ -650,37 +695,41 @@ fn root_surface_rsa_generated_key_end_to_end() { for oaep_profile in [RsaOaepProfile::Sha256, RsaOaepProfile::Sha384, RsaOaepProfile::Sha512] { public_key .encrypt_oaep(oaep_profile, label, plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption of the bounded plaintext must succeed"); public_key .encrypt_oaep_with_scratch(oaep_profile, label, plaintext, &mut ciphertext, &mut public_scratch) - .unwrap(); + .expect("scratch-backed OAEP encryption of the bounded plaintext must succeed"); let decrypted_len = key .decrypt_oaep(oaep_profile, label, &ciphertext, &mut decrypted) - .unwrap(); + .expect("OAEP decryption with the matching key and label must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); let decrypted_len = key .decrypt_oaep_with_scratch(oaep_profile, label, &ciphertext, &mut decrypted, &mut private_scratch) - .unwrap(); + .expect("scratch-backed OAEP decryption with the matching key and label must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); } let legacy_plaintext = b"root-surface-rsaes-pkcs1v15"; - public_key.encrypt_pkcs1v15(legacy_plaintext, &mut ciphertext).unwrap(); + public_key + .encrypt_pkcs1v15(legacy_plaintext, &mut ciphertext) + .expect("PKCS#1 v1.5 encryption of the bounded plaintext must succeed"); public_key .encrypt_pkcs1v15_with_scratch(legacy_plaintext, &mut ciphertext, &mut public_scratch) - .unwrap(); - let decrypted_len = key.decrypt_pkcs1v15(&ciphertext, &mut decrypted).unwrap(); + .expect("scratch-backed PKCS#1 v1.5 encryption of the bounded plaintext must succeed"); + let decrypted_len = key + .decrypt_pkcs1v15(&ciphertext, &mut decrypted) + .expect("PKCS#1 v1.5 decryption with the matching key must succeed"); assert_eq!(&decrypted[..decrypted_len], legacy_plaintext); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut private_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 decryption with the matching key must succeed"); assert_eq!(&decrypted[..decrypted_len], legacy_plaintext); } #[test] #[cfg(all(feature = "rsa", feature = "getrandom"))] fn root_surface_rsa_default_generated_key_end_to_end() { - let key = RsaPrivateKey::generate(3072).unwrap(); + let key = RsaPrivateKey::generate(3072).expect("the default policy must generate a 3072-bit RSA key"); assert_eq!(key.public_key().modulus_bits(), 3072); let public_key = key.public_key(); let message = b"root-surface-rsa-default-generated-key"; @@ -690,19 +739,25 @@ fn root_surface_rsa_default_generated_key_end_to_end() { let pkcs1_der = key.to_pkcs1_der(); let pkcs8_der = key.to_pkcs8_der(); assert_eq!( - RsaPrivateKey::from_pkcs1_der(&pkcs1_der).unwrap().public_key(), + RsaPrivateKey::from_pkcs1_der(&pkcs1_der) + .expect("the default-policy private key must round-trip through PKCS#1 DER") + .public_key(), public_key ); assert_eq!( - RsaPrivateKey::from_pkcs8_der(&pkcs8_der).unwrap().public_key(), + RsaPrivateKey::from_pkcs8_der(&pkcs8_der) + .expect("the default-policy private key must round-trip through PKCS#8 DER") + .public_key(), public_key ); assert_eq!( - RsaPublicKey::from_pkcs1_der(&public_key.to_pkcs1_der()).unwrap(), + RsaPublicKey::from_pkcs1_der(&public_key.to_pkcs1_der()) + .expect("the default-policy public key must round-trip through PKCS#1 DER"), *public_key ); assert_eq!( - RsaPublicKey::from_spki_der(&public_key.to_spki_der()).unwrap(), + RsaPublicKey::from_spki_der(&public_key.to_spki_der()) + .expect("the default-policy public key must round-trip through SPKI DER"), *public_key ); @@ -710,18 +765,18 @@ fn root_surface_rsa_default_generated_key_end_to_end() { let pkcs1v15_profile = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); key .sign_signature_with_scratch(pkcs1v15_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("default-policy scratch-backed PKCS#1 v1.5 signing must succeed"); public_key .verify_signature_with_scratch(pkcs1v15_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("default-policy scratch-backed PKCS#1 v1.5 verification must succeed"); let pss_profile = RsaSignatureProfile::pss(RsaPssProfile::Sha256); key .sign_signature_with_scratch(pss_profile, message, &mut signature, &mut private_scratch) - .unwrap(); + .expect("default-policy scratch-backed RSA-PSS signing must succeed"); public_key .verify_signature_with_scratch(pss_profile, message, &signature, &mut public_scratch) - .unwrap(); + .expect("default-policy scratch-backed RSA-PSS verification must succeed"); let label = b"root-surface-rsa-default-label"; let plaintext = b"root-surface-rsa-default-oaep"; @@ -737,7 +792,7 @@ fn root_surface_rsa_default_generated_key_end_to_end() { &mut public_scratch, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("deterministic scratch-backed OAEP encryption must succeed"); let decrypted_len = key .decrypt_oaep_with_scratch( RsaOaepProfile::Sha256, @@ -746,7 +801,7 @@ fn root_surface_rsa_default_generated_key_end_to_end() { &mut decrypted, &mut private_scratch, ) - .unwrap(); + .expect("scratch-backed OAEP decryption of the deterministic ciphertext must succeed"); assert_eq!(&decrypted[..decrypted_len], plaintext); let legacy_plaintext = b"root-surface-rsa-default-rsaes-pkcs1v15"; @@ -757,10 +812,10 @@ fn root_surface_rsa_default_generated_key_end_to_end() { &mut public_scratch, fill_rsa_random_with(0x5b), ) - .unwrap(); + .expect("deterministic scratch-backed PKCS#1 v1.5 encryption must succeed"); let decrypted_len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut private_scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 decryption of the deterministic ciphertext must succeed"); assert_eq!(&decrypted[..decrypted_len], legacy_plaintext); } @@ -771,13 +826,16 @@ fn root_surface_key_exchange_exports_compile() { let bob = X25519SecretKey::from_bytes([13u8; X25519SecretKey::LENGTH]); let alice_public: X25519PublicKey = (&alice).into(); let bob_public: X25519PublicKey = (&bob).into(); - let alice_shared = alice.diffie_hellman(&bob_public).unwrap(); - let bob_shared = X25519SharedSecret::diffie_hellman(&bob, &alice_public).unwrap(); + let alice_shared = alice + .diffie_hellman(&bob_public) + .expect("the fixed Alice and Bob X25519 keys must produce a shared secret"); + let bob_shared = X25519SharedSecret::diffie_hellman(&bob, &alice_public) + .expect("the fixed Bob and Alice X25519 keys must produce a shared secret"); assert_eq!(alice_public.as_bytes().len(), 32); assert_eq!(alice_shared.as_bytes().len(), 32); assert!(alice_shared.ct_eq(&bob_shared).declassify()); - let _ = X25519Error::new(); + let _error = X25519Error::new(); } #[test] @@ -801,6 +859,10 @@ fn root_surface_checksum_exports_compile() { assert_eq!(Crc32Ieee::checksum(data), Crc32::checksum(data)); assert_eq!(Crc32Castagnoli::checksum(data), Crc32C::checksum(data)); assert_eq!(Crc64Xz::checksum(data), Crc64::checksum(data)); + let _crc16_ccitt = Crc16Ccitt::checksum(data); + let _crc16_ibm = Crc16Ibm::checksum(data); + let _crc24_openpgp = Crc24OpenPgp::checksum(data); + let _crc64_nvme = Crc64Nvme::checksum(data); } #[test] @@ -831,38 +893,41 @@ fn root_surface_hash_exports_compile() { streaming.update(data); assert_eq!(oneshot, streaming.finalize()); - let mut xof = Blake3::xof(data); + let mut xof: Blake3XofReader = Blake3::xof(data); let mut out = [0u8; 16]; xof.squeeze(&mut out); - let mut shake = Shake256::xof(data); + let mut shake128: Shake128XofReader = Shake128::xof(data); + shake128.squeeze(&mut out); + + let mut shake: Shake256XofReader = Shake256::xof(data); shake.squeeze(&mut out); - let mut ascon = AsconXof::xof(data); + let mut ascon: AsconXofReader = AsconXof::xof(data); ascon.squeeze(&mut out); - let mut cshake = Cshake256::xof(b"", b"ctx=v1", data); + let mut cshake: Cshake256XofReader = Cshake256::xof(b"", b"ctx=v1", data); cshake.squeeze(&mut out); - let mut cshake128 = Cshake128::xof(b"", b"ctx=v1", data); + let mut cshake128: Cshake128XofReader = Cshake128::xof(b"", b"ctx=v1", data); cshake128.squeeze(&mut out); - let _: Option = None; - let mut cxof = AsconCxof128::xof(b"ctx=v1", data).unwrap(); + let mut cxof: AsconCxof128Reader = + AsconCxof128::xof(b"ctx=v1", data).expect("the short Ascon-CXOF customization must be accepted"); cxof.squeeze(&mut out); assert_eq!(Xxh3::hash(data), Xxh3_64::hash(data)); - let _ = RapidHash64::hash(data); + let _rapid_hash = RapidHash64::hash(data); let deterministic = RapidSeededState::new(42); - let mut collection = deterministic.build_hasher(); + let mut collection: RapidHasher = deterministic.build_hasher(); collection.write(data); - let _ = collection.finish(); + let _collection_hash = collection.finish(); let random = RapidRandomState::try_new_with(|seed| { seed.copy_from_slice(&42u64.to_le_bytes()); Ok::<_, ()>(()) }) - .unwrap(); - let _ = random.hash_one(data); + .expect("the deterministic seed filler must initialize RapidHash state"); + let _randomized_hash = random.hash_one(data); let mut stream = RapidStreamHasher::new(); stream.write(data); @@ -873,20 +938,22 @@ fn root_surface_hash_exports_compile() { #[test] #[cfg(all(feature = "hashes", feature = "std"))] fn digest_reader_writer_round_trip() { - use std::io::{Cursor, Read, Write}; + use std::io::Write; let data = b"hello digest reader writer"; let expected = Sha256::digest(data); // DigestReader: read data through and verify digest matches. - let mut reader = DigestReader::<_, Sha256>::new(Cursor::new(data.to_vec())); + let mut reader = DigestReader::<_, Sha256>::new(data.as_slice()); let mut sink = Vec::new(); - std::io::copy(&mut reader, &mut sink).unwrap(); + std::io::copy(&mut reader, &mut sink).expect("copying from an in-memory digest reader must succeed"); assert_eq!(reader.digest(), expected); // DigestWriter: write data through and verify digest matches. let mut writer = DigestWriter::<_, Sha256>::new(Vec::new()); - writer.write_all(data).unwrap(); + writer + .write_all(data) + .expect("writing to an in-memory digest writer must succeed"); let (out, digest) = writer.into_parts(); assert_eq!(&out, data); assert_eq!(digest, expected); @@ -897,12 +964,18 @@ fn digest_reader_writer_round_trip() { fn advanced_checksum_modules_compile() { fn assert_kernel_introspect() {} - let _: Crc32Config = Crc32::config(); - let _ = Crc32Force::Auto; - let _ = DispatchInfo::current(); - let _ = kernel_for::(64); - let _ = is_hardware_accelerated(); - let _ = rscrypto::platform::describe(); + let _crc16_config: Crc16Config = Crc16Ccitt::config(); + let _crc24_config: Crc24Config = Crc24OpenPgp::config(); + let _crc32_config: Crc32Config = Crc32::config(); + let _crc64_config: Crc64Config = Crc64::config(); + let _crc16_force = Crc16Force::Auto; + let _crc24_force = Crc24Force::Auto; + let _crc32_force = Crc32Force::Auto; + let _crc64_force = Crc64Force::Auto; + let _dispatch = DispatchInfo::current(); + let _kernel = kernel_for::(64); + let _accelerated = is_hardware_accelerated(); + let _platform = rscrypto::platform::describe(); assert_kernel_introspect::(); } @@ -911,14 +984,14 @@ fn advanced_checksum_modules_compile() { fn advanced_hash_modules_compile() { fn assert_hash_kernel_introspect() {} - let _ = HashDispatchInfo::current(); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); - let _ = hash_kernel_for::(64); + let _dispatch = HashDispatchInfo::current(); + let _sha256_kernel = hash_kernel_for::(64); + let _shake256_kernel = hash_kernel_for::(64); + let _blake3_kernel = hash_kernel_for::(64); + let _ascon_hash_kernel = hash_kernel_for::(64); + let _ascon_xof_kernel = hash_kernel_for::(64); + let _ascon_cxof_kernel = hash_kernel_for::(64); + let _xxh3_kernel = hash_kernel_for::(64); assert_hash_kernel_introspect::(); assert_hash_kernel_introspect::(); diff --git a/tests/rsa_allocations.rs b/tests/rsa_allocations.rs index 1e9d7369..f45d0edf 100644 --- a/tests/rsa_allocations.rs +++ b/tests/rsa_allocations.rs @@ -8,8 +8,8 @@ use std::alloc::System; use rsa::{BigUint, RsaPrivateKey as RustCryptoRsaPrivateKey, pkcs1::EncodeRsaPrivateKey}; use rscrypto::{ - RsaEncryptionError, RsaJwtAlgorithm, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, - RsaPublicKeyPolicy, RsaSignatureProfile, RsaX509PublicKey, + RsaBlindingPair, RsaEncryptionError, RsaJwtAlgorithm, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, + RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy, RsaSignatureProfile, RsaX509PublicKey, }; static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); @@ -74,41 +74,49 @@ const X509_PSS_DEFAULT_SHA1_ALGORITHM: &[u8] = &[ ]; fn hex_to_vec(hex: &str) -> Vec { - assert_eq!(hex.len() % 2, 0); - let mut out = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let (chunks, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty(), "hex input must contain complete byte pairs"); + let mut out = Vec::with_capacity(chunks.len()); + for &[hi, lo] in chunks { + let hi = hex_value(hi).expect("hex fixtures must contain only ASCII hexadecimal digits"); + let lo = hex_value(lo).expect("hex fixtures must contain only ASCII hexadecimal digits"); + out.push((hi << 4) | lo); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("a short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("a DER length of at least 128 must contain a nonzero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -148,7 +156,7 @@ fn bit_string(value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -167,7 +175,10 @@ fn algorithm_identifier(algorithm_oid: &[u8], params: Option<&[u8]>) -> Vec fn exponent_bytes(value: u64) -> Vec { let bytes = value.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); bytes[first_nonzero..].to_vec() } @@ -179,7 +190,7 @@ fn pkcs1_from_public_key(key: &RsaPublicKey) -> Vec { } fn spki_for_pkcs1_with_algorithm(pkcs1: &[u8], algorithm: &[u8]) -> Vec { - let mut subject_public_key = Vec::with_capacity(1 + pkcs1.len()); + let mut subject_public_key = Vec::with_capacity(1usize.strict_add(pkcs1.len())); subject_public_key.push(0); subject_public_key.extend_from_slice(pkcs1); @@ -196,11 +207,13 @@ fn pss_algorithm_spki_from_rsa_encryption_spki(spki: &[u8]) -> Vec { } fn legacy_public_key_from_spki(spki: &[u8]) -> RsaPublicKey { - RsaPublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the SPKI fixture must satisfy the legacy verification policy") } fn legacy_x509_public_key_from_spki(spki: &[u8]) -> RsaX509PublicKey { - RsaX509PublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaX509PublicKey::from_spki_der_with_policy(spki, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the X.509 SPKI fixture must satisfy the legacy verification policy") } fn minimal_tbs_certificate(signature_algorithm_der: &[u8]) -> Vec { @@ -211,7 +224,7 @@ fn minimal_tbs_certificate(signature_algorithm_der: &[u8]) -> Vec { } fn x509_certificate(tbs_certificate_der: &[u8], signature_algorithm_der: &[u8], signature: &[u8]) -> Vec { - let mut signature_value = Vec::with_capacity(signature.len() + 1); + let mut signature_value = Vec::with_capacity(signature.len().strict_add(1)); signature_value.push(0); signature_value.extend_from_slice(signature); @@ -282,32 +295,35 @@ fn rustcrypto_fixture_private_key() -> RustCryptoRsaPrivateKey { b"00d397b84d98a4c26138ed1b695a8106ead91d553bf06041b62d3fdc50a041e222b8f4529689c1b82c5e71554f5dd69fa2f4b6158cf0dbeb57811a0fc327e1f28e74fe74d3bc166c1eabdc1b8b57b934ca8be5b00b4f29975bcc99acaf415b59bb28a6782bb41a2c3c2976b3c18dbadef62f00c6bb226640095096c0cc60d22fe7ef987d75c6a81b10d96bf292028af110dc7cc1bbc43d22adab379a0cd5d8078cc780ff5cd6209dea34c922cf784f7717e428d75b5aec8ff30e5f0141510766e2e0ab8d473c84e8710b2b98227c3db095337ad3452f19e2b9bfbccdd8148abf6776fa552775e6e75956e45229ae5a9c46949bab1e622f0e48f56524a84ed3483b", 16, ) - .unwrap(), - BigUint::parse_bytes(b"010001", 16).unwrap(), + .expect("the fixed RSA modulus must be valid hexadecimal"), + BigUint::parse_bytes(b"010001", 16).expect("the fixed RSA exponent must be valid hexadecimal"), BigUint::parse_bytes( b"00c4e70c689162c94c660828191b52b4d8392115df486a9adbe831e458d73958320dc1b755456e93701e9702d76fb0b92f90e01d1fe248153281fe79aa9763a92fae69d8d7ecd144de29fa135bd14f9573e349e45031e3b76982f583003826c552e89a397c1a06bd2163488630d92e8c2bb643d7abef700da95d685c941489a46f54b5316f62b5d2c3a7f1bbd134cb37353a44683fdc9d95d36458de22f6c44057fe74a0a436c4308f73f4da42f35c47ac16a7138d483afc91e41dc3a1127382e0c0f5119b0221b4fc639d6b9c38177a6de9b526ebd88c38d7982c07f98a0efd877d508aae275b946915c02e2e1106d175d74ec6777f5e80d12c053d9c7be1e341", 16, ) - .unwrap(), + .expect("the fixed RSA private exponent must be valid hexadecimal"), vec![ BigUint::parse_bytes( b"00f827bbf3a41877c7cc59aebf42ed4b29c32defcb8ed96863d5b090a05a8930dd624a21c9dcf9838568fdfa0df65b8462a5f2ac913d6c56f975532bd8e78fb07bd405ca99a484bcf59f019bbddcb3933f2bce706300b4f7b110120c5df9018159067c35da3061a56c8635a52b54273b31271b4311f0795df6021e6355e1a42e61", 16, ) - .unwrap(), + .expect("the fixed first RSA prime must be valid hexadecimal"), BigUint::parse_bytes( b"00da4817ce0089dd36f2ade6a3ff410c73ec34bf1b4f6bda38431bfede11cef1f7f6efa70e5f8063a3b1f6e17296ffb15feefa0912a0325b8d1fd65a559e717b5b961ec345072e0ec5203d03441d29af4d64054a04507410cf1da78e7b6119d909ec66e6ad625bf995b279a4b3c5be7d895cd7c5b9c4c497fde730916fcdb4e41b", 16, ) - .unwrap(), + .expect("the fixed second RSA prime must be valid hexadecimal"), ], ) - .unwrap() + .expect("the fixed RSA components must form a valid private key") } fn private_key() -> RsaPrivateKey { - let der = rustcrypto_fixture_private_key().to_pkcs1_der().unwrap(); - RsaPrivateKey::from_pkcs1_der_with_policy(der.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()).unwrap() + let der = rustcrypto_fixture_private_key() + .to_pkcs1_der() + .expect("the RustCrypto private-key fixture must encode as PKCS#1"); + RsaPrivateKey::from_pkcs1_der_with_policy(der.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) + .expect("rscrypto must decode the RustCrypto private-key fixture") } fn factor_two_and_inverse(modulus: &[u8]) -> (Vec, Vec) { @@ -319,32 +335,32 @@ fn factor_two_and_inverse(modulus: &[u8]) -> (Vec, Vec) { let mut plus_one = modulus.to_vec(); let mut carry = 1u16; for byte in plus_one.iter_mut().rev() { - let sum = u16::from(*byte) + carry; - *byte = sum as u8; + let sum = u16::from(*byte).strict_add(carry); + *byte = sum.to_le_bytes()[0]; carry = sum >> 8; if carry == 0 { break; } } if carry != 0 { - plus_one.insert(0, carry as u8); + plus_one.insert(0, u8::try_from(carry).expect("addition carry must be at most one")); } let mut quotient = Vec::with_capacity(plus_one.len()); let mut remainder = 0u16; for byte in plus_one { let value = (remainder << 8) | u16::from(byte); - quotient.push((value / 2) as u8); + quotient.push(u8::try_from(value / 2).expect("a base-256 long-division digit must fit in one byte")); remainder = value % 2; } let first_nonzero = quotient .iter() .position(|&byte| byte != 0) - .unwrap_or(quotient.len() - 1); + .unwrap_or_else(|| quotient.len().strict_sub(1)); let inverse = quotient[first_nonzero..].to_vec(); let mut inverse_fixed = vec![0u8; modulus.len()]; - inverse_fixed[modulus.len() - inverse.len()..].copy_from_slice(&inverse); + inverse_fixed[modulus.len().strict_sub(inverse.len())..].copy_from_slice(&inverse); (factor, inverse_fixed) } @@ -379,12 +395,12 @@ fn reused_scratch_rsa_operations_do_not_allocate() { key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("the warm-up RSA public operation must succeed"); reset_allocations(); key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("the measured RSA public operation must succeed"); assert_eq!(allocation_count(), 0); let sig = pss_signature_sha256(); @@ -392,7 +408,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); key .verify_pss_with_scratch(RsaPssProfile::Sha256, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PSS verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); @@ -403,20 +419,20 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("typed scratch-backed PSS verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .jwt_verifier(RsaJwtAlgorithm::Ps256) .verify_with_scratch("PS256", MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PS256 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .verify_expected_cose_algorithm_id_with_scratch(-37, -37, pss_sha256, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PS256 COSE verification must succeed"); assert_eq!(allocation_count(), 0); let seed = [0x42; 32]; @@ -430,7 +446,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &mut scratch, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("scratch-backed OAEP encryption must succeed"); assert_eq!(allocation_count(), 0); let x509_key = legacy_x509_public_key_from_spki(&pss_spki()); @@ -438,13 +454,13 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); x509_key .verify_signature_from_x509_algorithm_der_with_scratch(X509_PSS_SHA256_ALGORITHM, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed X.509 PSS verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); x509_key .verify_expected_tls13_signature_scheme_with_scratch(0x0804, 0x0804, pss_sha256, MESSAGE_PSS, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed TLS 1.3 PSS verification must succeed"); assert_eq!(allocation_count(), 0); let key = legacy_public_key_from_spki(&pkcs1v15_spki()); @@ -454,7 +470,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); key .verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, MESSAGE_PKCS1V15, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); @@ -465,20 +481,20 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("typed scratch-backed PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .jwt_verifier(RsaJwtAlgorithm::Rs256) .verify_with_scratch("RS256", MESSAGE_PKCS1V15, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed RS256 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); key .verify_expected_cose_algorithm_id_with_scratch(-257, -257, pkcs1v15_sha256, MESSAGE_PKCS1V15, &sig, &mut scratch) - .unwrap(); + .expect("scratch-backed RS256 COSE verification must succeed"); assert_eq!(allocation_count(), 0); let message = b"scratch-backed RSAES-PKCS1-v1_5 encryption"; @@ -486,7 +502,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { reset_allocations(); key .encrypt_pkcs1v15_with_random_fill_and_scratch(message, &mut ciphertext, &mut scratch, fill_rsa_random_with(0x5d)) - .unwrap(); + .expect("scratch-backed PKCS#1 v1.5 encryption must succeed"); assert_eq!(allocation_count(), 0); let x509_key = legacy_x509_public_key_from_spki(&pkcs1v15_spki()); @@ -499,7 +515,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("scratch-backed X.509 PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); reset_allocations(); @@ -512,7 +528,7 @@ fn reused_scratch_rsa_operations_do_not_allocate() { &sig, &mut scratch, ) - .unwrap(); + .expect("scratch-backed TLS certificate PKCS#1 v1.5 verification must succeed"); assert_eq!(allocation_count(), 0); assert_one_shot_protocol_rejects_fail_before_scratch_allocation(); @@ -651,12 +667,11 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, b"private scratch allocation PKCS1v15", - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded PKCS#1 v1.5 signing must succeed"); assert_eq!(allocation_count(), 0); let salt = [0x7a; 32]; @@ -666,12 +681,11 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { RsaPssProfile::Sha256, b"private scratch allocation PSS", &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded PSS signing must succeed"); assert_eq!(allocation_count(), 0); let label = b"private-scratch-allocation"; @@ -687,7 +701,7 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut ciphertext, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("OAEP fixture encryption must succeed"); reset_allocations(); let len = key @@ -695,12 +709,11 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded OAEP decryption must succeed"); assert_eq!(&decrypted[..len], plaintext); assert_eq!(allocation_count(), 0); @@ -708,18 +721,17 @@ fn assert_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { key .public_key() .encrypt_pkcs1v15_with_random_fill(pkcs1v15_plaintext, &mut ciphertext, fill_rsa_random_with(0x5d)) - .unwrap(); + .expect("PKCS#1 v1.5 fixture encryption must succeed"); reset_allocations(); let len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut decrypted, &mut scratch, ) - .unwrap(); + .expect("scratch-backed blinded PKCS#1 v1.5 decryption must succeed"); assert_eq!(&decrypted[..len], pkcs1v15_plaintext); assert_eq!(allocation_count(), 0); } @@ -738,7 +750,7 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed randomized PKCS#1 v1.5 signing must succeed"); assert_eq!(allocation_count(), 0); key .public_key() @@ -747,7 +759,7 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { b"rng private scratch allocation PKCS1v15", &signature, ) - .unwrap(); + .expect("the randomized PKCS#1 v1.5 signature must verify"); reset_allocations(); key @@ -757,12 +769,12 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut signature, &mut scratch, ) - .unwrap(); + .expect("scratch-backed randomized PSS signing must succeed"); assert_eq!(allocation_count(), 0); key .public_key() .verify_pss(RsaPssProfile::Sha256, b"rng private scratch allocation PSS", &signature) - .unwrap(); + .expect("the randomized PSS signature must verify"); let label = b"rng-private-scratch-allocation"; let plaintext = b"rng private scratch allocation OAEP"; @@ -777,12 +789,12 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { &mut ciphertext, fill_rsa_random_from(&seed), ) - .unwrap(); + .expect("OAEP fixture encryption must succeed"); reset_allocations(); let len = key .decrypt_oaep_with_scratch(RsaOaepProfile::Sha256, label, &ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("scratch-backed randomized OAEP decryption must succeed"); assert_eq!(&decrypted[..len], plaintext); assert_eq!(allocation_count(), 0); @@ -790,12 +802,12 @@ fn assert_rng_private_scratch_operations_do_not_allocate(key: &RsaPrivateKey) { key .public_key() .encrypt_pkcs1v15_with_random_fill(pkcs1v15_plaintext, &mut ciphertext, fill_rsa_random_with(0x5d)) - .unwrap(); + .expect("PKCS#1 v1.5 fixture encryption must succeed"); reset_allocations(); let len = key .decrypt_pkcs1v15_with_scratch(&ciphertext, &mut decrypted, &mut scratch) - .unwrap(); + .expect("scratch-backed randomized PKCS#1 v1.5 decryption must succeed"); assert_eq!(&decrypted[..len], pkcs1v15_plaintext); assert_eq!(allocation_count(), 0); } diff --git a/tests/rsa_leakage.rs b/tests/rsa_leakage.rs index 59db0371..2f86cb87 100644 --- a/tests/rsa_leakage.rs +++ b/tests/rsa_leakage.rs @@ -4,7 +4,7 @@ use core::hint::black_box; use std::time::Instant; use rscrypto::{ - RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, + RsaBlindingPair, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, auth::rsa::diag_rsa_blinding_factor_inverse, }; use serde_json::Value; @@ -48,7 +48,7 @@ struct OnlineStats { impl OnlineStats { fn push(&mut self, value: f64) { - self.n += 1; + self.n = self.n.strict_add(1); let delta = value - self.mean; self.mean += delta / self.n as f64; let delta2 = value - self.mean; @@ -56,7 +56,11 @@ impl OnlineStats { } fn variance(&self) -> f64 { - if self.n > 1 { self.m2 / (self.n - 1) as f64 } else { 0.0 } + if self.n > 1 { + self.m2 / self.n.strict_sub(1) as f64 + } else { + 0.0 + } } } @@ -114,13 +118,13 @@ where }; let index = if use_random { let index = random_index; - random_index += 1; - random_remaining -= 1; + random_index = random_index.strict_add(1); + random_remaining = random_remaining.strict_sub(1); index } else { let index = fixed_index; - fixed_index += 1; - fixed_remaining -= 1; + fixed_index = fixed_index.strict_add(1); + fixed_remaining = fixed_remaining.strict_sub(1); index }; @@ -150,17 +154,19 @@ fn hex_to_vec(hex: &str) -> Vec { assert_eq!(hex.len() % 2, 0); let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("leakage fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("leakage fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -192,32 +198,32 @@ fn factor_two_and_inverse(modulus: &[u8]) -> (Vec, Vec) { let mut plus_one = modulus.to_vec(); let mut carry = 1u16; for byte in plus_one.iter_mut().rev() { - let sum = u16::from(*byte) + carry; - *byte = sum as u8; + let sum = u16::from(*byte).strict_add(carry); + *byte = u8::try_from(sum & 0xff).expect("masked addition result must fit u8"); carry = sum >> 8; if carry == 0 { break; } } if carry != 0 { - plus_one.insert(0, carry as u8); + plus_one.insert(0, u8::try_from(carry).expect("addition carry must fit u8")); } let mut quotient = Vec::with_capacity(plus_one.len()); let mut remainder = 0u16; for byte in plus_one { let value = (remainder << 8) | u16::from(byte); - quotient.push((value / 2) as u8); + quotient.push(u8::try_from(value / 2).expect("base-256 division quotient must fit u8")); remainder = value % 2; } let first_nonzero = quotient .iter() .position(|&byte| byte != 0) - .unwrap_or(quotient.len() - 1); + .unwrap_or_else(|| quotient.len().strict_sub(1)); let inverse = "ient[first_nonzero..]; let mut inverse_fixed = vec![0u8; modulus.len()]; - inverse_fixed[modulus.len() - inverse.len()..].copy_from_slice(inverse); + inverse_fixed[modulus.len().strict_sub(inverse.len())..].copy_from_slice(inverse); (factor, inverse_fixed) } @@ -274,7 +280,7 @@ fn fixed_and_random_oaep_ciphertexts(key: &RsaPrivateKey, count: usize) -> (Vec< fn fixed_and_random_pkcs1v15_ciphertexts(key: &RsaPrivateKey, count: usize) -> (Vec, Vec>) { let len = key.public_key().modulus().len(); let plaintext_len = b"fixed pkcs1v15 leakage text".len(); - let padding_len = len - plaintext_len - 3; + let padding_len = len.strict_sub(plaintext_len).strict_sub(3); let fixed_seed = vec![0x5a; padding_len]; let mut fixed = vec![0u8; len]; key @@ -288,7 +294,7 @@ fn fixed_and_random_pkcs1v15_ciphertexts(key: &RsaPrivateKey, count: usize) -> ( let mut seed = vec![0u8; padding_len]; for byte in &mut seed { while *byte == 0 { - *byte = rng.next_u64() as u8; + *byte = u8::try_from(rng.next_u64() & 0xff).expect("masked PRNG output must fit u8"); } } let mut plaintext = vec![0u8; plaintext_len]; @@ -361,8 +367,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, black_box(message), - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) @@ -389,8 +394,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { RsaPssProfile::Sha256, black_box(message), &pss_salt, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) @@ -417,8 +421,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { RsaOaepProfile::Sha256, b"leakage-label", black_box(ciphertext), - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) @@ -443,8 +446,7 @@ fn rsa_private_operations_do_not_show_first_order_timing_leakage() { let plaintext_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( black_box(ciphertext), - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), black_box(&mut out), &mut scratch, ) diff --git a/tests/rsa_nist_cavp.rs b/tests/rsa_nist_cavp.rs index b5ed390b..afe96076 100644 --- a/tests/rsa_nist_cavp.rs +++ b/tests/rsa_nist_cavp.rs @@ -5,8 +5,8 @@ extern crate alloc; use alloc::collections::BTreeMap; use rscrypto::{ - RsaKeyError, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy, - RsaSignatureProfile, + RsaBlindingPair, RsaKeyError, RsaPkcs1v15Profile, RsaPrivateKey, RsaPrivateKeyParts, RsaPssProfile, RsaPublicKey, + RsaPublicKeyPolicy, RsaSignatureProfile, }; use serde_json::Value; @@ -19,24 +19,41 @@ struct Counts { invalid: usize, } +#[derive(Clone, Copy)] +enum SigGenScheme { + Pkcs1v15, + Pss, +} + +#[derive(Clone, Copy)] +enum CavpResult { + Pass, + Fail, +} + fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("long DER length must contain a non-zero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("DER length-of-length must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -44,7 +61,7 @@ fn tlv(tag: u8, value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -72,7 +89,9 @@ fn hex_to_vec(hex: &str) -> Vec { let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("CAVP fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("CAVP fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } @@ -82,7 +101,7 @@ fn hex_to_canonical_vec(hex: &str) -> Vec { let first_nonzero = value .iter() .position(|&byte| byte != 0) - .unwrap_or(value.len().strict_sub(1)); + .unwrap_or_else(|| value.len().strict_sub(1)); if first_nonzero != 0 { value.drain(..first_nonzero); } @@ -99,44 +118,67 @@ fn hex_to_u64(hex: &str) -> u64 { value } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn field<'a>(value: &'a Value, name: &'static str) -> &'a str { value[name] .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("CAVP fixture must contain the requested string field") } fn pkcs1_profile(sha: &str) -> RsaPkcs1v15Profile { match sha { - "SHA256" => RsaPkcs1v15Profile::Sha256, - "SHA384" => RsaPkcs1v15Profile::Sha384, - "SHA512" => RsaPkcs1v15Profile::Sha512, - other => panic!("unsupported CAVP PKCS1v1.5 hash `{other}`"), + "SHA256" => Some(RsaPkcs1v15Profile::Sha256), + "SHA384" => Some(RsaPkcs1v15Profile::Sha384), + "SHA512" => Some(RsaPkcs1v15Profile::Sha512), + _ => None, } + .expect("CAVP fixture must use a supported PKCS#1 v1.5 hash") } fn pss_profile(sha: &str) -> RsaPssProfile { match sha { - "SHA256" => RsaPssProfile::Sha256, - "SHA384" => RsaPssProfile::Sha384, - "SHA512" => RsaPssProfile::Sha512, - other => panic!("unsupported CAVP PSS hash `{other}`"), + "SHA256" => Some(RsaPssProfile::Sha256), + "SHA384" => Some(RsaPssProfile::Sha384), + "SHA512" => Some(RsaPssProfile::Sha512), + _ => None, } + .expect("CAVP fixture must use a supported PSS hash") } -fn signature_profile(scheme: &str, sha: &str, salt_len: Option) -> RsaSignatureProfile { +fn siggen_scheme(scheme: &str) -> SigGenScheme { match scheme { - "pkcs1v15" => RsaSignatureProfile::pkcs1v15(pkcs1_profile(sha)), - "pss" => RsaSignatureProfile::pss_with_salt_len(pss_profile(sha), salt_len.unwrap() as usize), - other => panic!("unsupported CAVP RSA signature scheme `{other}`"), + "pkcs1v15" => Some(SigGenScheme::Pkcs1v15), + "pss" => Some(SigGenScheme::Pss), + _ => None, + } + .expect("CAVP fixture must use a supported RSA signature scheme") +} + +fn cavp_result(result: &str) -> CavpResult { + match result { + "P" => Some(CavpResult::Pass), + "F" => Some(CavpResult::Fail), + _ => None, + } + .expect("CAVP fixture result must be P or F") +} + +fn signature_profile(scheme: &str, sha: &str, salt_len: Option) -> RsaSignatureProfile { + match siggen_scheme(scheme) { + SigGenScheme::Pkcs1v15 => RsaSignatureProfile::pkcs1v15(pkcs1_profile(sha)), + SigGenScheme::Pss => RsaSignatureProfile::pss_with_salt_len( + pss_profile(sha), + usize::try_from(salt_len.expect("CAVP PSS case must provide a salt length")) + .expect("CAVP PSS salt length must fit usize"), + ), } } @@ -187,7 +229,8 @@ fn nist_cavp_odd_public_exponents_require_explicit_policy() { ); let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_odd_exponents(); - let key = RsaPublicKey::from_pkcs1_der_with_policy(&key_der, &policy).unwrap(); + let key = RsaPublicKey::from_pkcs1_der_with_policy(&key_der, &policy) + .expect("CAVP odd-exponent RSA key must parse under the explicit legacy policy"); assert_eq!(key.modulus_bits(), 2048); } @@ -242,15 +285,14 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { let blinding_factor = fixed_width_one(key.public_key().modulus().len()); let blinding_factor_inverse = fixed_width_one(key.public_key().modulus().len()); - match field(test, "scheme") { - "pkcs1v15" => { + match siggen_scheme(field(test, "scheme")) { + SigGenScheme::Pkcs1v15 => { pkcs1v15 = pkcs1v15.strict_add(1); key .sign_pkcs1v15_with_blinding_factor( pkcs1_profile(field(test, "sha")), &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) .expect("CAVP PKCS1v1.5 private signing must succeed"); @@ -259,8 +301,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { .sign_pkcs1v15_with_blinding_factor_and_scratch( pkcs1_profile(field(test, "sha")), &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_signature, &mut scratch, ) @@ -274,7 +315,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { .verify_pkcs1v15(pkcs1_profile(field(test, "sha")), &message, &signature) .expect("CAVP PKCS1v1.5 generated signature must verify"); } - "pss" => { + SigGenScheme::Pss => { pss = pss.strict_add(1); let salt = hex_to_vec(field(test, "salt")); key @@ -282,8 +323,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { pss_profile(field(test, "sha")), &message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) .expect("CAVP PSS private signing must succeed"); @@ -293,8 +333,7 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { pss_profile(field(test, "sha")), &message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_signature, &mut scratch, ) @@ -312,7 +351,6 @@ fn nist_cavp_sha2_siggen_private_operations_match_expected_signatures() { ) .expect("CAVP PSS generated signature must verify"); } - other => panic!("unsupported CAVP SigGen scheme `{other}`"), } } @@ -386,8 +424,7 @@ fn nist_cavp_same_width_private_scratch_rebinds_between_keys() { pss_profile(field(test_b, "sha")), &message, &salt, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, &mut scratch, ) @@ -411,8 +448,8 @@ fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { let mut scratch_signature = vec![0u8; key.public_key().modulus().len()]; let mut scratch = key.private_scratch(); - match field(test, "scheme") { - "pkcs1v15" => { + match siggen_scheme(field(test, "scheme")) { + SigGenScheme::Pkcs1v15 => { pkcs1v15 = pkcs1v15.strict_add(1); let profile = RsaSignatureProfile::pkcs1v15(pkcs1_profile(field(test, "sha"))); key @@ -430,7 +467,7 @@ fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { "CAVP PKCS1v1.5 scratch profile signature mismatch" ); } - "pss" => { + SigGenScheme::Pss => { pss = pss.strict_add(1); let salt_len = hex_to_vec(field(test, "salt")).len(); let profile = RsaSignatureProfile::pss_with_salt_len(pss_profile(field(test, "sha")), salt_len); @@ -449,7 +486,6 @@ fn nist_cavp_sha2_siggen_profile_signing_matches_expected_results() { .verify_signature(profile, &message, &scratch_signature) .expect("CAVP PSS scratch profile signature must verify"); } - other => panic!("unsupported CAVP SigGen scheme `{other}`"), } } @@ -515,12 +551,12 @@ fn nist_cavp_supported_sha2_sigver_subset_matches_expected_results() { ) .is_ok(); - match field(test, "result") { - "P" => { + match cavp_result(field(test, "result")) { + CavpResult::Pass => { counts.valid = counts.valid.strict_add(1); assert!(verified, "CAVP tcId {} rejected valid signature", test["tc_id"]); } - "F" => { + CavpResult::Fail => { counts.invalid = counts.invalid.strict_add(1); assert!( !verified, @@ -528,7 +564,6 @@ fn nist_cavp_supported_sha2_sigver_subset_matches_expected_results() { test["tc_id"], scheme ); } - other => panic!("unknown CAVP result `{other}`"), } } diff --git a/tests/rsa_profile_confusion.rs b/tests/rsa_profile_confusion.rs index 03bcd157..dc94afb5 100644 --- a/tests/rsa_profile_confusion.rs +++ b/tests/rsa_profile_confusion.rs @@ -17,16 +17,16 @@ fn assert_rejects(result: Result<(), VerificationError>) { #[test] fn rsa_signatures_reject_padding_profile_confusion() { - let key = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let key = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("RSA-3072 SPKI fixture must parse"); let pss = RsaSignatureProfile::pss(RsaPssProfile::Sha256); let pkcs1v15 = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); - assert!(key.verify_signature(pss, PSS_MESSAGE, RSA3072_PSS_SHA256).is_ok()); - assert!( - key - .verify_signature(pkcs1v15, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) - .is_ok() - ); + key + .verify_signature(pss, PSS_MESSAGE, RSA3072_PSS_SHA256) + .expect("RSA-PSS fixture signature must verify with its matching profile"); + key + .verify_signature(pkcs1v15, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) + .expect("PKCS#1 v1.5 fixture signature must verify with its matching profile"); assert_rejects(key.verify_signature(pkcs1v15, PSS_MESSAGE, RSA3072_PSS_SHA256)); assert_rejects(key.verify_signature(pss, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256)); @@ -36,19 +36,15 @@ fn rsa_signatures_reject_padding_profile_confusion() { #[test] fn rsa_protocol_scheme_helpers_reject_signature_profile_confusion() { - let key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).unwrap(); - let raw_key = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); - - assert!( - key - .verify_tls13_signature_scheme(0x0804, PSS_MESSAGE, RSA3072_PSS_SHA256) - .is_ok() - ); - assert!( - key - .verify_tls_certificate_signature_scheme(0x0401, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) - .is_ok() - ); + let key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).expect("RSA-3072 X.509 SPKI fixture must parse"); + let raw_key = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("RSA-3072 SPKI fixture must parse"); + + key + .verify_tls13_signature_scheme(0x0804, PSS_MESSAGE, RSA3072_PSS_SHA256) + .expect("TLS 1.3 rsa_pss_rsae_sha256 fixture must verify"); + key + .verify_tls_certificate_signature_scheme(0x0401, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256) + .expect("TLS rsa_pkcs1_sha256 certificate fixture must verify"); assert_rejects(key.verify_tls13_signature_scheme(0x0804, PKCS1V15_MESSAGE, RSA3072_PKCS1V15_SHA256)); assert_rejects(key.verify_tls_certificate_signature_scheme(0x0401, PSS_MESSAGE, RSA3072_PSS_SHA256)); diff --git a/tests/rsa_public_key.rs b/tests/rsa_public_key.rs index 87d72486..86e95e1c 100644 --- a/tests/rsa_public_key.rs +++ b/tests/rsa_public_key.rs @@ -9,7 +9,7 @@ use core::{ use std::{ fs, path::PathBuf, - process::{self, Command}, + process::{self, Command, Output}, sync::OnceLock, time::{SystemTime, UNIX_EPOCH}, }; @@ -71,22 +71,27 @@ const RSA8192_PKCS1V15_SHA256: &[u8] = include_bytes!("../benches/rsa_fixtures/r fn der_len(len: usize) -> Vec { if len < 128 { - return vec![len as u8]; + return vec![u8::try_from(len).expect("a short DER length must fit in one byte")]; } let bytes = len.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap(); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .expect("a DER length of at least 128 must contain a nonzero byte"); let len_bytes = &bytes[first_nonzero..]; - let mut out = Vec::with_capacity(1 + len_bytes.len()); - out.push(0x80 | len_bytes.len() as u8); + let mut out = Vec::with_capacity(1usize.strict_add(len_bytes.len())); + out.push(0x80 | u8::try_from(len_bytes.len()).expect("a usize DER length prefix must fit in one byte")); out.extend_from_slice(len_bytes); out } fn tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = Vec::with_capacity(1 + der_len(value.len()).len() + value.len()); + let encoded_len = der_len(value.len()); + let capacity = 1usize.strict_add(encoded_len.len()).strict_add(value.len()); + let mut out = Vec::with_capacity(capacity); out.push(tag); - out.extend_from_slice(&der_len(value.len())); + out.extend_from_slice(&encoded_len); out.extend_from_slice(value); out } @@ -99,7 +104,7 @@ fn tlv_with_leading_zero_long_len(der: &[u8]) -> Vec { let mut out = Vec::with_capacity(der.len().strict_add(1)); out.push(tag); - out.push(0x80 | (len_len.strict_add(1) as u8)); + out.push(0x80 | u8::try_from(len_len.strict_add(1)).expect("a DER length-of-length must fit in one byte")); out.push(0); out.extend_from_slice(&der[2..]); out @@ -170,7 +175,7 @@ fn context_constructed(index: u8, value: &[u8]) -> Vec { fn integer_unsigned(value: &[u8]) -> Vec { let first_nonzero = value.iter().position(|&byte| byte != 0); let value = first_nonzero.map_or(&[0u8][..], |index| &value[index..]); - let mut encoded = Vec::with_capacity(value.len() + usize::from(value[0] & 0x80 != 0)); + let mut encoded = Vec::with_capacity(value.len().strict_add(usize::from(value[0] & 0x80 != 0))); if value[0] & 0x80 != 0 { encoded.push(0); } @@ -188,11 +193,12 @@ fn algorithm_identifier(algorithm_oid: &[u8], params: Option<&[u8]>) -> Vec } fn hex_to_vec(hex: &str) -> Vec { - assert_eq!(hex.len() % 2, 0); - let mut out = Vec::with_capacity(hex.len() / 2); - for chunk in hex.as_bytes().chunks_exact(2) { - let hi = hex_value(chunk[0]); - let lo = hex_value(chunk[1]); + let (chunks, remainder) = hex.as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty(), "hex input must contain complete byte pairs"); + let mut out = Vec::with_capacity(chunks.len()); + for &[hi, lo] in chunks { + let hi = hex_value(hi).expect("hex fixtures must contain only ASCII hexadecimal digits"); + let lo = hex_value(lo).expect("hex fixtures must contain only ASCII hexadecimal digits"); out.push((hi << 4) | lo); } out @@ -211,12 +217,12 @@ fn cavp_hex_to_vec(hex: &str) -> Vec { hex_to_vec(hex) } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -246,20 +252,26 @@ fn valid_pkcs1_with_modulus_and_exponent(modulus: &[u8], exponent: &[u8]) -> Vec } fn legacy_public_key_from_pkcs1(der: &[u8]) -> RsaPublicKey { - RsaPublicKey::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPublicKey::from_pkcs1_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the PKCS#1 fixture must satisfy the legacy verification policy") } fn legacy_public_key_from_spki(der: &[u8]) -> RsaPublicKey { - RsaPublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the SPKI fixture must satisfy the legacy verification policy") } fn legacy_x509_public_key_from_spki(der: &[u8]) -> RsaX509PublicKey { - RsaX509PublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaX509PublicKey::from_spki_der_with_policy(der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("the X.509 SPKI fixture must satisfy the legacy verification policy") } fn exponent_bytes(exponent: u64) -> Vec { let bytes = exponent.to_be_bytes(); - let first_nonzero = bytes.iter().position(|&byte| byte != 0).unwrap_or(bytes.len() - 1); + let first_nonzero = bytes + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| bytes.len().strict_sub(1)); bytes[first_nonzero..].to_vec() } @@ -283,12 +295,16 @@ fn x509_pss_algorithm(profile: RsaPssProfile, salt_len: usize, trailer: Option Vec params.extend_from_slice(&context_constructed(1, &mgf1_sha256_without_params)); params.extend_from_slice(&context_constructed( 2, - &integer_unsigned(&exponent_bytes(u64::try_from(salt_len).unwrap())), + &integer_unsigned(&exponent_bytes( + u64::try_from(salt_len).expect("an X.509 PSS salt length must fit in u64"), + )), )); algorithm_identifier(ID_RSASSA_PSS_OID, Some(&sequence(¶ms))) } fn x509_certificate(tbs_certificate_der: &[u8], signature_algorithm_der: &[u8], signature: &[u8]) -> Vec { - let mut signature_value = Vec::with_capacity(signature.len() + 1); + let mut signature_value = Vec::with_capacity(signature.len().strict_add(1)); signature_value.push(0); signature_value.extend_from_slice(signature); @@ -386,27 +404,27 @@ fn rustcrypto_fixture_private_key() -> RustCryptoRsaPrivateKey { b"00d397b84d98a4c26138ed1b695a8106ead91d553bf06041b62d3fdc50a041e222b8f4529689c1b82c5e71554f5dd69fa2f4b6158cf0dbeb57811a0fc327e1f28e74fe74d3bc166c1eabdc1b8b57b934ca8be5b00b4f29975bcc99acaf415b59bb28a6782bb41a2c3c2976b3c18dbadef62f00c6bb226640095096c0cc60d22fe7ef987d75c6a81b10d96bf292028af110dc7cc1bbc43d22adab379a0cd5d8078cc780ff5cd6209dea34c922cf784f7717e428d75b5aec8ff30e5f0141510766e2e0ab8d473c84e8710b2b98227c3db095337ad3452f19e2b9bfbccdd8148abf6776fa552775e6e75956e45229ae5a9c46949bab1e622f0e48f56524a84ed3483b", 16, ) - .unwrap(), - BigUint::parse_bytes(b"010001", 16).unwrap(), + .expect("the fixed RSA modulus must be valid hexadecimal"), + BigUint::parse_bytes(b"010001", 16).expect("the fixed RSA exponent must be valid hexadecimal"), BigUint::parse_bytes( b"00c4e70c689162c94c660828191b52b4d8392115df486a9adbe831e458d73958320dc1b755456e93701e9702d76fb0b92f90e01d1fe248153281fe79aa9763a92fae69d8d7ecd144de29fa135bd14f9573e349e45031e3b76982f583003826c552e89a397c1a06bd2163488630d92e8c2bb643d7abef700da95d685c941489a46f54b5316f62b5d2c3a7f1bbd134cb37353a44683fdc9d95d36458de22f6c44057fe74a0a436c4308f73f4da42f35c47ac16a7138d483afc91e41dc3a1127382e0c0f5119b0221b4fc639d6b9c38177a6de9b526ebd88c38d7982c07f98a0efd877d508aae275b946915c02e2e1106d175d74ec6777f5e80d12c053d9c7be1e341", 16, ) - .unwrap(), + .expect("the fixed RSA private exponent must be valid hexadecimal"), vec![ BigUint::parse_bytes( b"00f827bbf3a41877c7cc59aebf42ed4b29c32defcb8ed96863d5b090a05a8930dd624a21c9dcf9838568fdfa0df65b8462a5f2ac913d6c56f975532bd8e78fb07bd405ca99a484bcf59f019bbddcb3933f2bce706300b4f7b110120c5df9018159067c35da3061a56c8635a52b54273b31271b4311f0795df6021e6355e1a42e61", 16, ) - .unwrap(), + .expect("the fixed first RSA prime must be valid hexadecimal"), BigUint::parse_bytes( b"00da4817ce0089dd36f2ade6a3ff410c73ec34bf1b4f6bda38431bfede11cef1f7f6efa70e5f8063a3b1f6e17296ffb15feefa0912a0325b8d1fd65a559e717b5b961ec345072e0ec5203d03441d29af4d64054a04507410cf1da78e7b6119d909ec66e6ad625bf995b279a4b3c5be7d895cd7c5b9c4c497fde730916fcdb4e41b", 16, ) - .unwrap(), + .expect("the fixed second RSA prime must be valid hexadecimal"), ], ) - .unwrap() + .expect("the fixed RSA components must form a valid private key") } fn x509_sha256_rsa_self_signed_certificate(params: Option<&[u8]>) -> (Vec, Vec) { @@ -552,7 +570,7 @@ eb10e16719b6b54d1768dc5278e6bcebc67d45226ab0164ede685b74a3d53eb14bcdfb", } fn spki_for_pkcs1_with_algorithm(pkcs1: &[u8], algorithm: &[u8]) -> Vec { - let mut subject_public_key = Vec::with_capacity(pkcs1.len() + 1); + let mut subject_public_key = Vec::with_capacity(pkcs1.len().strict_add(1)); subject_public_key.push(0); subject_public_key.extend_from_slice(pkcs1); @@ -600,58 +618,78 @@ fn assert_aws_lc_rs_pkcs1v15_sha256(spki: &[u8], message: &[u8], signature: &[u8 fn assert_ring_cavp(scheme: &str, sha: &str, pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { let actual = match (scheme, sha) { - ("pss", "SHA256") => ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA256, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA384") => ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA384, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA512") => ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA512, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA256") => { + ("pss", "SHA256") => Some( + ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA256, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA384") => Some( + ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA384, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA512") => Some( + ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PSS_2048_8192_SHA512, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA256") => Some( ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PKCS1_2048_8192_SHA256, pkcs1) .verify(message, signature) - .is_ok() - } - ("pkcs1v15", "SHA384") => { + .is_ok(), + ), + ("pkcs1v15", "SHA384") => Some( ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PKCS1_2048_8192_SHA384, pkcs1) .verify(message, signature) - .is_ok() - } - ("pkcs1v15", "SHA512") => { + .is_ok(), + ), + ("pkcs1v15", "SHA512") => Some( ring_signature::UnparsedPublicKey::new(&ring_signature::RSA_PKCS1_2048_8192_SHA512, pkcs1) .verify(message, signature) - .is_ok() - } - other => panic!("unsupported ring CAVP profile {other:?}"), - }; + .is_ok(), + ), + _ => None, + } + .expect("CAVP fixtures must use a ring-supported RSA/SHA-2 profile"); assert_eq!(actual, expected, "ring mismatch for {scheme}/{sha}"); } #[cfg(not(any(target_arch = "s390x", target_arch = "powerpc64")))] fn assert_aws_lc_rs_cavp(scheme: &str, sha: &str, pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { let actual = match (scheme, sha) { - ("pss", "SHA256") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA256, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA384") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA384, pkcs1) - .verify(message, signature) - .is_ok(), - ("pss", "SHA512") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA512, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA256") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA256, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA384") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA384, pkcs1) - .verify(message, signature) - .is_ok(), - ("pkcs1v15", "SHA512") => aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA512, pkcs1) - .verify(message, signature) - .is_ok(), - other => panic!("unsupported aws-lc-rs CAVP profile {other:?}"), - }; + ("pss", "SHA256") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA256, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA384") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA384, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pss", "SHA512") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PSS_2048_8192_SHA512, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA256") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA256, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA384") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA384, pkcs1) + .verify(message, signature) + .is_ok(), + ), + ("pkcs1v15", "SHA512") => Some( + aws_signature::UnparsedPublicKey::new(&aws_signature::RSA_PKCS1_2048_8192_SHA512, pkcs1) + .verify(message, signature) + .is_ok(), + ), + _ => None, + } + .expect("CAVP fixtures must use an aws-lc-rs-supported RSA/SHA-2 profile"); assert_eq!(actual, expected, "aws-lc-rs mismatch for {scheme}/{sha}"); } @@ -689,30 +727,30 @@ impl Drop for AwsLcMdCtx { } #[cfg(not(any(target_arch = "s390x", target_arch = "powerpc64")))] -fn aws_lc_md(sha: &str) -> *const aws_lc::EVP_MD { +fn aws_lc_md(sha: &str) -> Option<*const aws_lc::EVP_MD> { match sha { "SHA256" => { // SAFETY: Obtains AWS-LC's static SHA-256 EVP_MD descriptor because: // 1. The function takes no caller-owned pointers. // 2. AWS-LC returns a process-static descriptor that must not be freed. // 3. The descriptor is only passed back into AWS-LC verification APIs. - unsafe { aws_lc::EVP_sha256() } + Some(unsafe { aws_lc::EVP_sha256() }) } "SHA384" => { // SAFETY: Obtains AWS-LC's static SHA-384 EVP_MD descriptor because: // 1. The function takes no caller-owned pointers. // 2. AWS-LC returns a process-static descriptor that must not be freed. // 3. The descriptor is only passed back into AWS-LC verification APIs. - unsafe { aws_lc::EVP_sha384() } + Some(unsafe { aws_lc::EVP_sha384() }) } "SHA512" => { // SAFETY: Obtains AWS-LC's static SHA-512 EVP_MD descriptor because: // 1. The function takes no caller-owned pointers. // 2. AWS-LC returns a process-static descriptor that must not be freed. // 3. The descriptor is only passed back into AWS-LC verification APIs. - unsafe { aws_lc::EVP_sha512() } + Some(unsafe { aws_lc::EVP_sha512() }) } - other => panic!("unsupported AWS-LC sys hash `{other}`"), + _ => None, } } @@ -751,7 +789,7 @@ fn aws_lc_sys_verify( ) -> Option { let key = aws_lc_parse_public_key(spki)?; let ctx = aws_lc_new_md_ctx()?; - let md = aws_lc_md(sha); + let md = aws_lc_md(sha)?; let mut pctx = ptr::null_mut(); // SAFETY: Initializes an AWS-LC digest verification context because: @@ -759,7 +797,8 @@ fn aws_lc_sys_verify( // 2. `md` is a live process-static descriptor returned by AWS-LC for the selected SHA-2 digest. // 3. `pctx` points to stack storage where AWS-LC writes a context borrowed by `ctx`; it is not // freed separately. - let init_rc = unsafe { aws_lc::EVP_DigestVerifyInit(ctx.0.as_ptr(), &mut pctx, md, ptr::null_mut(), key.0.as_ptr()) }; + let init_rc = + unsafe { aws_lc::EVP_DigestVerifyInit(ctx.0.as_ptr(), &raw mut pctx, md, ptr::null_mut(), key.0.as_ptr()) }; if init_rc != 1 || pctx.is_null() { return None; } @@ -767,7 +806,7 @@ fn aws_lc_sys_verify( let padding = match scheme { "pss" => aws_lc::RSA_PKCS1_PSS_PADDING, "pkcs1v15" => aws_lc::RSA_PKCS1_PADDING, - other => panic!("unsupported AWS-LC sys RSA scheme `{other}`"), + _ => return None, }; // SAFETY: Configures RSA verification padding on AWS-LC's borrowed PKEY context because: @@ -824,7 +863,7 @@ fn assert_aws_lc_sys_cavp( ) { let spki = spki_for_pkcs1(pkcs1); let actual = aws_lc_sys_verify(scheme, sha, salt_len, &spki, message, signature) - .unwrap_or_else(|| panic!("AWS-LC sys setup failed for {scheme}/{sha} salt_len={salt_len:?}")); + .expect("CAVP fixtures must select supported AWS-LC RSA/SHA-2 parameters"); assert_eq!(actual, expected, "AWS-LC sys mismatch for {scheme}/{sha}"); } @@ -850,47 +889,58 @@ fn assert_rustcrypto_cavp( signature: &[u8], expected: bool, ) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the CAVP PKCS#1 public key must parse"); let actual = match (scheme, sha) { ("pss", "SHA256") => { - let key = RustCryptoPssVerifyingKey::::new_with_salt_len(key, salt_len.unwrap()); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let key = RustCryptoPssVerifyingKey::::new_with_salt_len( + key, + salt_len.expect("a CAVP PSS case must provide a salt length"), + ); + let signature = RustCryptoPssSignature::try_from(signature).expect("the CAVP PSS signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pss", "SHA384") => { - let key = RustCryptoPssVerifyingKey::::new_with_salt_len(key, salt_len.unwrap()); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let key = RustCryptoPssVerifyingKey::::new_with_salt_len( + key, + salt_len.expect("a CAVP PSS case must provide a salt length"), + ); + let signature = RustCryptoPssSignature::try_from(signature).expect("the CAVP PSS signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pss", "SHA512") => { - let key = RustCryptoPssVerifyingKey::::new_with_salt_len(key, salt_len.unwrap()); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let key = RustCryptoPssVerifyingKey::::new_with_salt_len( + key, + salt_len.expect("a CAVP PSS case must provide a salt length"), + ); + let signature = RustCryptoPssSignature::try_from(signature).expect("the CAVP PSS signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pkcs1v15", "SHA256") => { let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the CAVP PKCS#1 v1.5 signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pkcs1v15", "SHA384") => { let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the CAVP PKCS#1 v1.5 signature width must match"); + Some(key.verify(message, &signature).is_ok()) } ("pkcs1v15", "SHA512") => { let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); - key.verify(message, &signature).is_ok() + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the CAVP PKCS#1 v1.5 signature width must match"); + Some(key.verify(message, &signature).is_ok()) } - other => panic!("unsupported RustCrypto CAVP profile {other:?}"), - }; + _ => None, + } + .expect("CAVP fixtures must use a RustCrypto-supported RSA/SHA-2 profile"); assert_eq!(actual, expected, "RustCrypto mismatch for {scheme}/{sha}"); } fn cavp_field<'a>(value: &'a serde_json::Value, name: &'static str) -> &'a str { - value[name] - .as_str() - .unwrap_or_else(|| panic!("missing CAVP string field `{name}`")) + value[name].as_str().expect("a required CAVP field must be a string") } fn cavp_public_exponent(bytes: &[u8]) -> u64 { @@ -913,81 +963,92 @@ fn cavp_rscrypto_result( signature: &[u8], ) -> Result<(), rscrypto::VerificationError> { let profile = match (scheme, sha) { - ("pss", "SHA256") => RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, salt_len.unwrap()), - ("pss", "SHA384") => RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha384, salt_len.unwrap()), - ("pss", "SHA512") => RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha512, salt_len.unwrap()), - ("pkcs1v15", "SHA256") => RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - ("pkcs1v15", "SHA384") => RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha384), - ("pkcs1v15", "SHA512") => RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha512), - other => panic!("unsupported rscrypto CAVP profile {other:?}"), - }; + ("pss", "SHA256") => { + salt_len.map(|salt_len| RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, salt_len)) + } + ("pss", "SHA384") => { + salt_len.map(|salt_len| RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha384, salt_len)) + } + ("pss", "SHA512") => { + salt_len.map(|salt_len| RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha512, salt_len)) + } + ("pkcs1v15", "SHA256") => Some(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256)), + ("pkcs1v15", "SHA384") => Some(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha384)), + ("pkcs1v15", "SHA512") => Some(RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha512)), + _ => None, + } + .expect("CAVP fixtures must use a supported RSA/SHA-2 profile and provide a PSS salt length"); key.verify_signature(profile, message, signature) } fn assert_rustcrypto_pss_sha256(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pss_sha384(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pss_sha512(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } fn assert_rustcrypto_pkcs1v15_sha256(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pkcs1v15_sha384(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } #[cfg(feature = "getrandom")] fn assert_rustcrypto_pkcs1v15_sha512(pkcs1: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).unwrap(); + let key = RustCryptoRsaPublicKey::from_pkcs1_der(pkcs1).expect("the PKCS#1 public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } fn assert_rustcrypto_spki_pss_sha256(spki: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_public_key_der(spki).unwrap(); + let key = RustCryptoRsaPublicKey::from_public_key_der(spki).expect("the SPKI public-key fixture must parse"); let key = RustCryptoPssVerifyingKey::::new(key); - let signature = RustCryptoPssSignature::try_from(signature).unwrap(); + let signature = RustCryptoPssSignature::try_from(signature).expect("the PSS signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } fn assert_rustcrypto_spki_pkcs1v15_sha256(spki: &[u8], message: &[u8], signature: &[u8], expected: bool) { - let key = RustCryptoRsaPublicKey::from_public_key_der(spki).unwrap(); + let key = RustCryptoRsaPublicKey::from_public_key_der(spki).expect("the SPKI public-key fixture must parse"); let key = RustCryptoPkcs1v15VerifyingKey::::new(key); - let signature = RustCryptoPkcs1v15Signature::try_from(signature).unwrap(); + let signature = + RustCryptoPkcs1v15Signature::try_from(signature).expect("the PKCS#1 v1.5 signature width must match the key"); assert_eq!(key.verify(message, &signature).is_ok(), expected); } @@ -1012,7 +1073,16 @@ fn assert_openssl_signature( assert_eq!(actual, expected); } -#[allow(clippy::std_instead_of_core)] +fn openssl_output(command: &mut Command) -> Option { + match command.output() { + Ok(output) => Some(output), + Err(error) => { + eprintln!("OpenSSL oracle unavailable because the process could not start: {error}"); + None + } + } +} + fn openssl_verify( digest_arg: &'static str, spki: &[u8], @@ -1025,9 +1095,9 @@ fn openssl_verify( let msg_path = openssl_temp_path(&id, "msg.bin"); let sig_path = openssl_temp_path(&id, "sig.bin"); - fs::write(&key_path, spki).unwrap(); - fs::write(&msg_path, message).unwrap(); - fs::write(&sig_path, signature).unwrap(); + fs::write(&key_path, spki).expect("the OpenSSL SPKI fixture must be written"); + fs::write(&msg_path, message).expect("the OpenSSL message fixture must be written"); + fs::write(&sig_path, signature).expect("the OpenSSL signature fixture must be written"); let mut command = Command::new("openssl"); command @@ -1039,17 +1109,13 @@ fn openssl_verify( for sigopt in sigopts { command.args(["-sigopt", sigopt]); } - let output = command.arg(&msg_path).output(); + let output = openssl_output(command.arg(&msg_path)); - let _ = fs::remove_file(&key_path); - let _ = fs::remove_file(&msg_path); - let _ = fs::remove_file(&sig_path); + fs::remove_file(&key_path).expect("the temporary OpenSSL SPKI fixture must be removed"); + fs::remove_file(&msg_path).expect("the temporary OpenSSL message fixture must be removed"); + fs::remove_file(&sig_path).expect("the temporary OpenSSL signature fixture must be removed"); - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; + let output = output?; if output.status.success() { return Some(true); @@ -1057,18 +1123,15 @@ fn openssl_verify( let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - if stdout.contains("Verification failure") || stderr.contains("bad signature") { - return Some(false); - } - - panic!( + assert!( + stdout.contains("Verification failure") || stderr.contains("bad signature"), "openssl RSA verify failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(false) } #[cfg(feature = "getrandom")] -#[allow(clippy::std_instead_of_core)] fn openssl_oaep_crypt( operation: &'static str, key_der: &[u8], @@ -1080,8 +1143,8 @@ fn openssl_oaep_crypt( let key_path = openssl_temp_path(&id, "oaep-key.der"); let input_path = openssl_temp_path(&id, "oaep-input.bin"); - fs::write(&key_path, key_der).unwrap(); - fs::write(&input_path, input).unwrap(); + fs::write(&key_path, key_der).expect("the OpenSSL OAEP key fixture must be written"); + fs::write(&input_path, input).expect("the OpenSSL OAEP input fixture must be written"); let mut command = Command::new("openssl"); command @@ -1090,34 +1153,29 @@ fn openssl_oaep_crypt( if public_key { command.arg("-pubin"); } - let output = command - .arg("-in") - .arg(&input_path) - .args(["-pkeyopt", "rsa_padding_mode:oaep", "-pkeyopt"]) - .arg(format!("rsa_mgf1_md:{digest}")) - .arg("-pkeyopt") - .arg(format!("rsa_oaep_md:{digest}")) - .output(); - - let _ = fs::remove_file(&key_path); - let _ = fs::remove_file(&input_path); - - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; + let output = openssl_output( + command + .arg("-in") + .arg(&input_path) + .args(["-pkeyopt", "rsa_padding_mode:oaep", "-pkeyopt"]) + .arg(format!("rsa_mgf1_md:{digest}")) + .arg("-pkeyopt") + .arg(format!("rsa_oaep_md:{digest}")), + ); - if output.status.success() { - return Some(output.stdout); - } + fs::remove_file(&key_path).expect("the temporary OpenSSL OAEP key fixture must be removed"); + fs::remove_file(&input_path).expect("the temporary OpenSSL OAEP input fixture must be removed"); + + let output = output?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - panic!( + assert!( + output.status.success(), "openssl RSA OAEP {operation} failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(output.stdout) } #[cfg(feature = "getrandom")] @@ -1131,14 +1189,13 @@ fn openssl_oaep_decrypt(pkcs1_private_key: &[u8], ciphertext: &[u8], digest: &'s } #[cfg(feature = "getrandom")] -#[allow(clippy::std_instead_of_core)] fn openssl_pkcs1v15_crypt(operation: &'static str, key_der: &[u8], input: &[u8], public_key: bool) -> Option> { let id = openssl_temp_id(); let key_path = openssl_temp_path(&id, "pkcs1v15-key.der"); let input_path = openssl_temp_path(&id, "pkcs1v15-input.bin"); - fs::write(&key_path, key_der).unwrap(); - fs::write(&input_path, input).unwrap(); + fs::write(&key_path, key_der).expect("the OpenSSL PKCS#1 v1.5 key fixture must be written"); + fs::write(&input_path, input).expect("the OpenSSL PKCS#1 v1.5 input fixture must be written"); let mut command = Command::new("openssl"); command @@ -1147,31 +1204,26 @@ fn openssl_pkcs1v15_crypt(operation: &'static str, key_der: &[u8], input: &[u8], if public_key { command.arg("-pubin"); } - let output = command - .arg("-in") - .arg(&input_path) - .args(["-pkeyopt", "rsa_padding_mode:pkcs1"]) - .output(); - - let _ = fs::remove_file(&key_path); - let _ = fs::remove_file(&input_path); - - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; + let output = openssl_output( + command + .arg("-in") + .arg(&input_path) + .args(["-pkeyopt", "rsa_padding_mode:pkcs1"]), + ); - if output.status.success() { - return Some(output.stdout); - } + fs::remove_file(&key_path).expect("the temporary OpenSSL PKCS#1 v1.5 key fixture must be removed"); + fs::remove_file(&input_path).expect("the temporary OpenSSL PKCS#1 v1.5 input fixture must be removed"); + + let output = output?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - panic!( + assert!( + output.status.success(), "openssl RSA PKCS#1 v1.5 {operation} failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(output.stdout) } #[cfg(feature = "getrandom")] @@ -1199,10 +1251,12 @@ fn assert_generated_pkcs1v15_encryption_external_oracles( key .public_key() .encrypt_pkcs1v15_with_random_fill(plaintext, &mut ciphertext, fill_rsa_random_with(0x37)) - .unwrap(); + .expect("deterministic PKCS#1 v1.5 encryption must succeed"); let rustcrypto_decrypting_key = RustCryptoPkcs1v15DecryptingKey::new(rustcrypto_private_key.clone()); - let rustcrypto_decrypted = rustcrypto_decrypting_key.decrypt(&ciphertext).unwrap(); + let rustcrypto_decrypted = rustcrypto_decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching deterministic PKCS#1 v1.5 ciphertext"); assert_eq!(rustcrypto_decrypted, plaintext); if let Some(openssl_decrypted) = openssl_pkcs1v15_decrypt(pkcs1, &ciphertext) { @@ -1217,7 +1271,7 @@ fn assert_generated_pkcs1v15_encryption_external_oracles( let mut rscrypto_decrypted = vec![0u8; key.signature_len()]; let rscrypto_decrypted_len = key .decrypt_pkcs1v15(&openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL PKCS#1 v1.5 ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); } else { eprintln!( @@ -1226,69 +1280,69 @@ fn assert_generated_pkcs1v15_encryption_external_oracles( } } -#[allow(clippy::std_instead_of_core)] fn openssl_private_key_to_spki_der(private_key_der: &[u8]) -> Option> { let id = openssl_temp_id(); let key_path = openssl_temp_path(&id, "private-key.der"); - fs::write(&key_path, private_key_der).unwrap(); + fs::write(&key_path, private_key_der).expect("the OpenSSL private-key fixture must be written"); - let output = Command::new("openssl") - .args(["pkey", "-inform", "DER", "-in"]) - .arg(&key_path) - .args(["-pubout", "-outform", "DER"]) - .output(); + let output = openssl_output( + Command::new("openssl") + .args(["pkey", "-inform", "DER", "-in"]) + .arg(&key_path) + .args(["-pubout", "-outform", "DER"]), + ); - let _ = fs::remove_file(&key_path); + fs::remove_file(&key_path).expect("the temporary OpenSSL private-key fixture must be removed"); - let output = match output { - Ok(output) => output, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, - Err(error) => panic!("failed to run openssl: {error}"), - }; - - if output.status.success() { - return Some(output.stdout); - } + let output = output?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - panic!( + assert!( + output.status.success(), "openssl RSA private-key DER parse failed unexpectedly: status={:?} stdout={stdout:?} stderr={stderr:?}", output.status.code() ); + Some(output.stdout) } fn digest_salt_len(sha: &str) -> usize { match sha { - "SHA256" => 32, - "SHA384" => 48, - "SHA512" => 64, - other => panic!("unsupported SHA-2 profile `{other}`"), + "SHA256" => Some(32), + "SHA384" => Some(48), + "SHA512" => Some(64), + _ => None, } + .expect("CAVP fixtures must use a supported SHA-2 profile") } fn openssl_pss_sigopts(sha: &str, salt_len: usize) -> [&'static str; 3] { let salt_len = match salt_len { - 0 => "rsa_pss_saltlen:0", - 1 => "rsa_pss_saltlen:1", - 24 => "rsa_pss_saltlen:24", - 32 => "rsa_pss_saltlen:32", - 48 => "rsa_pss_saltlen:48", - 64 => "rsa_pss_saltlen:64", - other => panic!("unsupported CAVP PSS salt length `{other}`"), - }; + 0 => Some("rsa_pss_saltlen:0"), + 1 => Some("rsa_pss_saltlen:1"), + 24 => Some("rsa_pss_saltlen:24"), + 32 => Some("rsa_pss_saltlen:32"), + 48 => Some("rsa_pss_saltlen:48"), + 64 => Some("rsa_pss_saltlen:64"), + _ => None, + } + .expect("CAVP fixtures must use a supported PSS salt length"); let mgf1 = match sha { - "SHA256" => "rsa_mgf1_md:sha256", - "SHA384" => "rsa_mgf1_md:sha384", - "SHA512" => "rsa_mgf1_md:sha512", - other => panic!("unsupported OpenSSL PSS hash `{other}`"), - }; + "SHA256" => Some("rsa_mgf1_md:sha256"), + "SHA384" => Some("rsa_mgf1_md:sha384"), + "SHA512" => Some("rsa_mgf1_md:sha512"), + _ => None, + } + .expect("CAVP fixtures must use an OpenSSL-supported PSS hash"); ["rsa_padding_mode:pss", salt_len, mgf1] } fn openssl_temp_id() -> String { - let nanos = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("the system clock must be after the Unix epoch") + .as_nanos(); format!("rscrypto-rsa-{}-{nanos}", process::id()) } @@ -1333,7 +1387,10 @@ fn fixed_width_signature_candidate(material: &[u8], len: usize) -> Vec { return out; } for (index, byte) in out.iter_mut().enumerate() { - *byte = material[index % material.len()]; + let source_index = index + .checked_rem(material.len()) + .expect("nonempty signature material must have a valid remainder"); + *byte = material[source_index]; } out } @@ -1343,7 +1400,7 @@ fn assert_opaque_verification_failure(result: Result<(), rscrypto::VerificationE } fn assert_unsupported_protocol_algorithm(result: Result) { - let err = result.unwrap_err(); + let err = result.expect_err("the protocol algorithm must be rejected as unsupported"); assert_eq!(err, RsaProtocolAlgorithmError::UnsupportedAlgorithm); assert!(!err.is_malformed_algorithm_identifier()); assert!(err.is_unsupported_algorithm()); @@ -1351,7 +1408,7 @@ fn assert_unsupported_protocol_algorithm(result: Result(result: Result) { - let err = result.unwrap_err(); + let err = result.expect_err("the malformed protocol algorithm identifier must be rejected"); assert_eq!(err, RsaProtocolAlgorithmError::MalformedAlgorithmIdentifier); assert!(err.is_malformed_algorithm_identifier()); assert!(!err.is_unsupported_algorithm()); @@ -1389,15 +1446,17 @@ fn spki_public_key_accepts_rsa2048_with_legacy_policy() { #[test] fn private_key_der_exports_roundtrip_with_rustcrypto_rsa() { let rustcrypto_key = rustcrypto_fixture_private_key(); - let rustcrypto_pkcs1 = EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).unwrap(); - let rustcrypto_pkcs8 = EncodePrivateKey::to_pkcs8_der(&rustcrypto_key).unwrap(); + let rustcrypto_pkcs1 = + EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).expect("the RustCrypto fixture must encode as PKCS#1"); + let rustcrypto_pkcs8 = + EncodePrivateKey::to_pkcs8_der(&rustcrypto_key).expect("the RustCrypto fixture must encode as PKCS#8"); let rscrypto_from_pkcs1 = RsaPrivateKey::from_pkcs1_der_with_policy(rustcrypto_pkcs1.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) - .unwrap(); + .expect("rscrypto must decode the RustCrypto PKCS#1 private-key fixture"); let rscrypto_from_pkcs8 = RsaPrivateKey::from_pkcs8_der_with_policy(rustcrypto_pkcs8.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) - .unwrap(); + .expect("rscrypto must decode the RustCrypto PKCS#8 private-key fixture"); assert_eq!(rscrypto_from_pkcs1.public_key(), rscrypto_from_pkcs8.public_key()); let rscrypto_pkcs1 = rscrypto_from_pkcs1.to_pkcs1_der(); @@ -1448,7 +1507,8 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { impl RngCore for DeterministicRng { fn next_u32(&mut self) -> u32 { - self.next_u64() as u32 + let word = self.next_u64().to_le_bytes(); + u32::from_le_bytes([word[0], word[1], word[2], word[3]]) } fn next_u64(&mut self) -> u64 { @@ -1475,11 +1535,12 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { impl CryptoRng for DeterministicRng {} let rustcrypto_key = rustcrypto_fixture_private_key(); - let rustcrypto_pkcs1 = EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).unwrap(); + let rustcrypto_pkcs1 = + EncodeRsaPrivateKey::to_pkcs1_der(&rustcrypto_key).expect("the RustCrypto fixture must encode as PKCS#1"); let rustcrypto_public_key = rustcrypto_key.to_public_key(); let rscrypto_key = RsaPrivateKey::from_pkcs1_der_with_policy(rustcrypto_pkcs1.as_bytes(), &RsaPublicKeyPolicy::legacy_verification()) - .unwrap(); + .expect("rscrypto must decode the RustCrypto private-key fixture"); let rscrypto_public_pkcs1 = rscrypto_key.public_key().to_pkcs1_der(); let rscrypto_public_spki = rscrypto_key.public_key().to_spki_der(); let message = b"rscrypto private outputs verified by RustCrypto rsa"; @@ -1487,7 +1548,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut pkcs1v15_signature = vec![0u8; rscrypto_key.signature_len()]; rscrypto_key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 SHA-256 signing must succeed for the valid fixture"); assert_rustcrypto_pkcs1v15_sha256(&rscrypto_public_pkcs1, message, &pkcs1v15_signature, true); assert_openssl_signature( "-sha256", @@ -1499,7 +1560,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 SHA-384 signing must succeed for the valid fixture"); assert_rustcrypto_pkcs1v15_sha384(&rscrypto_public_pkcs1, message, &pkcs1v15_signature, true); assert_openssl_signature( "-sha384", @@ -1511,7 +1572,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pkcs1v15(RsaPkcs1v15Profile::Sha512, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("PKCS#1 v1.5 SHA-512 signing must succeed for the valid fixture"); assert_rustcrypto_pkcs1v15_sha512(&rscrypto_public_pkcs1, message, &pkcs1v15_signature, true); assert_openssl_signature( "-sha512", @@ -1525,7 +1586,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut pss_signature = vec![0u8; rscrypto_key.signature_len()]; rscrypto_key .sign_pss(RsaPssProfile::Sha256, message, &mut pss_signature) - .unwrap(); + .expect("PSS SHA-256 signing must succeed for the valid fixture"); assert_rustcrypto_pss_sha256(&rscrypto_public_pkcs1, message, &pss_signature, true); let openssl_pss_sha256 = openssl_pss_sigopts("SHA256", 32); assert_openssl_signature( @@ -1538,7 +1599,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pss(RsaPssProfile::Sha384, message, &mut pss_signature) - .unwrap(); + .expect("PSS SHA-384 signing must succeed for the valid fixture"); assert_rustcrypto_pss_sha384(&rscrypto_public_pkcs1, message, &pss_signature, true); let openssl_pss_sha384 = openssl_pss_sigopts("SHA384", 48); assert_openssl_signature( @@ -1551,7 +1612,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { ); rscrypto_key .sign_pss(RsaPssProfile::Sha512, message, &mut pss_signature) - .unwrap(); + .expect("PSS SHA-512 signing must succeed for the valid fixture"); assert_rustcrypto_pss_sha512(&rscrypto_public_pkcs1, message, &pss_signature, true); let openssl_pss_sha512 = openssl_pss_sigopts("SHA512", 64); assert_openssl_signature( @@ -1571,14 +1632,18 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { rscrypto_key .public_key() .encrypt_oaep($profile, label.as_bytes(), plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption must succeed for the valid RustCrypto round-trip fixture"); let decrypting_key = RustCryptoOaepDecryptingKey::<$digest>::new_with_label(rustcrypto_key.clone(), label); - let decrypted = decrypting_key.decrypt(&ciphertext).unwrap(); + let decrypted = decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching rscrypto OAEP ciphertext"); assert_eq!(decrypted, plaintext); let encrypting_key = RustCryptoOaepEncryptingKey::<$digest>::new_with_label(rustcrypto_public_key.clone(), label); let mut rng = DeterministicRng { state: $seed }; - let rustcrypto_ciphertext = encrypting_key.encrypt_with_rng(&mut rng, plaintext).unwrap(); + let rustcrypto_ciphertext = encrypting_key + .encrypt_with_rng(&mut rng, plaintext) + .expect("RustCrypto OAEP encryption must succeed for the valid fixture"); let mut rscrypto_decrypted = vec![0u8; rscrypto_key.signature_len()]; let rscrypto_decrypted_len = rscrypto_key .decrypt_oaep( @@ -1587,7 +1652,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { &rustcrypto_ciphertext, &mut rscrypto_decrypted, ) - .unwrap(); + .expect("rscrypto must decrypt the matching RustCrypto OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); }}; } @@ -1614,7 +1679,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { rscrypto_key .public_key() .encrypt_oaep($profile, b"", openssl_plaintext, &mut ciphertext) - .unwrap(); + .expect("OAEP encryption must succeed for the valid OpenSSL round-trip fixture"); if let Some(decrypted) = openssl_oaep_decrypt(rustcrypto_pkcs1.as_bytes(), &ciphertext, $digest) { assert_eq!(decrypted, openssl_plaintext); } else { @@ -1625,7 +1690,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut rscrypto_decrypted = vec![0u8; rscrypto_key.signature_len()]; let rscrypto_decrypted_len = rscrypto_key .decrypt_oaep($profile, b"", &openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], openssl_plaintext); } else { eprintln!("skipping OpenSSL RSA OAEP encrypt differential check because `openssl` is not available"); @@ -1641,7 +1706,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { rscrypto_key .public_key() .encrypt_pkcs1v15_with_random_fill(pkcs1v15_plaintext, &mut pkcs1v15_ciphertext, fill_rsa_random_with(0x6d)) - .unwrap(); + .expect("PKCS#1 v1.5 encryption must succeed for the valid OpenSSL round-trip fixture"); if let Some(decrypted) = openssl_pkcs1v15_decrypt(rustcrypto_pkcs1.as_bytes(), &pkcs1v15_ciphertext) { assert_eq!(decrypted, pkcs1v15_plaintext); } else { @@ -1652,7 +1717,7 @@ fn private_key_outputs_verify_and_decrypt_with_rustcrypto_rsa() { let mut rscrypto_decrypted = vec![0u8; rscrypto_key.signature_len()]; let rscrypto_decrypted_len = rscrypto_key .decrypt_pkcs1v15(&openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL PKCS#1 v1.5 ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], pkcs1v15_plaintext); } else { eprintln!("skipping OpenSSL RSA PKCS#1 v1.5 encrypt differential check because `openssl` is not available"); @@ -1709,7 +1774,7 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15($pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("generated-key PKCS#1 v1.5 signing must succeed"); $assert_pkcs1(&public_pkcs1, message, &pkcs1v15_signature, true); assert_ring_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); assert_aws_lc_rs_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); @@ -1725,7 +1790,9 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { assert_openssl_signature($digest_arg, &spki, message, &pkcs1v15_signature, &[], true); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_pss($pss_profile, message, &mut pss_signature).unwrap(); + key + .sign_pss($pss_profile, message, &mut pss_signature) + .expect("generated-key PSS signing must succeed"); $assert_pss(&public_pkcs1, message, &pss_signature, true); assert_ring_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); assert_aws_lc_rs_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); @@ -1753,10 +1820,12 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { &mut ciphertext, fill_rsa_random_from(&$seed), ) - .unwrap(); + .expect("generated-key OAEP encryption must succeed"); let rustcrypto_decrypting_key = RustCryptoOaepDecryptingKey::<$digest>::new_with_label(rustcrypto_from_pkcs1.clone(), label); - let rustcrypto_decrypted = rustcrypto_decrypting_key.decrypt(&ciphertext).unwrap(); + let rustcrypto_decrypted = rustcrypto_decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching generated-key OAEP ciphertext"); assert_eq!(rustcrypto_decrypted, plaintext); if let Some(openssl_decrypted) = openssl_oaep_decrypt(&pkcs1, &ciphertext, $oaep_digest) { @@ -1769,7 +1838,7 @@ fn generated_private_key_outputs_verify_and_decrypt_with_external_oracles() { let mut rscrypto_decrypted = vec![0u8; key.signature_len()]; let rscrypto_decrypted_len = key .decrypt_oaep($oaep_profile, b"", &openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); } else { eprintln!("skipping OpenSSL generated RSA OAEP encrypt check because `openssl` is not available"); @@ -1881,7 +1950,7 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles let mut pkcs1v15_signature = vec![0u8; key.signature_len()]; key .sign_pkcs1v15($pkcs1_profile, message, &mut pkcs1v15_signature) - .unwrap(); + .expect("generated modern-key PKCS#1 v1.5 signing must succeed"); $assert_pkcs1(&public_pkcs1, message, &pkcs1v15_signature, true); assert_ring_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); assert_aws_lc_rs_cavp("pkcs1v15", $pss_sha, &public_pkcs1, message, &pkcs1v15_signature, true); @@ -1897,7 +1966,9 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles assert_openssl_signature($digest_arg, &spki, message, &pkcs1v15_signature, &[], true); let mut pss_signature = vec![0u8; key.signature_len()]; - key.sign_pss($pss_profile, message, &mut pss_signature).unwrap(); + key + .sign_pss($pss_profile, message, &mut pss_signature) + .expect("generated modern-key PSS signing must succeed"); $assert_pss(&public_pkcs1, message, &pss_signature, true); assert_ring_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); assert_aws_lc_rs_cavp("pss", $pss_sha, &public_pkcs1, message, &pss_signature, true); @@ -1925,10 +1996,12 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles &mut ciphertext, fill_rsa_random_from(&$seed), ) - .unwrap(); + .expect("generated modern-key OAEP encryption must succeed"); let rustcrypto_decrypting_key = RustCryptoOaepDecryptingKey::<$digest>::new_with_label(rustcrypto_from_pkcs1.clone(), label); - let rustcrypto_decrypted = rustcrypto_decrypting_key.decrypt(&ciphertext).unwrap(); + let rustcrypto_decrypted = rustcrypto_decrypting_key + .decrypt(&ciphertext) + .expect("RustCrypto must decrypt the matching generated modern-key OAEP ciphertext"); assert_eq!(rustcrypto_decrypted, plaintext); if let Some(openssl_decrypted) = openssl_oaep_decrypt(&pkcs1, &ciphertext, $oaep_digest) { @@ -1941,7 +2014,7 @@ fn generated_modern_private_key_outputs_verify_and_decrypt_with_external_oracles let mut rscrypto_decrypted = vec![0u8; key.signature_len()]; let rscrypto_decrypted_len = key .decrypt_oaep($oaep_profile, b"", &openssl_ciphertext, &mut rscrypto_decrypted) - .unwrap(); + .expect("rscrypto must decrypt the matching OpenSSL OAEP ciphertext"); assert_eq!(&rscrypto_decrypted[..rscrypto_decrypted_len], plaintext); } else { eprintln!("skipping OpenSSL generated RSA-{bits} OAEP encrypt check because `openssl` is not available"); @@ -2007,12 +2080,10 @@ fn x509_spki_public_key_preserves_pss_key_algorithm_constraints() { ); let pss_only_key = legacy_x509_public_key_from_spki(&pss_only_spki); assert_eq!(pss_only_key.key_algorithm(), RsaX509PublicKeyAlgorithm::RsaPss); - assert!( - pss_only_key - .key_algorithm() - .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) - .is_ok() - ); + pss_only_key + .key_algorithm() + .permits_signature_profile(RsaSignatureProfile::pss(RsaPssProfile::Sha256)) + .expect("an unrestricted PSS key must permit PSS SHA-256"); assert_eq!( pss_only_key .key_algorithm() @@ -2029,12 +2100,10 @@ fn x509_spki_public_key_preserves_pss_key_algorithm_constraints() { minimum_salt_len: 32, } ); - assert!( - restricted_key - .key_algorithm() - .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) - .is_ok() - ); + restricted_key + .key_algorithm() + .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) + .expect("a SHA-256 PSS key with a 32-byte minimum salt must permit a 64-byte salt"); assert_eq!( restricted_key .key_algorithm() @@ -2105,16 +2174,12 @@ fn x509_restricted_pss_key_enforces_rfc4055_signature_parameter_validation() { minimum_salt_len: 32, }; - assert!( - key_algorithm - .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32)) - .is_ok() - ); - assert!( - key_algorithm - .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) - .is_ok() - ); + key_algorithm + .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32)) + .expect("a restricted PSS key must permit its minimum salt length"); + key_algorithm + .permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 64)) + .expect("a restricted PSS key must permit a salt longer than its minimum"); assert_eq!( key_algorithm.permits_signature_profile(RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 31)), Err(RsaProtocolAlgorithmError::UnsupportedAlgorithm) @@ -2184,11 +2249,9 @@ fn x509_spki_constraints_are_enforced_during_signature_verification() { let pss_algorithm = x509_pss_algorithm(RsaPssProfile::Sha256, 32, None); let pss_sig = pss_fixture_signature_sha256(); - assert!( - pss32_key - .verify_signature_from_x509_algorithm_der(&pss_algorithm, pss_fixture_message(), &pss_sig) - .is_ok() - ); + pss32_key + .verify_signature_from_x509_algorithm_der(&pss_algorithm, pss_fixture_message(), &pss_sig) + .expect("a restricted PSS key must verify a signature with matching parameters"); assert_eq!( pss32_key.verify_signature_from_x509_algorithm_der( &algorithm_identifier(SHA256_WITH_RSA_ENCRYPTION_OID, Some(&null())), @@ -2239,16 +2302,12 @@ fn x509_signature_verification_adapter_failures_are_opaque() { fn x509_certificate_signature_verification_accepts_real_rsa_certificates() { let issuer = legacy_x509_public_key_from_spki(&x509_certificate_fixture_public_key()); - assert!( - issuer - .verify_x509_certificate_signature_der(&x509_pkcs1v15_certificate_fixture()) - .is_ok() - ); - assert!( - issuer - .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) - .is_ok() - ); + issuer + .verify_x509_certificate_signature_der(&x509_pkcs1v15_certificate_fixture()) + .expect("the RSA issuer must verify the PKCS#1 v1.5 certificate fixture"); + issuer + .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) + .expect("the RSA issuer must verify the PSS certificate fixture"); let public_key = legacy_public_key_from_spki(&x509_certificate_fixture_public_key()); let pkcs1 = valid_pkcs1_with_modulus_and_exponent( @@ -2260,11 +2319,9 @@ fn x509_certificate_signature_verification_accepts_real_rsa_certificates() { &algorithm_identifier(ID_RSASSA_PSS_OID, None), )); - assert!( - pss_issuer - .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) - .is_ok() - ); + pss_issuer + .verify_x509_certificate_signature_der(&x509_pss_certificate_fixture()) + .expect("the PSS issuer must verify the PSS certificate fixture"); assert_eq!( pss_issuer.verify_x509_certificate_signature_der(&x509_pkcs1v15_certificate_fixture()), Err(VerificationError::new()) @@ -2275,22 +2332,20 @@ fn x509_certificate_signature_verification_accepts_real_rsa_certificates() { fn x509_certificate_signature_algorithms_accept_absent_and_null_sha2_rsa_params() { let (null_params_spki, null_params_certificate) = x509_sha256_rsa_self_signed_certificate(Some(&null())); let null_params_issuer = legacy_x509_public_key_from_spki(&null_params_spki); - assert!( - null_params_issuer - .verify_x509_certificate_signature_der(&null_params_certificate) - .is_ok() - ); + null_params_issuer + .verify_x509_certificate_signature_der(&null_params_certificate) + .expect("SHA-256 RSA parameters encoded as NULL must verify"); let (absent_params_spki, absent_params_certificate) = x509_sha256_rsa_self_signed_certificate(None); let absent_params_issuer = legacy_x509_public_key_from_spki(&absent_params_spki); - assert!( - absent_params_issuer - .verify_x509_certificate_signature_der(&absent_params_certificate) - .is_ok() - ); + absent_params_issuer + .verify_x509_certificate_signature_der(&absent_params_certificate) + .expect("absent SHA-256 RSA parameters must verify"); let mut tampered_absent_params_certificate = absent_params_certificate; - *tampered_absent_params_certificate.last_mut().unwrap() ^= 0x01; + *tampered_absent_params_certificate + .last_mut() + .expect("the certificate fixture must contain a signature byte") ^= 0x01; assert_eq!( absent_params_issuer.verify_x509_certificate_signature_der(&tampered_absent_params_certificate), Err(VerificationError::new()) @@ -2339,8 +2394,9 @@ fn x509_certificate_chain_signature_fixtures_cover_rsa_pss_and_pkcs1v15() { pss_certificate.as_slice(), ), ] { - assert!( - issuer.verify_x509_certificate_signature_der(certificate).is_ok(), + assert_eq!( + issuer.verify_x509_certificate_signature_der(certificate), + Ok(()), "{name}" ); } @@ -2365,7 +2421,9 @@ fn x509_certificate_chain_signature_fixtures_cover_rsa_pss_and_pkcs1v15() { } let mut tampered_pss_certificate = pss_certificate; - *tampered_pss_certificate.last_mut().unwrap() ^= 0x01; + *tampered_pss_certificate + .last_mut() + .expect("the PSS certificate fixture must contain a signature byte") ^= 0x01; assert_eq!( rsae_issuer.verify_x509_certificate_signature_der(&tampered_pss_certificate), Err(VerificationError::new()) @@ -2377,7 +2435,9 @@ fn x509_certificate_signature_verification_rejects_malformed_and_confused_certif let issuer = legacy_x509_public_key_from_spki(&x509_certificate_fixture_public_key()); let mut tampered = x509_pkcs1v15_certificate_fixture(); - *tampered.last_mut().unwrap() ^= 0x01; + *tampered + .last_mut() + .expect("the PKCS#1 v1.5 certificate fixture must contain a signature byte") ^= 0x01; assert_eq!( issuer.verify_x509_certificate_signature_der(&tampered), Err(VerificationError::new()) @@ -2524,18 +2584,14 @@ fn tls_signature_scheme_advertisement_matches_executable_key_constraints() { for algorithm in [rsae, pss, pss_sha256_32, pss_sha256_64] { for scheme in algorithm.advertised_tls13_signature_schemes().iter() { - assert!( - algorithm.signature_profile_from_tls13_signature_scheme(scheme).is_ok(), - "advertised TLS 1.3 scheme {scheme:#06x} must be executable by {algorithm:?}" - ); + let _profile = algorithm + .signature_profile_from_tls13_signature_scheme(scheme) + .expect("every advertised TLS 1.3 scheme must map to an executable profile"); } for scheme in algorithm.advertised_tls_certificate_signature_schemes().iter() { - assert!( - algorithm - .signature_profile_from_tls_certificate_signature_scheme(scheme) - .is_ok(), - "advertised TLS certificate scheme {scheme:#06x} must be executable by {algorithm:?}" - ); + let _profile = algorithm + .signature_profile_from_tls_certificate_signature_scheme(scheme) + .expect("every advertised TLS certificate scheme must map to an executable profile"); } for unsupported_legacy_scheme in [0x0101, 0x0201, 0x0301, 0x0420, 0x0520, 0x0620] { @@ -2555,8 +2611,8 @@ fn tls_signature_scheme_advertisement_matches_executable_key_constraints() { #[test] fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { - let rsae_key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).unwrap(); - let rsa3072_public = RsaPublicKey::from_spki_der(RSA3072_SPKI).unwrap(); + let rsae_key = RsaX509PublicKey::from_spki_der(RSA3072_SPKI).expect("the RSA-3072 X.509 fixture must parse"); + let rsa3072_public = RsaPublicKey::from_spki_der(RSA3072_SPKI).expect("the RSA-3072 SPKI fixture must parse"); let rsa3072_pkcs1 = valid_pkcs1_with_modulus_and_exponent( rsa3072_public.modulus(), &exponent_bytes(rsa3072_public.public_exponent().as_u64()), @@ -2565,12 +2621,12 @@ fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { &rsa3072_pkcs1, &algorithm_identifier(ID_RSASSA_PSS_OID, None), )) - .unwrap(); + .expect("the unrestricted RSA-PSS SPKI fixture must parse"); let restricted_pss_key = RsaX509PublicKey::from_spki_der(&spki_for_pkcs1_with_algorithm( &rsa3072_pkcs1, &x509_pss_algorithm(RsaPssProfile::Sha256, 32, None), )) - .unwrap(); + .expect("the restricted RSA-PSS SPKI fixture must parse"); assert!( rsae_key @@ -2584,53 +2640,45 @@ fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { .advertised_tls_certificate_signature_schemes() .contains(0x0401) ); - assert!( - rsae_key - .verify_expected_tls13_signature_scheme( - 0x0804, - 0x0804, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - ) - .is_ok() - ); + rsae_key + .verify_expected_tls13_signature_scheme( + 0x0804, + 0x0804, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + ) + .expect("the advertised RSAE TLS 1.3 PSS scheme must verify its fixture"); let mut rsae_scratch = rsae_key.public_key().public_scratch(); - assert!( - rsae_key - .verify_expected_tls13_signature_scheme_with_scratch( - 0x0804, - 0x0804, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - &mut rsae_scratch, - ) - .is_ok() - ); - assert!( - rsae_key - .verify_expected_tls_certificate_signature_scheme( - 0x0401, - 0x0401, - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - RSA3072_PKCS1V15_SHA256, - ) - .is_ok() - ); - assert!( - rsae_key - .verify_expected_tls_certificate_signature_scheme_with_scratch( - 0x0401, - 0x0401, - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - RSA3072_PKCS1V15_SHA256, - &mut rsae_scratch, - ) - .is_ok() - ); + rsae_key + .verify_expected_tls13_signature_scheme_with_scratch( + 0x0804, + 0x0804, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + &mut rsae_scratch, + ) + .expect("the advertised RSAE TLS 1.3 PSS scheme must verify with scratch reuse"); + rsae_key + .verify_expected_tls_certificate_signature_scheme( + 0x0401, + 0x0401, + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + RSA3072_PKCS1V15_SHA256, + ) + .expect("the advertised RSAE certificate PKCS#1 v1.5 scheme must verify its fixture"); + rsae_key + .verify_expected_tls_certificate_signature_scheme_with_scratch( + 0x0401, + 0x0401, + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + RSA3072_PKCS1V15_SHA256, + &mut rsae_scratch, + ) + .expect("the advertised RSAE certificate PKCS#1 v1.5 scheme must verify with scratch reuse"); for key in [&pss_key, &restricted_pss_key] { assert!( @@ -2645,53 +2693,45 @@ fn tls_signature_scheme_advertisement_executes_real_verification_helpers() { .advertised_tls_certificate_signature_schemes() .contains(0x0809) ); - assert!( - key - .verify_expected_tls13_signature_scheme( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - ) - .is_ok() - ); + key + .verify_expected_tls13_signature_scheme( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + ) + .expect("the advertised PSS TLS 1.3 scheme must verify its fixture"); let mut scratch = key.public_key().public_scratch(); - assert!( - key - .verify_expected_tls13_signature_scheme_with_scratch( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - &mut scratch, - ) - .is_ok() - ); - assert!( - key - .verify_expected_tls_certificate_signature_scheme( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - ) - .is_ok() - ); - assert!( - key - .verify_expected_tls_certificate_signature_scheme_with_scratch( - 0x0809, - 0x0809, - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - RSA3072_PSS_SHA256, - &mut scratch, - ) - .is_ok() - ); + key + .verify_expected_tls13_signature_scheme_with_scratch( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + &mut scratch, + ) + .expect("the advertised PSS TLS 1.3 scheme must verify with scratch reuse"); + key + .verify_expected_tls_certificate_signature_scheme( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + ) + .expect("the advertised PSS certificate scheme must verify its fixture"); + key + .verify_expected_tls_certificate_signature_scheme_with_scratch( + 0x0809, + 0x0809, + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + RSA3072_PSS_SHA256, + &mut scratch, + ) + .expect("the advertised PSS certificate scheme must verify with scratch reuse"); } } @@ -2708,26 +2748,18 @@ fn tls_signature_scheme_verification_rejects_key_algorithm_confusion() { let pss_signature = pss_fixture_signature_sha256(); let pss_profile = RsaSignatureProfile::pss(RsaPssProfile::Sha256); - assert!( - rsae_key - .verify_tls13_signature_scheme(0x0804, pss_fixture_message(), &pss_signature) - .is_ok() - ); - assert!( - rsae_key - .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_profile, pss_fixture_message(), &pss_signature) - .is_ok() - ); - assert!( - pss_key - .verify_tls13_signature_scheme(0x0809, pss_fixture_message(), &pss_signature) - .is_ok() - ); - assert!( - pss_key - .verify_expected_tls13_signature_scheme(0x0809, 0x0809, pss_profile, pss_fixture_message(), &pss_signature) - .is_ok() - ); + rsae_key + .verify_tls13_signature_scheme(0x0804, pss_fixture_message(), &pss_signature) + .expect("an RSAE key must verify its matching TLS 1.3 PSS fixture"); + rsae_key + .verify_expected_tls13_signature_scheme(0x0804, 0x0804, pss_profile, pss_fixture_message(), &pss_signature) + .expect("an RSAE key must verify matching expected and received TLS 1.3 schemes"); + pss_key + .verify_tls13_signature_scheme(0x0809, pss_fixture_message(), &pss_signature) + .expect("a PSS key must verify its matching TLS 1.3 PSS fixture"); + pss_key + .verify_expected_tls13_signature_scheme(0x0809, 0x0809, pss_profile, pss_fixture_message(), &pss_signature) + .expect("a PSS key must verify matching expected and received TLS 1.3 schemes"); assert_eq!( rsae_key.verify_expected_tls13_signature_scheme(0x0804, 0x0809, pss_profile, pss_fixture_message(), &pss_signature), Err(VerificationError::new()) @@ -2765,22 +2797,18 @@ fn tls_signature_scheme_verification_rejects_key_algorithm_confusion() { let rsae_pkcs1_key = legacy_x509_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_signature = pkcs1v15_fixture_signature_sha256(); let pkcs1_profile = RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256); - assert!( - rsae_pkcs1_key - .verify_tls_certificate_signature_scheme(0x0401, pkcs1v15_fixture_message(), &pkcs1_signature) - .is_ok() - ); - assert!( - rsae_pkcs1_key - .verify_expected_tls_certificate_signature_scheme( - 0x0401, - 0x0401, - pkcs1_profile, - pkcs1v15_fixture_message(), - &pkcs1_signature, - ) - .is_ok() - ); + rsae_pkcs1_key + .verify_tls_certificate_signature_scheme(0x0401, pkcs1v15_fixture_message(), &pkcs1_signature) + .expect("an RSAE key must verify its matching TLS certificate PKCS#1 v1.5 fixture"); + rsae_pkcs1_key + .verify_expected_tls_certificate_signature_scheme( + 0x0401, + 0x0401, + pkcs1_profile, + pkcs1v15_fixture_message(), + &pkcs1_signature, + ) + .expect("an RSAE key must verify matching expected and received TLS certificate schemes"); assert_eq!( rsae_pkcs1_key.verify_expected_tls_certificate_signature_scheme( 0x0401, @@ -3056,7 +3084,9 @@ fn protocol_verification_helpers_collapse_adapter_failures_to_opaque_error() { let issuer = legacy_x509_public_key_from_spki(&x509_certificate_fixture_public_key()); let mut certificate_scratch = issuer.public_key().public_scratch(); let mut tampered_certificate = x509_pkcs1v15_certificate_fixture(); - *tampered_certificate.last_mut().unwrap() ^= 0x01; + *tampered_certificate + .last_mut() + .expect("the certificate fixture must contain a signature byte") ^= 0x01; let certificate_with_short_signature = x509_certificate( &minimal_tbs_certificate(&pkcs1_algorithm), &pkcs1_algorithm, @@ -3112,55 +3142,49 @@ fn rsa_policy_boundary_separates_legacy_rsa2048_from_modern_rsa3072_verification let legacy_policy = RsaPublicKeyPolicy::legacy_verification(); let modern_policy = RsaPublicKeyPolicy::modern_verification(); - let legacy_pss_key = RsaPublicKey::from_spki_der_with_policy(&pss_fixture_public_key(), &legacy_policy).unwrap(); + let legacy_pss_key = RsaPublicKey::from_spki_der_with_policy(&pss_fixture_public_key(), &legacy_policy) + .expect("the RSA-2048 PSS fixture must satisfy the legacy policy"); assert_eq!(legacy_pss_key.modulus_bits(), 2048); - assert!( - legacy_pss_key - .verify_pss( - RsaPssProfile::Sha256, - pss_fixture_message(), - &pss_fixture_signature_sha256() - ) - .is_ok() - ); + legacy_pss_key + .verify_pss( + RsaPssProfile::Sha256, + pss_fixture_message(), + &pss_fixture_signature_sha256(), + ) + .expect("the RSA-2048 PSS fixture must verify under the legacy policy"); assert_eq!( RsaPublicKey::from_spki_der_with_policy(&pss_fixture_public_key(), &modern_policy), Err(RsaKeyError::InvalidModulus) ); - let legacy_pkcs1v15_key = - RsaPublicKey::from_spki_der_with_policy(&pkcs1v15_fixture_public_key(), &legacy_policy).unwrap(); + let legacy_pkcs1v15_key = RsaPublicKey::from_spki_der_with_policy(&pkcs1v15_fixture_public_key(), &legacy_policy) + .expect("the RSA-2048 PKCS#1 v1.5 fixture must satisfy the legacy policy"); assert_eq!(legacy_pkcs1v15_key.modulus_bits(), 2048); - assert!( - legacy_pkcs1v15_key - .verify_pkcs1v15( - RsaPkcs1v15Profile::Sha256, - pkcs1v15_fixture_message(), - &pkcs1v15_fixture_signature_sha256(), - ) - .is_ok() - ); + legacy_pkcs1v15_key + .verify_pkcs1v15( + RsaPkcs1v15Profile::Sha256, + pkcs1v15_fixture_message(), + &pkcs1v15_fixture_signature_sha256(), + ) + .expect("the RSA-2048 PKCS#1 v1.5 fixture must verify under the legacy policy"); assert_eq!( RsaPublicKey::from_spki_der_with_policy(&pkcs1v15_fixture_public_key(), &modern_policy), Err(RsaKeyError::InvalidModulus) ); - let modern_key = RsaPublicKey::from_spki_der_with_policy(RSA3072_SPKI, &modern_policy).unwrap(); + let modern_key = RsaPublicKey::from_spki_der_with_policy(RSA3072_SPKI, &modern_policy) + .expect("the RSA-3072 fixture must satisfy the modern policy"); assert_eq!(modern_key.modulus_bits(), 3072); - assert!( - modern_key - .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), RSA3072_PSS_SHA256) - .is_ok() - ); - assert!( - modern_key - .verify_pkcs1v15( - RsaPkcs1v15Profile::Sha256, - pkcs1v15_fixture_message(), - RSA3072_PKCS1V15_SHA256 - ) - .is_ok() - ); + modern_key + .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), RSA3072_PSS_SHA256) + .expect("the RSA-3072 PSS fixture must verify under the modern policy"); + modern_key + .verify_pkcs1v15( + RsaPkcs1v15Profile::Sha256, + pkcs1v15_fixture_message(), + RSA3072_PKCS1V15_SHA256, + ) + .expect("the RSA-3072 PKCS#1 v1.5 fixture must verify under the modern policy"); } #[test] @@ -3391,7 +3415,8 @@ fn pkcs1_public_key_can_accept_legacy_small_fermat_exponents_by_policy() { let policy = RsaPublicKeyPolicy::legacy_verification().allow_legacy_small_exponents(); let der = pkcs1_with_parts(integer_unsigned(&rsa2048_modulus()), integer_unsigned(&[0x03])); - let key = RsaPublicKey::from_pkcs1_der_with_policy(&der, &policy).unwrap(); + let key = RsaPublicKey::from_pkcs1_der_with_policy(&der, &policy) + .expect("the legacy policy must accept the RSA exponent 3 fixture"); assert_eq!(key.public_exponent().as_u64(), 3); } @@ -3429,14 +3454,14 @@ fn public_operation_boundary_representatives_match_independent_reference_across_ let key = if name == "rsa2048" { legacy_public_key_from_spki(spki) } else { - RsaPublicKey::from_spki_der(spki).unwrap() + RsaPublicKey::from_spki_der(spki).expect("the modern RSA boundary fixture must parse") }; let len = key.modulus().len(); let mut out = vec![0u8; len]; let mut scratch = key.public_scratch(); let mut one = vec![0u8; len]; - *one.last_mut().unwrap() = 1; + *one.last_mut().expect("a supported RSA representative must be nonempty") = 1; let mut leading_zero = vec![0u8; len]; let leading_zero_tail = b"rscrypto-rsa-boundary-reference"; leading_zero[len - leading_zero_tail.len()..].copy_from_slice(leading_zero_tail); @@ -3450,7 +3475,7 @@ fn public_operation_boundary_representatives_match_independent_reference_across_ for (representative_name, representative) in valid_representatives { key .public_operation_with_scratch(&representative, &mut out, &mut scratch) - .unwrap_or_else(|error| panic!("{name} {representative_name} public operation failed: {error:?}")); + .expect("a valid boundary representative must complete the RSA public operation"); assert_eq!( out, rsa_public_operation_reference(&key, &representative), @@ -3526,7 +3551,7 @@ ec34e8c72cc58fd5324fbe1ddd9714909caedfaa38706cfa66d9bc1026ba3ec1188092392a54a\ key .public_operation_with_scratch(&input, &mut out, &mut scratch) - .unwrap(); + .expect("the independent RSA-65537 public-operation vector must be valid"); assert_eq!(out, expected); } @@ -3578,16 +3603,20 @@ fc87eb5a2ea7a141091f3d2fa37a581ef39e6e496e5e476d43b6", let key_e3 = RsaPublicKey::from_pkcs1_der_with_policy(&valid_pkcs1_with_modulus_and_exponent(&modulus, &[0x03]), &policy) - .unwrap(); + .expect("the legacy policy must accept the exponent 3 vector"); let key_e17 = RsaPublicKey::from_pkcs1_der_with_policy(&valid_pkcs1_with_modulus_and_exponent(&modulus, &[0x11]), &policy) - .unwrap(); + .expect("the legacy policy must accept the exponent 17 vector"); let mut out = vec![0u8; modulus.len()]; - key_e3.public_operation(&input, &mut out).unwrap(); + key_e3 + .public_operation(&input, &mut out) + .expect("the exponent 3 public-operation vector must be valid"); assert_eq!(out, expected_e3); - key_e17.public_operation(&input, &mut out).unwrap(); + key_e17 + .public_operation(&input, &mut out) + .expect("the exponent 17 public-operation vector must be valid"); assert_eq!(out, expected_e17); } @@ -3605,7 +3634,7 @@ fn public_scratch_reuses_after_modulus_minus_one_operation() { core::hint::black_box(&mut out), &mut scratch, ) - .unwrap(); + .expect("the modulus-minus-one public operation must succeed across scratch reuse"); } key .verify_pss_with_scratch( @@ -3614,7 +3643,7 @@ fn public_scratch_reuses_after_modulus_minus_one_operation() { &pss_fixture_signature_sha256(), &mut scratch, ) - .unwrap(); + .expect("PSS verification must succeed after repeated scratch reuse"); } #[cfg(feature = "diag")] @@ -3651,7 +3680,7 @@ d5add90a8a212c10dd997b0a4efcb3df990808509dcb28c504e0649827a83ffd864395d1f62f2\ ] { let key = RsaPublicKey::from_pkcs1_der_with_policy(&valid_pkcs1_with_modulus_and_exponent(&modulus, exponent), &policy) - .unwrap(); + .expect("the diagnostic exponent fixture must satisfy its policy"); let mut current = vec![0u8; modulus.len()]; let mut product = vec![0u8; modulus.len()]; let mut comba = vec![0u8; modulus.len()]; @@ -3664,11 +3693,15 @@ d5add90a8a212c10dd997b0a4efcb3df990808509dcb28c504e0649827a83ffd864395d1f62f2\ key .public_operation_with_scratch(&input, &mut current, &mut scratch) - .unwrap(); - diag_rsa_public_operation_product(&key, &input, &mut product, &mut product_scratch).unwrap(); - diag_rsa_public_operation_comba_product(&key, &input, &mut comba, &mut comba_scratch).unwrap(); - diag_rsa_public_operation_window2_exponent(&key, &input, &mut window2, &mut window2_scratch).unwrap(); - diag_rsa_public_operation_bitserial(&key, &input, &mut bitserial).unwrap(); + .expect("the current Montgomery path must accept the diagnostic fixture"); + diag_rsa_public_operation_product(&key, &input, &mut product, &mut product_scratch) + .expect("the product Montgomery candidate must accept the diagnostic fixture"); + diag_rsa_public_operation_comba_product(&key, &input, &mut comba, &mut comba_scratch) + .expect("the Comba Montgomery candidate must accept the diagnostic fixture"); + diag_rsa_public_operation_window2_exponent(&key, &input, &mut window2, &mut window2_scratch) + .expect("the window-2 exponent candidate must accept the diagnostic fixture"); + diag_rsa_public_operation_bitserial(&key, &input, &mut bitserial) + .expect("the bit-serial candidate must accept the diagnostic fixture"); assert_eq!( product, current, @@ -3705,7 +3738,7 @@ fn public_operation_montgomery_candidates_match_current_path() { let key = if name == "rsa2048" { legacy_public_key_from_spki(spki) } else { - RsaPublicKey::from_spki_der(spki).unwrap() + RsaPublicKey::from_spki_der(spki).expect("the modern RSA diagnostic fixture must parse") }; let representative = modulus_minus_one(&key); let mut current = vec![0u8; key.modulus().len()]; @@ -3721,12 +3754,15 @@ fn public_operation_montgomery_candidates_match_current_path() { key .public_operation_with_scratch(&representative, &mut current, &mut scratch) - .unwrap(); - diag_rsa_public_operation_cios(&key, &representative, &mut cios, &mut cios_scratch).unwrap(); + .expect("the current Montgomery path must accept modulus minus one"); + diag_rsa_public_operation_cios(&key, &representative, &mut cios, &mut cios_scratch) + .expect("the CIOS candidate must accept modulus minus one"); diag_rsa_public_operation_cios_portable(&key, &representative, &mut cios_portable, &mut cios_portable_scratch) - .unwrap(); - diag_rsa_public_operation_comba_product(&key, &representative, &mut comba, &mut comba_scratch).unwrap(); - diag_rsa_public_operation_product(&key, &representative, &mut product, &mut product_scratch).unwrap(); + .expect("the portable CIOS candidate must accept modulus minus one"); + diag_rsa_public_operation_comba_product(&key, &representative, &mut comba, &mut comba_scratch) + .expect("the Comba candidate must accept modulus minus one"); + diag_rsa_public_operation_product(&key, &representative, &mut product, &mut product_scratch) + .expect("the product candidate must accept modulus minus one"); assert_eq!(cios, current, "CIOS mismatch for modulus-minus-one representative"); assert_eq!( cios_portable, current, @@ -3743,11 +3779,15 @@ fn public_operation_montgomery_candidates_match_current_path() { key .public_operation_with_scratch(signature, &mut current, &mut scratch) - .unwrap(); - diag_rsa_public_operation_cios(&key, signature, &mut cios, &mut cios_scratch).unwrap(); - diag_rsa_public_operation_cios_portable(&key, signature, &mut cios_portable, &mut cios_portable_scratch).unwrap(); - diag_rsa_public_operation_comba_product(&key, signature, &mut comba, &mut comba_scratch).unwrap(); - diag_rsa_public_operation_product(&key, signature, &mut product, &mut product_scratch).unwrap(); + .expect("the current Montgomery path must accept the signature fixture"); + diag_rsa_public_operation_cios(&key, signature, &mut cios, &mut cios_scratch) + .expect("the CIOS candidate must accept the signature fixture"); + diag_rsa_public_operation_cios_portable(&key, signature, &mut cios_portable, &mut cios_portable_scratch) + .expect("the portable CIOS candidate must accept the signature fixture"); + diag_rsa_public_operation_comba_product(&key, signature, &mut comba, &mut comba_scratch) + .expect("the Comba candidate must accept the signature fixture"); + diag_rsa_public_operation_product(&key, signature, &mut product, &mut product_scratch) + .expect("the product candidate must accept the signature fixture"); assert_eq!(cios, current, "CIOS mismatch for fixture signature representative"); assert_eq!( cios_portable, current, @@ -3800,9 +3840,15 @@ aec5205b05ff989176db1199e8d34341380f501c34973526d024ef9fd87108e041c16625937a\ c2b691954f9bd86140e31acf6a8a2b9d28cba358e509dfc234c1e33e223c", ); - assert!(key.verify_pss(RsaPssProfile::Sha256, message, &sig256).is_ok()); - assert!(key.verify_pss(RsaPssProfile::Sha384, message, &sig384).is_ok()); - assert!(key.verify_pss(RsaPssProfile::Sha512, message, &sig512).is_ok()); + key + .verify_pss(RsaPssProfile::Sha256, message, &sig256) + .expect("the OpenSSL PSS SHA-256 vector must verify"); + key + .verify_pss(RsaPssProfile::Sha384, message, &sig384) + .expect("the OpenSSL PSS SHA-384 vector must verify"); + key + .verify_pss(RsaPssProfile::Sha512, message, &sig512) + .expect("the OpenSSL PSS SHA-512 vector must verify"); } #[test] @@ -3832,25 +3878,21 @@ fn typed_signature_profile_dispatches_and_rejects_algorithm_confusion() { let pss_key = legacy_public_key_from_spki(&pss_fixture_public_key()); let pss_sig = pss_fixture_signature_sha256(); let mut pss_scratch = pss_key.public_scratch(); - assert!( - pss_key - .verify_signature( - RsaSignatureProfile::pss(RsaPssProfile::Sha256), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); - assert!( - pss_key - .verify_signature_with_scratch( - RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32), - pss_fixture_message(), - &pss_sig, - &mut pss_scratch, - ) - .is_ok() - ); + pss_key + .verify_signature( + RsaSignatureProfile::pss(RsaPssProfile::Sha256), + pss_fixture_message(), + &pss_sig, + ) + .expect("the typed PSS profile must verify its matching fixture"); + pss_key + .verify_signature_with_scratch( + RsaSignatureProfile::pss_with_salt_len(RsaPssProfile::Sha256, 32), + pss_fixture_message(), + &pss_sig, + &mut pss_scratch, + ) + .expect("the typed PSS profile must verify with scratch reuse"); assert_eq!( pss_key.verify_signature( RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), @@ -3871,25 +3913,21 @@ fn typed_signature_profile_dispatches_and_rejects_algorithm_confusion() { let pkcs1_key = legacy_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_sig = pkcs1v15_fixture_signature_sha256(); let mut pkcs1_scratch = pkcs1_key.public_scratch(); - assert!( - pkcs1_key - .verify_signature( - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); - assert!( - pkcs1_key - .verify_signature_with_scratch( - RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), - pkcs1v15_fixture_message(), - &pkcs1_sig, - &mut pkcs1_scratch, - ) - .is_ok() - ); + pkcs1_key + .verify_signature( + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("the typed PKCS#1 v1.5 profile must verify its matching fixture"); + pkcs1_key + .verify_signature_with_scratch( + RsaSignatureProfile::pkcs1v15(RsaPkcs1v15Profile::Sha256), + pkcs1v15_fixture_message(), + &pkcs1_sig, + &mut pkcs1_scratch, + ) + .expect("the typed PKCS#1 v1.5 profile must verify with scratch reuse"); assert_eq!( pkcs1_key.verify_signature( RsaSignatureProfile::pss(RsaPssProfile::Sha256), @@ -4053,27 +4091,23 @@ fn provider_facing_legacy_algorithm_rejections_are_typed() { fn protocol_mapped_profiles_verify_and_reject_algorithm_confusion() { let pss_key = legacy_public_key_from_spki(&pss_fixture_public_key()); let pss_sig = pss_fixture_signature_sha256(); - assert!( - pss_key - .verify_signature( - RsaSignatureProfile::from_tls13_signature_scheme(0x0804).unwrap(), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); - assert!( - pss_key - .verify_signature( - RsaJwtAlgorithm::Ps256.signature_profile(), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); + pss_key + .verify_signature( + RsaSignatureProfile::from_tls13_signature_scheme(0x0804).expect("TLS scheme 0x0804 must map to RSA-PSS SHA-256"), + pss_fixture_message(), + &pss_sig, + ) + .expect("the TLS-mapped PSS profile must verify its matching fixture"); + pss_key + .verify_signature( + RsaJwtAlgorithm::Ps256.signature_profile(), + pss_fixture_message(), + &pss_sig, + ) + .expect("the JWT-mapped PSS profile must verify its matching fixture"); assert_eq!( pss_key.verify_signature( - RsaSignatureProfile::from_cose_algorithm_id(-257).unwrap(), + RsaSignatureProfile::from_cose_algorithm_id(-257).expect("COSE algorithm -257 must map to RS256"), pss_fixture_message(), &pss_sig, ), @@ -4082,27 +4116,24 @@ fn protocol_mapped_profiles_verify_and_reject_algorithm_confusion() { let pkcs1_key = legacy_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_sig = pkcs1v15_fixture_signature_sha256(); - assert!( - pkcs1_key - .verify_signature( - RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0401).unwrap(), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); - assert!( - pkcs1_key - .verify_signature( - RsaJwtAlgorithm::Rs256.signature_profile(), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); + pkcs1_key + .verify_signature( + RsaSignatureProfile::from_tls_certificate_signature_scheme(0x0401) + .expect("TLS certificate scheme 0x0401 must map to PKCS#1 v1.5 SHA-256"), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("the TLS-mapped PKCS#1 v1.5 profile must verify its matching fixture"); + pkcs1_key + .verify_signature( + RsaJwtAlgorithm::Rs256.signature_profile(), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("the JWT-mapped PKCS#1 v1.5 profile must verify its matching fixture"); assert_eq!( pkcs1_key.verify_signature( - RsaSignatureProfile::from_cose_algorithm_id(-37).unwrap(), + RsaSignatureProfile::from_cose_algorithm_id(-37).expect("COSE algorithm -37 must map to PS256"), pkcs1v15_fixture_message(), &pkcs1_sig, ), @@ -4120,13 +4151,13 @@ fn jwt_policy_and_cose_helpers_reject_algorithm_confusion() { let pss_jwt = pss_key.jwt_verifier(RsaJwtAlgorithm::Ps256); assert_eq!(pss_jwt.algorithm(), RsaJwtAlgorithm::Ps256); - assert!(core::ptr::eq(pss_jwt.key(), &pss_key)); - assert!(pss_jwt.verify("PS256", pss_fixture_message(), &pss_sig).is_ok()); - assert!( - pss_jwt - .verify_with_scratch("PS256", pss_fixture_message(), &pss_sig, &mut scratch) - .is_ok() - ); + assert!(core::ptr::eq(pss_jwt.key(), &raw const pss_key)); + pss_jwt + .verify("PS256", pss_fixture_message(), &pss_sig) + .expect("the PS256 JWT verifier must accept its matching fixture"); + pss_jwt + .verify_with_scratch("PS256", pss_fixture_message(), &pss_sig, &mut scratch) + .expect("the PS256 JWT verifier must accept its matching fixture with scratch reuse"); for header_alg in [ "", "none", @@ -4147,23 +4178,19 @@ fn jwt_policy_and_cose_helpers_reject_algorithm_confusion() { "accepted noncanonical or mismatched JWT alg {header_alg:?}", ); } - assert!( - pss_key - .verify_expected_cose_algorithm_id(-37, -37, pss_profile, pss_fixture_message(), &pss_sig) - .is_ok() - ); - assert!( - pss_key - .verify_expected_cose_algorithm_id_with_scratch( - -37, - -37, - pss_profile, - pss_fixture_message(), - &pss_sig, - &mut scratch, - ) - .is_ok() - ); + pss_key + .verify_expected_cose_algorithm_id(-37, -37, pss_profile, pss_fixture_message(), &pss_sig) + .expect("matching PS256 COSE identifiers must verify"); + pss_key + .verify_expected_cose_algorithm_id_with_scratch( + -37, + -37, + pss_profile, + pss_fixture_message(), + &pss_sig, + &mut scratch, + ) + .expect("matching PS256 COSE identifiers must verify with scratch reuse"); for result in [ pss_key .jwt_verifier(RsaJwtAlgorithm::Rs256) @@ -4181,33 +4208,25 @@ fn jwt_policy_and_cose_helpers_reject_algorithm_confusion() { let mut scratch = pkcs1_key.public_scratch(); let pkcs1_jwt = pkcs1_key.jwt_verifier(RsaJwtAlgorithm::Rs256); - assert!( - pkcs1_jwt - .verify("RS256", pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); - assert!( - pkcs1_jwt - .verify_with_scratch("RS256", pkcs1v15_fixture_message(), &pkcs1_sig, &mut scratch) - .is_ok() - ); - assert!( - pkcs1_key - .verify_expected_cose_algorithm_id(-257, -257, pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); - assert!( - pkcs1_key - .verify_expected_cose_algorithm_id_with_scratch( - -257, - -257, - pkcs1_profile, - pkcs1v15_fixture_message(), - &pkcs1_sig, - &mut scratch, - ) - .is_ok() - ); + pkcs1_jwt + .verify("RS256", pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("the RS256 JWT verifier must accept its matching fixture"); + pkcs1_jwt + .verify_with_scratch("RS256", pkcs1v15_fixture_message(), &pkcs1_sig, &mut scratch) + .expect("the RS256 JWT verifier must accept its matching fixture with scratch reuse"); + pkcs1_key + .verify_expected_cose_algorithm_id(-257, -257, pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("matching RS256 COSE identifiers must verify"); + pkcs1_key + .verify_expected_cose_algorithm_id_with_scratch( + -257, + -257, + pkcs1_profile, + pkcs1v15_fixture_message(), + &pkcs1_sig, + &mut scratch, + ) + .expect("matching RS256 COSE identifiers must verify with scratch reuse"); for result in [ pkcs1_key .jwt_verifier(RsaJwtAlgorithm::Ps256) @@ -4353,16 +4372,14 @@ fn x509_signature_algorithm_mapping_is_strict_and_rejects_sha1_defaults() { fn x509_mapped_profiles_verify_real_fixtures_and_reject_padding_mismatch() { let pss_key = legacy_public_key_from_spki(&pss_fixture_public_key()); let pss_sig = pss_fixture_signature_sha256(); - assert!( - pss_key - .verify_signature( - RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss_algorithm(RsaPssProfile::Sha256, 32, None)) - .unwrap(), - pss_fixture_message(), - &pss_sig, - ) - .is_ok() - ); + pss_key + .verify_signature( + RsaSignatureProfile::from_x509_signature_algorithm_der(&x509_pss_algorithm(RsaPssProfile::Sha256, 32, None)) + .expect("the X.509 PSS SHA-256 algorithm fixture must map to a signature profile"), + pss_fixture_message(), + &pss_sig, + ) + .expect("the X.509-mapped PSS profile must verify its matching fixture"); let pkcs1_key = legacy_public_key_from_spki(&pkcs1v15_fixture_public_key()); let pkcs1_sig = pkcs1v15_fixture_signature_sha256(); @@ -4370,22 +4387,18 @@ fn x509_mapped_profiles_verify_real_fixtures_and_reject_padding_mismatch() { SHA256_WITH_RSA_ENCRYPTION_OID, Some(&null()), )) - .unwrap(); - assert!( - pkcs1_key - .verify_signature(pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); + .expect("the X.509 PKCS#1 v1.5 SHA-256 algorithm fixture must map to a signature profile"); + pkcs1_key + .verify_signature(pkcs1_profile, pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("the X.509-mapped PKCS#1 v1.5 profile must verify its matching fixture"); let pkcs1_x509_key = legacy_x509_public_key_from_spki(&pkcs1v15_fixture_public_key()); - assert!( - pkcs1_x509_key - .verify_signature_from_x509_algorithm_der( - &algorithm_identifier(SHA256_WITH_RSA_ENCRYPTION_OID, None), - pkcs1v15_fixture_message(), - &pkcs1_sig, - ) - .is_ok() - ); + pkcs1_x509_key + .verify_signature_from_x509_algorithm_der( + &algorithm_identifier(SHA256_WITH_RSA_ENCRYPTION_OID, None), + pkcs1v15_fixture_message(), + &pkcs1_sig, + ) + .expect("an X.509 PKCS#1 v1.5 algorithm with absent parameters must verify"); assert_eq!( pss_key.verify_signature(pkcs1_profile, pss_fixture_message(), &pss_sig), Err(VerificationError::new()) @@ -4428,21 +4441,15 @@ addc98f3b49437df55aad5e96a6b7db196f9d30de7173dda79944f51fc7a339655cd2727d47\ 4630a322103c344bc4c65add2214f60155b3819869210f19730544989fed6921bf", ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &sig256) - .is_ok() - ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &sig384) - .is_ok() - ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha512, message, &sig512) - .is_ok() - ); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, message, &sig256) + .expect("the OpenSSL PKCS#1 v1.5 SHA-256 vector must verify"); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha384, message, &sig384) + .expect("the OpenSSL PKCS#1 v1.5 SHA-384 vector must verify"); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha512, message, &sig512) + .expect("the OpenSSL PKCS#1 v1.5 SHA-512 vector must verify"); } #[test] @@ -4478,13 +4485,16 @@ fn pss_encoded_message_oracle_failures_are_opaque() { let mut encoded = vec![0u8; key.modulus().len()]; key .public_operation(&pss_fixture_signature_sha256(), &mut encoded) - .unwrap(); + .expect("the PSS signature fixture must decode to its encoded message"); let em_bits = key.modulus_bits().strict_sub(1); - assert!(diag_rsa_verify_pss_encoded(RsaPssProfile::Sha256, pss_fixture_message(), &encoded, em_bits).is_ok()); + diag_rsa_verify_pss_encoded(RsaPssProfile::Sha256, pss_fixture_message(), &encoded, em_bits) + .expect("the valid PSS encoded message must verify"); let mut bad_trailer = encoded.clone(); - *bad_trailer.last_mut().unwrap() ^= 0x01; + *bad_trailer + .last_mut() + .expect("the PSS encoded message must contain a trailer byte") ^= 0x01; assert_opaque_verification_failure(diag_rsa_verify_pss_encoded( RsaPssProfile::Sha256, pss_fixture_message(), @@ -4555,9 +4565,10 @@ fn pkcs1v15_encoded_message_oracle_failures_are_opaque() { let mut encoded = vec![0u8; key.modulus().len()]; key .public_operation(&pkcs1v15_fixture_signature_sha256(), &mut encoded) - .unwrap(); + .expect("the PKCS#1 v1.5 signature fixture must decode to its encoded message"); - assert!(diag_rsa_verify_pkcs1v15_encoded(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &encoded).is_ok()); + diag_rsa_verify_pkcs1v15_encoded(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &encoded) + .expect("the valid PKCS#1 v1.5 encoded message must verify"); let mut bad_prefix = encoded.clone(); bad_prefix[0] = 0x01; @@ -4602,7 +4613,9 @@ fn pkcs1v15_encoded_message_oracle_failures_are_opaque() { )); let mut bad_digest = encoded.clone(); - *bad_digest.last_mut().unwrap() ^= 0x80; + *bad_digest + .last_mut() + .expect("the PKCS#1 v1.5 encoded message must contain a digest byte") ^= 0x80; assert_opaque_verification_failure(diag_rsa_verify_pkcs1v15_encoded( RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), @@ -4629,11 +4642,9 @@ fn sha256_signature_verification_matches_external_oracles_for_valid_and_tampered let pss_pkcs1 = valid_pkcs1_with_modulus_and_exponent(pss_key.modulus(), &exponent_bytes(pss_key.public_exponent().as_u64())); - assert!( - pss_key - .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), &pss_sig) - .is_ok() - ); + pss_key + .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), &pss_sig) + .expect("the valid PSS SHA-256 fixture must verify before external-oracle comparison"); assert_ring_pss_sha256(&pss_pkcs1, pss_fixture_message(), &pss_sig, true); assert_aws_lc_rs_pss_sha256(&pss_pkcs1, pss_fixture_message(), &pss_sig, true); assert_rustcrypto_pss_sha256(&pss_pkcs1, pss_fixture_message(), &pss_sig, true); @@ -4645,7 +4656,7 @@ fn sha256_signature_verification_matches_external_oracles_for_valid_and_tampered true, ); - let mut tampered = pss_sig.clone(); + let mut tampered = pss_sig; tampered[0] ^= 0x80; assert!( pss_key @@ -4671,17 +4682,15 @@ fn sha256_signature_verification_matches_external_oracles_for_valid_and_tampered &exponent_bytes(pkcs1_key.public_exponent().as_u64()), ); - assert!( - pkcs1_key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &pkcs1_sig) - .is_ok() - ); + pkcs1_key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), &pkcs1_sig) + .expect("the valid PKCS#1 v1.5 SHA-256 fixture must verify before external-oracle comparison"); assert_ring_pkcs1v15_sha256(&pkcs1_der, pkcs1v15_fixture_message(), &pkcs1_sig, true); assert_aws_lc_rs_pkcs1v15_sha256(&pkcs1_der, pkcs1v15_fixture_message(), &pkcs1_sig, true); assert_rustcrypto_pkcs1v15_sha256(&pkcs1_der, pkcs1v15_fixture_message(), &pkcs1_sig, true); assert_openssl_sha256(&pkcs1_spki, pkcs1v15_fixture_message(), &pkcs1_sig, &[], true); - let mut tampered = pkcs1_sig.clone(); + let mut tampered = pkcs1_sig; tampered[0] ^= 0x80; assert!( pkcs1_key @@ -4703,13 +4712,16 @@ fn nist_cavp_sha2_signatures_match_rust_external_oracles() { for test in tests { let scheme = cavp_field(test, "scheme"); let sha = cavp_field(test, "sha"); - let salt_len = - (scheme == "pss").then(|| test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric") as usize); + let salt_len = (scheme == "pss").then(|| { + usize::try_from(test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric")) + .expect("a CAVP PSS salt length must fit in usize") + }); let expected = match cavp_field(test, "result") { - "P" => true, - "F" => false, - other => panic!("unsupported CAVP result `{other}`"), - }; + "P" => Some(true), + "F" => Some(false), + _ => None, + } + .expect("CAVP result fields must be `P` or `F`"); let modulus = cavp_hex_to_vec(cavp_field(test, "n")); let exponent = cavp_hex_to_vec(cavp_field(test, "e")); @@ -4745,13 +4757,16 @@ fn nist_cavp_sha2_signatures_match_openssl_cli_when_available() { for test in tests { let scheme = cavp_field(test, "scheme"); let sha = cavp_field(test, "sha"); - let salt_len = - (scheme == "pss").then(|| test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric") as usize); + let salt_len = (scheme == "pss").then(|| { + usize::try_from(test["salt_len"].as_u64().expect("CAVP PSS salt length must be numeric")) + .expect("a CAVP PSS salt length must fit in usize") + }); let expected = match cavp_field(test, "result") { - "P" => true, - "F" => false, - other => panic!("unsupported CAVP result `{other}`"), - }; + "P" => Some(true), + "F" => Some(false), + _ => None, + } + .expect("CAVP result fields must be `P` or `F`"); let pkcs1 = valid_pkcs1_with_modulus_and_exponent( &cavp_hex_to_vec(cavp_field(test, "n")), &cavp_hex_to_vec(cavp_field(test, "e")), @@ -4760,14 +4775,15 @@ fn nist_cavp_sha2_signatures_match_openssl_cli_when_available() { let message = cavp_hex_to_vec(cavp_field(test, "msg")); let signature = cavp_hex_to_vec(cavp_field(test, "sig")); let digest_arg = match sha { - "SHA256" => "-sha256", - "SHA384" => "-sha384", - "SHA512" => "-sha512", - other => panic!("unsupported OpenSSL CAVP hash `{other}`"), - }; + "SHA256" => Some("-sha256"), + "SHA384" => Some("-sha384"), + "SHA512" => Some("-sha512"), + _ => None, + } + .expect("CAVP fixtures must use an OpenSSL-supported SHA-2 profile"); let pss_sigopts; let sigopts = if scheme == "pss" { - pss_sigopts = openssl_pss_sigopts(sha, salt_len.unwrap()); + pss_sigopts = openssl_pss_sigopts(sha, salt_len.expect("a CAVP PSS case must provide a salt length")); &pss_sigopts[..] } else { &[] @@ -4795,19 +4811,15 @@ fn generated_rsa_size_fixtures_verify_for_benchmark_matrix() { (4096, RSA4096_SPKI, RSA4096_PSS_SHA256, RSA4096_PKCS1V15_SHA256), (8192, RSA8192_SPKI, RSA8192_PSS_SHA256, RSA8192_PKCS1V15_SHA256), ] { - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("the generated RSA benchmark fixture must parse"); assert_eq!(key.modulus_bits(), bits); - assert!( - key - .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig) - .is_ok() - ); - assert!( - key - .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), pkcs1_sig) - .is_ok() - ); + key + .verify_pss(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig) + .expect("the generated RSA benchmark PSS fixture must verify"); + key + .verify_pkcs1v15(RsaPkcs1v15Profile::Sha256, pkcs1v15_fixture_message(), pkcs1_sig) + .expect("the generated RSA benchmark PKCS#1 v1.5 fixture must verify"); let pkcs1_der = valid_pkcs1_with_modulus_and_exponent(key.modulus(), &exponent_bytes(key.public_exponent().as_u64())); @@ -4841,35 +4853,29 @@ fn aarch64_linux_dispatch_verifies_generated_rsa_fixtures() { (4096, RSA4096_SPKI, RSA4096_PSS_SHA256, RSA4096_PKCS1V15_SHA256), (8192, RSA8192_SPKI, RSA8192_PSS_SHA256, RSA8192_PKCS1V15_SHA256), ] { - let key = RsaPublicKey::from_spki_der(spki).unwrap(); + let key = RsaPublicKey::from_spki_der(spki).expect("the generated AArch64 Linux RSA fixture must parse"); let mut scratch = key.public_scratch(); assert_eq!(key.modulus_bits(), bits); - assert!( - key - .verify_pss_with_scratch(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig, &mut scratch) - .is_ok(), - "RSA-{bits} PSS fixture failed through AArch64 Linux dispatch" - ); - assert!( - key - .verify_pkcs1v15_with_scratch( - RsaPkcs1v15Profile::Sha256, - pkcs1v15_fixture_message(), - pkcs1_sig, - &mut scratch, - ) - .is_ok(), - "RSA-{bits} PKCS1v15 fixture failed through AArch64 Linux dispatch" - ); + key + .verify_pss_with_scratch(RsaPssProfile::Sha256, pss_fixture_message(), pss_sig, &mut scratch) + .expect("the PSS fixture must verify through AArch64 Linux dispatch"); + key + .verify_pkcs1v15_with_scratch( + RsaPkcs1v15Profile::Sha256, + pkcs1v15_fixture_message(), + pkcs1_sig, + &mut scratch, + ) + .expect("the PKCS#1 v1.5 fixture must verify through AArch64 Linux dispatch"); } } proptest! { #[test] fn arbitrary_der_inputs_do_not_panic(input in proptest::collection::vec(any::(), 0..4096)) { - let _ = RsaPublicKey::from_pkcs1_der(&input); - let _ = RsaPublicKey::from_spki_der(&input); + let _pkcs1_result = RsaPublicKey::from_pkcs1_der(&input); + let _spki_result = RsaPublicKey::from_spki_der(&input); } #[test] @@ -4880,12 +4886,18 @@ proptest! { let key = arbitrary_verify_key(); let mut scratch = key.public_scratch(); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); + let _pss_sha256_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); + let _pss_sha384_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); + let _pss_sha512_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); + let _pkcs1_sha256_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); + let _pkcs1_sha384_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); + let _pkcs1_sha512_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); } #[test] @@ -4897,11 +4909,17 @@ proptest! { let signature = fixed_width_signature_candidate(&material, key.modulus().len()); let mut scratch = key.public_scratch(); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); - let _ = key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); + let _pss_sha256_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha256, &message, &signature, &mut scratch); + let _pss_sha384_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha384, &message, &signature, &mut scratch); + let _pss_sha512_result = + key.verify_pss_with_scratch(RsaPssProfile::Sha512, &message, &signature, &mut scratch); + let _pkcs1_sha256_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha256, &message, &signature, &mut scratch); + let _pkcs1_sha384_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha384, &message, &signature, &mut scratch); + let _pkcs1_sha512_result = + key.verify_pkcs1v15_with_scratch(RsaPkcs1v15Profile::Sha512, &message, &signature, &mut scratch); } } diff --git a/tests/rsa_wycheproof.rs b/tests/rsa_wycheproof.rs index 5daded75..3b8ed77d 100644 --- a/tests/rsa_wycheproof.rs +++ b/tests/rsa_wycheproof.rs @@ -2,7 +2,9 @@ use core::ops::Range; -use rscrypto::{RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy}; +use rscrypto::{ + RsaBlindingPair, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKey, RsaPublicKeyPolicy, +}; use serde_json::Value; const PKCS1_SHA256: &str = include_str!("../testdata/rsa/wycheproof/rsa_signature_2048_sha256_test.json"); @@ -55,21 +57,30 @@ struct ExpectedCounts { invalid: usize, } +#[derive(Clone, Copy)] +enum WycheproofResult { + Valid, + Acceptable, + Invalid, +} + fn hex_to_vec(hex: &str) -> Vec { assert_eq!(hex.len() % 2, 0); let mut out = Vec::with_capacity(hex.len() / 2); for chunk in hex.as_bytes().chunks_exact(2) { - out.push((hex_value(chunk[0]) << 4) | hex_value(chunk[1])); + let high = hex_value(chunk[0]).expect("Wycheproof fixture must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("Wycheproof fixture must contain hexadecimal digits"); + out.push((high << 4) | low); } out } -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -87,7 +98,17 @@ fn test_cases(group: &Value) -> &[Value] { fn field<'a>(value: &'a Value, name: &str) -> &'a str { value[name] .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof fixture must contain the requested string field") +} + +fn wycheproof_result(result: &str) -> WycheproofResult { + match result { + "valid" => Some(WycheproofResult::Valid), + "acceptable" => Some(WycheproofResult::Acceptable), + "invalid" => Some(WycheproofResult::Invalid), + _ => None, + } + .expect("Wycheproof result must be valid, acceptable, or invalid") } fn public_key_from_wycheproof_spki(der: &[u8], key_size: u64, context: &str) -> RsaPublicKey { @@ -133,8 +154,8 @@ fn assert_pkcs1v15_wycheproof_vectors( .verify_pkcs1v15_with_scratch(profile, &msg, &sig, &mut scratch) .is_ok(); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); assert!( verified, @@ -142,7 +163,7 @@ fn assert_pkcs1v15_wycheproof_vectors( test["tcId"] ); } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( !verified, @@ -151,10 +172,9 @@ fn assert_pkcs1v15_wycheproof_vectors( field(test, "comment") ); } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } } @@ -199,8 +219,8 @@ fn assert_pss_wycheproof_vectors( .verify_pss_with_salt_len_and_scratch(profile, expected_salt_len, &msg, &sig, &mut scratch) .is_ok(); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); assert!( verified, @@ -208,7 +228,7 @@ fn assert_pss_wycheproof_vectors( test["tcId"] ); } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( !verified, @@ -217,10 +237,9 @@ fn assert_pss_wycheproof_vectors( field(test, "comment") ); } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } } @@ -295,20 +314,14 @@ fn assert_oaep_wycheproof_vectors( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut plaintext, ); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); - let plaintext_len = decrypted.unwrap_or_else(|error| { - panic!( - "Wycheproof OAEP tcId {} rejected valid ciphertext: {error}", - test["tcId"] - ) - }); + let plaintext_len = decrypted.expect("Wycheproof valid OAEP ciphertext must decrypt"); assert_eq!( &plaintext[..plaintext_len], hex_to_vec(field(test, "msg")).as_slice(), @@ -322,8 +335,7 @@ fn assert_oaep_wycheproof_vectors( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -337,7 +349,7 @@ fn assert_oaep_wycheproof_vectors( scratch_valid_checked = true; } } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( decrypted.is_err(), @@ -358,8 +370,7 @@ fn assert_oaep_wycheproof_vectors( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -376,10 +387,9 @@ fn assert_oaep_wycheproof_vectors( scratch_invalid_checked = true; } } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } if group_has_valid { @@ -447,8 +457,7 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut plaintext, ); let mut scratch_plaintext = vec![0xa5; key.public_key().modulus().len()]; @@ -456,14 +465,13 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( profile, &label, &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); assert!( decrypted.is_err(), @@ -476,7 +484,7 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( test["tcId"] ); } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( decrypted.is_err(), @@ -491,7 +499,7 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( field(test, "comment") ); } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); assert!( decrypted.is_err(), @@ -504,7 +512,6 @@ fn assert_oaep_mgf1sha1_vectors_are_rejected( test["tcId"] ); } - other => panic!("unknown Wycheproof result `{other}`"), } assert!( plaintext.iter().all(|&byte| byte == 0), @@ -583,20 +590,14 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( let mut plaintext = vec![0xa5; key.public_key().modulus().len()]; let decrypted = key.decrypt_pkcs1v15_with_blinding_factor( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut plaintext, ); - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); - let plaintext_len = decrypted.unwrap_or_else(|error| { - panic!( - "Wycheproof RSAES-PKCS1-v1_5 tcId {} rejected valid ciphertext: {error}", - test["tcId"] - ) - }); + let plaintext_len = decrypted.expect("Wycheproof valid RSAES-PKCS1-v1_5 ciphertext must decrypt"); assert_eq!( &plaintext[..plaintext_len], hex_to_vec(field(test, "msg")).as_slice(), @@ -608,8 +609,7 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( let scratch_plaintext_len = key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -623,7 +623,7 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( scratch_valid_checked = true; } } - "invalid" => { + WycheproofResult::Invalid => { invalid = invalid.strict_add(1); assert!( decrypted.is_err(), @@ -642,8 +642,7 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( key .decrypt_pkcs1v15_with_blinding_factor_and_scratch( &ciphertext, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_plaintext, &mut scratch, ) @@ -660,10 +659,9 @@ fn assert_rsaes_pkcs1v15_wycheproof_vectors( scratch_invalid_checked = true; } } - "acceptable" => { + WycheproofResult::Acceptable => { acceptable = acceptable.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } if group_has_valid { @@ -716,8 +714,8 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 let blinding_factor_inverse = fixed_width_one(key.public_key().modulus().len()); for test in test_cases(group) { - match field(test, "result") { - "valid" => { + match wycheproof_result(field(test, "result")) { + WycheproofResult::Valid => { valid = valid.strict_add(1); let message = hex_to_vec(field(test, "msg")); let expected_signature = hex_to_vec(field(test, "sig")); @@ -728,8 +726,7 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 .sign_pkcs1v15_with_blinding_factor( profile, &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut signature, ) .expect("Wycheproof PKCS1v1.5 private signing must succeed"); @@ -742,8 +739,7 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 .sign_pkcs1v15_with_blinding_factor_and_scratch( profile, &message, - &blinding_factor, - &blinding_factor_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_factor_inverse), &mut scratch_signature, &mut scratch, ) @@ -758,10 +754,9 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 .verify_pkcs1v15(profile, &message, &signature) .expect("Wycheproof PKCS1v1.5 generated signature must verify"); } - "acceptable" | "invalid" => { + WycheproofResult::Acceptable | WycheproofResult::Invalid => { skipped = skipped.strict_add(1); } - other => panic!("unknown Wycheproof result `{other}`"), } } } @@ -769,7 +764,12 @@ fn assert_pkcs1v15_sig_gen_wycheproof_vectors(json: &str, expected_key_size: u64 assert_eq!(valid, expected_valid); assert_eq!( valid.strict_add(skipped), - suite["numberOfTests"].as_u64().unwrap() as usize + usize::try_from( + suite["numberOfTests"] + .as_u64() + .expect("Wycheproof numberOfTests must be numeric") + ) + .expect("Wycheproof numberOfTests must fit usize") ); } diff --git a/tests/scrypt_differential.rs b/tests/scrypt_differential.rs index f02a4001..ca5792db 100644 --- a/tests/scrypt_differential.rs +++ b/tests/scrypt_differential.rs @@ -11,16 +11,16 @@ use proptest::{prelude::*, test_runner::Config as ProptestConfig}; use rscrypto::{Scrypt, ScryptParams}; fn oracle_scrypt(password: &[u8], salt: &[u8], log_n: u8, r: u32, p: u32, out_len: usize) -> Vec { - let params = scrypt::Params::new(log_n, r, p).unwrap(); + let params = scrypt::Params::new(log_n, r, p).expect("RustCrypto must accept generated scrypt parameters"); let mut out = vec![0u8; out_len]; - scrypt::scrypt(password, salt, ¶ms, &mut out).unwrap(); + scrypt::scrypt(password, salt, ¶ms, &mut out).expect("RustCrypto scrypt derivation must succeed"); out } fn rs_hash(password: &[u8], salt: &[u8], log_n: u8, r: u32, p: u32, out_len: usize) -> Vec { - let params = ScryptParams::new(log_n, r, p).unwrap(); + let params = ScryptParams::new(log_n, r, p).expect("generated rscrypto scrypt parameters must be valid"); let mut out = vec![0u8; out_len]; - Scrypt::derive(¶ms, password, salt, &mut out).unwrap(); + Scrypt::derive(¶ms, password, salt, &mut out).expect("rscrypto scrypt derivation must succeed"); out } @@ -54,10 +54,12 @@ proptest! { p in 1u32..=2, ) { let params = ScryptParams::new(log_n, r, p) - .unwrap(); + .expect("generated scrypt verification parameters must be valid"); let mut hash = [0u8; 32]; - Scrypt::derive(¶ms, &password, &salt, &mut hash).unwrap(); - prop_assert!(Scrypt::verify(¶ms, &password, &salt, &hash).is_ok()); + Scrypt::derive(¶ms, &password, &salt, &mut hash) + .expect("scrypt verification fixture must derive"); + Scrypt::verify(¶ms, &password, &salt, &hash) + .expect("fresh scrypt hash must verify"); } } @@ -83,29 +85,25 @@ fn scrypt_short_dklen_is_prefix_of_wide() { // `Scrypt::derive(.., out_len=1)` must equal the first byte of // `Scrypt::derive(.., out_len=64)` for the same inputs. Exercises the // `out_len=1` PBKDF2 single-byte tail that the oracle rejects. - let params_short = rscrypto::ScryptParams::new(6, 2, 1).unwrap(); - let params_wide = rscrypto::ScryptParams::new(6, 2, 1).unwrap(); + let params = ScryptParams::new(6, 2, 1).expect("short-output scrypt parameters must be valid"); let mut short_out = [0u8; 1]; let mut wide_out = [0u8; 64]; - Scrypt::derive(¶ms_short, b"pw", b"salty-salty-salt", &mut short_out).unwrap(); - Scrypt::derive(¶ms_wide, b"pw", b"salty-salty-salt", &mut wide_out).unwrap(); + Scrypt::derive(¶ms, b"pw", b"salty-salty-salt", &mut short_out).expect("one-byte scrypt derivation must succeed"); + Scrypt::derive(¶ms, b"pw", b"salty-salty-salt", &mut wide_out).expect("64-byte scrypt derivation must succeed"); assert_eq!(short_out[0], wide_out[0]); } #[test] fn scrypt_verify_rejects_byte_flip_at_every_position() { - let params = ScryptParams::new(6, 2, 1).unwrap(); + let params = ScryptParams::new(6, 2, 1).expect("byte-flip scrypt parameters must be valid"); let password = b"correct horse battery staple"; let salt = b"random-salt-1234"; let mut hash = [0u8; 32]; - Scrypt::derive(¶ms, password, salt, &mut hash).unwrap(); + Scrypt::derive(¶ms, password, salt, &mut hash).expect("byte-flip scrypt fixture must derive"); for pos in 0..hash.len() { let mut tampered = hash; tampered[pos] ^= 0x01; - assert!( - Scrypt::verify(¶ms, password, salt, &tampered).is_err(), - "verify must reject flip at byte {pos}", - ); + Scrypt::verify(¶ms, password, salt, &tampered).expect_err("scrypt must reject a hash with any flipped byte"); } } diff --git a/tests/secret_redaction.rs b/tests/secret_redaction.rs index 227766eb..96c1df68 100644 --- a/tests/secret_redaction.rs +++ b/tests/secret_redaction.rs @@ -11,7 +11,21 @@ fn generic_secret_owner_debug_is_redacted() { #[test] fn keyed_state_debug_snapshots_are_redacted() { + #[cfg(any(feature = "aes-gcm", feature = "aes-gcm-siv", feature = "ascon-aead"))] const KEY_16: [u8; 16] = [0x53; 16]; + #[cfg(any( + feature = "aegis256", + feature = "aes-gcm", + feature = "aes-gcm-siv", + feature = "chacha20poly1305", + feature = "hkdf", + feature = "hmac", + feature = "hmac-sha3", + feature = "kmac", + feature = "pbkdf2", + feature = "poly1305", + feature = "xchacha20poly1305" + ))] const KEY_32: [u8; 32] = [0x53; 32]; #[cfg(feature = "aes-gcm")] @@ -132,11 +146,13 @@ fn keyed_state_debug_snapshots_are_redacted() { #[test] fn keyed_hash_debug_snapshots_are_redacted() { + #[cfg(any(feature = "blake2b", feature = "blake2s", feature = "blake3"))] const KEY: [u8; 32] = [0x53; 32]; #[cfg(feature = "blake2s")] { - let params = rscrypto::Blake2sParams::new().key(rscrypto::Blake2sKey::new(&KEY).unwrap()); + let key = rscrypto::Blake2sKey::new(&KEY).expect("32-byte BLAKE2s key must be valid"); + let params = rscrypto::Blake2sParams::new().key(key); assert_debug_snapshot( ¶ms, "Blake2sParams { key_len: 32, salt: [0, 0, 0, 0, 0, 0, 0, 0], personal: [0, 0, 0, 0, 0, 0, 0, 0] }", @@ -147,7 +163,8 @@ fn keyed_hash_debug_snapshots_are_redacted() { #[cfg(feature = "blake2b")] { - let params = rscrypto::Blake2bParams::new().key(rscrypto::Blake2bKey::new(&KEY).unwrap()); + let key = rscrypto::Blake2bKey::new(&KEY).expect("32-byte BLAKE2b key must be valid"); + let params = rscrypto::Blake2bParams::new().key(key); assert_debug_snapshot( ¶ms, "Blake2bParams { key_len: 32, salt: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], personal: [0, 0, 0, 0, 0, \ @@ -155,7 +172,8 @@ fn keyed_hash_debug_snapshots_are_redacted() { ); assert_debug_snapshot(¶ms.build_256(), "Blake2b256 { .. }"); assert_debug_snapshot(¶ms.build_512(), "Blake2b512 { .. }"); - assert_debug_snapshot(¶ms.build(32).unwrap(), "Blake2b { output_len: 32, .. }"); + let variable = params.build(32).expect("32-byte BLAKE2b output must be valid"); + assert_debug_snapshot(&variable, "Blake2b { output_len: 32, .. }"); } #[cfg(feature = "blake3")] @@ -243,7 +261,8 @@ fn private_key_and_shared_secret_debug_snapshots_are_redacted() { let (_, key) = <$profile>::generate_keypair(|out| { for (index, byte) in out.iter_mut().enumerate() { - *byte = 0x53u8.wrapping_add(index as u8); + let index = u8::try_from(index).expect("ML-KEM entropy index must fit in u8"); + *byte = 0x53u8.wrapping_add(index); } Ok::<(), rscrypto::MlKemError>(()) }) @@ -297,7 +316,8 @@ fn secret_input_error_snapshots_do_not_echo_input_bytes() { impl core::error::Error for EntropyStateError {} - let passwords = rscrypto::Argon2idPassword::new(rscrypto::Argon2Params::new(32, 2, 1).unwrap()).unwrap(); + let params = rscrypto::Argon2Params::new(32, 2, 1).expect("test Argon2 parameters must be valid"); + let passwords = rscrypto::Argon2idPassword::new(params).expect("test Argon2 policy must be valid"); let error = passwords .hash_password_with(b"password", |_| Err(EntropyStateError)) .expect_err("entropy-source failure must be preserved as an opaque error"); diff --git a/tests/serde_roundtrip.rs b/tests/serde_roundtrip.rs index aedb01b4..ecbe9c7c 100644 --- a/tests/serde_roundtrip.rs +++ b/tests/serde_roundtrip.rs @@ -13,7 +13,8 @@ macro_rules! serde_roundtrip { let bytes = { let mut b = [0u8; $len]; for (i, v) in b.iter_mut().enumerate() { - *v = (i as u8).wrapping_mul(0x37).wrapping_add(0x11); + let i = u8::try_from(i).expect("serde fixture index must fit in one byte"); + *v = i.wrapping_mul(0x37).wrapping_add(0x11); } b }; @@ -154,10 +155,10 @@ fn wrong_length_bytes_rejected() { use rscrypto::aead::Nonce96; // Nonce96 is 12 bytes; feeding 11 or 13 must fail. - let short = serde_json::to_string(&[0u8; 11]).unwrap(); - let long = serde_json::to_string(&[0u8; 13]).unwrap(); - assert!(serde_json::from_str::(&short).is_err()); - assert!(serde_json::from_str::(&long).is_err()); + let short = serde_json::to_string(&[0u8; 11]).expect("short nonce fixture must serialize"); + let long = serde_json::to_string(&[0u8; 13]).expect("long nonce fixture must serialize"); + serde_json::from_str::(&short).expect_err("11-byte nonce must be rejected"); + serde_json::from_str::(&long).expect_err("13-byte nonce must be rejected"); } #[cfg(all(feature = "serde-secrets", feature = "aes-gcm"))] @@ -165,11 +166,12 @@ fn wrong_length_bytes_rejected() { fn secret_sequence_rejects_partial_and_wrong_length_inputs() { use rscrypto::Aes256GcmKey; - assert!(serde_json::from_str::("[17,34]").is_err()); - assert!(serde_json::from_str::("[17,\"not-a-byte\"]").is_err()); + serde_json::from_str::("[17,34]").expect_err("partial AES-256-GCM key must be rejected"); + serde_json::from_str::("[17,\"not-a-byte\"]") + .expect_err("non-byte AES-256-GCM key element must be rejected"); let exact = vec![0x42u8; Aes256GcmKey::LENGTH]; - let encoded = serde_json::to_string(&exact).unwrap(); - let key = serde_json::from_str::(&encoded).unwrap(); + let encoded = serde_json::to_string(&exact).expect("exact-length key fixture must serialize"); + let key = serde_json::from_str::(&encoded).expect("exact-length AES-256-GCM key must deserialize"); assert_eq!(key.as_bytes(), exact.as_slice()); } diff --git a/tests/sha256_differential.rs b/tests/sha256_differential.rs index 5ab6860c..27bc3e3e 100644 --- a/tests/sha256_differential.rs +++ b/tests/sha256_differential.rs @@ -35,8 +35,8 @@ proptest! { let mut h = Sha256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -56,8 +56,8 @@ proptest! { let mut h = Sha224::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/sha256_official_vectors.rs b/tests/sha256_official_vectors.rs index a72bcb6f..4cce9e21 100644 --- a/tests/sha256_official_vectors.rs +++ b/tests/sha256_official_vectors.rs @@ -3,16 +3,16 @@ mod support; use rscrypto::hashes::crypto::Sha256; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; #[test] fn sha256_official_vectors() { let data = include_bytes!("../testdata/sha2/sha256.blb"); - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha256 vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("sha256 vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-256 vector row must decode"); let actual = Sha256::digest(input); assert_eq!( &actual[..], diff --git a/tests/sha2_official_vectors.rs b/tests/sha2_official_vectors.rs index 8310d429..03ff0970 100644 --- a/tests/sha2_official_vectors.rs +++ b/tests/sha2_official_vectors.rs @@ -3,14 +3,14 @@ mod support; use rscrypto::hashes::crypto::{Sha224, Sha384, Sha512, Sha512_256}; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; fn run_fixed_vectors(data: &'static [u8], name: &str, mut digest: impl FnMut(&[u8]) -> [u8; OUT]) { - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha2 vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-2 vector row must decode"); let actual = digest(input); assert_eq!( &actual[..], diff --git a/tests/sha3_differential.rs b/tests/sha3_differential.rs index 28873415..dd6175f2 100644 --- a/tests/sha3_differential.rs +++ b/tests/sha3_differential.rs @@ -66,8 +66,8 @@ proptest! { let mut h = Sha3_224::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -82,8 +82,8 @@ proptest! { let mut h = Sha3_256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -98,8 +98,8 @@ proptest! { let mut h = Sha3_384::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -114,8 +114,8 @@ proptest! { let mut h = Sha3_512::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/sha3_official_vectors.rs b/tests/sha3_official_vectors.rs index 298d067b..e6b2263c 100644 --- a/tests/sha3_official_vectors.rs +++ b/tests/sha3_official_vectors.rs @@ -7,14 +7,14 @@ use rscrypto::{ hashes::crypto::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}, traits::Xof as _, }; -use support::blobby_compat::Blob2Iterator; +use support::blobby_compat::BlobIterator; fn run_fixed_vectors(data: &'static [u8], name: &str, mut digest: impl FnMut(&[u8]) -> [u8; OUT]) { - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha3 vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-3 vector row must decode"); let actual = digest(input); assert_eq!( &actual[..], @@ -50,11 +50,11 @@ fn sha3_384_official_vectors() { } fn run_xof_vectors(data: &'static [u8], name: &str, mut xof: impl FnMut(&[u8], &mut [u8])) { - for (i, row) in Blob2Iterator::new(data) + for (i, row) in BlobIterator::<2>::new(data) .expect("sha3 xof vector corpus must parse") .enumerate() { - let [input, output] = row.unwrap_or_else(|err| panic!("{name} vector row decode failed at case {i}: {err:?}")); + let [input, output] = row.expect("SHA-3 XOF vector row must decode"); let mut out = vec![0u8; output.len()]; xof(input, &mut out); assert_eq!( diff --git a/tests/sha512_differential.rs b/tests/sha512_differential.rs index 2c4e1857..04f425cc 100644 --- a/tests/sha512_differential.rs +++ b/tests/sha512_differential.rs @@ -43,8 +43,8 @@ proptest! { let mut h = Sha512::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -64,8 +64,8 @@ proptest! { let mut h = Sha384::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -85,8 +85,8 @@ proptest! { let mut h = Sha512_256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = (usize::from(data[i]) % 97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } diff --git a/tests/shake128_differential.rs b/tests/shake128_differential.rs index f30075b7..153dc2cf 100644 --- a/tests/shake128_differential.rs +++ b/tests/shake128_differential.rs @@ -36,8 +36,8 @@ proptest! { let mut h = Shake128::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = usize::from(data[i]).strict_rem(97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -55,7 +55,7 @@ proptest! { out_len in 0usize..2048, split in any::(), ) { - let split = split % (out_len + 1); + let split = split.strict_rem(out_len.strict_add(1)); let mut expected = vec![0u8; out_len]; { diff --git a/tests/shake256_differential.rs b/tests/shake256_differential.rs index 64a558da..22bfd94e 100644 --- a/tests/shake256_differential.rs +++ b/tests/shake256_differential.rs @@ -36,8 +36,8 @@ proptest! { let mut h = Shake256::new(); let mut i = 0usize; while i < data.len() { - let step = (data[i] as usize % 97) + 1; - let end = core::cmp::min(data.len(), i + step); + let step = usize::from(data[i]).strict_rem(97).strict_add(1); + let end = core::cmp::min(data.len(), i.strict_add(step)); h.update(&data[i..end]); i = end; } @@ -55,7 +55,7 @@ proptest! { out_len in 0usize..2048, split in any::(), ) { - let split = split % (out_len + 1); + let split = split.strict_rem(out_len.strict_add(1)); let mut expected = vec![0u8; out_len]; { diff --git a/tests/support/blobby_compat.rs b/tests/support/blobby_compat.rs index 3424fa0e..fed8cfcf 100644 --- a/tests/support/blobby_compat.rs +++ b/tests/support/blobby_compat.rs @@ -1,7 +1,5 @@ -#![allow(dead_code)] - #[derive(Debug, Eq, PartialEq, Copy, Clone)] -pub enum Error { +pub(crate) enum Error { InvalidVlq, InvalidIndex, UnexpectedEnd, @@ -13,9 +11,9 @@ const VAL_MASK: u8 = 0b0111_1111; fn read_vlq(data: &[u8], pos: &mut usize) -> Result { let b = *data.get(*pos).ok_or(Error::UnexpectedEnd)?; - *pos += 1; + *pos = (*pos).strict_add(1); let mut next = b & NEXT_MASK; - let mut val = (b & VAL_MASK) as usize; + let mut val = usize::from(b & VAL_MASK); macro_rules! step { () => { @@ -23,10 +21,10 @@ fn read_vlq(data: &[u8], pos: &mut usize) -> Result { return Ok(val); } let b = *data.get(*pos).ok_or(Error::UnexpectedEnd)?; - *pos += 1; + *pos = (*pos).strict_add(1); next = b & NEXT_MASK; - let t = (b & VAL_MASK) as usize; - val = ((val + 1) << 7) + t; + let t = usize::from(b & VAL_MASK); + val = val.strict_add(1).strict_mul(128).strict_add(t); }; } @@ -41,14 +39,18 @@ fn read_vlq(data: &[u8], pos: &mut usize) -> Result { Ok(val) } -pub struct BlobIterator<'a> { +pub(crate) struct BlobIterator<'a, const N: usize> { data: &'a [u8], dedup: Box<[&'a [u8]]>, pos: usize, } -impl<'a> BlobIterator<'a> { - pub fn new(data: &'a [u8]) -> Result { +impl<'a, const N: usize> BlobIterator<'a, N> { + pub(crate) fn new(data: &'a [u8]) -> Result { + if N == 0 { + return Err(Error::NotEnoughElements); + } + let mut pos = 0; let dedup_n = read_vlq(data, &mut pos)?; @@ -61,7 +63,7 @@ impl<'a> BlobIterator<'a> { } Ok(Self { - data: &data[pos..], + data: data.get(pos..).ok_or(Error::UnexpectedEnd)?, dedup: dedup.into_boxed_slice(), pos: 0, }) @@ -87,58 +89,28 @@ impl<'a> BlobIterator<'a> { } } -impl<'a> Iterator for BlobIterator<'a> { - type Item = Result<&'a [u8], Error>; +impl<'a, const N: usize> Iterator for BlobIterator<'a, N> { + type Item = Result<[&'a [u8]; N], Error>; fn next(&mut self) -> Option { if self.pos >= self.data.len() { return None; } - let value = self.read(); - if value.is_err() { - self.error_block(); - } - Some(value) - } -} - -macro_rules! blob_iter { - ($name:ident, $n:expr) => { - pub struct $name<'a> { - inner: BlobIterator<'a>, - } - - impl<'a> $name<'a> { - pub fn new(data: &'a [u8]) -> Result { - BlobIterator::new(data).map(|inner| Self { inner }) + let mut out = [&[][..]; N]; + for slot in &mut out { + if self.pos >= self.data.len() { + self.error_block(); + return Some(Err(Error::NotEnoughElements)); } - } - - impl<'a> Iterator for $name<'a> { - type Item = Result<[&'a [u8]; $n], Error>; - - fn next(&mut self) -> Option { - let mut out = [&[][..]; $n]; - - for (i, slot) in out.iter_mut().enumerate() { - *slot = match self.inner.next() { - Some(Ok(value)) => value, - Some(Err(err)) => return Some(Err(err)), - None if i == 0 => return None, - None => { - self.inner.error_block(); - return Some(Err(Error::NotEnoughElements)); - } - }; + *slot = match self.read() { + Ok(value) => value, + Err(err) => { + self.error_block(); + return Some(Err(err)); } - - Some(Ok(out)) - } + }; } - }; + Some(Ok(out)) + } } - -blob_iter!(Blob2Iterator, 2); -blob_iter!(Blob3Iterator, 3); -blob_iter!(Blob6Iterator, 6); diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 65835047..db0af257 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -3,4 +3,4 @@ //! This stays intentionally small. Right now it only exposes vector-corpus //! parsing helpers used by the official test suites. -pub mod blobby_compat; +pub(crate) mod blobby_compat; diff --git a/tests/x25519_vectors.rs b/tests/x25519_vectors.rs index 29dfa028..652a91f8 100644 --- a/tests/x25519_vectors.rs +++ b/tests/x25519_vectors.rs @@ -4,9 +4,11 @@ use rscrypto::{X25519PublicKey, X25519SecretKey, X25519SharedSecret}; use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret as DalekStaticSecret}; mod common; +#[path = "common/array.rs"] +mod hex_array; fn decode_hex_32(hex: &str) -> [u8; 32] { - common::decode_hex_array::<32>(hex) + hex_array::decode_hex_array::<32>(hex) } fn iterative_x25519(iterations: usize) -> [u8; 32] { @@ -14,7 +16,11 @@ fn iterative_x25519(iterations: usize) -> [u8; 32] { let mut u = X25519PublicKey::basepoint(); for _ in 0..iterations { - let next = k.diffie_hellman(&u).unwrap().expose_secret().expose(); + let next = k + .diffie_hellman(&u) + .expect("RFC 7748 iterative ladder must produce a nonzero secret") + .expose_secret() + .expose(); u = X25519PublicKey::from_bytes(*k.as_bytes()); k = X25519SecretKey::from_bytes(next); } @@ -32,7 +38,10 @@ fn rfc_7748_scalar_multiplication_vectors_match() { )); let expected = decode_hex_32("c3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552"); - assert_eq!(*scalar.diffie_hellman(&public).unwrap().as_bytes(), expected); + let shared = scalar + .diffie_hellman(&public) + .expect("first RFC 7748 scalar-multiplication vector must produce a nonzero secret"); + assert_eq!(*shared.as_bytes(), expected); let scalar = X25519SecretKey::from_bytes(decode_hex_32( "4b66e9d4d1b4673c5ad22691957d6af5c11b6421e0ea01d42ca4169e7918ba0d", @@ -42,7 +51,10 @@ fn rfc_7748_scalar_multiplication_vectors_match() { )); let expected = decode_hex_32("95cbde9476e8907d7aade45cb4b873f88b595a68799fa152e6f8f7647aac7957"); - assert_eq!(*scalar.diffie_hellman(&public).unwrap().as_bytes(), expected); + let shared = scalar + .diffie_hellman(&public) + .expect("second RFC 7748 scalar-multiplication vector must produce a nonzero secret"); + assert_eq!(*shared.as_bytes(), expected); } #[test] @@ -73,8 +85,12 @@ fn rfc_7748_diffie_hellman_vector_matches() { )); let expected = decode_hex_32("4a5d9d5ba4ce2de1728e3bf480350f25e07e21c947d19e3376f09b3c1e161742"); - let alice_shared = alice.diffie_hellman(&bob_public).unwrap(); - let bob_shared = bob.diffie_hellman(&alice_public).unwrap(); + let alice_shared = alice + .diffie_hellman(&bob_public) + .expect("Alice RFC 7748 exchange must produce a nonzero secret"); + let bob_shared = bob + .diffie_hellman(&alice_public) + .expect("Bob RFC 7748 exchange must produce a nonzero secret"); assert_eq!(alice.public_key(), alice_public); assert_eq!(bob.public_key(), bob_public); @@ -87,8 +103,11 @@ fn low_order_points_return_all_zero_error() { let secret = X25519SecretKey::from_bytes([0x42; X25519SecretKey::LENGTH]); let low_order = X25519PublicKey::from_bytes([0u8; X25519PublicKey::LENGTH]); - assert!(secret.diffie_hellman(&low_order).is_err()); - assert!(X25519SharedSecret::diffie_hellman(&secret, &low_order).is_err()); + secret + .diffie_hellman(&low_order) + .expect_err("X25519 must reject a low-order point"); + X25519SharedSecret::diffie_hellman(&secret, &low_order) + .expect_err("X25519 shared-secret API must reject a low-order point"); } #[test] @@ -98,10 +117,9 @@ fn non_canonical_public_inputs_are_accepted_and_reduced() { "edffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f", )); - assert!( - secret.diffie_hellman(&public).is_err(), - "u = p should reduce to zero and fail the all-zero check" - ); + secret + .diffie_hellman(&public) + .expect_err("u = p must reduce to zero and fail the all-zero check"); assert_eq!( public.to_bytes(), decode_hex_32("edffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f") @@ -113,7 +131,8 @@ fn public_keys_and_shared_secrets_match_x25519_dalek() { for seed in 0u8..32 { let mut secret_bytes = [0u8; 32]; for (index, byte) in secret_bytes.iter_mut().enumerate() { - *byte = seed.wrapping_mul(17).wrapping_add((index as u8).wrapping_mul(29)); + let index = u8::try_from(index).expect("32-byte key index must fit in one byte"); + *byte = seed.wrapping_mul(17).wrapping_add(index.wrapping_mul(29)); } let ours_secret = X25519SecretKey::from_bytes(secret_bytes); @@ -125,9 +144,10 @@ fn public_keys_and_shared_secrets_match_x25519_dalek() { let mut peer_bytes = [0u8; 32]; for (index, byte) in peer_bytes.iter_mut().enumerate() { + let index = u8::try_from(index).expect("32-byte peer-key index must fit in one byte"); *byte = seed .wrapping_mul(23) - .wrapping_add((index as u8).wrapping_mul(11)) + .wrapping_add(index.wrapping_mul(11)) .wrapping_add(5); } peer_bytes[31] |= 0x80; @@ -138,9 +158,10 @@ fn public_keys_and_shared_secrets_match_x25519_dalek() { let dalek_shared = dalek_secret.diffie_hellman(&dalek_peer).to_bytes(); if dalek_shared.iter().all(|&byte| byte == 0) { - assert!(ours_shared.is_err()); + ours_shared.expect_err("rscrypto must reject every all-zero X25519 shared secret"); } else { - assert_eq!(*ours_shared.unwrap().as_bytes(), dalek_shared); + let ours_shared = ours_shared.expect("nonzero x25519-dalek shared secret must be accepted"); + assert_eq!(*ours_shared.as_bytes(), dalek_shared); } } } @@ -150,12 +171,15 @@ fn public_key_matches_basepoint_diffie_hellman() { for seed in 0u8..32 { let mut secret_bytes = [0u8; 32]; for (index, byte) in secret_bytes.iter_mut().enumerate() { - *byte = seed.wrapping_mul(41).wrapping_add((index as u8).wrapping_mul(7)); + let index = u8::try_from(index).expect("32-byte key index must fit in one byte"); + *byte = seed.wrapping_mul(41).wrapping_add(index.wrapping_mul(7)); } let secret = X25519SecretKey::from_bytes(secret_bytes); let public = secret.public_key(); - let via_ladder = secret.diffie_hellman(&X25519PublicKey::basepoint()).unwrap(); + let via_ladder = secret + .diffie_hellman(&X25519PublicKey::basepoint()) + .expect("basepoint exchange must produce a nonzero X25519 secret"); assert_eq!(public.to_bytes(), *via_ladder.as_bytes()); } @@ -165,7 +189,9 @@ fn public_key_matches_basepoint_diffie_hellman() { fn secret_and_shared_debug_are_masked() { let secret = X25519SecretKey::from_bytes([0x42; 32]); let public = secret.public_key(); - let shared = secret.diffie_hellman(&public).unwrap(); + let shared = secret + .diffie_hellman(&public) + .expect("self-derived public key must produce a nonzero X25519 secret"); assert_eq!(format!("{secret:?}"), "X25519SecretKey(****)"); assert_eq!(format!("{shared:?}"), "X25519SharedSecret(****)"); diff --git a/tests/x25519_wycheproof.rs b/tests/x25519_wycheproof.rs index 38dde501..43012b98 100644 --- a/tests/x25519_wycheproof.rs +++ b/tests/x25519_wycheproof.rs @@ -4,7 +4,9 @@ use rscrypto::{X25519PublicKey, X25519SecretKey}; use serde_json::Value; mod common; -use common::decode_hex_array; +#[path = "common/array.rs"] +mod hex_array; +use hex_array::decode_hex_array; const X25519: &str = include_str!("../testdata/auth/wycheproof/x25519_test.json"); @@ -16,9 +18,11 @@ struct Counts { } fn field<'a>(value: &'a Value, name: &str) -> &'a str { - value[name] + value + .get(name) + .expect("Wycheproof field must be present") .as_str() - .unwrap_or_else(|| panic!("missing string field `{name}`")) + .expect("Wycheproof field must be a string") } fn groups(suite: &Value) -> &[Value] { @@ -52,26 +56,22 @@ fn wycheproof_x25519_vectors_match_or_reject_all_zero_shared_secret() { let expected_shared: [u8; 32] = decode_hex_array(field(test, "shared")); let result = secret.diffie_hellman(&public); - match field(test, "result") { - "valid" => counts.valid = counts.valid.strict_add(1), - "acceptable" => counts.acceptable = counts.acceptable.strict_add(1), - other => panic!("unsupported Wycheproof X25519 result `{other}`"), + let disposition = field(test, "result"); + assert!( + matches!(disposition, "valid" | "acceptable"), + "unsupported Wycheproof X25519 result `{disposition}`" + ); + if disposition == "valid" { + counts.valid = counts.valid.strict_add(1); + } else { + counts.acceptable = counts.acceptable.strict_add(1); } if expected_shared == [0u8; 32] { counts.zero_shared_rejected = counts.zero_shared_rejected.strict_add(1); - assert!( - result.is_err(), - "Wycheproof X25519 tcId {} must reject all-zero shared secret", - test["tcId"] - ); + result.expect_err("Wycheproof X25519 all-zero shared secret must be rejected"); } else { - let shared = result.unwrap_or_else(|error| { - panic!( - "Wycheproof X25519 tcId {} unexpectedly rejected non-zero shared secret: {error}", - test["tcId"] - ) - }); + let shared = result.expect("Wycheproof X25519 nonzero shared secret must be accepted"); assert_eq!( *shared.as_bytes(), expected_shared, diff --git a/tests/xchacha20poly1305.rs b/tests/xchacha20poly1305.rs index 816d16f7..355ce3e5 100644 --- a/tests/xchacha20poly1305.rs +++ b/tests/xchacha20poly1305.rs @@ -15,9 +15,10 @@ use common::decode_hex_vec as decode_hex; fn pattern_bytes(len: usize, seed: u8) -> Vec { let mut out = vec![0u8; len]; for (index, byte) in out.iter_mut().enumerate() { + let index = index.to_le_bytes()[0]; *byte = seed - .wrapping_add((index as u8).wrapping_mul(19)) - .wrapping_add((index as u8).rotate_left(2)); + .wrapping_add(index.wrapping_mul(19)) + .wrapping_add(index.rotate_left(2)); } out } @@ -35,18 +36,26 @@ fn xchacha20poly1305_matches_draft_vector() { ); let expected_tag = decode_hex("c0875924c1c7987947deafd8780acf49"); - let key = XChaCha20Poly1305Key::from_bytes(key.try_into().unwrap()); - let nonce = Nonce192::from_bytes(nonce.try_into().unwrap()); + let key = XChaCha20Poly1305Key::from_bytes(key.try_into().expect("draft XChaCha20-Poly1305 key must be 32 bytes")); + let nonce = Nonce192::from_bytes( + nonce + .try_into() + .expect("draft XChaCha20-Poly1305 nonce must be 24 bytes"), + ); let cipher = XChaCha20Poly1305::new(&key); let mut sealed = vec![0u8; plaintext.len() + XChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut sealed).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut sealed) + .expect("draft XChaCha20-Poly1305 seal buffer must fit plaintext and tag"); assert_eq!(&sealed[..plaintext.len()], expected_ciphertext.as_slice()); assert_eq!(&sealed[plaintext.len()..], expected_tag.as_slice()); let mut opened = vec![0u8; plaintext.len()]; - cipher.decrypt(&nonce, &aad, &sealed, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &sealed, &mut opened) + .expect("draft XChaCha20-Poly1305 ciphertext and tag must authenticate"); assert_eq!(opened, plaintext); } @@ -65,18 +74,22 @@ fn xchacha20poly1305_matches_rustcrypto_oracle() { let oracle_nonce = Array(nonce_bytes); let mut ours = plaintext.to_vec(); - let tag = cipher.encrypt_in_place(&nonce, aad, &mut ours).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, aad, &mut ours) + .expect("rscrypto XChaCha20-Poly1305 oracle input must seal"); let mut oracle_buffer = plaintext.to_vec(); let oracle_tag = oracle .encrypt_inout_detached(&oracle_nonce, aad, oracle_buffer.as_mut_slice().into()) - .unwrap(); + .expect("RustCrypto XChaCha20-Poly1305 oracle input must seal"); assert_eq!(ours, oracle_buffer); assert_eq!(tag.as_bytes(), oracle_tag.as_slice()); let typed_tag = XChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); - cipher.decrypt_in_place(&nonce, aad, &mut ours, &typed_tag).unwrap(); + cipher + .decrypt_in_place(&nonce, aad, &mut ours, &typed_tag) + .expect("fresh rscrypto XChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!(ours, plaintext); } @@ -87,22 +100,23 @@ fn xchacha20poly1305_rejects_modified_tag() { let cipher = XChaCha20Poly1305::new(&key); let mut buffer = *b"forgery-check"; - let mut tag = cipher.encrypt_in_place(&nonce, b"aad", &mut buffer).unwrap().to_bytes(); + let mut tag = cipher + .encrypt_in_place(&nonce, b"aad", &mut buffer) + .expect("XChaCha20-Poly1305 forgery fixture must seal") + .to_bytes(); tag[0] ^= 1; - assert!( - cipher - .decrypt_in_place(&nonce, b"aad", &mut buffer, &XChaCha20Poly1305Tag::from_bytes(tag)) - .is_err() - ); + cipher + .decrypt_in_place(&nonce, b"aad", &mut buffer, &XChaCha20Poly1305Tag::from_bytes(tag)) + .expect_err("modified XChaCha20-Poly1305 tag must fail authentication"); } #[test] fn xchacha20poly1305_rejects_wrong_tag_length() { - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 0]).is_err()); - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 15]).is_err()); - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 17]).is_err()); - assert!(XChaCha20Poly1305::tag_from_slice(&[0u8; 16]).is_ok()); + XChaCha20Poly1305::tag_from_slice(&[0u8; 0]).expect_err("empty XChaCha20-Poly1305 tag must be rejected"); + XChaCha20Poly1305::tag_from_slice(&[0u8; 15]).expect_err("short XChaCha20-Poly1305 tag must be rejected"); + XChaCha20Poly1305::tag_from_slice(&[0u8; 17]).expect_err("long XChaCha20-Poly1305 tag must be rejected"); + let _tag = XChaCha20Poly1305::tag_from_slice(&[0u8; 16]).expect("16-byte XChaCha20-Poly1305 tag must be accepted"); } #[test] @@ -128,7 +142,9 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { let aad = pattern_bytes(aad_len, 0xc4); let mut combined = vec![0u8; plaintext_len + XChaCha20Poly1305::TAG_SIZE]; - cipher.encrypt(&nonce, &aad, &plaintext, &mut combined).unwrap(); + cipher + .encrypt(&nonce, &aad, &plaintext, &mut combined) + .expect("rscrypto combined XChaCha20-Poly1305 oracle input must seal"); let oracle_combined = oracle .encrypt( @@ -138,14 +154,16 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("RustCrypto combined XChaCha20-Poly1305 oracle input must seal"); assert_eq!( combined, oracle_combined, "combined ciphertext mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut opened = vec![0u8; plaintext_len]; - cipher.decrypt(&nonce, &aad, &oracle_combined, &mut opened).unwrap(); + cipher + .decrypt(&nonce, &aad, &oracle_combined, &mut opened) + .expect("RustCrypto XChaCha20-Poly1305 ciphertext must open in rscrypto"); assert_eq!( opened, plaintext, "combined decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" @@ -159,14 +177,16 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { aad: &aad, }, ) - .unwrap(); + .expect("rscrypto XChaCha20-Poly1305 ciphertext must open in RustCrypto"); assert_eq!( oracle_opened, plaintext, "oracle decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" ); let mut detached = plaintext.clone(); - let tag = cipher.encrypt_in_place(&nonce, &aad, &mut detached).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, &aad, &mut detached) + .expect("rscrypto detached XChaCha20-Poly1305 oracle input must seal"); assert_eq!( detached, oracle_combined[..plaintext_len], @@ -181,7 +201,7 @@ fn xchacha20poly1305_boundary_and_large_inputs_match_oracle() { let typed_tag = XChaCha20Poly1305Tag::from_bytes(tag.to_bytes()); cipher .decrypt_in_place(&nonce, &aad, &mut detached, &typed_tag) - .unwrap(); + .expect("fresh detached XChaCha20-Poly1305 ciphertext must authenticate"); assert_eq!( detached, plaintext, "detached decrypt mismatch pt_len={plaintext_len} aad_len={aad_len}" diff --git a/tools/ct-binsec-harness/src/main.rs b/tools/ct-binsec-harness/src/main.rs index d000e60f..c6ac58c1 100644 --- a/tools/ct-binsec-harness/src/main.rs +++ b/tools/ct-binsec-harness/src/main.rs @@ -2,10 +2,9 @@ use core::ptr; use rscrypto::aead::expert::AeadWithNonce; use rscrypto::{ - Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, - Aes256GcmSiv, Aes256GcmSivKey, AsconAead128, AsconAead128Key, Blake3KeyedHash, ChaCha20Poly1305, - ChaCha20Poly1305Key, HmacSha384Tag, HmacSha512Tag, Kmac256, X25519SecretKey, XChaCha20Poly1305, - XChaCha20Poly1305Key, + Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, Aes256GcmSiv, + Aes256GcmSivKey, AsconAead128, AsconAead128Key, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, + HmacSha384Tag, HmacSha512Tag, Kmac256, X25519SecretKey, XChaCha20Poly1305, XChaCha20Poly1305Key, aead::{Nonce96, Nonce128, Nonce192, Nonce256}, }; @@ -20,20 +19,31 @@ const RSA_WINDOW_TABLE_LIMBS: usize = 16 * RSA_WINDOW_LIMBS; #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible volatile byte fill for the analysis harness. +/// +/// # Safety +/// +/// `dst` must be valid and writable for `len` bytes. pub unsafe extern "C" fn memset(dst: *mut u8, value: i32, len: usize) -> *mut u8 { let mut offset = 0usize; while offset < len { // SAFETY: C callers require `dst..dst+len` to be writable. Volatile writes // keep zeroization visible to BINSEC and prevent this shim from becoming a // recursive compiler intrinsic. - unsafe { ptr::write_volatile(dst.add(offset), value as u8) }; - offset += 1; + unsafe { ptr::write_volatile(dst.add(offset), value.to_le_bytes()[0]) }; + offset = offset.strict_add(1); } dst } #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible overlapping byte copy for the analysis harness. +/// +/// # Safety +/// +/// `src` must be readable and `dst` writable for `len` bytes. Both pointers +/// must satisfy Rust's pointer provenance requirements for those ranges. pub unsafe extern "C" fn memmove(dst: *mut u8, src: *const u8, len: usize) -> *mut u8 { if (dst as usize) <= (src as usize) { let mut offset = 0usize; @@ -42,12 +52,12 @@ pub unsafe extern "C" fn memmove(dst: *mut u8, src: *const u8, len: usize) -> *m let byte = unsafe { ptr::read(src.add(offset)) }; // SAFETY: C callers require source and destination ranges to be valid. unsafe { ptr::write(dst.add(offset), byte) }; - offset += 1; + offset = offset.strict_add(1); } } else { let mut remaining = len; while remaining != 0 { - remaining -= 1; + remaining = remaining.strict_sub(1); // SAFETY: C callers require source and destination ranges to be valid. let byte = unsafe { ptr::read(src.add(remaining)) }; // SAFETY: C callers require source and destination ranges to be valid. @@ -59,6 +69,12 @@ pub unsafe extern "C" fn memmove(dst: *mut u8, src: *const u8, len: usize) -> *m #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible non-overlapping byte copy for the analysis harness. +/// +/// # Safety +/// +/// `src` must be readable and `dst` writable for `len` bytes, and the two +/// ranges must not overlap. pub unsafe extern "C" fn memcpy(dst: *mut u8, src: *const u8, len: usize) -> *mut u8 { // SAFETY: `memcpy` has the same validity contract as `memmove`, with the // additional non-overlap precondition. The `memmove` implementation is valid @@ -68,6 +84,11 @@ pub unsafe extern "C" fn memcpy(dst: *mut u8, src: *const u8, len: usize) -> *mu #[unsafe(no_mangle)] #[inline(never)] +/// C-compatible byte comparison for the analysis harness. +/// +/// # Safety +/// +/// `lhs` and `rhs` must each be readable for `len` bytes. pub unsafe extern "C" fn bcmp(lhs: *const u8, rhs: *const u8, len: usize) -> i32 { let mut acc = 0u8; let mut offset = 0usize; @@ -77,7 +98,7 @@ pub unsafe extern "C" fn bcmp(lhs: *const u8, rhs: *const u8, len: usize) -> i32 // SAFETY: C callers require both ranges to be readable for `len` bytes. let r = unsafe { ptr::read(rhs.add(offset)) }; acc |= l ^ r; - offset += 1; + offset = offset.strict_add(1); } i32::from(acc) } @@ -243,6 +264,12 @@ pub static mut CT_BINSEC_RSA_OTHER_32: [u8; 32] = [0u8; 32]; pub static mut CT_BINSEC_RESULT: u8 = 0; #[inline(always)] +/// Read a fixed-size byte array from a harness global. +/// +/// # Safety +/// +/// `ptr` must point to a valid, initialized array that remains readable for +/// the duration of the volatile read. unsafe fn array_from_global(ptr: *const [u8; N]) -> [u8; N] { // SAFETY: BINSEC harness globals are fixed-size byte arrays with static // storage duration. `read_volatile` keeps the symbolic bytes observable. @@ -250,6 +277,12 @@ unsafe fn array_from_global(ptr: *const [u8; N]) -> [u8; N] { } #[inline(always)] +/// Read a fixed-size limb array from a harness global. +/// +/// # Safety +/// +/// `ptr` must point to a valid, initialized array that remains readable for +/// the duration of the volatile read. unsafe fn limbs_from_global(ptr: *const [u64; N]) -> [u64; N] { // SAFETY: BINSEC harness globals are fixed-size limb arrays with static // storage duration. `read_volatile` keeps the symbolic limbs observable. @@ -555,8 +588,22 @@ macro_rules! aead_open_entry { }; } -aead_seal_entry!(ct_binsec_aes128gcm_seal, Aes128Gcm, Aes128GcmKey, Nonce96, CT_BINSEC_KEY_16, 0xA1); -aead_seal_entry!(ct_binsec_aes256gcm_seal, Aes256Gcm, Aes256GcmKey, Nonce96, CT_BINSEC_KEY_32, 0xA2); +aead_seal_entry!( + ct_binsec_aes128gcm_seal, + Aes128Gcm, + Aes128GcmKey, + Nonce96, + CT_BINSEC_KEY_16, + 0xA1 +); +aead_seal_entry!( + ct_binsec_aes256gcm_seal, + Aes256Gcm, + Aes256GcmKey, + Nonce96, + CT_BINSEC_KEY_32, + 0xA2 +); aead_seal_entry!( ct_binsec_aes128gcmsiv_seal, Aes128GcmSiv, @@ -589,7 +636,14 @@ aead_seal_entry!( CT_BINSEC_KEY_32, 0xA6 ); -aead_seal_entry!(ct_binsec_aegis256_seal, Aegis256, Aegis256Key, Nonce256, CT_BINSEC_KEY_32, 0xA7); +aead_seal_entry!( + ct_binsec_aegis256_seal, + Aegis256, + Aegis256Key, + Nonce256, + CT_BINSEC_KEY_32, + 0xA7 +); aead_seal_entry!( ct_binsec_ascon_aead128_seal, AsconAead128, @@ -599,8 +653,22 @@ aead_seal_entry!( 0xA8 ); -aead_open_entry!(ct_binsec_aes128gcm_open, Aes128Gcm, Aes128GcmKey, Nonce96, CT_BINSEC_KEY_16, 0xB1); -aead_open_entry!(ct_binsec_aes256gcm_open, Aes256Gcm, Aes256GcmKey, Nonce96, CT_BINSEC_KEY_32, 0xB2); +aead_open_entry!( + ct_binsec_aes128gcm_open, + Aes128Gcm, + Aes128GcmKey, + Nonce96, + CT_BINSEC_KEY_16, + 0xB1 +); +aead_open_entry!( + ct_binsec_aes256gcm_open, + Aes256Gcm, + Aes256GcmKey, + Nonce96, + CT_BINSEC_KEY_32, + 0xB2 +); aead_open_entry!( ct_binsec_aes128gcmsiv_open, Aes128GcmSiv, @@ -633,7 +701,14 @@ aead_open_entry!( CT_BINSEC_KEY_32, 0xB6 ); -aead_open_entry!(ct_binsec_aegis256_open, Aegis256, Aegis256Key, Nonce256, CT_BINSEC_KEY_32, 0xB7); +aead_open_entry!( + ct_binsec_aegis256_open, + Aegis256, + Aegis256Key, + Nonce256, + CT_BINSEC_KEY_32, + 0xB7 +); aead_open_entry!( ct_binsec_ascon_aead128_open, AsconAead128, @@ -658,7 +733,8 @@ pub extern "C" fn ct_binsec_argon2i_hash() -> ! { for word in dst { acc ^= word; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -726,7 +802,8 @@ pub extern "C" fn ct_binsec_curve25519_conditional_swap() -> ! { for limb in lhs.into_iter().chain(rhs) { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -740,7 +817,8 @@ pub extern "C" fn ct_binsec_ed25519_select_basepoint_cached() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -754,7 +832,8 @@ pub extern "C" fn ct_binsec_ecdsa_p256_select_signing_generator_affine() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -768,7 +847,8 @@ pub extern "C" fn ct_binsec_ecdsa_p384_select_signing_generator_affine() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -784,7 +864,8 @@ pub extern "C" fn ct_binsec_rsa_private_select_window_power_4() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] @@ -804,6 +885,9 @@ pub extern "C" fn ct_binsec_rsa_private_component_validation_32() -> ! { #[unsafe(no_mangle)] #[inline(never)] #[target_feature(enable = "avx2")] +/// # Safety +/// +/// The caller must ensure AVX2 is available before invoking this entrypoint. pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_avx2() -> ! { // SAFETY: This pointer references a fixed harness global with static storage. let digit = unsafe { ptr::read_volatile(ptr::addr_of!(CT_BINSEC_ED25519_DIGIT)) }; @@ -815,13 +899,18 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_avx2() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[cfg(target_arch = "x86_64")] #[unsafe(no_mangle)] #[inline(never)] #[target_feature(enable = "avx2,avx512ifma,avx512vl")] +/// # Safety +/// +/// The caller must ensure AVX2, AVX-512 IFMA, and AVX-512 VL are available +/// before invoking this entrypoint. pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_ifma() -> ! { // SAFETY: This pointer references a fixed harness global with static storage. let digit = unsafe { ptr::read_volatile(ptr::addr_of!(CT_BINSEC_ED25519_DIGIT)) }; @@ -833,7 +922,8 @@ pub unsafe extern "C" fn ct_binsec_ed25519_select_basepoint_cached_ifma() -> ! { for limb in limbs { acc ^= limb; } - ct_binsec_done((acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56)) as u8) + let folded = acc | (acc >> 8) | (acc >> 16) | (acc >> 24) | (acc >> 32) | (acc >> 40) | (acc >> 48) | (acc >> 56); + ct_binsec_done(folded.to_le_bytes()[0]) } #[unsafe(no_mangle)] diff --git a/tools/ct-dudect/Cargo.lock b/tools/ct-dudect/Cargo.lock index 541d50d3..58b12237 100644 --- a/tools/ct-dudect/Cargo.lock +++ b/tools/ct-dudect/Cargo.lock @@ -396,18 +396,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", diff --git a/tools/ct-dudect/src/main.rs b/tools/ct-dudect/src/main.rs index f900b79d..479a2e95 100644 --- a/tools/ct-dudect/src/main.rs +++ b/tools/ct-dudect/src/main.rs @@ -1,21 +1,20 @@ use dudect_bencher::{BenchRng, Class, CtRunner, ctbench_main_with_seeds, rand::RngExt}; use rscrypto::aead::expert::AeadWithNonce; use rscrypto::{ - Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, - Aes256GcmSiv, Aes256GcmSivKey, Argon2Params, Argon2i, AsconAead128, AsconAead128Key, Blake2b256, Blake2b512, - Blake2bKey, Blake2s128, Blake2s256, Blake2sKey, Blake3, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, - EcdsaP256SecretKey, EcdsaP384SecretKey, Ed25519Keypair, Ed25519SecretKey, HkdfSha256, - HkdfSha384, HmacSha256, HmacSha256Tag, HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, Kmac256, MlKem512, - MlKem512Ciphertext, MlKem512DecapsulationKey, MlKem768, MlKem768Ciphertext, MlKem768DecapsulationKey, MlKem1024, - MlKem1024Ciphertext, MlKem1024DecapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaOaepProfile, - RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, SecretBytes, Sha512, X25519SecretKey, XChaCha20Poly1305, - XChaCha20Poly1305Key, + Aegis256, Aegis256Key, Aes128Gcm, Aes128GcmKey, Aes128GcmSiv, Aes128GcmSivKey, Aes256Gcm, Aes256GcmKey, Aes256GcmSiv, + Aes256GcmSivKey, Argon2Params, Argon2i, AsconAead128, AsconAead128Key, Blake2b256, Blake2b512, Blake2bKey, + Blake2s128, Blake2s256, Blake2sKey, Blake3, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, + EcdsaP256SecretKey, EcdsaP384SecretKey, Ed25519Keypair, Ed25519SecretKey, HkdfSha256, HkdfSha384, HmacSha256, + HmacSha256Tag, HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, Kmac256, MlKem512, MlKem512Ciphertext, + MlKem512DecapsulationKey, MlKem768, MlKem768Ciphertext, MlKem768DecapsulationKey, MlKem1024, MlKem1024Ciphertext, + MlKem1024DecapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaBlindingPair, RsaEncryptionError, + RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, SecretBytes, Sha512, + X25519SecretKey, XChaCha20Poly1305, XChaCha20Poly1305Key, aead::{ - Nonce96, Nonce128, Nonce192, Nonce256, diag_aes128gcm_ctr32_be, diag_aes128gcm_ghash, - diag_aes128gcm_tag_aes, diag_aes128gcmsiv_ctr32, diag_aes128gcmsiv_derive_keys, - diag_aes128gcmsiv_polyval_digest, diag_aes128gcmsiv_raw_tag_aes, diag_aes256gcm_ctr32_be, - diag_aes256gcm_ghash, diag_aes256gcm_tag_aes, diag_aes256gcmsiv_ctr32, diag_aes256gcmsiv_derive_keys, - diag_aes256gcmsiv_raw_tag_aes, + Nonce96, Nonce128, Nonce192, Nonce256, diag_aes128gcm_ctr32_be, diag_aes128gcm_ghash, diag_aes128gcm_tag_aes, + diag_aes128gcmsiv_ctr32, diag_aes128gcmsiv_derive_keys, diag_aes128gcmsiv_polyval_digest, + diag_aes128gcmsiv_raw_tag_aes, diag_aes256gcm_ctr32_be, diag_aes256gcm_ghash, diag_aes256gcm_tag_aes, + diag_aes256gcmsiv_ctr32, diag_aes256gcmsiv_derive_keys, diag_aes256gcmsiv_raw_tag_aes, }, auth::{ diag_ecdsa_p256_basepoint_blinded_limb_digest, diag_ecdsa_p256_final_multiply_limb_digest, @@ -34,7 +33,6 @@ use rscrypto::{ diag_rsa_validate_pkcs8_private_key_der, diag_rsa_validate_pkcs8_private_key_der_stage, }, traits::Kem as _, - RsaEncryptionError, RsaPublicKeyPolicy, }; const DEFAULT_SAMPLES: usize = 20_000; @@ -76,9 +74,9 @@ fn balanced_classes(rng: &mut BenchRng, count: usize) -> Vec { let mut classes = Vec::with_capacity(count); while classes.len() < count { let start = classes.len(); - let len = (count - start).min(BLOCK_LEN); - let extra_left = len % 2 == 1 && rng.random::(); - let left_len = len / 2 + usize::from(extra_left); + let len = count.strict_sub(start).min(BLOCK_LEN); + let extra_left = len.rem_euclid(2) == 1 && rng.random::(); + let left_len = len.div_euclid(2).strict_add(usize::from(extra_left)); classes.extend((0..len).map(|index| if index < left_len { Class::Left } else { Class::Right })); for index in (1..len).rev() { @@ -92,7 +90,8 @@ fn balanced_classes(rng: &mut BenchRng, count: usize) -> Vec { fn mlkem_poly_from_seed(seed: u16) -> [u16; 256] { let mut out = [0u16; 256]; for (i, coeff) in out.iter_mut().enumerate() { - *coeff = seed.wrapping_add((i as u16).wrapping_mul(73)) % MLKEM_Q; + let index = u16::try_from(i).expect("ML-KEM polynomial indices fit u16"); + *coeff = seed.wrapping_add(index.wrapping_mul(73)).rem_euclid(MLKEM_Q); } out } @@ -100,7 +99,7 @@ fn mlkem_poly_from_seed(seed: u16) -> [u16; 256] { fn random_mlkem_poly(rng: &mut BenchRng) -> [u16; 256] { let mut out = [0u16; 256]; for coeff in &mut out { - *coeff = rng.random::() % MLKEM_Q; + *coeff = rng.random::().rem_euclid(MLKEM_Q); } out } @@ -126,24 +125,30 @@ fn random_mlkem_polyvec4(rng: &mut BenchRng) -> [[u16; 256]; 4] { fn json_string_value_n<'a>(json: &'a str, key: &str, n: usize) -> &'a str { let needle = format!("\"{key}\": \""); let mut rest = json; - for index in 0..=n { - let start = rest.find(&needle).unwrap_or_else(|| panic!("missing JSON string key {key} at index {index}")); - let value = &rest[start + needle.len()..]; - let end = value.find('"').unwrap_or_else(|| panic!("unterminated JSON string key {key} at index {index}")); - if index == n { - return &value[..end]; - } - rest = &value[end + 1..]; - } - unreachable!() -} - -fn hex_nibble(byte: u8) -> u8 { + for _ in 0..n { + let (_, value_and_rest) = rest + .split_once(&needle) + .expect("RSA fixture must contain the requested JSON string key"); + let (_, tail) = value_and_rest + .split_once('"') + .expect("RSA fixture JSON string must be terminated"); + rest = tail; + } + let (_, value_and_rest) = rest + .split_once(&needle) + .expect("RSA fixture must contain the requested JSON string key"); + let (value, _) = value_and_rest + .split_once('"') + .expect("RSA fixture JSON string must be terminated"); + value +} + +fn hex_nibble(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex byte"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } @@ -152,7 +157,11 @@ fn hex_to_vec(hex: &str) -> Vec { assert!(bytes.len().is_multiple_of(2), "hex string must have even length"); bytes .chunks_exact(2) - .map(|pair| (hex_nibble(pair[0]) << 4) | hex_nibble(pair[1])) + .map(|pair| { + let high = hex_nibble(pair[0]).expect("RSA fixture must contain hexadecimal bytes"); + let low = hex_nibble(pair[1]).expect("RSA fixture must contain hexadecimal bytes"); + high.strict_shl(4) | low + }) .collect() } @@ -162,51 +171,59 @@ fn rsa_pkcs8_der(index: usize) -> Vec { fn rsa_ct_fixture_key(index: usize) -> RsaPrivateKey { let der = rsa_pkcs8_der(index); - RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap() + RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("bundled RSA fixture must contain a valid private key") } fn rsa_blinding_pair(key: &RsaPrivateKey) -> (Vec, Vec) { let modulus = key.public_key().modulus(); let mut factor = vec![0u8; modulus.len()]; - factor[modulus.len() - 1] = 2; + factor[modulus.len().strict_sub(1)] = 2; let mut plus_one = modulus.to_vec(); let mut carry = 1u16; for byte in plus_one.iter_mut().rev() { - let sum = u16::from(*byte) + carry; - *byte = sum as u8; + let sum = u16::from(*byte).strict_add(carry); + *byte = sum.to_le_bytes()[0]; carry = sum >> 8; if carry == 0 { break; } } if carry != 0 { - plus_one.insert(0, carry as u8); + plus_one.insert(0, carry.to_le_bytes()[0]); } let mut quotient = Vec::with_capacity(plus_one.len()); let mut remainder = 0u16; for byte in plus_one { let value = (remainder << 8) | u16::from(byte); - quotient.push((value / 2) as u8); - remainder = value % 2; + quotient.push(value.div_euclid(2).to_le_bytes()[0]); + remainder = value.rem_euclid(2); } - let first_nonzero = quotient.iter().position(|&byte| byte != 0).unwrap_or(quotient.len() - 1); - let inverse = "ient[first_nonzero..]; + let first_nonzero = quotient + .iter() + .position(|&byte| byte != 0) + .unwrap_or_else(|| quotient.len().strict_sub(1)); + let inverse = quotient + .get(first_nonzero..) + .expect("the first nonzero quotient byte is in bounds"); let mut inverse_fixed = vec![0u8; modulus.len()]; - inverse_fixed[modulus.len() - inverse.len()..].copy_from_slice(inverse); + let inverse_start = modulus.len().strict_sub(inverse.len()); + inverse_fixed + .get_mut(inverse_start..) + .expect("the RSA inverse fits the modulus width") + .copy_from_slice(inverse); (factor, inverse_fixed) } fn argon2i_params() -> Argon2Params { - Argon2Params::new(32, 1, 1) - .unwrap() + Argon2Params::new(32, 1, 1).expect("fixed Argon2i profile is valid") } fn argon2i_parallel_params() -> Argon2Params { - Argon2Params::new(512, 1, 4) - .unwrap() + Argon2Params::new(512, 1, 4).expect("fixed parallel Argon2i profile is valid") } macro_rules! fixed_owner_eq_case { @@ -224,7 +241,11 @@ macro_rules! fixed_owner_eq_case { } for (class, left, right) in inputs { - runner.run_one(class, || $entry(left.as_ptr(), right.as_ptr()) == 1); + runner.run_one(class, || { + // SAFETY: left and right are initialized [$len; u8] values whose storage remains alive + // and immutable for the entrypoint's fixed-size reads. + unsafe { $entry(left.as_ptr(), right.as_ptr()) == 1 } + }); } } }; @@ -303,7 +324,10 @@ fn hmac_sha256_truncated_64_valid_vs_invalid_tag(runner: &mut CtRunner, rng: &mu let class = random_class(rng); let key = rand_array::<32>(rng); let full_tag = HmacSha256::mac(&key, MESSAGE); - let mut expected = *full_tag.as_bytes().first_chunk::<8>().unwrap(); + let mut expected = *full_tag + .as_bytes() + .first_chunk::<8>() + .expect("HMAC-SHA-256 tags contain eight bytes"); if matches!(class, Class::Right) { expected[0] ^= 1; } @@ -311,7 +335,9 @@ fn hmac_sha256_truncated_64_valid_vs_invalid_tag(runner: &mut CtRunner, rng: &mu } for (class, key, expected) in inputs { - runner.run_one(class, || HmacSha256::verify_truncated_tag_64(&key, MESSAGE, &expected).is_ok()); + runner.run_one(class, || { + HmacSha256::verify_truncated_tag_64(&key, MESSAGE, &expected).is_ok() + }); } } @@ -365,7 +391,9 @@ macro_rules! aead_fixed_vs_random_key_open { let key = <$key>::from_bytes(key); let cipher = <$cipher>::new(&key); let mut ciphertext = AEAD_PLAINTEXT; - let tag = cipher.encrypt_in_place(&nonce, AAD, &mut ciphertext).unwrap(); + let tag = cipher + .encrypt_in_place(&nonce, AAD, &mut ciphertext) + .expect("fixed-size AEAD timing fixture must encrypt"); inputs.push((class, key, nonce, ciphertext, tag)); } @@ -502,7 +530,9 @@ fn gcm_siv_diag_polyval_fixed_vs_random_auth_key(runner: &mut CtRunner, rng: &mu } for (class, auth_key) in inputs { - runner.run_one(class, || diag_aes128gcmsiv_polyval_digest(&auth_key, AAD, &AEAD_PLAINTEXT)); + runner.run_one(class, || { + diag_aes128gcmsiv_polyval_digest(&auth_key, AAD, &AEAD_PLAINTEXT) + }); } } @@ -766,7 +796,7 @@ macro_rules! mlkem_dudect_profile { } for (class, sigma) in inputs { - runner.run_one(class, || std::hint::black_box($diag(rho, sigma))[0]); + runner.run_one(class, || core::hint::black_box($diag(rho, sigma))[0]); } } @@ -775,7 +805,7 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&[0x51; <$profile>::KEY_GENERATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM key-generation entropy must produce a keypair"); let mut inputs = Vec::with_capacity(samples()); for _ in 0..samples() { let class = random_class(rng); @@ -793,8 +823,8 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap(); - std::hint::black_box(ciphertext.as_bytes()[0] ^ shared_secret.as_bytes()[0]) + .expect("fixed ML-KEM encapsulation entropy must produce a ciphertext"); + core::hint::black_box(ciphertext.as_bytes()[0] ^ shared_secret.as_bytes()[0]) }); } } @@ -804,12 +834,12 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&[0x61; <$profile>::KEY_GENERATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM key-generation entropy must produce a keypair"); let (ciphertext, _) = <$profile>::encapsulate(&fixed_encapsulation_key, |out| { out.copy_from_slice(&[0x62; <$profile>::ENCAPSULATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM encapsulation entropy must produce a ciphertext"); let secret_key_len = <$profile>::DECAPSULATION_KEY_SIZE - <$profile>::ENCAPSULATION_KEY_SIZE - 64; let mut inputs = Vec::with_capacity(samples()); for _ in 0..samples() { @@ -821,7 +851,7 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&random); Ok::<(), MlKemError>(()) }) - .unwrap() + .expect("random ML-KEM key-generation entropy must produce a keypair") .1; decapsulation_key_bytes[..secret_key_len] .copy_from_slice(&random_decapsulation_key.as_bytes()[..secret_key_len]); @@ -831,8 +861,9 @@ macro_rules! mlkem_dudect_profile { for (class, decapsulation_key) in inputs { runner.run_one(class, || { - let shared_secret = <$profile>::decapsulate(&decapsulation_key, &ciphertext).unwrap(); - std::hint::black_box(shared_secret.as_bytes()[0]) + let shared_secret = <$profile>::decapsulate(&decapsulation_key, &ciphertext) + .expect("well-formed ML-KEM ciphertext must decapsulate"); + core::hint::black_box(shared_secret.as_bytes()[0]) }); } } @@ -842,12 +873,12 @@ macro_rules! mlkem_dudect_profile { out.copy_from_slice(&[0x71; <$profile>::KEY_GENERATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM key-generation entropy must produce a keypair"); let (ciphertext, _) = <$profile>::encapsulate(&encapsulation_key, |out| { out.copy_from_slice(&[0x72; <$profile>::ENCAPSULATION_RANDOM_SIZE]); Ok::<(), MlKemError>(()) }) - .unwrap(); + .expect("fixed ML-KEM encapsulation entropy must produce a ciphertext"); let mut rejected_ciphertext = ciphertext.to_bytes(); rejected_ciphertext[0] ^= 1; let rejected_ciphertext = <$ciphertext>::from_bytes(rejected_ciphertext); @@ -868,8 +899,9 @@ macro_rules! mlkem_dudect_profile { for (class, decapsulation_key) in inputs { runner.run_one(class, || { - let shared_secret = <$profile>::decapsulate(&decapsulation_key, &rejected_ciphertext).unwrap(); - std::hint::black_box(shared_secret.as_bytes()[0]) + let shared_secret = <$profile>::decapsulate(&decapsulation_key, &rejected_ciphertext) + .expect("ML-KEM implicit rejection must produce a shared secret"); + core::hint::black_box(shared_secret.as_bytes()[0]) }); } } @@ -920,7 +952,7 @@ fn mlkem_arithmetic_ntt_fixed_vs_random_poly(runner: &mut CtRunner, rng: &mut Be } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_ntt_input_digest(poly))); + runner.run_one(class, || core::hint::black_box(diag_mlkem_ntt_input_digest(poly))); } } @@ -937,7 +969,9 @@ fn mlkem_arithmetic_inverse_ntt_fixed_vs_random_poly(runner: &mut CtRunner, rng: } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_inverse_ntt_montgomery_product_input_digest(poly))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_inverse_ntt_montgomery_product_input_digest(poly)) + }); } } @@ -954,7 +988,9 @@ fn mlkem_arithmetic_to_product_domain_fixed_vs_random_poly(runner: &mut CtRunner } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_to_montgomery_product_domain_input_digest(poly))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_to_montgomery_product_domain_input_digest(poly)) + }); } } @@ -971,7 +1007,9 @@ fn mlkem_arithmetic_from_product_domain_fixed_vs_random_poly(runner: &mut CtRunn } for (class, poly) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_from_montgomery_product_domain_input_digest(poly))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_from_montgomery_product_domain_input_digest(poly)) + }); } } @@ -992,7 +1030,9 @@ fn mlkem_arithmetic_basemul_fixed_vs_random_operands(runner: &mut CtRunner, rng: } for (class, a, b, acc) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem_multiply_ntts_add_assign_input_digest(a, b, acc))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem_multiply_ntts_add_assign_input_digest(a, b, acc)) + }); } } @@ -1007,13 +1047,19 @@ fn mlkem1024_arithmetic_dot_fixed_vs_random_operands(runner: &mut CtRunner, rng: mlkem_poly_from_seed(0xA01), ) } else { - (random_mlkem_polyvec4(rng), random_mlkem_polyvec4(rng), random_mlkem_poly(rng)) + ( + random_mlkem_polyvec4(rng), + random_mlkem_polyvec4(rng), + random_mlkem_poly(rng), + ) }; inputs.push((class, a, b, acc)); } for (class, a, b, acc) in inputs { - runner.run_one(class, || std::hint::black_box(diag_mlkem1024_multiply_ntts_accumulate_input_digest(a, b, acc))); + runner.run_one(class, || { + core::hint::black_box(diag_mlkem1024_multiply_ntts_accumulate_input_digest(a, b, acc)) + }); } } @@ -1079,7 +1125,10 @@ fn ed25519_keypair_sign_fixed_vs_random_secret(runner: &mut CtRunner, rng: &mut } else { rand_array::<{ Ed25519SecretKey::LENGTH }>(rng) }; - inputs.push((class, Ed25519Keypair::from_secret_key(Ed25519SecretKey::from_bytes(secret)))); + inputs.push(( + class, + Ed25519Keypair::from_secret_key(Ed25519SecretKey::from_bytes(secret)), + )); } for (class, keypair) in inputs { @@ -1117,9 +1166,14 @@ fn ecdsa_p256_sign_fixed_vs_random_secret(runner: &mut CtRunner, rng: &mut Bench } for (class, secret, blind) in inputs { - let key = EcdsaP256SecretKey::from_bytes(secret).unwrap(); + let key = EcdsaP256SecretKey::from_bytes(secret).expect("generated P-256 secret scalar is valid"); runner.run_one(class, || { - std::hint::black_box(key.try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)).unwrap().to_bytes()); + core::hint::black_box( + key + .try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)) + .expect("valid P-256 key and blinding input must sign") + .to_bytes(), + ); }); } } @@ -1136,9 +1190,14 @@ fn ecdsa_p384_sign_fixed_vs_random_secret(runner: &mut CtRunner, rng: &mut Bench } for (class, secret, blind) in inputs { - let key = EcdsaP384SecretKey::from_bytes(secret).unwrap(); + let key = EcdsaP384SecretKey::from_bytes(secret).expect("generated P-384 secret scalar is valid"); runner.run_one(class, || { - std::hint::black_box(key.try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)).unwrap().to_bytes()); + core::hint::black_box( + key + .try_sign_blinded(MESSAGE, |out| out.copy_from_slice(&blind)) + .expect("valid P-384 key and blinding input must sign") + .to_bytes(), + ); }); } } @@ -1156,7 +1215,7 @@ fn ecdsa_p256_diag_nonce_reduce_fixed_vs_random_secret(runner: &mut CtRunner, rn for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_nonce_reduce_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_nonce_reduce_limb_digest(secret, MESSAGE))[0] }); } } @@ -1174,7 +1233,7 @@ fn ecdsa_p256_diag_reduce_wide_fixed_vs_random_input(runner: &mut CtRunner, rng: for (class, wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_reduce_wide_order_limb_digest(wide))[0] + core::hint::black_box(diag_ecdsa_p256_reduce_wide_order_limb_digest(wide))[0] }); } } @@ -1192,7 +1251,7 @@ fn ecdsa_p256_diag_basepoint_blinded_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret, blind) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] }); } } @@ -1210,7 +1269,7 @@ fn ecdsa_p256_diag_scalar_finish_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1228,7 +1287,7 @@ fn ecdsa_p256_diag_order_mul_fixed_r_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret))[0] + core::hint::black_box(diag_ecdsa_p256_order_mul_fixed_r_limb_digest(secret))[0] }); } } @@ -1246,7 +1305,7 @@ fn ecdsa_p256_diag_blinded_order_mul_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret, blind) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret, blind))[0] + core::hint::black_box(diag_ecdsa_p256_order_mul_blinded_fixed_r_limb_digest(secret, blind))[0] }); } } @@ -1264,7 +1323,7 @@ fn ecdsa_p256_diag_nonce_inverse_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_nonce_inverse_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_nonce_inverse_limb_digest(secret, MESSAGE))[0] }); } } @@ -1282,7 +1341,7 @@ fn ecdsa_p256_diag_final_multiply_fixed_vs_random_secret(runner: &mut CtRunner, for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p256_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p256_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1300,7 +1359,7 @@ fn ecdsa_p384_diag_nonce_reduce_fixed_vs_random_secret(runner: &mut CtRunner, rn for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_nonce_reduce_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_nonce_reduce_limb_digest(secret, MESSAGE))[0] }); } } @@ -1318,7 +1377,7 @@ fn ecdsa_p384_diag_reduce_wide_fixed_vs_random_input(runner: &mut CtRunner, rng: for (class, wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_reduce_wide_order_limb_digest(wide))[0] + core::hint::black_box(diag_ecdsa_p384_reduce_wide_order_limb_digest(wide))[0] }); } } @@ -1336,7 +1395,7 @@ fn ecdsa_p384_diag_basepoint_blinded_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret, blind) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_basepoint_blinded_limb_digest(secret, blind, MESSAGE))[0] }); } } @@ -1354,7 +1413,7 @@ fn ecdsa_p384_diag_scalar_finish_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_scalar_finish_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1372,7 +1431,7 @@ fn ecdsa_p384_diag_order_mul_fixed_r_fixed_vs_random_secret(runner: &mut CtRunne for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret))[0] + core::hint::black_box(diag_ecdsa_p384_order_mul_fixed_r_limb_digest(secret))[0] }); } } @@ -1390,7 +1449,7 @@ fn ecdsa_p384_diag_nonce_inverse_fixed_vs_random_secret(runner: &mut CtRunner, r for (class, secret) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_nonce_inverse_limb_digest(secret, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_nonce_inverse_limb_digest(secret, MESSAGE))[0] }); } } @@ -1408,7 +1467,7 @@ fn ecdsa_p384_diag_final_multiply_fixed_vs_random_secret(runner: &mut CtRunner, for (class, secret, nonce_wide) in inputs { runner.run_one(class, || { - std::hint::black_box(diag_ecdsa_p384_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] + core::hint::black_box(diag_ecdsa_p384_final_multiply_limb_digest(secret, nonce_wide, MESSAGE))[0] }); } } @@ -1436,8 +1495,7 @@ fn rsa_pkcs1v15_fixed_vs_random_message(runner: &mut CtRunner, rng: &mut BenchRn .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, &message, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), &mut out, ) .is_ok() @@ -1470,8 +1528,7 @@ fn rsa_pss_fixed_vs_random_message(runner: &mut CtRunner, rng: &mut BenchRng) { RsaPssProfile::Sha256, &message, &salt, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), &mut out, ) .is_ok() @@ -1498,8 +1555,7 @@ fn rsa_pkcs1v15_full_width_vs_short_canonical_crt_exponent(runner: &mut CtRunner .sign_pkcs1v15_with_blinding_factor( RsaPkcs1v15Profile::Sha256, &message, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), &mut out, ) .is_ok() @@ -1532,7 +1588,7 @@ fn rsa_oaep_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &mut B out.copy_from_slice(&seed); Ok(()) }) - .unwrap(); + .expect("fixed OAEP fixture must encrypt"); inputs.push((class, ciphertext)); } @@ -1544,8 +1600,7 @@ fn rsa_oaep_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &mut B RsaOaepProfile::Sha256, label, &ciphertext, - &blinding_factor, - &blinding_inverse, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), &mut out, ) .is_ok() @@ -1573,7 +1628,7 @@ fn rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &m out.fill(0x5d); Ok(()) }) - .unwrap(); + .expect("fixed PKCS#1 v1.5 fixture must encrypt"); inputs.push((class, ciphertext)); } @@ -1581,7 +1636,11 @@ fn rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext(runner: &mut CtRunner, rng: &m runner.run_one(class, || { let mut out = vec![0u8; sig_len]; key - .decrypt_pkcs1v15_with_blinding_factor(&ciphertext, &blinding_factor, &blinding_inverse, &mut out) + .decrypt_pkcs1v15_with_blinding_factor( + &ciphertext, + RsaBlindingPair::new(&blinding_factor, &blinding_inverse), + &mut out, + ) .is_ok() }); } @@ -1612,7 +1671,9 @@ fn rsa_private_component_validation_fixed_vs_random_component(runner: &mut CtRun } for (class, component) in inputs { - runner.run_one(class, || diag_rsa_private_component_validation_32(&component, &upper_bound, &other)); + runner.run_one(class, || { + diag_rsa_private_component_validation_32(&component, &upper_bound, &other) + }); } } @@ -1629,7 +1690,8 @@ fn rsa_private_key_pkcs8_import_key_a_vs_key_b(runner: &mut CtRunner, rng: &mut let selected = if matches!(class, Class::Left) { &der_a } else { &der_b }; der.copy_from_slice(selected); runner.run_one(class, || { - let key = RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()).unwrap(); + let key = RsaPrivateKey::from_pkcs8_der_with_policy(&der, &RsaPublicKeyPolicy::legacy_verification()) + .expect("selected bundled RSA fixture must import"); key.signature_len() }); } @@ -1716,7 +1778,10 @@ fn rsa_private_key_pkcs8_import_stage_key_a_vs_key_b(runner: &mut CtRunner, rng: for class in inputs { let selected = if matches!(class, Class::Left) { &der_a } else { &der_b }; der.copy_from_slice(selected); - runner.run_one(class, || diag_rsa_import_pkcs8_private_key_der_stage(&der, &policy, stage).unwrap()); + runner.run_one(class, || { + diag_rsa_import_pkcs8_private_key_der_stage(&der, &policy, stage) + .expect("selected bundled RSA fixture must reach the requested import stage") + }); } } @@ -1735,9 +1800,10 @@ fn rsa_private_key_pkcs8_validate_stage_key_a_vs_key_b(runner: &mut CtRunner, rn der.copy_from_slice(selected); runner.run_one(class, || { if stage == u8::MAX { - diag_rsa_validate_pkcs8_private_key_der(&der, &policy).unwrap() + diag_rsa_validate_pkcs8_private_key_der(&der, &policy).expect("selected bundled RSA fixture must validate") } else { - diag_rsa_validate_pkcs8_private_key_der_stage(&der, &policy, stage).unwrap() + diag_rsa_validate_pkcs8_private_key_der_stage(&der, &policy, stage) + .expect("selected bundled RSA fixture must reach the requested validation stage") } }); } @@ -1962,7 +2028,7 @@ macro_rules! blake2_keyed_fixed_vs_random { } for (class, key) in inputs { - let key = <$key_ty>::new(&key).unwrap(); + let key = <$key_ty>::new(&key).expect("32-byte BLAKE2 timing key is valid"); runner.run_one(class, || <$ty>::keyed_digest(key, MESSAGE)[0]); } } @@ -2040,10 +2106,7 @@ ctbench_main_with_seeds!( (owner_eq_64_equal_vs_first_diff, Some(0x6f776e657236345f)), (secret_wrappers_debug_fixed_vs_random, Some(0x7365637265745f77)), (hmac_sha256_valid_vs_invalid_tag, Some(0x686d61635f736861)), - ( - hmac_sha256_truncated_64_valid_vs_invalid_tag, - Some(0x686d61635f743634) - ), + (hmac_sha256_truncated_64_valid_vs_invalid_tag, Some(0x686d61635f743634)), (hmac_sha384_valid_vs_invalid_tag, Some(0x686d61633338345f)), (hmac_sha512_valid_vs_invalid_tag, Some(0x686d61633531325f)), (kmac256_valid_vs_invalid_tag, Some(0x6b6d61633235365f)), @@ -2055,8 +2118,14 @@ ctbench_main_with_seeds!( (aes128_gcm_siv_diag_derive_fixed_vs_random_key, Some(0x6131323867646572)), (aes256_gcm_siv_diag_derive_fixed_vs_random_key, Some(0x6132353667646572)), (gcm_siv_diag_polyval_fixed_vs_random_auth_key, Some(0x6763736976706f6c)), - (aes128_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, Some(0x6131323867746167)), - (aes256_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, Some(0x6132353667746167)), + ( + aes128_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, + Some(0x6131323867746167) + ), + ( + aes256_gcm_siv_diag_raw_tag_aes_fixed_vs_random_key, + Some(0x6132353667746167) + ), (aes128_gcm_diag_ctr32_be_fixed_vs_random_key, Some(0x6731323863747262)), (aes256_gcm_diag_ctr32_be_fixed_vs_random_key, Some(0x6732353663747262)), (aes128_gcm_diag_ghash_fixed_vs_random_h, Some(0x6731323867686173)), @@ -2074,43 +2143,124 @@ ctbench_main_with_seeds!( (x25519_fixed_vs_random_scalar, Some(0x7832353531395f63)), (mlkem512_keygen_secret_noise_fixed_vs_random, Some(0x6d6b3531326b676e)), (mlkem512_encapsulate_fixed_vs_random_coins, Some(0x6d6b353132656e63)), - (mlkem512_decapsulate_fixed_vs_random_secret_key, Some(0x6d6b353132646563)), - (mlkem512_decapsulate_rejection_seed_fixed_vs_random, Some(0x6d6b35313272656a)), + ( + mlkem512_decapsulate_fixed_vs_random_secret_key, + Some(0x6d6b353132646563) + ), + ( + mlkem512_decapsulate_rejection_seed_fixed_vs_random, + Some(0x6d6b35313272656a) + ), (mlkem768_keygen_secret_noise_fixed_vs_random, Some(0x6d6c6b656d6b676e)), (mlkem768_encapsulate_fixed_vs_random_coins, Some(0x6d6c6b656d656e63)), - (mlkem768_decapsulate_fixed_vs_random_secret_key, Some(0x6d6c6b656d646563)), - (mlkem768_decapsulate_rejection_seed_fixed_vs_random, Some(0x6d6c6b656d72656a)), + ( + mlkem768_decapsulate_fixed_vs_random_secret_key, + Some(0x6d6c6b656d646563) + ), + ( + mlkem768_decapsulate_rejection_seed_fixed_vs_random, + Some(0x6d6c6b656d72656a) + ), (mlkem1024_keygen_secret_noise_fixed_vs_random, Some(0x6d6b313032346b67)), (mlkem1024_encapsulate_fixed_vs_random_coins, Some(0x6d6b31303234656e)), - (mlkem1024_decapsulate_fixed_vs_random_secret_key, Some(0x6d6b313032346465)), - (mlkem1024_decapsulate_rejection_seed_fixed_vs_random, Some(0x6d6b31303234726a)), + ( + mlkem1024_decapsulate_fixed_vs_random_secret_key, + Some(0x6d6b313032346465) + ), + ( + mlkem1024_decapsulate_rejection_seed_fixed_vs_random, + Some(0x6d6b31303234726a) + ), (mlkem_arithmetic_ntt_fixed_vs_random_poly, Some(0x6d6c6b6e7474706f)), - (mlkem_arithmetic_inverse_ntt_fixed_vs_random_poly, Some(0x6d6c6b696e747470)), - (mlkem_arithmetic_to_product_domain_fixed_vs_random_poly, Some(0x6d6c6b746f70726f)), - (mlkem_arithmetic_from_product_domain_fixed_vs_random_poly, Some(0x6d6c6b667270726f)), - (mlkem_arithmetic_basemul_fixed_vs_random_operands, Some(0x6d6c6b626173656d)), - (mlkem1024_arithmetic_dot_fixed_vs_random_operands, Some(0x6d6c6b313032646f)), + ( + mlkem_arithmetic_inverse_ntt_fixed_vs_random_poly, + Some(0x6d6c6b696e747470) + ), + ( + mlkem_arithmetic_to_product_domain_fixed_vs_random_poly, + Some(0x6d6c6b746f70726f) + ), + ( + mlkem_arithmetic_from_product_domain_fixed_vs_random_poly, + Some(0x6d6c6b667270726f) + ), + ( + mlkem_arithmetic_basemul_fixed_vs_random_operands, + Some(0x6d6c6b626173656d) + ), + ( + mlkem1024_arithmetic_dot_fixed_vs_random_operands, + Some(0x6d6c6b313032646f) + ), (ed25519_sign_fixed_vs_random_secret, Some(0x656432353531395f)), (ed25519_public_key_fixed_vs_random_secret, Some(0x6564323535313950)), - (ed25519_sha512_secret_expand_fixed_vs_random_secret, Some(0x6564323535314853)), + ( + ed25519_sha512_secret_expand_fixed_vs_random_secret, + Some(0x6564323535314853) + ), (ed25519_keypair_sign_fixed_vs_random_secret, Some(0x656432353531394b)), (ecdsa_p256_sign_fixed_vs_random_secret, Some(0x703235365f736967)), (ecdsa_p384_sign_fixed_vs_random_secret, Some(0x703338345f736967)), - (ecdsa_p256_diag_nonce_reduce_fixed_vs_random_secret, Some(0x703235366e6f6e63)), - (ecdsa_p256_diag_reduce_wide_fixed_vs_random_input, Some(0x7032353672656475)), - (ecdsa_p256_diag_basepoint_blinded_fixed_vs_random_secret, Some(0x7032353662617365)), - (ecdsa_p256_diag_scalar_finish_fixed_vs_random_secret, Some(0x7032353666696e73)), - (ecdsa_p256_diag_order_mul_fixed_r_fixed_vs_random_secret, Some(0x703235366d756c72)), - (ecdsa_p256_diag_blinded_order_mul_fixed_vs_random_secret, Some(0x703235366d756c62)), - (ecdsa_p256_diag_nonce_inverse_fixed_vs_random_secret, Some(0x70323536696e766b)), - (ecdsa_p256_diag_final_multiply_fixed_vs_random_secret, Some(0x703235366d756c73)), - (ecdsa_p384_diag_nonce_reduce_fixed_vs_random_secret, Some(0x703338346e6f6e63)), - (ecdsa_p384_diag_reduce_wide_fixed_vs_random_input, Some(0x7033383472656475)), - (ecdsa_p384_diag_basepoint_blinded_fixed_vs_random_secret, Some(0x7033383462617365)), - (ecdsa_p384_diag_scalar_finish_fixed_vs_random_secret, Some(0x7033383466696e73)), - (ecdsa_p384_diag_order_mul_fixed_r_fixed_vs_random_secret, Some(0x703338346d756c72)), - (ecdsa_p384_diag_nonce_inverse_fixed_vs_random_secret, Some(0x70333834696e766b)), - (ecdsa_p384_diag_final_multiply_fixed_vs_random_secret, Some(0x703338346d756c73)), + ( + ecdsa_p256_diag_nonce_reduce_fixed_vs_random_secret, + Some(0x703235366e6f6e63) + ), + ( + ecdsa_p256_diag_reduce_wide_fixed_vs_random_input, + Some(0x7032353672656475) + ), + ( + ecdsa_p256_diag_basepoint_blinded_fixed_vs_random_secret, + Some(0x7032353662617365) + ), + ( + ecdsa_p256_diag_scalar_finish_fixed_vs_random_secret, + Some(0x7032353666696e73) + ), + ( + ecdsa_p256_diag_order_mul_fixed_r_fixed_vs_random_secret, + Some(0x703235366d756c72) + ), + ( + ecdsa_p256_diag_blinded_order_mul_fixed_vs_random_secret, + Some(0x703235366d756c62) + ), + ( + ecdsa_p256_diag_nonce_inverse_fixed_vs_random_secret, + Some(0x70323536696e766b) + ), + ( + ecdsa_p256_diag_final_multiply_fixed_vs_random_secret, + Some(0x703235366d756c73) + ), + ( + ecdsa_p384_diag_nonce_reduce_fixed_vs_random_secret, + Some(0x703338346e6f6e63) + ), + ( + ecdsa_p384_diag_reduce_wide_fixed_vs_random_input, + Some(0x7033383472656475) + ), + ( + ecdsa_p384_diag_basepoint_blinded_fixed_vs_random_secret, + Some(0x7033383462617365) + ), + ( + ecdsa_p384_diag_scalar_finish_fixed_vs_random_secret, + Some(0x7033383466696e73) + ), + ( + ecdsa_p384_diag_order_mul_fixed_r_fixed_vs_random_secret, + Some(0x703338346d756c72) + ), + ( + ecdsa_p384_diag_nonce_inverse_fixed_vs_random_secret, + Some(0x70333834696e766b) + ), + ( + ecdsa_p384_diag_final_multiply_fixed_vs_random_secret, + Some(0x703338346d756c73) + ), (rsa_pkcs1v15_fixed_vs_random_message, Some(0x7273615f7369676e)), (rsa_pss_fixed_vs_random_message, Some(0x7273615f70737373)), ( @@ -2119,25 +2269,76 @@ ctbench_main_with_seeds!( ), (rsa_oaep_decrypt_fixed_vs_random_plaintext, Some(0x7273615f6f616570)), (rsa_pkcs1v15_decrypt_fixed_vs_random_plaintext, Some(0x7273615f64656331)), - (rsa_private_component_validation_fixed_vs_random_component, Some(0x7273615f636f6d70)), + ( + rsa_private_component_validation_fixed_vs_random_component, + Some(0x7273615f636f6d70) + ), (rsa_private_key_pkcs8_import_key_a_vs_key_b, Some(0x7273615f6b657969)), (rsa_private_key_pkcs8_validate_key_a_vs_key_b, Some(0x7273615f76616c69)), - (rsa_private_key_pkcs8_validate_stage0_key_a_vs_key_b, Some(0x7273615f76733030)), - (rsa_private_key_pkcs8_validate_stage1_key_a_vs_key_b, Some(0x7273615f76733031)), - (rsa_private_key_pkcs8_validate_stage2_key_a_vs_key_b, Some(0x7273615f76733032)), - (rsa_private_key_pkcs8_validate_stage3_key_a_vs_key_b, Some(0x7273615f76733033)), - (rsa_private_key_pkcs8_validate_stage4_key_a_vs_key_b, Some(0x7273615f76733034)), - (rsa_private_key_pkcs8_validate_stage30_key_a_vs_key_b, Some(0x7273615f76333030)), - (rsa_private_key_pkcs8_validate_stage31_key_a_vs_key_b, Some(0x7273615f76333031)), - (rsa_private_key_pkcs8_validate_stage32_key_a_vs_key_b, Some(0x7273615f76333032)), - (rsa_private_key_pkcs8_validate_stage40_key_a_vs_key_b, Some(0x7273615f76343030)), - (rsa_private_key_pkcs8_validate_stage41_key_a_vs_key_b, Some(0x7273615f76343031)), - (rsa_private_key_pkcs8_validate_stage42_key_a_vs_key_b, Some(0x7273615f76343032)), - (rsa_private_key_pkcs8_import_stage50_key_a_vs_key_b, Some(0x7273615f69353030)), - (rsa_private_key_pkcs8_import_stage51_key_a_vs_key_b, Some(0x7273615f69353031)), - (rsa_private_key_pkcs8_import_stage52_key_a_vs_key_b, Some(0x7273615f69353032)), - (rsa_private_key_pkcs8_import_stage53_key_a_vs_key_b, Some(0x7273615f69353033)), - (rsa_private_key_pkcs8_import_stage54_key_a_vs_key_b, Some(0x7273615f69353034)), + ( + rsa_private_key_pkcs8_validate_stage0_key_a_vs_key_b, + Some(0x7273615f76733030) + ), + ( + rsa_private_key_pkcs8_validate_stage1_key_a_vs_key_b, + Some(0x7273615f76733031) + ), + ( + rsa_private_key_pkcs8_validate_stage2_key_a_vs_key_b, + Some(0x7273615f76733032) + ), + ( + rsa_private_key_pkcs8_validate_stage3_key_a_vs_key_b, + Some(0x7273615f76733033) + ), + ( + rsa_private_key_pkcs8_validate_stage4_key_a_vs_key_b, + Some(0x7273615f76733034) + ), + ( + rsa_private_key_pkcs8_validate_stage30_key_a_vs_key_b, + Some(0x7273615f76333030) + ), + ( + rsa_private_key_pkcs8_validate_stage31_key_a_vs_key_b, + Some(0x7273615f76333031) + ), + ( + rsa_private_key_pkcs8_validate_stage32_key_a_vs_key_b, + Some(0x7273615f76333032) + ), + ( + rsa_private_key_pkcs8_validate_stage40_key_a_vs_key_b, + Some(0x7273615f76343030) + ), + ( + rsa_private_key_pkcs8_validate_stage41_key_a_vs_key_b, + Some(0x7273615f76343031) + ), + ( + rsa_private_key_pkcs8_validate_stage42_key_a_vs_key_b, + Some(0x7273615f76343032) + ), + ( + rsa_private_key_pkcs8_import_stage50_key_a_vs_key_b, + Some(0x7273615f69353030) + ), + ( + rsa_private_key_pkcs8_import_stage51_key_a_vs_key_b, + Some(0x7273615f69353031) + ), + ( + rsa_private_key_pkcs8_import_stage52_key_a_vs_key_b, + Some(0x7273615f69353032) + ), + ( + rsa_private_key_pkcs8_import_stage53_key_a_vs_key_b, + Some(0x7273615f69353033) + ), + ( + rsa_private_key_pkcs8_import_stage54_key_a_vs_key_b, + Some(0x7273615f69353034) + ), (rsa_private_key_pkcs8_export_key_a_vs_key_b, Some(0x7273615f6b657978)), (hkdf_sha2_fixed_vs_random_ikm, Some(0x686b64665f736861)), (pbkdf2_sha2_fixed_vs_random_password, Some(0x70626b6466325f73)), diff --git a/tools/ct-harness/src/lib.rs b/tools/ct-harness/src/lib.rs index 8e3e8c4b..14acccae 100644 --- a/tools/ct-harness/src/lib.rs +++ b/tools/ct-harness/src/lib.rs @@ -6,7 +6,6 @@ // C ABI harness functions must remain plain `extern "C"` symbols. Pointer validity is documented // on the shared helpers and each entrypoint rejects null shapes before copying fixed-size inputs. -#![allow(clippy::not_unsafe_ptr_arg_deref)] use core::{ptr, slice}; use std::format; @@ -17,12 +16,11 @@ use rscrypto::{ Blake2bKey, Blake2s128, Blake2s256, Blake2sKey, Blake3, Blake3KeyedHash, ChaCha20Poly1305, ChaCha20Poly1305Key, Crc32, EcdsaP256SecretKey, EcdsaP384SecretKey, Ed25519PublicKey, Ed25519SecretKey, Ed25519Signature, HkdfSha256, HkdfSha384, HmacSha3_224Tag, HmacSha256, HmacSha256Tag, HmacSha384, HmacSha384Tag, HmacSha512, HmacSha512Tag, - Kmac256, MlKem512, - MlKem512Ciphertext, MlKem512DecapsulationKey, MlKem512EncapsulationKey, MlKem768, MlKem768Ciphertext, - MlKem768DecapsulationKey, MlKem768EncapsulationKey, MlKem1024, MlKem1024Ciphertext, MlKem1024DecapsulationKey, - MlKem1024EncapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, - RsaPssProfile, RsaPublicKeyPolicy, Scrypt, ScryptParams, SecretBytes, Sha256, X25519PublicKey, X25519SecretKey, - XChaCha20Poly1305, XChaCha20Poly1305Key, + Kmac256, MlKem512, MlKem512Ciphertext, MlKem512DecapsulationKey, MlKem512EncapsulationKey, MlKem768, + MlKem768Ciphertext, MlKem768DecapsulationKey, MlKem768EncapsulationKey, MlKem1024, MlKem1024Ciphertext, + MlKem1024DecapsulationKey, MlKem1024EncapsulationKey, MlKemError, Pbkdf2Sha256, Pbkdf2Sha512, RsaBlindingPair, + RsaOaepProfile, RsaPkcs1v15Profile, RsaPrivateKey, RsaPssProfile, RsaPublicKeyPolicy, Scrypt, ScryptParams, + SecretBytes, Sha256, X25519PublicKey, X25519SecretKey, XChaCha20Poly1305, XChaCha20Poly1305Key, aead::{Nonce96, Nonce128, Nonce192, Nonce256}, checksum::Checksum, traits::Kem as _, @@ -122,13 +120,16 @@ unsafe fn read_mlkem_polyvec(ptr: *const u16) -> Option<[[u16; 2 } let mut out = [[0u16; 256]; K]; + let coefficient_count = K.strict_mul(256); // SAFETY: Copies a fixed-size ML-KEM polyvec from FFI memory because: - // 1. The caller contract requires `ptr` to be valid for reads of `K * 256` initialized `u16` - // values. - // 2. The caller contract requires `ptr` to be aligned for `u16`. - // 3. `out` is stack-owned storage valid for writes of `K * 256` `u16` values. - // 4. Source and destination cannot overlap because `out` is newly allocated stack storage. - unsafe { ptr::copy_nonoverlapping(ptr, out.as_mut_ptr().cast::(), K * 256) }; + // 1. `coefficient_count` is exactly `K * 256`; strict multiplication rejects an impossible + // destination layout instead of wrapping the copy length. + // 2. The caller contract requires `ptr` to be valid for reads of `coefficient_count` initialized + // `u16` values. + // 3. The caller contract requires `ptr` to be aligned for `u16`. + // 4. `out` is stack-owned storage valid for writes of `coefficient_count` `u16` values. + // 5. Source and destination cannot overlap because `out` is newly allocated stack storage. + unsafe { ptr::copy_nonoverlapping(ptr, out.as_mut_ptr().cast::(), coefficient_count) }; Some(out) } @@ -157,23 +158,36 @@ fn rsa_ct_fixture_key(pkcs8_der: &[u8]) -> Option { } /// HMAC-SHA256 tag verification harness. +/// +/// # Safety +/// +/// - When non-null, `key` and `data` must each reference a single allocation containing the +/// corresponding number of initialized bytes. Each byte length must be no greater than +/// `isize::MAX`, and neither range may be mutated during the call. +/// - When non-null, `expected_tag` must be valid for reads of 32 initialized bytes and must not be +/// mutated while copied. +/// - Null key and data pointers represent empty slices only when their corresponding lengths are +/// zero. Other null shapes are rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_hmac_sha256_verify( +pub unsafe extern "C" fn ct_entry_hmac_sha256_verify( key: *const u8, key_len: usize, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow a non-null `key`; `input_slice` handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `data`; `input_slice` handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly 32 readable bytes. + // SAFETY: The function contract requires a non-null `expected_tag` to reference exactly 32 + // readable, initialized bytes that remain immutable while copied into owned storage. let Some(expected_tag) = (unsafe { read_array::<32>(expected_tag) }) else { return STATUS_ERR; }; @@ -185,23 +199,36 @@ pub extern "C" fn ct_entry_hmac_sha256_verify( } /// HMAC-SHA256 64-bit truncated-tag verification harness. +/// +/// # Safety +/// +/// - When non-null, `key` and `data` must each reference a single allocation containing the +/// corresponding number of initialized bytes. Each byte length must be no greater than +/// `isize::MAX`, and neither range may be mutated during the call. +/// - When non-null, `expected_tag` must be valid for reads of 8 initialized bytes and must not be +/// mutated while copied. +/// - Null key and data pointers represent empty slices only when their corresponding lengths are +/// zero. Other null shapes are rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_hmac_sha256_verify_truncated_64( +pub unsafe extern "C" fn ct_entry_hmac_sha256_verify_truncated_64( key: *const u8, key_len: usize, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow a non-null `key`; `input_slice` handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `data`; `input_slice` handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly 8 readable bytes. + // SAFETY: The function contract requires a non-null `expected_tag` to reference exactly 8 + // readable, initialized bytes that remain immutable while copied into owned storage. let Some(expected_tag) = (unsafe { read_array::<8>(expected_tag) }) else { return STATUS_ERR; }; @@ -212,22 +239,35 @@ pub extern "C" fn ct_entry_hmac_sha256_verify_truncated_64( } /// BLAKE3 keyed-tag verification harness. +/// +/// # Safety +/// +/// - When non-null, `key` and `expected_tag` must each be valid for reads of 32 initialized bytes +/// and must not be mutated while copied. Null pointers are rejected. +/// - When non-null, `data` must reference a single allocation containing `data_len` initialized +/// bytes. `data_len` must be no greater than `isize::MAX`, and the range must not be mutated +/// during the call. +/// - A null `data` pointer represents an empty slice only when `data_len == 0`; other null shapes +/// are rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_blake3_verify_keyed( +pub unsafe extern "C" fn ct_entry_blake3_verify_keyed( key: *const u8, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: The key pointer must reference exactly 32 readable bytes. + // SAFETY: The function contract requires a non-null `key` to reference exactly 32 readable, + // initialized bytes that remain immutable while copied into owned storage. let Some(key) = (unsafe { read_array::<32>(key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `data`; `input_slice` handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly 32 readable bytes. + // SAFETY: The function contract requires a non-null `expected_tag` to reference exactly 32 + // readable, initialized bytes that remain immutable while copied into owned storage. let Some(expected_tag) = (unsafe { read_array::<32>(expected_tag) }) else { return STATUS_ERR; }; @@ -239,8 +279,23 @@ pub extern "C" fn ct_entry_blake3_verify_keyed( } /// ChaCha20-Poly1305 open/authentication harness. +/// +/// # Safety +/// +/// - When non-null, `key`, `nonce`, and `tag` must be valid for reads of 32, 12, and 16 initialized +/// bytes respectively and must not be mutated while copied. Null pointers are rejected. +/// - When non-null, `aad` must reference a single allocation containing `aad_len` initialized +/// bytes. `aad_len` must be no greater than `isize::MAX`, and the range must not be mutated +/// during the call. +/// - When non-null, `buffer` must reference a single allocation containing `buffer_len` initialized +/// bytes. `buffer_len` must be no greater than `isize::MAX`, and the range must remain exclusively +/// accessible for reads and writes during the call. +/// - Null `aad` and `buffer` pointers represent empty slices only when their corresponding lengths +/// are zero; other null shapes are rejected. +/// - The buffer range must not overlap the AAD or tag ranges. It may overlap the key or nonce ranges +/// because those inputs are copied before the mutable buffer borrow begins. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_chacha20poly1305_open( +pub unsafe extern "C" fn ct_entry_chacha20poly1305_open( key: *const u8, nonce: *const u8, aad: *const u8, @@ -249,23 +304,30 @@ pub extern "C" fn ct_entry_chacha20poly1305_open( buffer_len: usize, tag: *const u8, ) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires a non-null `key` to reference exactly 32 readable, + // initialized bytes that remain immutable while copied into owned storage. let Some(key) = (unsafe { read_array::<32>(key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires a non-null `nonce` to reference exactly 12 readable, + // initialized bytes that remain immutable while copied into owned storage. let Some(nonce) = (unsafe { read_array::<12>(nonce) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, immutability, + // and lifetime required to borrow non-null `aad`; `input_slice` handles both null shapes. let Some(aad) = (unsafe { input_slice(aad, aad_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI output pointer is validated by `output_slice`. + // SAFETY: The function contract establishes the allocation, bounds, initialization, exclusivity, + // lifetime, and disjointness from the live AAD and tag ranges required to borrow `buffer` + // mutably; `output_slice` handles both null shapes. let Some(buffer) = (unsafe { output_slice(buffer, buffer_len) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null `tag` to reference exactly 16 readable, + // initialized bytes that remain immutable while copied and do not overlap the live mutable + // buffer borrow. let Some(tag) = (unsafe { read_array::<16>(tag) }) else { return STATUS_ERR; }; @@ -284,13 +346,23 @@ pub extern "C" fn ct_entry_chacha20poly1305_open( } /// X25519 scalar multiplication / shared-secret harness. +/// +/// # Safety +/// +/// - When non-null, `scalar` and `point` must each be valid for reads of 32 initialized bytes and +/// must not be mutated while copied. +/// - When non-null, `out` must be valid for writes of 32 bytes and must not be accessed while +/// written. +/// - Every non-null pointer must remain valid for the duration of the call. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_x25519(out: *mut u8, scalar: *const u8, point: *const u8) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. +pub unsafe extern "C" fn ct_entry_x25519(out: *mut u8, scalar: *const u8, point: *const u8) -> u8 { + // SAFETY: The function contract requires a non-null `scalar` to reference exactly 32 readable, + // initialized bytes that remain valid while `read_array` copies them into owned storage. let Some(scalar) = (unsafe { read_array::<32>(scalar) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires a non-null `point` to reference exactly 32 readable, + // initialized bytes that remain valid while `read_array` copies them into owned storage. let Some(point) = (unsafe { read_array::<32>(point) }) else { return STATUS_ERR; }; @@ -300,7 +372,8 @@ pub extern "C" fn ct_entry_x25519(out: *mut u8, scalar: *const u8, point: *const let Ok(shared) = secret.diffie_hellman(&public) else { return STATUS_ERR; }; - // SAFETY: The output pointer must reference exactly 32 writable bytes. + // SAFETY: The function contract requires a non-null `out` to reference exactly 32 writable bytes + // that remain valid and unaccessed while `write_array` copies the initialized shared secret. if unsafe { write_array(out, shared.as_bytes()) } { STATUS_OK } else { @@ -318,8 +391,24 @@ macro_rules! mlkem_ct_harness { $decapsulation_key:ty, $ciphertext:ty ) => { + /// Generate an ML-KEM key pair through the constant-time C ABI harness. + /// + /// # Safety + /// + /// - `encapsulation_key_out` must be valid for writes of + /// `<$profile>::ENCAPSULATION_KEY_SIZE` bytes. + /// - `decapsulation_key_out` must be valid for writes of + /// `<$profile>::DECAPSULATION_KEY_SIZE` bytes. + /// - `random` must be valid for reads of `<$profile>::KEY_GENERATION_RANDOM_SIZE` initialized + /// bytes. + /// - Each pointer must remain valid for the duration of the call, and writable regions must not + /// be accessed concurrently. #[unsafe(no_mangle)] - pub extern "C" fn $keygen(encapsulation_key_out: *mut u8, decapsulation_key_out: *mut u8, random: *const u8) -> u8 { + pub unsafe extern "C" fn $keygen( + encapsulation_key_out: *mut u8, + decapsulation_key_out: *mut u8, + random: *const u8, + ) -> u8 { // SAFETY: Copies ML-KEM key-generation randomness from FFI memory because: // 1. `read_array` rejects a null pointer before reading. // 2. The required length is exactly `<$profile>::KEY_GENERATION_RANDOM_SIZE`. @@ -353,18 +442,31 @@ macro_rules! mlkem_ct_harness { STATUS_OK } + /// Encapsulate an ML-KEM shared secret through the constant-time C ABI harness. + /// + /// # Safety + /// + /// - `ciphertext_out` must be valid for writes of `<$profile>::CIPHERTEXT_SIZE` bytes. + /// - `shared_secret_out` must be valid for writes of `<$profile>::SHARED_SECRET_SIZE` bytes. + /// - `encapsulation_key` must be valid for reads of `<$profile>::ENCAPSULATION_KEY_SIZE` + /// initialized bytes. + /// - `random` must be valid for reads of `<$profile>::ENCAPSULATION_RANDOM_SIZE` initialized + /// bytes. + /// - Each pointer must remain valid for the duration of the call, and writable regions must not + /// be accessed concurrently. #[unsafe(no_mangle)] - pub extern "C" fn $encapsulate( + pub unsafe extern "C" fn $encapsulate( ciphertext_out: *mut u8, shared_secret_out: *mut u8, encapsulation_key: *const u8, random: *const u8, ) -> u8 { - // SAFETY: Copies the ML-KEM encapsulation key from FFI memory because: - // 1. `read_array` rejects a null pointer before reading. - // 2. The required length is exactly `<$profile>::ENCAPSULATION_KEY_SIZE`. - // 3. The returned array is owned by this harness function before validation. let Some(encapsulation_key) = + // SAFETY: Copies the ML-KEM encapsulation key from FFI memory because: + // 1. `read_array` rejects a null pointer before reading. + // 2. The function contract requires exactly `<$profile>::ENCAPSULATION_KEY_SIZE` readable + // bytes. + // 3. The returned array is owned by this harness function before validation. (unsafe { read_array::<{ <$profile>::ENCAPSULATION_KEY_SIZE }>(encapsulation_key) }) else { return STATUS_ERR; @@ -405,17 +507,28 @@ macro_rules! mlkem_ct_harness { STATUS_OK } + /// Decapsulate an ML-KEM shared secret through the constant-time C ABI harness. + /// + /// # Safety + /// + /// - `shared_secret_out` must be valid for writes of `<$profile>::SHARED_SECRET_SIZE` bytes. + /// - `decapsulation_key` must be valid for reads of `<$profile>::DECAPSULATION_KEY_SIZE` + /// initialized bytes. + /// - `ciphertext` must be valid for reads of `<$profile>::CIPHERTEXT_SIZE` initialized bytes. + /// - Each pointer must remain valid for the duration of the call, and `shared_secret_out` must + /// not be accessed concurrently. #[unsafe(no_mangle)] - pub extern "C" fn $decapsulate( + pub unsafe extern "C" fn $decapsulate( shared_secret_out: *mut u8, decapsulation_key: *const u8, ciphertext: *const u8, ) -> u8 { - // SAFETY: Copies the ML-KEM decapsulation key from FFI memory because: - // 1. `read_array` rejects a null pointer before reading. - // 2. The required length is exactly `<$profile>::DECAPSULATION_KEY_SIZE`. - // 3. The returned array is owned by this harness function before use. let Some(decapsulation_key) = + // SAFETY: Copies the ML-KEM decapsulation key from FFI memory because: + // 1. `read_array` rejects a null pointer before reading. + // 2. The function contract requires exactly `<$profile>::DECAPSULATION_KEY_SIZE` readable + // bytes. + // 3. The returned array is owned by this harness function before use. (unsafe { read_array::<{ <$profile>::DECAPSULATION_KEY_SIZE }>(decapsulation_key) }) else { return STATUS_ERR; @@ -474,9 +587,16 @@ mlkem_ct_harness!( MlKem1024Ciphertext ); +/// Run the ML-KEM forward-NTT diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_ntt(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_ntt(poly: *const u16) -> u16 { + // SAFETY: The function contract requires a non-null `poly` to be aligned for `u16` and valid for + // reads of exactly 256 initialized coefficients that remain immutable while copied. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -493,9 +613,16 @@ pub extern "C" fn ct_entry_mlkem_diag_ntt(poly: *const u16) -> u16 { rscrypto::auth::mlkem::diag_mlkem_ntt_input_digest(poly) } +/// Run the ML-KEM inverse-NTT diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_inverse_ntt(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_inverse_ntt(poly: *const u16) -> u16 { + // SAFETY: The function contract requires a non-null `poly` to be aligned for `u16` and valid for + // reads of exactly 256 initialized coefficients that remain immutable while copied. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -512,9 +639,16 @@ pub extern "C" fn ct_entry_mlkem_diag_inverse_ntt(poly: *const u16) -> u16 { rscrypto::auth::mlkem::diag_mlkem_inverse_ntt_montgomery_product_input_digest(poly) } +/// Run the ML-KEM to-product-domain diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_to_product_domain(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_to_product_domain(poly: *const u16) -> u16 { + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // source whenever poly is non-null; read_u16_array rejects null before copying. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -531,9 +665,16 @@ pub extern "C" fn ct_entry_mlkem_diag_to_product_domain(poly: *const u16) -> u16 rscrypto::auth::mlkem::diag_mlkem_to_montgomery_product_domain_input_digest(poly) } +/// Run the ML-KEM from-product-domain diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `poly` must be aligned for `u16`, valid for reads of 256 initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_from_product_domain(poly: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. +pub unsafe extern "C" fn ct_entry_mlkem_diag_from_product_domain(poly: *const u16) -> u16 { + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // source whenever poly is non-null; read_u16_array rejects null before copying. let Some(poly) = (unsafe { read_u16_array::<256>(poly) }) else { return 0; }; @@ -550,21 +691,31 @@ pub extern "C" fn ct_entry_mlkem_diag_from_product_domain(poly: *const u16) -> u rscrypto::auth::mlkem::diag_mlkem_from_montgomery_product_domain_input_digest(poly) } +/// Run the ML-KEM NTT multiply/add diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, each of `a`, `b`, and `acc` must be aligned for `u16`, valid for reads +/// of 256 initialized coefficients, and immutable while copied. Null pointers are rejected. +/// The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_add_assign( +pub unsafe extern "C" fn ct_entry_mlkem_diag_multiply_ntts_add_assign( a: *const u16, b: *const u16, acc: *const u16, ) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // source whenever a is non-null; read_u16_array rejects null before copying. let Some(a) = (unsafe { read_u16_array::<256>(a) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract provides the same guarantees for b. Its range may overlap a + // because both operations only read and each result is independently owned. let Some(b) = (unsafe { read_u16_array::<256>(b) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract provides the same guarantees for acc. Its range may overlap a + // or b because all three operations only read and each result is independently owned. let Some(acc) = (unsafe { read_u16_array::<256>(acc) }) else { return 0; }; @@ -581,21 +732,31 @@ pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_add_assign( rscrypto::auth::mlkem::diag_mlkem_multiply_ntts_add_assign_input_digest(a, b, acc) } +/// Run the ML-KEM K=3 NTT accumulation diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `a` and `b` must each be aligned for `u16`, valid for reads of 768 +/// initialized coefficients, and immutable while copied. `acc` has the same obligations for +/// 256 coefficients. Null pointers are rejected. The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k3( +pub unsafe extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k3( a: *const u16, b: *const u16, acc: *const u16, ) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly three readable ML-KEM polynomials. + // SAFETY: The function contract requires an aligned, initialized, immutable 768-coefficient + // source whenever a is non-null; read_mlkem_polyvec rejects null before copying. let Some(a) = (unsafe { read_mlkem_polyvec::<3>(a) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly three readable ML-KEM polynomials. + // SAFETY: The function contract provides the same guarantees for b. Its range may overlap a + // because both operations only read and each result is independently owned. let Some(b) = (unsafe { read_mlkem_polyvec::<3>(b) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // acc source. Its range may overlap a or b because all three operations only read. let Some(acc) = (unsafe { read_u16_array::<256>(acc) }) else { return 0; }; @@ -611,21 +772,31 @@ pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k3( rscrypto::auth::mlkem::diag_mlkem768_multiply_ntts_accumulate_input_digest(a, b, acc) } +/// Run the ML-KEM K=4 NTT accumulation diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `a` and `b` must each be aligned for `u16`, valid for reads of 1,024 +/// initialized coefficients, and immutable while copied. `acc` has the same obligations for +/// 256 coefficients. Null pointers are rejected. The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k4( +pub unsafe extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k4( a: *const u16, b: *const u16, acc: *const u16, ) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly four readable ML-KEM polynomials. + // SAFETY: The function contract requires an aligned, initialized, immutable 1,024-coefficient + // source whenever a is non-null; read_mlkem_polyvec rejects null before copying. let Some(a) = (unsafe { read_mlkem_polyvec::<4>(a) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly four readable ML-KEM polynomials. + // SAFETY: The function contract provides the same guarantees for b. Its range may overlap a + // because both operations only read and each result is independently owned. let Some(b) = (unsafe { read_mlkem_polyvec::<4>(b) }) else { return 0; }; - // SAFETY: The diagnostic pointer must reference exactly one readable ML-KEM polynomial. + // SAFETY: The function contract requires an aligned, initialized, immutable 256-coefficient + // acc source. Its range may overlap a or b because all three operations only read. let Some(acc) = (unsafe { read_u16_array::<256>(acc) }) else { return 0; }; @@ -641,9 +812,16 @@ pub extern "C" fn ct_entry_mlkem_diag_multiply_ntts_accumulate_k4( rscrypto::auth::mlkem::diag_mlkem1024_multiply_ntts_accumulate_input_digest(a, b, acc) } +/// Run the ML-KEM compression/decompression diagnostic through the constant-time C ABI harness. +/// +/// # Safety +/// +/// When non-null, `values` must be aligned for `u16`, valid for reads of four initialized +/// coefficients, and immutable while copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_mlkem_diag_compress_decompress(values: *const u16) -> u16 { - // SAFETY: The diagnostic pointer must reference exactly four readable ML-KEM coefficient values. +pub unsafe extern "C" fn ct_entry_mlkem_diag_compress_decompress(values: *const u16) -> u16 { + // SAFETY: The function contract requires an aligned, initialized, immutable four-coefficient + // source whenever values is non-null; read_u16_array rejects null before copying. let Some(values) = (unsafe { read_u16_array::<4>(values) }) else { return 0; }; @@ -660,26 +838,42 @@ pub extern "C" fn ct_entry_mlkem_diag_compress_decompress(values: *const u16) -> rscrypto::auth::mlkem::diag_mlkem_compress_decompress_values_digest(values) } -/// Ed25519 signing harness. +/// Sign a message with Ed25519 through the constant-time C ABI harness. +/// +/// # Safety +/// +/// - When non-null, `secret_key` must be valid for reads of 32 initialized bytes and immutable +/// while copied. A null pointer is rejected. +/// - A null `message` represents an empty slice only when `message_len` is zero. When non-null, +/// `message` must reference one allocation of `message_len` initialized bytes, `message_len` +/// must not exceed `isize::MAX`, and the range must remain immutable while signing reads it. +/// - When non-null, `out` must be valid for writes of 64 bytes and must not be accessed +/// concurrently while written. A null pointer is rejected. +/// - The output may overlap either input because both input reads finish before the output write. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ed25519_sign( +pub unsafe extern "C" fn ct_entry_ed25519_sign( out: *mut u8, secret_key: *const u8, message: *const u8, message_len: usize, ) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null secret_key to expose 32 initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(secret_key) = (unsafe { read_array::<32>(secret_key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; let secret_key = Ed25519SecretKey::from_bytes(secret_key); let signature = secret_key.sign(message); - // SAFETY: The output pointer must reference exactly 64 writable bytes. + // SAFETY: The function contract requires a non-null out to expose 64 writable bytes without + // concurrent access. Both caller-memory reads have finished, so out may overlap either input; + // write_array rejects null before writing from the independently owned signature. if unsafe { write_array(out, signature.as_bytes()) } { STATUS_OK } else { @@ -687,24 +881,40 @@ pub extern "C" fn ct_entry_ed25519_sign( } } -/// ECDSA/P-256 signing harness with caller-supplied projective blinding. +/// Sign a message with ECDSA/P-256 and caller-supplied projective blinding. +/// +/// # Safety +/// +/// - When non-null, `secret_key` and `blind` must be valid for reads of 32 and 64 initialized +/// bytes, respectively, and immutable while copied. Null pointers are rejected. +/// - A null `message` represents an empty slice only when `message_len` is zero. When non-null, +/// `message` must reference one allocation of `message_len` initialized bytes, `message_len` +/// must not exceed `isize::MAX`, and the range must remain immutable while signing reads it. +/// - When non-null, `out` must be valid for writes of 64 bytes and must not be accessed +/// concurrently while written. A null pointer is rejected. +/// - The read-only inputs may overlap. The output may overlap any input because all input reads +/// finish before the output write. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ecdsa_p256_sign( +pub unsafe extern "C" fn ct_entry_ecdsa_p256_sign( out: *mut u8, secret_key: *const u8, blind: *const u8, message: *const u8, message_len: usize, ) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null secret_key to expose 32 initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(secret_key) = (unsafe { read_array::<32>(secret_key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null blind to expose 64 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(blind) = (unsafe { read_array::<64>(blind) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; @@ -717,7 +927,9 @@ pub extern "C" fn ct_entry_ecdsa_p256_sign( }; let signature = signature.to_bytes(); - // SAFETY: The output pointer must reference exactly 64 writable bytes. + // SAFETY: The function contract requires a non-null out to expose 64 writable bytes without + // concurrent access. All caller-memory reads have finished, so out may overlap any input; + // write_array rejects null before writing from the independently owned signature. if unsafe { write_array(out, &signature) } { STATUS_OK } else { @@ -725,24 +937,40 @@ pub extern "C" fn ct_entry_ecdsa_p256_sign( } } -/// ECDSA/P-384 signing harness with caller-supplied projective blinding. +/// Sign a message with ECDSA/P-384 and caller-supplied projective blinding. +/// +/// # Safety +/// +/// - When non-null, `secret_key` and `blind` must be valid for reads of 48 and 96 initialized +/// bytes, respectively, and immutable while copied. Null pointers are rejected. +/// - A null `message` represents an empty slice only when `message_len` is zero. When non-null, +/// `message` must reference one allocation of `message_len` initialized bytes, `message_len` +/// must not exceed `isize::MAX`, and the range must remain immutable while signing reads it. +/// - When non-null, `out` must be valid for writes of 96 bytes and must not be accessed +/// concurrently while written. A null pointer is rejected. +/// - The read-only inputs may overlap. The output may overlap any input because all input reads +/// finish before the output write. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ecdsa_p384_sign( +pub unsafe extern "C" fn ct_entry_ecdsa_p384_sign( out: *mut u8, secret_key: *const u8, blind: *const u8, message: *const u8, message_len: usize, ) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null secret_key to expose 48 initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(secret_key) = (unsafe { read_array::<48>(secret_key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires a non-null blind to expose 96 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(blind) = (unsafe { read_array::<96>(blind) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; @@ -755,7 +983,9 @@ pub extern "C" fn ct_entry_ecdsa_p384_sign( }; let signature = signature.to_bytes(); - // SAFETY: The output pointer must reference exactly 96 writable bytes. + // SAFETY: The function contract requires a non-null out to expose 96 writable bytes without + // concurrent access. All caller-memory reads have finished, so out may overlap any input; + // write_array rejects null before writing from the independently owned signature. if unsafe { write_array(out, &signature) } { STATUS_OK } else { @@ -763,9 +993,21 @@ pub extern "C" fn ct_entry_ecdsa_p384_sign( } } -/// PBKDF2-HMAC-SHA256 verification harness. +/// Verify a PBKDF2-HMAC-SHA256 password through the constant-time C ABI harness. +/// +/// # Safety +/// +/// Each `(pointer, length)` pair for `password`, `salt`, and `expected` must satisfy all of the +/// following: +/// +/// - a null pointer is used only with a zero length; +/// - a non-null pointer references one allocation of `length` initialized bytes; +/// - `length` does not exceed `isize::MAX`; and +/// - the referenced bytes remain immutable for the call. +/// +/// The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_pbkdf2_sha256_verify( +pub unsafe extern "C" fn ct_entry_pbkdf2_sha256_verify( password: *const u8, password_len: usize, salt: *const u8, @@ -774,15 +1016,21 @@ pub extern "C" fn ct_entry_pbkdf2_sha256_verify( expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null password; input_slice handles both null + // shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null salt; input_slice handles both null + // shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null expected value; input_slice handles both + // null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; @@ -794,23 +1042,38 @@ pub extern "C" fn ct_entry_pbkdf2_sha256_verify( macro_rules! fixed_tag_verify_entry { ($name:ident, $ty:ty, $tag_ty:ty, $tag_len:literal) => { + #[doc = "Verify a fixed-size authentication tag through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `key` and `data` pointers are permitted only with matching zero lengths. Each"] + #[doc = " non-null pointer must reference one allocation of initialized bytes, its length must"] + #[doc = " not exceed `isize::MAX`, and the range must remain immutable for the call."] + #[doc = "- When non-null, `expected_tag` must be valid for reads of the fixed tag length,"] + #[doc = " contain initialized bytes, and remain immutable while copied. Null is rejected."] + #[doc = "- The three read-only ranges may overlap."] #[unsafe(no_mangle)] - pub extern "C" fn $name( + pub unsafe extern "C" fn $name( key: *const u8, key_len: usize, data: *const u8, data_len: usize, expected_tag: *const u8, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for a non-null key; input_slice + // handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice` / `read_array`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null data; input_slice + // handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: The expected tag pointer must reference exactly `$tag_len` readable bytes. + // SAFETY: The generated function contract requires a non-null expected_tag to expose + // $tag_len initialized, readable bytes that remain immutable while read_array copies them. let Some(expected_tag) = (unsafe { read_array::<$tag_len>(expected_tag) }) else { return STATUS_ERR; }; @@ -828,14 +1091,23 @@ fixed_tag_verify_entry!(ct_entry_hmac_sha512_verify, HmacSha512, HmacSha512Tag, macro_rules! fixed_owner_eq_entry { ($name:ident, $type:ty, $len:expr) => { + #[doc = "Compare two fixed-size secret-owning values through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "When non-null, `a` and `b` must each be valid for reads of the selected fixed length,"] + #[doc = "contain initialized bytes, and remain immutable while copied. Null is rejected. The"] + #[doc = "two read-only ranges may overlap."] #[inline(never)] #[unsafe(no_mangle)] - pub extern "C" fn $name(a: *const u8, b: *const u8) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + pub unsafe extern "C" fn $name(a: *const u8, b: *const u8) -> u8 { + // SAFETY: The generated function contract requires non-null a to expose $len initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(a) = (unsafe { read_array::<$len>(a) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The generated function contract requires non-null b to expose $len initialized, + // readable bytes that remain immutable while read_array copies them into owned storage. let Some(b) = (unsafe { read_array::<$len>(b) }) else { return STATUS_ERR; }; @@ -854,9 +1126,16 @@ fixed_owner_eq_entry!(ct_entry_owner_eq_1632, MlKem512DecapsulationKey, 1632); fixed_owner_eq_entry!(ct_entry_owner_eq_2400, MlKem768DecapsulationKey, 2400); fixed_owner_eq_entry!(ct_entry_owner_eq_3168, MlKem1024DecapsulationKey, 3168); +/// Check that a 32-byte secret's `Debug` representation remains redacted. +/// +/// # Safety +/// +/// When non-null, `secret` must be valid for reads of 32 initialized bytes and immutable while +/// copied. A null pointer is rejected. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_secret_bytes32_debug_masked(secret: *const u8) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. +pub unsafe extern "C" fn ct_entry_secret_bytes32_debug_masked(secret: *const u8) -> u8 { + // SAFETY: The function contract requires a non-null secret to expose 32 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(secret) = (unsafe { read_array::<32>(secret) }) else { return STATUS_ERR; }; @@ -865,8 +1144,21 @@ pub extern "C" fn ct_entry_secret_bytes32_debug_masked(secret: *const u8) -> u8 u8::from(formatted == "SecretBytes(****)") } +/// Verify a KMAC256 tag through the constant-time C ABI harness. +/// +/// # Safety +/// +/// Each `(pointer, length)` pair for `key`, `customization`, `data`, and `expected_tag` must +/// satisfy all of the following: +/// +/// - a null pointer is used only with a zero length; +/// - a non-null pointer references one allocation of `length` initialized bytes; +/// - `length` does not exceed `isize::MAX`; and +/// - the referenced bytes remain immutable for the call. +/// +/// The four read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_kmac256_verify( +pub unsafe extern "C" fn ct_entry_kmac256_verify( key: *const u8, key_len: usize, customization: *const u8, @@ -876,19 +1168,25 @@ pub extern "C" fn ct_entry_kmac256_verify( expected_tag: *const u8, expected_tag_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null key; input_slice handles both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null customization; input_slice handles both + // null shapes. let Some(customization) = (unsafe { input_slice(customization, customization_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for a non-null expected_tag; input_slice handles both + // null shapes. let Some(expected_tag) = (unsafe { input_slice(expected_tag, expected_tag_len) }) else { return STATUS_ERR; }; @@ -900,8 +1198,26 @@ pub extern "C" fn ct_entry_kmac256_verify( macro_rules! aead_open_entry { ($name:ident, $cipher:ty, $key:ty, $nonce:ty, $key_len:literal, $nonce_len:literal) => { + #[doc = "Authenticate and decrypt a buffer in place through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Non-null `key` and `nonce` pointers must expose the selected fixed lengths of"] + #[doc = " initialized readable bytes and remain immutable while copied. Null is rejected."] + #[doc = "- A null `aad` pointer is permitted only when `aad_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `aad_len` initialized bytes, `aad_len` must not"] + #[doc = " exceed `isize::MAX`, and the range must remain immutable for the call."] + #[doc = "- A null `buffer` pointer is permitted only when `buffer_len` is zero. A non-null"] + #[doc = " pointer must reference one allocation of `buffer_len` initialized writable bytes,"] + #[doc = " `buffer_len` must not exceed `isize::MAX`, and the range must be exclusively"] + #[doc = " accessed for the call."] + #[doc = "- When non-null, `tag` must expose 16 initialized readable bytes and remain immutable"] + #[doc = " while copied. Null is rejected."] + #[doc = "- Nonempty `buffer` ranges must be disjoint from `aad` and `tag`. The buffer may"] + #[doc = " overlap `key` or `nonce` because both are copied before its mutable borrow begins."] + #[doc = " Read-only ranges may overlap one another."] #[unsafe(no_mangle)] - pub extern "C" fn $name( + pub unsafe extern "C" fn $name( key: *const u8, nonce: *const u8, aad: *const u8, @@ -910,23 +1226,30 @@ macro_rules! aead_open_entry { buffer_len: usize, tag: *const u8, ) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The generated function contract requires non-null key to expose $key_len + // initialized, readable bytes that remain immutable while read_array copies them. let Some(key) = (unsafe { read_array::<$key_len>(key) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The generated function contract requires non-null nonce to expose $nonce_len + // initialized, readable bytes that remain immutable while read_array copies them. let Some(nonce) = (unsafe { read_array::<$nonce_len>(nonce) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null aad; input_slice handles + // both null shapes. let Some(aad) = (unsafe { input_slice(aad, aad_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI output pointer is validated by `output_slice`. + // SAFETY: The generated function contract establishes buffer's allocation, length bound, + // initialization, exclusivity, and lifetime. It also requires disjointness from the live aad + // and tag reads; output_slice handles both null shapes. let Some(buffer) = (unsafe { output_slice(buffer, buffer_len) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The generated function contract requires non-null tag to expose 16 initialized, + // readable bytes, remain immutable while copied, and be disjoint from the live buffer. let Some(tag) = (unsafe { read_array::<16>(tag) }) else { return STATUS_ERR; }; @@ -984,8 +1307,22 @@ aead_open_entry!( macro_rules! hkdf_derive_entry { ($name:ident, $ty:ty) => { + #[doc = "Derive HKDF output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `salt`, `ikm`, and `info` pointers are permitted only with matching zero"] + #[doc = " lengths. Each non-null pointer must reference one allocation of initialized bytes,"] + #[doc = " its length must not exceed `isize::MAX`, and the range must remain immutable for"] + #[doc = " the call."] + #[doc = "- A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `out_len` initialized writable bytes, `out_len`"] + #[doc = " must not exceed `isize::MAX`, and the range must be exclusively accessed for the"] + #[doc = " call."] + #[doc = "- Input ranges may overlap one another. A nonempty output range must be disjoint from"] + #[doc = " every input range because the shared and mutable borrows remain live together."] #[unsafe(no_mangle)] - pub extern "C" fn $name( + pub unsafe extern "C" fn $name( salt: *const u8, salt_len: usize, ikm: *const u8, @@ -995,19 +1332,27 @@ macro_rules! hkdf_derive_entry { out: *mut u8, out_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null ikm; input_slice handles + // both null shapes. let Some(ikm) = (unsafe { input_slice(ikm, ikm_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null info; input_slice handles + // both null shapes. let Some(info) = (unsafe { input_slice(info, info_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes out's allocation, length bound, + // initialization, exclusivity, and lifetime, including disjointness from all live inputs; + // output_slice handles both null shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; @@ -1024,8 +1369,22 @@ hkdf_derive_entry!(ct_entry_hkdf_sha384_derive, HkdfSha384); macro_rules! pbkdf2_entry { ($derive_name:ident, $verify_name:ident, $ty:ty) => { + #[doc = "Derive PBKDF2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `password` and `salt` pointers are permitted only with matching zero lengths."] + #[doc = " Each non-null pointer must reference one allocation of initialized bytes, its"] + #[doc = " length must not exceed `isize::MAX`, and the range must remain immutable for the"] + #[doc = " call."] + #[doc = "- A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `out_len` initialized writable bytes, `out_len`"] + #[doc = " must not exceed `isize::MAX`, and the range must be exclusively accessed for the"] + #[doc = " call."] + #[doc = "- Password and salt may overlap. A nonempty output range must be disjoint from both"] + #[doc = " input ranges because their shared and mutable borrows remain live together."] #[unsafe(no_mangle)] - pub extern "C" fn $derive_name( + pub unsafe extern "C" fn $derive_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1034,15 +1393,21 @@ macro_rules! pbkdf2_entry { out: *mut u8, out_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes out's allocation, length bound, + // initialization, exclusivity, and lifetime, including disjointness from both live inputs; + // output_slice handles both null shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; @@ -1052,8 +1417,16 @@ macro_rules! pbkdf2_entry { .unwrap_or(STATUS_ERR) } + #[doc = "Verify PBKDF2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "Null `password`, `salt`, and `expected` pointers are permitted only with matching"] + #[doc = "zero lengths. Each non-null pointer must reference one allocation of initialized"] + #[doc = "bytes, its length must not exceed `isize::MAX`, and the range must remain immutable"] + #[doc = "for the call. The three read-only ranges may overlap."] #[unsafe(no_mangle)] - pub extern "C" fn $verify_name( + pub unsafe extern "C" fn $verify_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1062,15 +1435,21 @@ macro_rules! pbkdf2_entry { expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null expected output; + // input_slice handles both null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; @@ -1095,8 +1474,22 @@ pbkdf2_entry!( macro_rules! argon2_entry { ($derive_name:ident, $verify_name:ident, $ty:ty) => { + #[doc = "Derive Argon2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "- Null `password` and `salt` pointers are permitted only with matching zero lengths."] + #[doc = " Each non-null pointer must reference one allocation of initialized bytes, its"] + #[doc = " length must not exceed `isize::MAX`, and the range must remain immutable for the"] + #[doc = " call."] + #[doc = "- A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer"] + #[doc = " must reference one allocation of `out_len` initialized writable bytes, `out_len`"] + #[doc = " must not exceed `isize::MAX`, and the range must be exclusively accessed for the"] + #[doc = " call."] + #[doc = "- Password and salt may overlap. A nonempty output range must be disjoint from both"] + #[doc = " input ranges because their shared and mutable borrows remain live together."] #[unsafe(no_mangle)] - pub extern "C" fn $derive_name( + pub unsafe extern "C" fn $derive_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1106,15 +1499,21 @@ macro_rules! argon2_entry { out: *mut u8, out_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes out's allocation, length bound, + // initialization, exclusivity, and lifetime, including disjointness from both live inputs; + // output_slice handles both null shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; @@ -1127,8 +1526,16 @@ macro_rules! argon2_entry { .unwrap_or(STATUS_ERR) } + #[doc = "Verify Argon2 output through the constant-time C ABI harness."] + #[doc = ""] + #[doc = "# Safety"] + #[doc = ""] + #[doc = "Null `password`, `salt`, and `expected` pointers are permitted only with matching"] + #[doc = "zero lengths. Each non-null pointer must reference one allocation of initialized"] + #[doc = "bytes, its length must not exceed `isize::MAX`, and the range must remain immutable"] + #[doc = "for the call. The three read-only ranges may overlap."] #[unsafe(no_mangle)] - pub extern "C" fn $verify_name( + pub unsafe extern "C" fn $verify_name( password: *const u8, password_len: usize, salt: *const u8, @@ -1138,15 +1545,21 @@ macro_rules! argon2_entry { expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null password; input_slice + // handles both null shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null salt; input_slice handles + // both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null expected output; + // input_slice handles both null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; @@ -1165,8 +1578,21 @@ argon2_entry!(ct_entry_argon2i_hash, ct_entry_argon2i_verify, Argon2i); argon2_entry!(ct_entry_argon2d_hash, ct_entry_argon2d_verify, Argon2d); argon2_entry!(ct_entry_argon2id_hash, ct_entry_argon2id_verify, Argon2id); +/// Verify scrypt output through the constant-time C ABI harness. +/// +/// # Safety +/// +/// Each `(pointer, length)` pair for `password`, `salt`, and `expected` must satisfy all of the +/// following: +/// +/// - a null pointer is used only with a zero length; +/// - a non-null pointer references one allocation of `length` initialized bytes; +/// - `length` does not exceed `isize::MAX`; and +/// - the referenced bytes remain immutable for the call. +/// +/// The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_scrypt_verify( +pub unsafe extern "C" fn ct_entry_scrypt_verify( password: *const u8, password_len: usize, salt: *const u8, @@ -1177,20 +1603,24 @@ pub extern "C" fn ct_entry_scrypt_verify( expected: *const u8, expected_len: usize, ) -> u8 { - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null password; input_slice handles both null + // shapes. let Some(password) = (unsafe { input_slice(password, password_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null salt; input_slice handles both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null expected output; input_slice handles both + // null shapes. let Some(expected) = (unsafe { input_slice(expected, expected_len) }) else { return STATUS_ERR; }; - let Ok(params) = ScryptParams::new(log_n, r, p) - else { + let Ok(params) = ScryptParams::new(log_n, r, p) else { return STATUS_ERR; }; @@ -1199,22 +1629,36 @@ pub extern "C" fn ct_entry_scrypt_verify( .unwrap_or(STATUS_ERR) } +/// Verify an Ed25519 signature through the constant-time C ABI harness. +/// +/// # Safety +/// +/// - When non-null, `public_key` and `signature` must be valid for reads of 32 and 64 initialized +/// bytes, respectively, and immutable while copied. Null pointers are rejected. +/// - A null `message` pointer is permitted only when `message_len` is zero. A non-null pointer +/// must reference one allocation of `message_len` initialized bytes, `message_len` must not +/// exceed `isize::MAX`, and the range must remain immutable for the call. +/// - The three read-only ranges may overlap. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_ed25519_verify( +pub unsafe extern "C" fn ct_entry_ed25519_verify( public_key: *const u8, message: *const u8, message_len: usize, signature: *const u8, ) -> u8 { - // SAFETY: Fixed-size FFI inputs are copied by value after null checks. + // SAFETY: The function contract requires non-null public_key to expose 32 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(public_key) = (unsafe { read_array::<32>(public_key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null message; input_slice handles both null + // shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; - // SAFETY: Fixed-size FFI input is copied by value after null check. + // SAFETY: The function contract requires non-null signature to expose 64 initialized, readable + // bytes that remain immutable while read_array copies them into owned storage. let Some(signature) = (unsafe { read_array::<64>(signature) }) else { return STATUS_ERR; }; @@ -1225,8 +1669,21 @@ pub extern "C" fn ct_entry_ed25519_verify( .unwrap_or(STATUS_ERR) } +/// Sign with RSA PKCS#1 v1.5 and caller-supplied blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `message`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1238,23 +1695,33 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null message; input_slice + // handles both null shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return STATUS_ERR; }; @@ -1267,8 +1734,7 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( .sign_pkcs1v15_with_blinding_factor_and_scratch( RsaPkcs1v15Profile::Sha256, message, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), out, &mut scratch, ) @@ -1276,8 +1742,21 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_sign_fixed_blinding( .unwrap_or(STATUS_ERR) } +/// Sign with RSA-PSS and caller-supplied salt and blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `message`, `salt`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1291,27 +1770,39 @@ pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> u8 { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null message; input_slice + // handles both null shapes. let Some(message) = (unsafe { input_slice(message, message_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null salt; input_slice + // handles both null shapes. let Some(salt) = (unsafe { input_slice(salt, salt_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return STATUS_ERR; }; @@ -1325,8 +1816,7 @@ pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( RsaPssProfile::Sha256, message, salt, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), out, &mut scratch, ) @@ -1334,8 +1824,21 @@ pub extern "C" fn ct_entry_rsa_pss_sign_fixed_blinding( .unwrap_or(STATUS_ERR) } +/// Decrypt RSA-OAEP with caller-supplied blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `label`, `ciphertext`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1349,27 +1852,39 @@ pub extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> usize { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null label; input_slice + // handles both null shapes. let Some(label) = (unsafe { input_slice(label, label_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null ciphertext; input_slice + // handles both null shapes. let Some(ciphertext) = (unsafe { input_slice(ciphertext, ciphertext_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return usize::MAX; }; @@ -1383,16 +1898,28 @@ pub extern "C" fn ct_entry_rsa_oaep_decrypt_fixed_blinding( RsaOaepProfile::Sha256, label, ciphertext, - blinding_factor, - blinding_inverse, + RsaBlindingPair::new(blinding_factor, blinding_inverse), out, &mut scratch, ) .unwrap_or(usize::MAX) } +/// Decrypt RSAES-PKCS1-v1_5 with caller-supplied blinding through the CT harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - Each `(pointer, length)` pair for `pkcs8_der`, `ciphertext`, `blinding_factor`, and +/// `blinding_inverse` may use a null pointer only with a zero length. Every non-null pointer must +/// reference one allocation of initialized bytes, its length must not exceed `isize::MAX`, and +/// the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from every input range. Read-only input ranges may +/// overlap one another. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( +pub unsafe extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( out: *mut u8, out_len: usize, pkcs8_der: *const u8, @@ -1404,23 +1931,33 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( blinding_inverse: *const u8, blinding_inverse_len: usize, ) -> usize { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from every live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null ciphertext; input_slice + // handles both null shapes. let Some(ciphertext) = (unsafe { input_slice(ciphertext, ciphertext_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_factor; + // input_slice handles both null shapes. let Some(blinding_factor) = (unsafe { input_slice(blinding_factor, blinding_factor_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null blinding_inverse; + // input_slice handles both null shapes. let Some(blinding_inverse) = (unsafe { input_slice(blinding_inverse, blinding_inverse_len) }) else { return usize::MAX; }; @@ -1430,22 +1967,42 @@ pub extern "C" fn ct_entry_rsa_pkcs1v15_decrypt_fixed_blinding( }; let mut scratch = key.private_scratch(); key - .decrypt_pkcs1v15_with_blinding_factor_and_scratch(ciphertext, blinding_factor, blinding_inverse, out, &mut scratch) + .decrypt_pkcs1v15_with_blinding_factor_and_scratch( + ciphertext, + RsaBlindingPair::new(blinding_factor, blinding_inverse), + out, + &mut scratch, + ) .unwrap_or(usize::MAX) } +/// Parse and re-encode the CT fixture RSA private key through the C ABI harness. +/// +/// # Safety +/// +/// - A null `out` pointer is permitted only when `out_len` is zero. A non-null pointer must +/// reference one allocation of `out_len` initialized writable bytes, `out_len` must not exceed +/// `isize::MAX`, and the range must be exclusively accessed for the call. +/// - A null `pkcs8_der` pointer is permitted only when `pkcs8_der_len` is zero. A non-null pointer +/// must reference one allocation of `pkcs8_der_len` initialized bytes, `pkcs8_der_len` must not +/// exceed `isize::MAX`, and the range must remain immutable for the call. +/// - A nonempty output range must be disjoint from the input range. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_rsa_private_key_pkcs8_roundtrip( +pub unsafe extern "C" fn ct_entry_rsa_private_key_pkcs8_roundtrip( out: *mut u8, out_len: usize, pkcs8_der: *const u8, pkcs8_der_len: usize, ) -> usize { - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes out's allocation, length bound, initialization, + // exclusivity, lifetime, and disjointness from the live input; output_slice handles both null + // shapes. let Some(out) = (unsafe { output_slice(out, out_len) }) else { return usize::MAX; }; - // SAFETY: FFI input/output pointers are validated by slice helpers. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, lifetime, and output disjointness required for non-null pkcs8_der; input_slice + // handles both null shapes. let Some(pkcs8_der) = (unsafe { input_slice(pkcs8_der, pkcs8_der_len) }) else { return usize::MAX; }; @@ -1463,13 +2020,37 @@ pub extern "C" fn ct_entry_rsa_private_key_pkcs8_roundtrip( macro_rules! blake2_keyed_entry { ($name:ident, $ty:ty, $key_ty:ty, $out_len:literal) => { + #[doc = concat!("Compute a keyed `", stringify!($ty), "` digest through the CT harness.")] + /// + /// # Safety + /// + /// - A null `key` or `data` pointer represents an empty slice only when its corresponding + /// length is zero. Every non-null pointer must reference one allocation of initialized + /// bytes, its length must not exceed `isize::MAX`, and the range must remain immutable + /// through its last use. The two read-only ranges may overlap. + #[doc = concat!( + "- When non-null, `out` must be valid for writes of ", + stringify!($out_len), + " bytes. A null output is rejected. The output may overlap caller input because hashing and input borrows end ", + "before the digest is copied out; it must not overlap the stack-owned digest." + )] #[unsafe(no_mangle)] - pub extern "C" fn $name(key: *const u8, key_len: usize, data: *const u8, data_len: usize, out: *mut u8) -> u8 { - // SAFETY: FFI input pointers are validated by slice helpers. + pub unsafe extern "C" fn $name( + key: *const u8, + key_len: usize, + data: *const u8, + data_len: usize, + out: *mut u8, + ) -> u8 { + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null key; input_slice handles + // both null shapes. let Some(key) = (unsafe { input_slice(key, key_len) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointers are validated by slice helpers. + // SAFETY: The generated function contract establishes the allocation, length bound, + // initialization, immutability, and lifetime required for non-null data; input_slice handles + // both null shapes. Shared key and data ranges may overlap. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; @@ -1478,7 +2059,9 @@ macro_rules! blake2_keyed_entry { return STATUS_ERR; }; let digest = <$ty>::keyed_digest(key, data); - // SAFETY: The output pointer must reference exactly `$out_len` writable bytes. + // SAFETY: The generated function contract requires non-null out to be valid for writes of + // the selected digest width. Hashing and the caller-input borrows have ended, and the + // contract prevents overlap with the stack-owned digest copied by write_array. if unsafe { write_array::<$out_len>(out, &digest) } { STATUS_OK } else { @@ -1493,19 +2076,41 @@ blake2_keyed_entry!(ct_entry_blake2b512_keyed_digest, Blake2b512, Blake2bKey, 64 blake2_keyed_entry!(ct_entry_blake2s128_keyed_digest, Blake2s128, Blake2sKey, 16); blake2_keyed_entry!(ct_entry_blake2s256_keyed_digest, Blake2s256, Blake2sKey, 32); +/// Compute a keyed BLAKE3 digest through the CT harness. +/// +/// # Safety +/// +/// - When non-null, `key` must be valid for reads of 32 initialized bytes and remain immutable +/// while copied. A null key is rejected. +/// - A null `data` pointer represents an empty slice only when `data_len` is zero. A non-null +/// pointer must reference one allocation of `data_len` initialized bytes, `data_len` must not +/// exceed `isize::MAX`, and the range must remain immutable through its last use. The key and +/// data ranges may overlap. +/// - When non-null, `out` must be valid for writes of 32 bytes. A null output is rejected. Output +/// may overlap caller input because hashing and input borrows end before the digest is copied +/// out; it must not overlap the stack-owned digest. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_blake3_keyed_digest(key: *const u8, data: *const u8, data_len: usize, out: *mut u8) -> u8 { - // SAFETY: Fixed-size FFI input is copied by value after null check. +pub unsafe extern "C" fn ct_entry_blake3_keyed_digest( + key: *const u8, + data: *const u8, + data_len: usize, + out: *mut u8, +) -> u8 { + // SAFETY: The function contract requires non-null key to expose 32 initialized, readable bytes + // that remain immutable while read_array copies them into owned storage. let Some(key) = (unsafe { read_array::<32>(key) }) else { return STATUS_ERR; }; - // SAFETY: FFI input pointer is validated by `input_slice`. + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; let digest = Blake3::keyed_digest(&key, data); - // SAFETY: The output pointer must reference exactly 32 writable bytes. + // SAFETY: The function contract requires non-null out to be valid for a 32-byte write. Hashing + // and caller-input borrows have ended, and the contract prevents overlap with the stack-owned + // digest copied by write_array. if unsafe { write_array(out, digest.as_bytes()) } { STATUS_OK } else { @@ -1513,14 +2118,27 @@ pub extern "C" fn ct_entry_blake3_keyed_digest(key: *const u8, data: *const u8, } } +/// Compute a public-data SHA-256 digest through the CT harness. +/// +/// # Safety +/// +/// - A null `data` pointer represents an empty slice only when `data_len` is zero. A non-null +/// pointer must reference one allocation of `data_len` initialized bytes, `data_len` must not +/// exceed `isize::MAX`, and the range must remain immutable through its last use. +/// - When non-null, `out` must be valid for writes of 32 bytes. A null output is rejected. Output +/// may overlap the input because hashing and the input borrow end before the digest is copied +/// out; it must not overlap the stack-owned digest. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_public_sha256_digest(data: *const u8, data_len: usize, out: *mut u8) -> u8 { - // SAFETY: FFI input pointer is validated by `input_slice`. +pub unsafe extern "C" fn ct_entry_public_sha256_digest(data: *const u8, data_len: usize, out: *mut u8) -> u8 { + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return STATUS_ERR; }; let digest = Sha256::digest(data); - // SAFETY: The output pointer must reference exactly 32 writable bytes. + // SAFETY: The function contract requires non-null out to be valid for a 32-byte write. Hashing + // and the caller-input borrow have ended, and the contract prevents overlap with the + // stack-owned digest copied by write_array. if unsafe { write_array(out, &digest) } { STATUS_OK } else { @@ -1528,9 +2146,17 @@ pub extern "C" fn ct_entry_public_sha256_digest(data: *const u8, data_len: usize } } +/// Compute a public-data IEEE CRC-32 checksum through the CT harness. +/// +/// # Safety +/// +/// A null `data` pointer represents an empty slice only when `data_len` is zero. A non-null +/// pointer must reference one allocation of `data_len` initialized bytes, `data_len` must not +/// exceed `isize::MAX`, and the range must remain immutable for the call. #[unsafe(no_mangle)] -pub extern "C" fn ct_entry_public_crc32_checksum(data: *const u8, data_len: usize) -> u32 { - // SAFETY: FFI input pointer is validated by `input_slice`. +pub unsafe extern "C" fn ct_entry_public_crc32_checksum(data: *const u8, data_len: usize) -> u32 { + // SAFETY: The function contract establishes the allocation, length bound, initialization, + // immutability, and lifetime required for non-null data; input_slice handles both null shapes. let Some(data) = (unsafe { input_slice(data, data_len) }) else { return 0; }; diff --git a/tools/ct-harness/src/main.rs b/tools/ct-harness/src/main.rs index b5426873..e31f4041 100644 --- a/tools/ct-harness/src/main.rs +++ b/tools/ct-harness/src/main.rs @@ -3,7 +3,7 @@ fn main() { macro_rules! retain { ($($entry:path),+ $(,)?) => { - $(let _ = std::hint::black_box($entry as *const ());)+ + $(let _ = core::hint::black_box($entry as *const ());)+ }; } diff --git a/tools/wasm-runtime-vectors/Cargo.lock b/tools/wasm-runtime-vectors/Cargo.lock index 290a790e..5ce59554 100644 --- a/tools/wasm-runtime-vectors/Cargo.lock +++ b/tools/wasm-runtime-vectors/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "rscrypto" -version = "0.7.8" +version = "0.8.1" [[package]] name = "rscrypto-wasm-runtime-vectors" diff --git a/tools/wasm-runtime-vectors/src/main.rs b/tools/wasm-runtime-vectors/src/main.rs index 1443bc25..4854c89a 100644 --- a/tools/wasm-runtime-vectors/src/main.rs +++ b/tools/wasm-runtime-vectors/src/main.rs @@ -1,25 +1,31 @@ use rscrypto::{Blake2b512, Blake3, Digest, Sha256, Sha512}; -fn hex_value(byte: u8) -> u8 { +fn hex_value(byte: u8) -> Option { match byte { - b'0'..=b'9' => byte - b'0', - b'a'..=b'f' => byte - b'a' + 10, - b'A'..=b'F' => byte - b'A' + 10, - _ => panic!("invalid hex digit"), + b'0'..=b'9' => Some(byte.strict_sub(b'0')), + b'a'..=b'f' => Some(byte.strict_sub(b'a').strict_add(10)), + b'A'..=b'F' => Some(byte.strict_sub(b'A').strict_add(10)), + _ => None, } } fn assert_hex(actual: &[u8], expected: &str) { assert_eq!(actual.len().strict_mul(2), expected.len()); for (i, chunk) in expected.as_bytes().chunks_exact(2).enumerate() { - let byte = (hex_value(chunk[0]) << 4) | hex_value(chunk[1]); + let high = hex_value(chunk[0]).expect("known hash vector must contain hexadecimal digits"); + let low = hex_value(chunk[1]).expect("known hash vector must contain hexadecimal digits"); + let byte = high.strict_shl(4) | low; assert_eq!(actual[i], byte, "hex mismatch at byte {i}"); } } fn patterned_bytes(len: usize) -> Vec { (0..len) - .map(|i| (i as u8).wrapping_mul(37).wrapping_add((i >> 8) as u8)) + .map(|i| { + i.to_le_bytes()[0] + .wrapping_mul(37) + .wrapping_add(i.strict_shr(8).to_le_bytes()[0]) + }) .collect() }