Qsafe post-quantum encryption, signed manifests, streaming AES v2, manifest hardening

.github/workflows/ci.yml

@@ -57,10 +57,23 @@ jobs:
           python -m pip install --upgrade pip
           pip install -e ".[dev,test]"
 
-      - name: Mypy
+      - name: Mypy (whole tree, advisory)
         run: mypy src
         continue-on-error: true  # gradual adoption
 
+      - name: Mypy (strict leaves, blocking)
+        # These modules opt in to strict checking via tool.mypy.overrides.
+        # Adding to that list above gates the new module on full typing.
+        run: |
+          mypy \
+            src/backup_handler/_paths.py \
+            src/backup_handler/encryption.py \
+            src/backup_handler/lock.py \
+            src/backup_handler/manifest.py \
+            src/backup_handler/qsafe_backend.py \
+            src/backup_handler/ssh_client.py \
+            src/backup_handler/status.py
+
   test:
     name: Test (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
@@ -84,8 +97,11 @@ jobs:
           pip install -e ".[test]"
 
       - name: Run tests with coverage
+        # --cov-fail-under reads the floor from pyproject [tool.coverage.report].
+        # CI fails when coverage drops below the floor; raise the floor as
+        # coverage grows.
         run: |
-          pytest --cov=src --cov-report=xml --cov-report=term-missing
+          pytest --cov=backup_handler --cov-report=xml --cov-report=term-missing
 
       - name: Upload coverage
         if: matrix.python-version == '3.12'
@@ -95,6 +111,69 @@ jobs:
           path: coverage.xml
           retention-days: 7
 
+  qsafe-integration:
+    # The roundtrip/tamper/signature tests in test_qsafe_backend.py skip
+    # when the qsafe CLI is absent — this job builds it from source so
+    # they actually run on every PR instead of silently skipping.
+    name: Qsafe integration (real CLI)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install build dependencies
+        run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev cmake git
+
+      - name: Cache liboqs
+        id: cache-liboqs
+        uses: actions/cache@v4
+        with:
+          path: ~/liboqs-install
+          key: liboqs-0.12.0-${{ runner.os }}
+
+      - name: Build liboqs
+        if: steps.cache-liboqs.outputs.cache-hit != 'true'
+        run: |
+          git clone --depth 1 --branch 0.12.0 https://github.com/open-quantum-safe/liboqs.git /tmp/liboqs
+          cmake -S /tmp/liboqs -B /tmp/liboqs/build \
+            -DCMAKE_BUILD_TYPE=Release -DOQS_BUILD_ONLY_LIB=ON \
+            -DCMAKE_INSTALL_PREFIX="$HOME/liboqs-install"
+          cmake --build /tmp/liboqs/build -j"$(nproc)"
+          cmake --install /tmp/liboqs/build
+
+      - name: Install liboqs system-wide
+        run: |
+          sudo cp -r "$HOME/liboqs-install/"* /usr/local/
+          sudo ldconfig
+
+      - name: Build and install qsafe
+        # SP1R4/Qsafe is private: set the QSAFE_REPO_TOKEN repo secret to a
+        # fine-grained PAT with read access to it. The anonymous fallback
+        # works if the repo is ever made public.
+        env:
+          QSAFE_TOKEN: ${{ secrets.QSAFE_REPO_TOKEN }}
+        run: |
+          if [ -n "$QSAFE_TOKEN" ]; then
+            git clone --depth 1 "https://x-access-token:${QSAFE_TOKEN}@github.com/SP1R4/Qsafe.git" /tmp/qsafe
+          else
+            git clone --depth 1 https://github.com/SP1R4/Qsafe.git /tmp/qsafe
+          fi
+          make -C /tmp/qsafe
+          sudo make -C /tmp/qsafe install
+          qsafe --version
+
+      - name: Install package
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[test]"
+
+      - name: Run qsafe integration tests
+        run: pytest tests/test_qsafe_backend.py -v --no-cov
+
   security:
     name: Security scan
     runs-on: ubuntu-latest
@@ -115,12 +194,19 @@ jobs:
         run: bandit -r src -c pyproject.toml
 
       - name: pip-audit (dependency scan)
-        run: pip-audit --strict --skip-editable
-        continue-on-error: true  # deps-only findings warn but don't block
+        # Audits the lockfile, not the installed environment — the project
+        # itself is not on PyPI, and --strict treats anything it cannot
+        # audit (editable installs, unpublished dists) as fatal.
+        run: pip-audit --strict -r requirements.txt
 
   secrets-scan:
     name: Secrets scan
     runs-on: ubuntu-latest
+    # Private repo: the action lists PR commits via the API, which needs
+    # explicit pull-requests read (public repos got this implicitly).
+    permissions:
+      contents: read
+      pull-requests: read
     steps:
       - uses: actions/checkout@v4
         with:

.gitignore

@@ -17,3 +17,4 @@ config/config.ini
 config/.bot_users.json
 .backup-handler.lock
 .claude/
+.coverage

.gitleaks.toml

@@ -0,0 +1,20 @@
+# Gitleaks configuration — extends the default ruleset.
+#
+# The project's own documentation discusses key management at length
+# ("key ceremony", "--key-file backup.key", "escrow/Shamir", ...), which
+# trips the generic-api-key rule on prose and CLI examples — including in
+# historical commits, where rewording cannot help. Docs are allowlisted;
+# code, config, and everything else remain fully scanned.
+
+[extend]
+useDefault = true
+
+[allowlist]
+description = "Project documentation — key-management prose and qsafe CLI placeholder examples"
+paths = [
+    '''ReadMe\.md''',
+    '''RUNBOOK\.md''',
+    '''CHANGELOG\.md''',
+    '''SECURITY\.md''',
+    '''docs/.+\.md''',
+]

CHANGELOG.md

@@ -7,8 +7,118 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Security
+
+- SSH host keys are now pinned via `paramiko.RejectPolicy()` + a known_hosts
+  file. Unknown hosts are refused with an actionable error instead of silently
+  accepted (was `WarningPolicy` — TOFU). New config: `[SSH] known_hosts`.
+- Encrypted file format **v1**: 4-byte magic + 1-byte KDF id bound into the
+  AES-GCM associated_data prevents downgrade attacks. Legacy `.enc` files auto-
+  detected and remain decryptable.
+- Argon2id KDF available alongside PBKDF2-HMAC-SHA256 (opt-in via
+  `[ENCRYPTION] kdf = argon2id` and the `[argon2]` extra).
+- Atomic `.enc` writes (tmp + fsync + os.replace) — crashes mid-encrypt no
+  longer leave a half-written ciphertext alongside the plaintext.
+- `assert_config_safe_for_hooks` refuses to run shell hooks when the config
+  file is group/world-writable. Bypass with `BACKUP_HANDLER_TRUST_CONFIG=1`.
+- `pip-audit` no longer `continue-on-error` in CI — known CVEs in cryptography
+  or paramiko now block merges.
+- Compression: replaced unmaintained `pyminizip` (ZipCrypto, weak) with
+  `pyzipper` (WinZip AES-256, pure Python).
+
+### Changed
+
+- Package layout: `src/` is now a src-layout shell containing
+  `src/backup_handler/`. The wheel installs as `backup_handler/`, not the old
+  `src/` at the site-packages root. `main.py` -> `backup_handler.cli`; the 1,314-
+  line cli split into `cli.py` (entrypoint), `dispatch.py` (early-exit handlers),
+  `orchestrator.py` (backup pipeline), `status.py`, `lock.py`, `_paths.py`.
+  Entry point: `backup-handler = backup_handler.cli:main`.
+- Filesystem paths are now XDG-aware. Config search:
+  `$BACKUP_HANDLER_CONFIG_DIR` -> `$XDG_CONFIG_HOME/backup-handler` ->
+  `/etc/backup-handler` -> bundled. Data (Logs, BackupTimestamp, snapshots)
+  follows the same shape via `$XDG_DATA_HOME` / `/var/lib/backup-handler`,
+  preserving existing package-root installs first.
+- Banner output gated to `sys.stdout.isatty()` — cron firings no longer log
+  decorative ANSI escapes.
+- `_copy_single_file` hashes source and destination once each instead of three
+  times (verify_backup re-hashed, then we re-hashed). Halves I/O on local copy.
+- Backup timestamp persistence (`update_last_backup_time`,
+  `update_last_full_backup_time`) writes atomically (tmp + fsync + rename).
+- Manifest filename validation tightened to `backup_manifest_YYYYMMDD_HHMMSS.json`
+  via regex; previously a prefix-strip happily included foreign or truncated files.
+- `requirements.txt` is now generated by `pip-compile` from `requirements.in`;
+  every transitive is annotated with its source. Dependency floors bumped:
+  cryptography 43->45, paramiko 3.4->3.5, boto3 1.28->1.34.
+- Schema version bumped to **4** for the new `[SSH] known_hosts`,
+  `[ENCRYPTION] kdf` keys, then to **5** for the new `[ENCRYPTION]`
+  Qsafe keys: `backend`, `qsafe_recipients`, `qsafe_secret_key`,
+  `qsafe_sign_key`, `qsafe_sign_pub`, `qsafe_sign_passphrase`.
+
 ### Added
 
+- **Qsafe post-quantum encryption backend** (`[ENCRYPTION] backend = qsafe`):
+  encrypts backups to one or more Qsafe recipient public keys using hybrid
+  X25519 + ML-KEM-1024 + AES-256-GCM (NIST FIPS 203, Level 5). The backup
+  host needs only *public* keys — the passphrase-wrapped secret key
+  (`qsafe_secret_key`) is required only for restore/verify and can live
+  off-host. Multi-recipient escrow supported via comma-separated
+  `qsafe_recipients`. `.enc` files self-describe their format by magic
+  bytes, so AES and Qsafe files coexist and old backups keep restoring.
+  Uses the `qsafe` CLI on PATH or the `libqsafe` Python bindings.
+- **Signed manifests** (`[ENCRYPTION] qsafe_sign_key` / `qsafe_sign_pub`):
+  each `backup_manifest_*.json` gets a detached post-quantum ML-DSA-87
+  signature at backup time. With `qsafe_sign_pub` set, `--verify` marks a
+  manifest with an invalid signature as corrupted, and a point-in-time
+  `--restore` refuses to replay it. Missing signatures (pre-signing
+  backups) only warn. `.sig` files are excluded from encryption. Works
+  with either encryption backend.
+- `--verify` authenticates Qsafe-encrypted files **in place** via the AEAD
+  tag — no plaintext is written to disk during verification (AES files
+  still decrypt to a temp directory for the size check).
+- Qsafe readiness preflight: a backup run aborts (exit 2, with critical
+  alert + status sentinel) before any files are copied when the qsafe
+  engine or configured key files are missing — previously encryption
+  failed *after* the copy, leaving a plaintext backup on disk.
+- RUNBOOK section 6: Qsafe key inventory, key ceremony, Shamir escrow,
+  rotation rules, and escrow-key drill requirements.
+- **Streaming AES format v2** (magic `BHE2`, written by default): files
+  are sealed as chunked AES-256-GCM with a per-chunk counter nonce and a
+  final-chunk flag, so encryption and decryption run in constant memory
+  (a 300 MB file roundtrips in ~34 MB RSS vs whole-file-in-RAM before)
+  and truncating, extending, or reordering chunks fails authentication.
+  v1 and legacy `.enc` files remain fully decryptable.
+- Manifests record each file's **backup-relative path** (`rel_path`).
+  Restore and verify resolve entries exactly instead of guessing by
+  filename — previously two same-named files in different subdirectories
+  could restore the wrong content silently.
+- Manifests record each file's **ciphertext SHA-256** (`enc_checksum`)
+  after encryption: `--verify` now proves encrypted-backup integrity with
+  no keys at all, and detects two validly-encrypted files being swapped —
+  which AEAD authentication alone cannot catch. Manifest signing runs
+  after this pass so signatures cover the checksums.
+- Point-in-time restore compares every restored file against the
+  manifest's recorded plaintext checksum and fails on mismatch — backup
+  content altered after the manifest was written can no longer restore
+  silently.
+- CI: new `qsafe-integration` job builds liboqs (cached) and the Qsafe
+  CLI from source, so the roundtrip/tamper/signature tests actually run
+  on every PR instead of skipping.
+- S3 sync sweeps multipart uploads older than 24 hours under the configured
+  prefix at the start of each run. Pairs with a bucket lifecycle rule to keep
+  storage costs bounded after crash-killed runs.
+- End-to-end test (`tests/test_e2e_backup.py`): full -> verify -> restore
+  roundtrip against a temp tree, encrypted and unencrypted variants.
+- New tests for sync, S3 multipart cleanup, manifest validation, XDG path
+  resolution, and the self-overwriting-restore refusal. 170 total.
+- Coverage gate via `[tool.coverage.report] fail_under = 25` and strict-mypy
+  job on the leaf modules (`_paths`, `encryption`, `lock`, `manifest`,
+  `ssh_client`, `status`).
+- `docs/` directory: deep-dive content (preflight, snapshots, install,
+  restore drill) moved out of the README so the README stays scannable.
+- README: "Why this vs. restic / borg / duplicity?" section.
+- `handle_restore` refuses when `--from-dir` and `--to-dir` resolve to the
+  same path or `--to-dir` is nested under `--from-dir`.
 - `--install` (`src/installer.py`): one-shot system bootstrap that
   performs every OS-level prerequisite the preflight self-heal needs.
   Single privileged invocation (`sudo -E backup-handler --install`)

CONTRIBUTING.md

@@ -54,11 +54,16 @@ Run everything locally before pushing:
 ruff check .
 ruff format --check .
 black --check .
-mypy src
-pytest --cov=src
+mypy src                                            # advisory
+mypy src/backup_handler/{_paths,encryption,lock,manifest,ssh_client,status}.py  # strict
+pytest --cov=backup_handler
 bandit -r src -c pyproject.toml
+pip-audit --strict --skip-editable
 ```
 
+The `pre-commit` hooks installed above run the lint and format checks on
+every commit, so you should rarely need to invoke them manually.
+
 ## Commit messages
 
 Follow the Conventional Commits convention:
@@ -111,7 +116,7 @@ All new features need tests. Update `tests/` alongside the code change.
 
 Maintainers only.
 
-1. Update `src/__version__.py`.
+1. Update `src/backup_handler/__version__.py`.
 2. Move `[Unreleased]` entries in `CHANGELOG.md` into a new versioned
    section with today's date.
 3. Commit: `chore(release): v<x.y.z>`.

Dockerfile

@@ -18,7 +18,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 COPY pyproject.toml ReadMe.md LICENSE ./
 COPY src/ ./src/
-COPY main.py ./
 
 RUN pip install --upgrade pip build \
     && python -m build --wheel --outdir /wheels

RUNBOOK.md

@@ -45,6 +45,10 @@ or a fresh VM.
 - The encryption passphrase or key file, if encryption is enabled. **The key
   is NOT stored on the destination.** Retrieve it from your secrets manager
   (Vault path: fill in before deployment).
+- For Qsafe-encrypted backups (`backend = qsafe`): the `qsafe_secret_key`
+  file **and** its passphrase — see section 6 for where each lives. Any one
+  recipient key decrypts, so the offline escrow key works if the ops key
+  is lost with the host.
 - Root/sudo on the restore target.
 
 **Procedure**
@@ -382,3 +386,75 @@ is misconfigured — fix it before relying on the next scheduled run.
    silent failures survive for months.
 5. If a command in this runbook doesn't match the system's current
    behavior, the runbook is wrong — update it in the same PR as the fix.
+
+---
+
+## 6. Qsafe key management (post-quantum encryption & signed manifests)
+
+When `[ENCRYPTION] backend = qsafe`, backups are encrypted to recipient
+*public* keys and the decryption secret never touches the backup host.
+That inverts the usual key-handling rules, so this section is normative.
+
+### 6.1 Key inventory
+
+| Key | Lives on | Compromise impact |
+|:--|:--|:--|
+| `ops.pub`, `escrow.pub` (encryption public keys) | Backup host (`qsafe_recipients`) | None — public by design |
+| `ops.key` (day-to-day decryption secret) | Restore/verify host or secrets manager. **Never the backup host.** | Read every backup |
+| `escrow.key` (recovery decryption secret) | Offline: sealed envelope, HSM, or Shamir shares | Read every backup |
+| `manifest-sign.key` (ML-DSA-87 signing secret) | Backup host (`qsafe_sign_key`) | Forge manifests — cannot read backups |
+| `manifest-sign.pub` (signing public key) | Restore/verify host (`qsafe_sign_pub`) | None |
+| Passphrases (wrap each secret key) | Secrets manager only (Vault path: fill in before deployment) | Unwraps the matching key |
+
+### 6.2 Key ceremony (one-time)
+
+Run on a trusted admin workstation, **not** on the backup host:
+
+```bash
+# Day-to-day keypair
+qsafe keygen --key-file ops.key --pub-file ops.pub
+# Offline escrow keypair (second recipient — any one key decrypts)
+qsafe keygen --key-file escrow.key --pub-file escrow.pub
+# Manifest signing keypair
+qsafe sign-keygen --key-file manifest-sign.key --pub-file manifest-sign.pub
+```
+
+Then distribute:
+
+1. Copy `ops.pub`, `escrow.pub`, and `manifest-sign.key` to the backup
+   host; reference them in `[ENCRYPTION]`.
+2. Move `ops.key` to the designated restore host or secrets manager.
+3. Take `escrow.key` offline. With Qsafe >= 7, prefer Shamir shares over a
+   single artifact — any 2 of 3 shares recover the key if the passphrase
+   custodian is unavailable:
+   ```bash
+   qsafe split-key --threshold 2 --shares 3 escrow
+   # -> escrow.share1..3: hand to separate custodians, then destroy escrow.key
+   ```
+4. Store every passphrase in the secrets manager. A secret key file and
+   its passphrase must never share a storage location.
+
+### 6.3 Rotation
+
+Adding or replacing a recipient only affects **future** backups — old
+`.enc` files remain decryptable solely by the keys they were encrypted to.
+Therefore:
+
+- To rotate: generate the new keypair, update `qsafe_recipients`, and
+  **retain the old secret key until every backup encrypted to it has aged
+  out of retention.** Deleting an old secret key orphans every backup that
+  listed only that recipient.
+- Signing-key rotation is cheaper: old manifests verify against the old
+  public key; keep it alongside the new one until those backups expire.
+
+### 6.4 Drill requirements
+
+The weekly restore drill (docs/restore-drill.md) must exercise the real
+recovery path, not the convenient one:
+
+- At least quarterly, run the drill decrypting with the **escrow** key
+  (reassemble from Shamir shares if split). An escrow key that has never
+  been used in a drill does not count as recovery insurance.
+- If manifest signing is enabled, the drill host must have
+  `qsafe_sign_pub` configured so a tampered manifest fails the drill —
+  restore aborts on an invalid signature by design.