Fix is_secret three-state check and use [[ ]] in detect_secrets_baseline.sh

.detect-secrets-ignore

@@ -9,3 +9,36 @@
 # Documentation files contain placeholder/example credentials only
 README\.md
 CHANGELOG\.md
+
+# Python artifacts and caches
+\.mypy_cache
+\.pytest_cache
+\.tox
+\.venv
+venv
+.*\.egg-info
+__pycache__
+.*\.pyc
+.*\.pyo
+.*\.pyd
+.*\.so
+.*\.egg
+.*\.whl
+.*\.pth
+pip-wheel-metadata
+\.Python
+
+# Environment / config files (may contain local paths but not real secrets)
+.*\.env\.example
+.*\.env\.sample
+.*\.env\.template
+.*\.env\.test
+
+# Editor and IDE artifacts
+\.vscode
+\.idea
+.*\.sublime-workspace
+.*\.sublime-project
+.*\.swp
+.*\.swo
+.*~

.gitignore

@@ -140,3 +140,4 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+.envrc

.secrets.baseline

@@ -159,5 +159,5 @@
       }
     ]
   },
-  "generated_at": "2026-07-03T00:22:23Z"
+  "generated_at": "2026-08-04T17:17:22Z"
 }

scripts/detect_secrets_baseline.sh

@@ -1,114 +1,211 @@
 #!/bin/bash
-# Single source of truth for detect-secrets arguments.
-# Per-repo exclusions go in .detect-secrets-ignore (one regex per line, # for comments).
+#
+# detect_secrets_baseline.sh — manage the detect-secrets baseline for this repo.
 #
 # Usage:
-#   scripts/detect_secrets_baseline.sh scan   # Regenerate .secrets.baseline
-#   scripts/detect_secrets_baseline.sh audit  # Interactively audit .secrets.baseline
-#   scripts/detect_secrets_baseline.sh        # Check for new secrets vs baseline
-set -e
-
-# Prefer venv's detect-secrets over system install
-if [ -f "venv/bin/detect-secrets" ]; then
-    DETECT_SECRETS="venv/bin/detect-secrets"
-elif [ -f ".venv/bin/detect-secrets" ]; then
-    DETECT_SECRETS=".venv/bin/detect-secrets"
-else
-    DETECT_SECRETS="detect-secrets"
-fi
-
-# Global excludes applied in every repo
+#   scripts/detect_secrets_baseline.sh          # CI mode: verify nothing new or unresolved
+#   scripts/detect_secrets_baseline.sh scan     # Regenerate .secrets.baseline from scratch
+#   scripts/detect_secrets_baseline.sh audit    # Interactively classify each finding
+#
+# Per-repo path exclusions go in .detect-secrets-ignore (one regex per line; # comments ok).
+
+set -euo pipefail
+
+BASELINE=".secrets.baseline"
+BASELINE_IGNORE=".detect-secrets-ignore"
+
+# ---------------------------------------------------------------------------
+# Locate detect-secrets binary (prefer local venv over system install)
+# ---------------------------------------------------------------------------
+
+find_detect_secrets() {
+    if [[ -f "venv/bin/detect-secrets" ]]; then
+        echo "venv/bin/detect-secrets"
+    elif [[ -f ".venv/bin/detect-secrets" ]]; then
+        echo ".venv/bin/detect-secrets"
+    else
+        echo "detect-secrets"
+    fi
+}
+
+DETECT_SECRETS="$(find_detect_secrets)"
+
+# ---------------------------------------------------------------------------
+# Build --exclude-files arguments from global + per-repo patterns
+# ---------------------------------------------------------------------------
+
+# Paths that are always excluded, regardless of repo type.
 GLOBAL_EXCLUDES=(
-    '\.secrets\..*'
-    '\.git.*'
-    '\.pre-commit-config\.yaml'
-    '\.mypy_cache'
-    '\.pytest_cache'
-    '\.tox'
-    '\.venv'
+    '\.secrets\..*'             # the baseline files themselves
+    '(^|/)\.git/'               # git internals (path-segment anchored to avoid blocking .github/)
+    '\.pre-commit-config\.yaml' # pre-commit config (often contains hook refs)
+    'node_modules'              # JS dependencies
+    'dist'                      # build output
+    'build'                     # build output
+    '\.venv'                    # Python virtual envs
     'venv'
-    'dist'
-    'build'
-    '.*\.egg-info'
-    'scripts/detect_secrets_baseline\.sh'
+    'scripts/detect_secrets_baseline\.sh'  # this script
 )
 
-EXCLUDE_ARGS=()
-for pat in "${GLOBAL_EXCLUDES[@]}"; do
-    EXCLUDE_ARGS+=(--exclude-files "$pat")
-done
-
-# Per-repo excludes from .detect-secrets-ignore (one regex per line, # comments ok)
-if [ -f .detect-secrets-ignore ]; then
-    while IFS= read -r line || [ -n "$line" ]; do
-        [[ "$line" =~ ^[[:space:]]*# ]] && continue
-        [[ -z "${line// }" ]] && continue
-        EXCLUDE_ARGS+=(--exclude-files "$line")
-    done < .detect-secrets-ignore
-fi
-
-compare_secrets() {
-    diff \
-        <(python3 -c "
-import json, sys
-with open(sys.argv[1]) as f: data = json.load(f)
-lines = [f\"{k},{s['hashed_secret']}\" for k, v in data.get('results', {}).items() for s in v]
-print('\n'.join(sorted(lines)))
-" "$1") \
-        <(python3 -c "
+build_exclude_args() {
+    local args=()
+
+    for pattern in "${GLOBAL_EXCLUDES[@]}"; do
+        args+=(--exclude-files "$pattern")
+    done
+
+    # Per-repo additions from .detect-secrets-ignore
+    if [[ -f "$BASELINE_IGNORE" ]]; then
+        while IFS= read -r line || [[ -n "$line" ]]; do
+            [[ "$line" =~ ^[[:space:]]*#  ]] && continue  # skip comments
+            [[ -z "${line// }"            ]] && continue  # skip blank lines
+            args+=(--exclude-files "$line")
+        done < "$BASELINE_IGNORE"
+    fi
+
+    echo "${args[@]}"
+}
+
+# Store as an array so it expands correctly when passed to detect-secrets.
+read -ra EXCLUDE_ARGS <<< "$(build_exclude_args)"
+
+# ---------------------------------------------------------------------------
+# Helper: extract a sorted "file,hash" list from a baseline JSON file.
+# Used to compare two baselines without caring about key ordering.
+# ---------------------------------------------------------------------------
+
+baseline_fingerprints() {
+    local file="$1"
+    python3 - "$file" <<'PYTHON'
 import json, sys
-with open(sys.argv[1]) as f: data = json.load(f)
-lines = [f\"{k},{s['hashed_secret']}\" for k, v in data.get('results', {}).items() for s in v]
-print('\n'.join(sorted(lines)))
-" "$2") \
-        >/dev/null
+
+with open(sys.argv[1]) as fh:
+    data = json.load(fh)
+
+fingerprints = [
+    f"{filename},{secret['hashed_secret']}"
+    for filename, secrets in data.get("results", {}).items()
+    for secret in secrets
+]
+
+print("\n".join(sorted(fingerprints)))
+PYTHON
 }
 
-if [ "$1" = "scan" ]; then
-    $DETECT_SECRETS scan "${EXCLUDE_ARGS[@]}" > .secrets.baseline
-    echo "Updated .secrets.baseline"
-    echo "Next step: run 'scripts/detect_secrets_baseline.sh audit' to review and classify detected secrets."
-elif [ "$1" = "audit" ]; then
-    $DETECT_SECRETS audit .secrets.baseline
-else
-    if [ ! -f .secrets.baseline ]; then
-        echo "❌ .secrets.baseline not found. Run 'scripts/detect_secrets_baseline.sh scan' to generate it." >&2
-        exit 1
-    fi
+baselines_match() {
+    local a="$1" b="$2"
+    diff <(baseline_fingerprints "$a") <(baseline_fingerprints "$b") > /dev/null
+}
+
+# ---------------------------------------------------------------------------
+# Helper: count findings in a baseline that match a given Python condition.
+# $1 = baseline file, $2 = Python expression that evaluates to True/False
+#      per secret dict (variable name: `s`)
+# ---------------------------------------------------------------------------
 
-    # Check 1: Fail if any secrets in the baseline have not been audited
-    unaudited=$(python3 -c "
+count_findings() {
+    local file="$1"
+    local condition="$2"
+    python3 - "$file" "$condition" <<'PYTHON'
 import json, sys
-with open('.secrets.baseline') as f: data = json.load(f)
-count = sum(1 for v in data.get('results', {}).values() for s in v if 'is_secret' not in s)
+
+with open(sys.argv[1]) as fh:
+    data = json.load(fh)
+
+condition = sys.argv[2]
+count = sum(
+    1
+    for secrets in data.get("results", {}).values()
+    for s in secrets
+    if eval(condition)
+)
+
 print(count)
-")
-    if [ "$unaudited" -gt 0 ]; then
-        echo "⚠️ Attention Required! ⚠️" >&2
-        echo "$unaudited secret(s) in .secrets.baseline have not been audited." >&2
-        echo "Run 'scripts/detect_secrets_baseline.sh audit' to review and classify each detected secret." >&2
-        exit 1
+PYTHON
+}
+
+# ---------------------------------------------------------------------------
+# CI checks (default mode — no argument)
+# ---------------------------------------------------------------------------
+
+check_unaudited_findings() {
+    local count
+    count="$(count_findings "$BASELINE" '"is_secret" not in s')"
+
+    if [[ "$count" -gt 0 ]]; then
+        echo "⚠️  $count finding(s) in $BASELINE have not been audited yet." >&2
+        echo "   Run: scripts/detect_secrets_baseline.sh audit" >&2
+        return 1
     fi
+}
 
-    # Check 2: Fail if any new secrets are detected that are not in the baseline
-    cp .secrets.baseline .secrets.new
-    trap 'rm -f .secrets.new' EXIT
-    $DETECT_SECRETS scan "${EXCLUDE_ARGS[@]}" --baseline .secrets.new
-
-    if ! compare_secrets .secrets.baseline .secrets.new; then
-        echo "⚠️ Attention Required! ⚠️" >&2
-        echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2
-        echo "" >&2
-        echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2
-        echo "" >&2
-        echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2
-        echo "" >&2
-        echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2
-        echo "" >&2
-        echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2
-        rm -f .secrets.new
-        exit 1
+check_confirmed_secrets() {
+    local count
+    count="$(count_findings "$BASELINE" 's.get("is_secret") is True')"
+
+    if [[ "$count" -gt 0 ]]; then
+        echo "⚠️  $count confirmed secret(s) are still present in $BASELINE." >&2
+        echo "   Remove them from the codebase, then re-run: scripts/detect_secrets_baseline.sh scan" >&2
+        return 1
+    fi
+}
+
+check_new_secrets() {
+    local scratch="$BASELINE.new"
+    cp "$BASELINE" "$scratch"
+    # shellcheck disable=SC2064
+    trap "rm -f '$scratch'" RETURN
+
+    "$DETECT_SECRETS" scan "${EXCLUDE_ARGS[@]}" --baseline "$scratch" > /dev/null
+
+    if ! baselines_match "$BASELINE" "$scratch"; then
+        cat >&2 <<EOF
+
+⚠️  New secrets detected that are not in $BASELINE.
+
+To investigate and resolve on your local machine:
+
+  1. Install detect-secrets:
+       pip install detect-secrets
+
+  2. Scan and review findings:
+       scripts/detect_secrets_baseline.sh scan
+       scripts/detect_secrets_baseline.sh audit
+
+  3. Commit the updated $BASELINE and re-push.
+
+Reference: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets
+
+EOF
+        return 1
     fi
+}
 
-    rm -f .secrets.new
-fi
+run_ci_checks() {
+    check_unaudited_findings
+    check_confirmed_secrets
+    check_new_secrets
+}
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+case "${1:-}" in
+    scan)
+        "$DETECT_SECRETS" scan "${EXCLUDE_ARGS[@]}" > "$BASELINE"
+        echo "✅ $BASELINE updated."
+        echo "   Next: scripts/detect_secrets_baseline.sh audit"
+        ;;
+    audit)
+        "$DETECT_SECRETS" audit "$BASELINE"
+        ;;
+    "")
+        run_ci_checks
+        echo "✅ No new secrets detected."
+        ;;
+    *)
+        echo "Usage: $0 [scan|audit]" >&2
+        exit 1
+        ;;
+esac