diff --git a/.detect-secrets-ignore b/.detect-secrets-ignore index 904a107..8b5cba4 100644 --- a/.detect-secrets-ignore +++ b/.detect-secrets-ignore @@ -9,3 +9,36 @@ # Documentation files contain placeholder/example credentials only README\.md CHANGELOG\.md + +# Python artifacts and caches +\.mypy_cache +\.pytest_cache +\.tox +\.venv +venv +.*\.egg-info +__pycache__ +.*\.pyc +.*\.pyo +.*\.pyd +.*\.so +.*\.egg +.*\.whl +.*\.pth +pip-wheel-metadata +\.Python + +# Environment / config files (may contain local paths but not real secrets) +.*\.env\.example +.*\.env\.sample +.*\.env\.template +.*\.env\.test + +# Editor and IDE artifacts +\.vscode +\.idea +.*\.sublime-workspace +.*\.sublime-project +.*\.swp +.*\.swo +.*~ \ No newline at end of file diff --git a/.gitignore b/.gitignore index c41a155..5043c91 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,4 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc +.envrc diff --git a/.secrets.baseline b/.secrets.baseline index 32cf433..f555a74 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -159,5 +159,5 @@ } ] }, - "generated_at": "2026-07-03T00:22:23Z" + "generated_at": "2026-08-04T17:17:22Z" } diff --git a/scripts/detect_secrets_baseline.sh b/scripts/detect_secrets_baseline.sh index b6c3e34..6411da0 100755 --- a/scripts/detect_secrets_baseline.sh +++ b/scripts/detect_secrets_baseline.sh @@ -1,114 +1,211 @@ #!/bin/bash -# Single source of truth for detect-secrets arguments. -# Per-repo exclusions go in .detect-secrets-ignore (one regex per line, # for comments). +# +# detect_secrets_baseline.sh — manage the detect-secrets baseline for this repo. # # Usage: -# scripts/detect_secrets_baseline.sh scan # Regenerate .secrets.baseline -# scripts/detect_secrets_baseline.sh audit # Interactively audit .secrets.baseline -# scripts/detect_secrets_baseline.sh # Check for new secrets vs baseline -set -e - -# Prefer venv's detect-secrets over system install -if [ -f "venv/bin/detect-secrets" ]; then - DETECT_SECRETS="venv/bin/detect-secrets" -elif [ -f ".venv/bin/detect-secrets" ]; then - DETECT_SECRETS=".venv/bin/detect-secrets" -else - DETECT_SECRETS="detect-secrets" -fi - -# Global excludes applied in every repo +# scripts/detect_secrets_baseline.sh # CI mode: verify nothing new or unresolved +# scripts/detect_secrets_baseline.sh scan # Regenerate .secrets.baseline from scratch +# scripts/detect_secrets_baseline.sh audit # Interactively classify each finding +# +# Per-repo path exclusions go in .detect-secrets-ignore (one regex per line; # comments ok). + +set -euo pipefail + +BASELINE=".secrets.baseline" +BASELINE_IGNORE=".detect-secrets-ignore" + +# --------------------------------------------------------------------------- +# Locate detect-secrets binary (prefer local venv over system install) +# --------------------------------------------------------------------------- + +find_detect_secrets() { + if [[ -f "venv/bin/detect-secrets" ]]; then + echo "venv/bin/detect-secrets" + elif [[ -f ".venv/bin/detect-secrets" ]]; then + echo ".venv/bin/detect-secrets" + else + echo "detect-secrets" + fi +} + +DETECT_SECRETS="$(find_detect_secrets)" + +# --------------------------------------------------------------------------- +# Build --exclude-files arguments from global + per-repo patterns +# --------------------------------------------------------------------------- + +# Paths that are always excluded, regardless of repo type. GLOBAL_EXCLUDES=( - '\.secrets\..*' - '\.git.*' - '\.pre-commit-config\.yaml' - '\.mypy_cache' - '\.pytest_cache' - '\.tox' - '\.venv' + '\.secrets\..*' # the baseline files themselves + '(^|/)\.git/' # git internals (path-segment anchored to avoid blocking .github/) + '\.pre-commit-config\.yaml' # pre-commit config (often contains hook refs) + 'node_modules' # JS dependencies + 'dist' # build output + 'build' # build output + '\.venv' # Python virtual envs 'venv' - 'dist' - 'build' - '.*\.egg-info' - 'scripts/detect_secrets_baseline\.sh' + 'scripts/detect_secrets_baseline\.sh' # this script ) -EXCLUDE_ARGS=() -for pat in "${GLOBAL_EXCLUDES[@]}"; do - EXCLUDE_ARGS+=(--exclude-files "$pat") -done - -# Per-repo excludes from .detect-secrets-ignore (one regex per line, # comments ok) -if [ -f .detect-secrets-ignore ]; then - while IFS= read -r line || [ -n "$line" ]; do - [[ "$line" =~ ^[[:space:]]*# ]] && continue - [[ -z "${line// }" ]] && continue - EXCLUDE_ARGS+=(--exclude-files "$line") - done < .detect-secrets-ignore -fi - -compare_secrets() { - diff \ - <(python3 -c " -import json, sys -with open(sys.argv[1]) as f: data = json.load(f) -lines = [f\"{k},{s['hashed_secret']}\" for k, v in data.get('results', {}).items() for s in v] -print('\n'.join(sorted(lines))) -" "$1") \ - <(python3 -c " +build_exclude_args() { + local args=() + + for pattern in "${GLOBAL_EXCLUDES[@]}"; do + args+=(--exclude-files "$pattern") + done + + # Per-repo additions from .detect-secrets-ignore + if [[ -f "$BASELINE_IGNORE" ]]; then + while IFS= read -r line || [[ -n "$line" ]]; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue # skip comments + [[ -z "${line// }" ]] && continue # skip blank lines + args+=(--exclude-files "$line") + done < "$BASELINE_IGNORE" + fi + + echo "${args[@]}" +} + +# Store as an array so it expands correctly when passed to detect-secrets. +read -ra EXCLUDE_ARGS <<< "$(build_exclude_args)" + +# --------------------------------------------------------------------------- +# Helper: extract a sorted "file,hash" list from a baseline JSON file. +# Used to compare two baselines without caring about key ordering. +# --------------------------------------------------------------------------- + +baseline_fingerprints() { + local file="$1" + python3 - "$file" <<'PYTHON' import json, sys -with open(sys.argv[1]) as f: data = json.load(f) -lines = [f\"{k},{s['hashed_secret']}\" for k, v in data.get('results', {}).items() for s in v] -print('\n'.join(sorted(lines))) -" "$2") \ - >/dev/null + +with open(sys.argv[1]) as fh: + data = json.load(fh) + +fingerprints = [ + f"{filename},{secret['hashed_secret']}" + for filename, secrets in data.get("results", {}).items() + for secret in secrets +] + +print("\n".join(sorted(fingerprints))) +PYTHON } -if [ "$1" = "scan" ]; then - $DETECT_SECRETS scan "${EXCLUDE_ARGS[@]}" > .secrets.baseline - echo "Updated .secrets.baseline" - echo "Next step: run 'scripts/detect_secrets_baseline.sh audit' to review and classify detected secrets." -elif [ "$1" = "audit" ]; then - $DETECT_SECRETS audit .secrets.baseline -else - if [ ! -f .secrets.baseline ]; then - echo "❌ .secrets.baseline not found. Run 'scripts/detect_secrets_baseline.sh scan' to generate it." >&2 - exit 1 - fi +baselines_match() { + local a="$1" b="$2" + diff <(baseline_fingerprints "$a") <(baseline_fingerprints "$b") > /dev/null +} + +# --------------------------------------------------------------------------- +# Helper: count findings in a baseline that match a given Python condition. +# $1 = baseline file, $2 = Python expression that evaluates to True/False +# per secret dict (variable name: `s`) +# --------------------------------------------------------------------------- - # Check 1: Fail if any secrets in the baseline have not been audited - unaudited=$(python3 -c " +count_findings() { + local file="$1" + local condition="$2" + python3 - "$file" "$condition" <<'PYTHON' import json, sys -with open('.secrets.baseline') as f: data = json.load(f) -count = sum(1 for v in data.get('results', {}).values() for s in v if 'is_secret' not in s) + +with open(sys.argv[1]) as fh: + data = json.load(fh) + +condition = sys.argv[2] +count = sum( + 1 + for secrets in data.get("results", {}).values() + for s in secrets + if eval(condition) +) + print(count) -") - if [ "$unaudited" -gt 0 ]; then - echo "⚠️ Attention Required! ⚠️" >&2 - echo "$unaudited secret(s) in .secrets.baseline have not been audited." >&2 - echo "Run 'scripts/detect_secrets_baseline.sh audit' to review and classify each detected secret." >&2 - exit 1 +PYTHON +} + +# --------------------------------------------------------------------------- +# CI checks (default mode — no argument) +# --------------------------------------------------------------------------- + +check_unaudited_findings() { + local count + count="$(count_findings "$BASELINE" '"is_secret" not in s')" + + if [[ "$count" -gt 0 ]]; then + echo "⚠️ $count finding(s) in $BASELINE have not been audited yet." >&2 + echo " Run: scripts/detect_secrets_baseline.sh audit" >&2 + return 1 fi +} - # Check 2: Fail if any new secrets are detected that are not in the baseline - cp .secrets.baseline .secrets.new - trap 'rm -f .secrets.new' EXIT - $DETECT_SECRETS scan "${EXCLUDE_ARGS[@]}" --baseline .secrets.new - - if ! compare_secrets .secrets.baseline .secrets.new; then - echo "⚠️ Attention Required! ⚠️" >&2 - echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2 - echo "" >&2 - echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2 - echo "" >&2 - echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2 - echo "" >&2 - echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2 - echo "" >&2 - echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2 - rm -f .secrets.new - exit 1 +check_confirmed_secrets() { + local count + count="$(count_findings "$BASELINE" 's.get("is_secret") is True')" + + if [[ "$count" -gt 0 ]]; then + echo "⚠️ $count confirmed secret(s) are still present in $BASELINE." >&2 + echo " Remove them from the codebase, then re-run: scripts/detect_secrets_baseline.sh scan" >&2 + return 1 + fi +} + +check_new_secrets() { + local scratch="$BASELINE.new" + cp "$BASELINE" "$scratch" + # shellcheck disable=SC2064 + trap "rm -f '$scratch'" RETURN + + "$DETECT_SECRETS" scan "${EXCLUDE_ARGS[@]}" --baseline "$scratch" > /dev/null + + if ! baselines_match "$BASELINE" "$scratch"; then + cat >&2 < "$BASELINE" + echo "✅ $BASELINE updated." + echo " Next: scripts/detect_secrets_baseline.sh audit" + ;; + audit) + "$DETECT_SECRETS" audit "$BASELINE" + ;; + "") + run_ci_checks + echo "✅ No new secrets detected." + ;; + *) + echo "Usage: $0 [scan|audit]" >&2 + exit 1 + ;; +esac