security-scan #47
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2021 Collate | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: security-scan | |
| on: | |
| schedule: | |
| - cron: "0 0 */2 * *" | |
| workflow_dispatch: | |
| inputs: | |
| SEND_SLACK_NOTIFICATION: | |
| description: "Post results to Slack (uncheck for test runs)" | |
| type: boolean | |
| required: false | |
| default: true | |
| env: | |
| # Manual runs may opt out of Slack; schedule (where inputs are unset) always sends. | |
| # Explicit empty-check, since `||` would treat the string 'false' inconsistently when the | |
| # input is typed `boolean` — only an empty string should fall back to 'true'. | |
| SEND_SLACK_NOTIFICATION: ${{ github.event.inputs.SEND_SLACK_NOTIFICATION == '' && 'true' || github.event.inputs.SEND_SLACK_NOTIFICATION }} | |
| jobs: | |
| vulnerability-scan: | |
| runs-on: ubuntu-latest | |
| environment: security-scan | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc" | |
| - name: Enable yarn | |
| run: corepack enable | |
| - name: Install UI dependencies | |
| working-directory: openmetadata-ui/src/main/resources/ui | |
| run: yarn install --frozen-lockfile --ignore-scripts | |
| - name: Run Retire.js scan | |
| id: retire-scan | |
| continue-on-error: true | |
| working-directory: openmetadata-ui/src/main/resources/ui | |
| run: | | |
| npx retire@5 \ | |
| --path node_modules/ \ | |
| --severity high \ | |
| --outputformat json \ | |
| --outputpath retire-report.json | |
| - name: Verify report was generated | |
| working-directory: openmetadata-ui/src/main/resources/ui | |
| run: | | |
| if [ ! -f retire-report.json ]; then | |
| echo '::error::retire-report.json was not generated — retire scan may have crashed' | |
| exit 1 | |
| fi | |
| - name: Upload Retire.js Report | |
| if: success() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: retire-js-report | |
| path: openmetadata-ui/src/main/resources/ui/retire-report.json | |
| retention-days: 30 | |
| - name: Generate Retire.js Slack summary | |
| if: always() | |
| run: | | |
| mkdir -p retire-report | |
| python3 scripts/retire_slack_summary.py \ | |
| openmetadata-ui/src/main/resources/ui/retire-report.json \ | |
| --counts-file retire-report/_retire_counts.json \ | |
| --slack-file retire-report/_retire_slack.txt \ | |
| > /dev/null | |
| - name: Upload Retire.js Slack artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: retire-slack | |
| path: | | |
| retire-report/_retire_slack.txt | |
| retire-report/_retire_counts.json | |
| retention-days: 30 | |
| - name: Publish Retire.js Summary | |
| if: success() | |
| working-directory: openmetadata-ui/src/main/resources/ui | |
| run: | | |
| python3 - << 'EOF' >> $GITHUB_STEP_SUMMARY | |
| import json | |
| SEVERITY_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"} | |
| SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3} | |
| NM = "node_modules/" | |
| def escape(text): | |
| return str(text).replace('|', '\\|').replace('`', "'") | |
| try: | |
| with open("retire-report.json") as f: | |
| data = json.load(f) | |
| except FileNotFoundError: | |
| print("## Retire.js Scan Results\n\n> Report file not found — scan may not have run.") | |
| raise SystemExit(0) | |
| findings = data.get("data", []) | |
| libs = {} | |
| for item in findings: | |
| filepath = item.get("file", "") | |
| short = filepath[filepath.find(NM) + len(NM):] if NM in filepath else filepath | |
| for result in item.get("results", []): | |
| key = (result.get("component", ""), result.get("version", "")) | |
| if key not in libs: | |
| libs[key] = {"files": [], "vulns": result.get("vulnerabilities", [])} | |
| if short not in libs[key]["files"]: | |
| libs[key]["files"].append(short) | |
| print("## Retire.js Scan Results\n") | |
| if not libs: | |
| print("✅ No vulnerable libraries found.") | |
| else: | |
| total_vulns = sum(len(v["vulns"]) for v in libs.values()) | |
| print(f"> **{len(libs)} vulnerable librar{'y' if len(libs) == 1 else 'ies'} · {total_vulns} CVE{'s' if total_vulns != 1 else ''} found**\n") | |
| for (component, version), info in sorted(libs.items(), key=lambda x: min( | |
| (SEVERITY_ORDER.get(v.get("severity", "low"), 3) for v in x[1]["vulns"]), default=3)): | |
| top_sev = min(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)) | |
| icon = SEVERITY_ICON.get(top_sev.get("severity", "low"), "⚪") | |
| print(f"### {icon} {component} {version}\n") | |
| print("| Severity | CVE | Summary |") | |
| print("|---|---|---|") | |
| for vuln in sorted(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)): | |
| sev = vuln.get("severity", "") | |
| ids = vuln.get("identifiers", {}) | |
| cves = ids.get("CVE", []) | |
| summary = ids.get("summary", "").split("\n")[0][:120] | |
| cve_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves) if cves else ids.get("githubID", "—") | |
| print(f"| {SEVERITY_ICON.get(sev, '')} {sev} | {escape(cve_str)} | {escape(summary)} |") | |
| print("\n**Bundled in:**") | |
| for f in info["files"]: | |
| print(f"- `{f}`") | |
| print() | |
| EOF | |
| - name: Force failure on vulnerabilities found | |
| if: steps.retire-scan.outcome == 'failure' | |
| run: exit 1 | |
| security-scan: | |
| runs-on: ubuntu-latest | |
| environment: security-scan | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| SNYK_ORGANIZATION: ${{ secrets.SNYK_ORGANIZATION_ID }} | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: false | |
| docker-images: true | |
| swap-storage: true | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: "21" | |
| distribution: "temurin" | |
| - name: Install Ubuntu dependencies | |
| run: | | |
| # stop relying on apt cache of GitHub runners | |
| sudo apt-get update | |
| sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ | |
| librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev | |
| # Install and Authenticate to Snyk | |
| - name: Install Snyk & Authenticate | |
| run: | | |
| sudo make install_antlr_cli | |
| sudo npm install -g snyk | |
| snyk auth ${SNYK_TOKEN} | |
| snyk config set org=${SNYK_ORGANIZATION} | |
| - name: Install Python dependencies | |
| run: | | |
| python3 -m venv env | |
| source env/bin/activate | |
| make install_all install_apis | |
| - name: Maven build | |
| id: maven-build | |
| continue-on-error: true | |
| run: mvn -DskipTests clean install | |
| - name: Run Scan | |
| id: security-report | |
| if: steps.maven-build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| source env/bin/activate | |
| rm -rf security-report | |
| mkdir -p security-report | |
| # Run snyk subtargets directly; skip `export-snyk-pdf-report` which deletes JSONs after PDF conversion. | |
| make snyk-ingestion-report || true | |
| make snyk-ingestion-base-slim-report || true | |
| make snyk-airflow-apis-report || true | |
| make snyk-server-report || true | |
| make snyk-ui-report || true | |
| - name: Publish Snyk Summary | |
| id: snyk-summary | |
| if: always() && steps.maven-build.outcome == 'success' | |
| run: | | |
| python3 scripts/snyk_summary.py security-report \ | |
| --counts-file security-report/_counts.json \ | |
| --slack-file security-report/_slack.txt \ | |
| >> $GITHUB_STEP_SUMMARY | |
| # Expose counts as step output for downstream gating. | |
| counts=$(cat security-report/_counts.json) | |
| echo "counts=$counts" >> $GITHUB_OUTPUT | |
| high=$(jq '.high + .critical' security-report/_counts.json) | |
| echo "high_critical=$high" >> $GITHUB_OUTPUT | |
| - name: Fail on high/critical Snyk findings | |
| if: always() && steps.snyk-summary.outputs.high_critical != '' && steps.snyk-summary.outputs.high_critical != '0' | |
| run: | | |
| echo "::error::Snyk found ${{ steps.snyk-summary.outputs.high_critical }} high/critical vulnerabilities (see Job Summary)" | |
| exit 1 | |
| - name: Generate Snyk HTML/PDF | |
| if: always() && steps.maven-build.outcome == 'success' | |
| run: | | |
| # Back up JSONs because html_to_pdf.py deletes them after PDF conversion. | |
| mkdir -p /tmp/snyk-json-backup | |
| cp security-report/*.json /tmp/snyk-json-backup/ 2>/dev/null || true | |
| make export-snyk-pdf-report || true | |
| # Restore JSONs alongside generated PDFs/HTMLs. | |
| cp /tmp/snyk-json-backup/*.json security-report/ 2>/dev/null || true | |
| - name: Upload Snyk Reports | |
| if: always() && steps.maven-build.outcome == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: security-report | |
| path: security-report | |
| retention-days: 30 | |
| - name: Force failure | |
| if: steps.maven-build.outcome != 'success' || steps.security-report.outcome != 'success' | |
| run: | | |
| exit 1 | |
| notify: | |
| runs-on: ubuntu-latest | |
| environment: security-scan | |
| needs: [vulnerability-scan, security-scan] | |
| if: always() | |
| steps: | |
| - name: Download Snyk artifact | |
| if: needs.security-scan.result != 'skipped' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: security-report | |
| path: security-report | |
| continue-on-error: true | |
| - name: Download Retire.js Slack artifact | |
| if: needs.vulnerability-scan.result != 'skipped' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: retire-slack | |
| path: retire-report | |
| continue-on-error: true | |
| - name: Build Slack header payload | |
| id: build-header | |
| env: | |
| RETIRE_RESULT: ${{ needs.vulnerability-scan.result }} | |
| SNYK_RESULT: ${{ needs.security-scan.result }} | |
| REF_NAME: ${{ github.ref_name }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| python3 - <<'PY' > header.json | |
| import json, os, pathlib | |
| retire = os.environ["RETIRE_RESULT"] | |
| snyk = os.environ["SNYK_RESULT"] | |
| ref_name = os.environ["REF_NAME"] | |
| run_url = os.environ["RUN_URL"] | |
| def status_icon(s): | |
| return {"success": "✅", "cancelled": "⚠️ (cancelled)", "skipped": "⚠️ (skipped)"}.get(s, "❌") | |
| def load_counts(path): | |
| p = pathlib.Path(path) | |
| if not p.exists(): | |
| return None | |
| try: | |
| return json.loads(p.read_text()) | |
| except Exception: | |
| return None | |
| def totals_line(counts): | |
| if not counts: | |
| return "" | |
| return ( | |
| f"\n🚨 {counts.get('critical', 0)} critical · " | |
| f"🔴 {counts.get('high', 0)} high · " | |
| f"🟠 {counts.get('medium', 0)} medium · " | |
| f"🟡 {counts.get('low', 0)} low" | |
| ) | |
| retire_counts = load_counts("retire-report/_retire_counts.json") | |
| snyk_counts = load_counts("security-report/_counts.json") | |
| if retire == "success" and snyk == "success": | |
| overall = "🟢" | |
| elif retire == "failure" or snyk == "failure": | |
| overall = "🚨" | |
| else: | |
| overall = "⚠️" | |
| header = ( | |
| f"{overall} *Security scan* — *OpenMetadata Repo*\n" | |
| f"_branch_ `{ref_name}` · <{run_url}|Open run details>\n" | |
| f"• Vulnerability scan (Retire.js): {status_icon(retire)}" | |
| f"{totals_line(retire_counts)}\n" | |
| f"• Security scan (Snyk): {status_icon(snyk)}" | |
| f"{totals_line(snyk_counts)}" | |
| ) | |
| fallback = f"{overall} Security scan — OpenMetadata Repo on {ref_name}. {run_url}" | |
| payload = { | |
| "text": fallback, | |
| "blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": header}}], | |
| } | |
| print(json.dumps(payload)) | |
| PY | |
| - name: Send Slack — header | |
| id: send-header | |
| if: env.SEND_SLACK_NOTIFICATION == 'true' | |
| uses: slackapi/slack-github-action@v1.27.1 | |
| with: | |
| channel-id: ${{ secrets.SLACK_CHANNEL_IDS }} | |
| payload-file-path: header.json | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| - name: Build Slack thread payload | |
| id: build-thread | |
| if: always() && steps.send-header.outputs.ts != '' | |
| env: | |
| THREAD_TS: ${{ steps.send-header.outputs.ts }} | |
| run: | | |
| python3 - <<'PY' > thread.json | |
| import json, os, pathlib | |
| thread_ts = os.environ["THREAD_TS"] | |
| def read_sections(path): | |
| p = pathlib.Path(path) | |
| if not p.exists(): | |
| return [] | |
| text = p.read_text().strip() | |
| return [s.strip() for s in text.split("\n\n") if s.strip()] | |
| retire_sections = read_sections("retire-report/_retire_slack.txt") | |
| snyk_sections = read_sections("security-report/_slack.txt") | |
| blocks = [] | |
| MAX_TEXT = 2850 | |
| def push(section): | |
| text = section if len(section) <= MAX_TEXT else section[:MAX_TEXT].rstrip() + "\n_…truncated, see Job Summary_" | |
| blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}}) | |
| for section in retire_sections[:24]: | |
| push(section) | |
| if retire_sections and snyk_sections: | |
| blocks.append({"type": "divider"}) | |
| for section in snyk_sections[:23]: | |
| push(section) | |
| if not blocks: | |
| push("_No detailed scan output was produced._") | |
| payload = { | |
| "thread_ts": thread_ts, | |
| "text": "Security scan details (thread reply)", | |
| "blocks": blocks, | |
| } | |
| print(json.dumps(payload)) | |
| PY | |
| - name: Send Slack — thread reply | |
| if: always() && env.SEND_SLACK_NOTIFICATION == 'true' && steps.send-header.outputs.ts != '' && steps.build-thread.outcome == 'success' | |
| uses: slackapi/slack-github-action@v1.27.1 | |
| with: | |
| channel-id: ${{ secrets.SLACK_CHANNEL_IDS }} | |
| payload-file-path: thread.json | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} |