security-scan.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: security-scan
on:
  schedule:
    - cron: "0 0 */2 * *"
  workflow_dispatch:
    inputs:
      SEND_SLACK_NOTIFICATION:
        description: "Post results to Slack (uncheck for test runs)"
        type: boolean
        required: false
        default: true

env:
  # Manual runs may opt out of Slack; schedule (where inputs are unset) always sends.
  # Explicit empty-check, since `||` would treat the string 'false' inconsistently when the
  # input is typed `boolean` — only an empty string should fall back to 'true'.
  SEND_SLACK_NOTIFICATION: ${{ github.event.inputs.SEND_SLACK_NOTIFICATION == '' && 'true' || github.event.inputs.SEND_SLACK_NOTIFICATION }}

jobs:
  vulnerability-scan:
    runs-on: ubuntu-latest
    environment: security-scan
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"

      - name: Enable yarn
        run: corepack enable

      - name: Install UI dependencies
        working-directory: openmetadata-ui/src/main/resources/ui
        run: yarn install --frozen-lockfile --ignore-scripts

      - name: Run Retire.js scan
        id: retire-scan
        continue-on-error: true
        working-directory: openmetadata-ui/src/main/resources/ui
        run: |
          npx retire@5 \
            --path node_modules/ \
            --severity high \
            --outputformat json \
            --outputpath retire-report.json

      - name: Verify report was generated
        working-directory: openmetadata-ui/src/main/resources/ui
        run: |
          if [ ! -f retire-report.json ]; then
            echo '::error::retire-report.json was not generated — retire scan may have crashed'
            exit 1
          fi

      - name: Upload Retire.js Report
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: retire-js-report
          path: openmetadata-ui/src/main/resources/ui/retire-report.json
          retention-days: 30

      - name: Generate Retire.js Slack summary
        if: always()
        run: |
          mkdir -p retire-report
          python3 scripts/retire_slack_summary.py \
            openmetadata-ui/src/main/resources/ui/retire-report.json \
            --counts-file retire-report/_retire_counts.json \
            --slack-file retire-report/_retire_slack.txt \
            > /dev/null

      - name: Upload Retire.js Slack artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: retire-slack
          path: |
            retire-report/_retire_slack.txt
            retire-report/_retire_counts.json
          retention-days: 30

      - name: Publish Retire.js Summary
        if: success()
        working-directory: openmetadata-ui/src/main/resources/ui
        run: |
          python3 - << 'EOF' >> $GITHUB_STEP_SUMMARY
          import json

          SEVERITY_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
          SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
          NM = "node_modules/"

          def escape(text):
              return str(text).replace('|', '\\|').replace('`', "'")

          try:
              with open("retire-report.json") as f:
                  data = json.load(f)
          except FileNotFoundError:
              print("## Retire.js Scan Results\n\n> Report file not found — scan may not have run.")
              raise SystemExit(0)

          findings = data.get("data", [])
          libs = {}
          for item in findings:
              filepath = item.get("file", "")
              short = filepath[filepath.find(NM) + len(NM):] if NM in filepath else filepath
              for result in item.get("results", []):
                  key = (result.get("component", ""), result.get("version", ""))
                  if key not in libs:
                      libs[key] = {"files": [], "vulns": result.get("vulnerabilities", [])}
                  if short not in libs[key]["files"]:
                      libs[key]["files"].append(short)

          print("## Retire.js Scan Results\n")

          if not libs:
              print("✅ No vulnerable libraries found.")
          else:
              total_vulns = sum(len(v["vulns"]) for v in libs.values())
              print(f"> **{len(libs)} vulnerable librar{'y' if len(libs) == 1 else 'ies'} · {total_vulns} CVE{'s' if total_vulns != 1 else ''} found**\n")

              for (component, version), info in sorted(libs.items(), key=lambda x: min(
                      (SEVERITY_ORDER.get(v.get("severity", "low"), 3) for v in x[1]["vulns"]), default=3)):
                  top_sev = min(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3))
                  icon = SEVERITY_ICON.get(top_sev.get("severity", "low"), "⚪")
                  print(f"### {icon} {component} {version}\n")
                  print("| Severity | CVE | Summary |")
                  print("|---|---|---|")
                  for vuln in sorted(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)):
                      sev = vuln.get("severity", "")
                      ids = vuln.get("identifiers", {})
                      cves = ids.get("CVE", [])
                      summary = ids.get("summary", "").split("\n")[0][:120]
                      cve_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves) if cves else ids.get("githubID", "—")
                      print(f"| {SEVERITY_ICON.get(sev, '')} {sev} | {escape(cve_str)} | {escape(summary)} |")
                  print("\n**Bundled in:**")
                  for f in info["files"]:
                      print(f"- `{f}`")
                  print()
          EOF

      - name: Force failure on vulnerabilities found
        if: steps.retire-scan.outcome == 'failure'
        run: exit 1

  security-scan:
    runs-on: ubuntu-latest
    environment: security-scan
    env:
      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
      SNYK_ORGANIZATION: ${{ secrets.SNYK_ORGANIZATION_ID }}

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: true
          swap-storage: true
      - uses: actions/checkout@v4

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: "21"
          distribution: "temurin"

      - name: Install Ubuntu dependencies
        run: |
          # stop relying on apt cache of GitHub runners
          sudo apt-get update
          sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
          librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev

      # Install and Authenticate to Snyk
      - name: Install Snyk & Authenticate
        run: |
          sudo make install_antlr_cli
          sudo npm install -g snyk
          snyk auth ${SNYK_TOKEN}
          snyk config set org=${SNYK_ORGANIZATION}

      - name: Install Python dependencies
        run: |
          python3 -m venv env
          source env/bin/activate
          make install_all install_apis

      - name: Maven build
        id: maven-build
        continue-on-error: true
        run: mvn -DskipTests clean install

      - name: Run Scan
        id: security-report
        if: steps.maven-build.outcome == 'success'
        continue-on-error: true
        run: |
          source env/bin/activate
          rm -rf security-report
          mkdir -p security-report
          # Run snyk subtargets directly; skip `export-snyk-pdf-report` which deletes JSONs after PDF conversion.
          make snyk-ingestion-report || true
          make snyk-ingestion-base-slim-report || true
          make snyk-airflow-apis-report || true
          make snyk-server-report || true
          make snyk-ui-report || true

      - name: Publish Snyk Summary
        id: snyk-summary
        if: always() && steps.maven-build.outcome == 'success'
        run: |
          python3 scripts/snyk_summary.py security-report \
            --counts-file security-report/_counts.json \
            --slack-file security-report/_slack.txt \
            >> $GITHUB_STEP_SUMMARY
          # Expose counts as step output for downstream gating.
          counts=$(cat security-report/_counts.json)
          echo "counts=$counts" >> $GITHUB_OUTPUT
          high=$(jq '.high + .critical' security-report/_counts.json)
          echo "high_critical=$high" >> $GITHUB_OUTPUT

      - name: Fail on high/critical Snyk findings
        if: always() && steps.snyk-summary.outputs.high_critical != '' && steps.snyk-summary.outputs.high_critical != '0'
        run: |
          echo "::error::Snyk found ${{ steps.snyk-summary.outputs.high_critical }} high/critical vulnerabilities (see Job Summary)"
          exit 1

      - name: Generate Snyk HTML/PDF
        if: always() && steps.maven-build.outcome == 'success'
        run: |
          # Back up JSONs because html_to_pdf.py deletes them after PDF conversion.
          mkdir -p /tmp/snyk-json-backup
          cp security-report/*.json /tmp/snyk-json-backup/ 2>/dev/null || true
          make export-snyk-pdf-report || true
          # Restore JSONs alongside generated PDFs/HTMLs.
          cp /tmp/snyk-json-backup/*.json security-report/ 2>/dev/null || true

      - name: Upload Snyk Reports
        if: always() && steps.maven-build.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: security-report
          path: security-report
          retention-days: 30

      - name: Force failure
        if: steps.maven-build.outcome != 'success' || steps.security-report.outcome != 'success'
        run: |
          exit 1

  notify:
    runs-on: ubuntu-latest
    environment: security-scan
    needs: [vulnerability-scan, security-scan]
    if: always()
    steps:
      - name: Download Snyk artifact
        if: needs.security-scan.result != 'skipped'
        uses: actions/download-artifact@v4
        with:
          name: security-report
          path: security-report
        continue-on-error: true

      - name: Download Retire.js Slack artifact
        if: needs.vulnerability-scan.result != 'skipped'
        uses: actions/download-artifact@v4
        with:
          name: retire-slack
          path: retire-report
        continue-on-error: true

      - name: Build Slack header payload
        id: build-header
        env:
          RETIRE_RESULT: ${{ needs.vulnerability-scan.result }}
          SNYK_RESULT: ${{ needs.security-scan.result }}
          REF_NAME: ${{ github.ref_name }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          python3 - <<'PY' > header.json
          import json, os, pathlib

          retire = os.environ["RETIRE_RESULT"]
          snyk = os.environ["SNYK_RESULT"]
          ref_name = os.environ["REF_NAME"]
          run_url = os.environ["RUN_URL"]

          def status_icon(s):
              return {"success": "✅", "cancelled": "⚠️ (cancelled)", "skipped": "⚠️ (skipped)"}.get(s, "❌")

          def load_counts(path):
              p = pathlib.Path(path)
              if not p.exists():
                  return None
              try:
                  return json.loads(p.read_text())
              except Exception:
                  return None

          def totals_line(counts):
              if not counts:
                  return ""
              return (
                  f"\n🚨 {counts.get('critical', 0)} critical  ·  "
                  f"🔴 {counts.get('high', 0)} high  ·  "
                  f"🟠 {counts.get('medium', 0)} medium  ·  "
                  f"🟡 {counts.get('low', 0)} low"
              )

          retire_counts = load_counts("retire-report/_retire_counts.json")
          snyk_counts = load_counts("security-report/_counts.json")

          if retire == "success" and snyk == "success":
              overall = "🟢"
          elif retire == "failure" or snyk == "failure":
              overall = "🚨"
          else:
              overall = "⚠️"

          header = (
              f"{overall}  *Security scan* — *OpenMetadata Repo*\n"
              f"_branch_ `{ref_name}`  ·  <{run_url}|Open run details>\n"
              f"• Vulnerability scan (Retire.js): {status_icon(retire)}"
              f"{totals_line(retire_counts)}\n"
              f"• Security scan (Snyk): {status_icon(snyk)}"
              f"{totals_line(snyk_counts)}"
          )
          fallback = f"{overall} Security scan — OpenMetadata Repo on {ref_name}. {run_url}"
          payload = {
              "text": fallback,
              "blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": header}}],
          }
          print(json.dumps(payload))
          PY

      - name: Send Slack — header
        id: send-header
        if: env.SEND_SLACK_NOTIFICATION == 'true'
        uses: slackapi/slack-github-action@v1.27.1
        with:
          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
          payload-file-path: header.json
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

      - name: Build Slack thread payload
        id: build-thread
        if: always() && steps.send-header.outputs.ts != ''
        env:
          THREAD_TS: ${{ steps.send-header.outputs.ts }}
        run: |
          python3 - <<'PY' > thread.json
          import json, os, pathlib

          thread_ts = os.environ["THREAD_TS"]

          def read_sections(path):
              p = pathlib.Path(path)
              if not p.exists():
                  return []
              text = p.read_text().strip()
              return [s.strip() for s in text.split("\n\n") if s.strip()]

          retire_sections = read_sections("retire-report/_retire_slack.txt")
          snyk_sections = read_sections("security-report/_slack.txt")

          blocks = []
          MAX_TEXT = 2850

          def push(section):
              text = section if len(section) <= MAX_TEXT else section[:MAX_TEXT].rstrip() + "\n_…truncated, see Job Summary_"
              blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}})

          for section in retire_sections[:24]:
              push(section)
          if retire_sections and snyk_sections:
              blocks.append({"type": "divider"})
          for section in snyk_sections[:23]:
              push(section)

          if not blocks:
              push("_No detailed scan output was produced._")

          payload = {
              "thread_ts": thread_ts,
              "text": "Security scan details (thread reply)",
              "blocks": blocks,
          }
          print(json.dumps(payload))
          PY

      - name: Send Slack — thread reply
        if: always() && env.SEND_SLACK_NOTIFICATION == 'true' && steps.send-header.outputs.ts != '' && steps.build-thread.outcome == 'success'
        uses: slackapi/slack-github-action@v1.27.1
        with:
          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
          payload-file-path: thread.json
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}