acc: mutate-based invariant fuzzing for bundle configs

.github/workflows/push.yml

@@ -408,6 +408,77 @@ jobs:
         run: |
           go tool -modfile=tools/task/go.mod task test-sandbox
 
+  test-fuzz:
+    needs:
+      - cleanups
+
+    # Nightly drift-on exploration; PRs rely on the committed acceptance/bundle/fuzz test.
+    if: ${{ github.event_name == 'schedule' }}
+    name: "task test-fuzz"
+    runs-on:
+      group: databricks-protected-runner-group-large
+      labels: linux-ubuntu-latest-large
+
+    defaults:
+      run:
+        shell: bash
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-fuzz
+
+      - name: Run tests
+        run: |
+          go tool -modfile=tools/task/go.mod task test-fuzz
+
+      - name: Summarize failure for triage
+        if: ${{ failure() }}
+        run: |
+          # Resolve once and hand it to the upload step: -keeptmp writes under os.TempDir().
+          fuzz_tmp_dir="${TMPDIR:-/tmp}/acceptance"
+          echo "FUZZ_TMP_DIR=$fuzz_tmp_dir" >> "$GITHUB_ENV"
+          {
+            echo "## Fuzz nightly failed"
+            echo
+            echo "Exact drift is on (\`FUZZ_CHECK_DRIFT=1\`); a red nightly is a finding to triage."
+            echo "Use the failing variant's \`LOG.repro\`, or:"
+            echo
+            echo '```'
+            echo 'ENVFILTER=FUZZ_TARGET=<target> FUZZ_SEED_START=<seed> FUZZ_SEED_COUNT=1 FUZZ_CHECK_DRIFT=1 ./task test-fuzz'
+            echo '```'
+            echo
+            echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            # Paste each failing variant's LOG.repro from the kept workdirs.
+            echo
+            while IFS= read -r repro; do
+              echo
+              echo "### \`${repro}\`"
+              echo
+              echo '```'
+              cat "$repro"
+              echo '```'
+            done < <(find "$fuzz_tmp_dir" -name LOG.repro 2>/dev/null | sort)
+          } | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload fuzz triage logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fuzz-triage-logs
+          # Same workdir root the summary step listed; path is not shell-expanded.
+          path: ${{ env.FUZZ_TMP_DIR }}/**/LOG.*
+          if-no-files-found: warn
+          retention-days: 14
+
   # This job groups the result of all the above test jobs.
   # It is a required check, so it blocks auto-merge and the merge queue.
   #
@@ -417,13 +488,16 @@ jobs:
   #
   # The step checks `contains(needs.*.result, 'failure')` to fail if any dependency failed.
   # Reference: https://github.com/orgs/community/discussions/25970
+  #
+  # test-fuzz is schedule-only (skipped on PRs); listed so nightlies gate test-result.
   test-result:
     needs:
       - test
       - test-exp-aitools
       - test-exp-ssh
       - test-pipelines
       - test-sandbox
+      - test-fuzz
 
     if: ${{ always() }}
     name: test-result

Taskfile.yml

@@ -733,6 +733,20 @@ tasks:
           --packages ./acceptance/... \
           -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox"
 
+  test-fuzz:
+    desc: Run invariant fuzz tests (mutated configs, direct engine)
+    # No sources fingerprint: the window depends on FUZZ_* env vars Task can't see.
+    cmds:
+      - |
+        # Ceiling; run_fuzz.py stops at FUZZ_TIME_BUDGET.
+        export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-10000}"
+        # Day-of-epoch start keeps consecutive nightly windows disjoint.
+        export FUZZ_SEED_START="${FUZZ_SEED_START:-$(( $(date -u +%s) / 86400 * FUZZ_SEED_COUNT ))}"
+        export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}"
+        export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}"
+        # go test (not gotestsum): -keeptmp must follow the package list.
+        go test ./acceptance -count=1 -keeptmp -timeout=${LOCAL_TIMEOUT:-90m} -run "TestAccept/bundle/fuzz"
+
   # --- Integration tests ---
 
   integration:

acceptance/acceptance_test.go

@@ -346,6 +346,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	t.Setenv("CLI", execPath)
 	repls.SetPath(execPath, "[CLI]")
 
+	// Fuzzer mutator is stdlib-only Python; yaml2json parses bases the way the bundle does.
+	yaml2jsonPath := BuildYaml2Json(t, buildDir, runtime.GOOS, runtime.GOARCH)
+	t.Setenv("YAML2JSON", yaml2jsonPath)
+	repls.SetPath(yaml2jsonPath, "[YAML2JSON]")
+
 	if !inprocessMode {
 		cli293Path := DownloadCLI(t, buildDir, "0.293.0")
 		t.Setenv("CLI_293", cli293Path)
@@ -1300,6 +1305,24 @@ func BuildCLI(t *testing.T, buildDir, coverDir, osName, arch string) string {
 	return execPath
 }
 
+// BuildYaml2Json builds the acceptance-only yaml2json helper and returns its path.
+func BuildYaml2Json(t *testing.T, buildDir, osName, arch string) string {
+	execPath := filepath.Join(buildDir, "yaml2json")
+	if osName == "windows" {
+		execPath += ".exe"
+	}
+
+	args := []string{"go", "build", "-o", execPath}
+	if osName == "windows" {
+		// Same as BuildCLI: "error obtaining VCS status: exit status 128" without this.
+		args = append(args, "-buildvcs=false")
+	}
+	args = append(args, "./acceptance/cmd/yaml2json")
+
+	RunCommand(t, args, "..", []string{"GOOS=" + osName, "GOARCH=" + arch})
+	return execPath
+}
+
 // CreateReleaseArtifacts builds release artifacts for the given OS using amd64 and arm64 architectures,
 // archives them into zip files, and returns the directory containing the release artifacts.
 func CreateReleaseArtifacts(t *testing.T, cwd, coverDir, osName string) string {