diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b4c5238326b..861bb9f4cca 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -408,6 +408,77 @@ jobs: run: | go tool -modfile=tools/task/go.mod task test-sandbox + test-fuzz: + needs: + - cleanups + + # Nightly drift-on exploration; PRs rely on the committed acceptance/bundle/fuzz test. + if: ${{ github.event_name == 'schedule' }} + name: "task test-fuzz" + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: test-fuzz + + - name: Run tests + run: | + go tool -modfile=tools/task/go.mod task test-fuzz + + - name: Summarize failure for triage + if: ${{ failure() }} + run: | + # Resolve once and hand it to the upload step: -keeptmp writes under os.TempDir(). + fuzz_tmp_dir="${TMPDIR:-/tmp}/acceptance" + echo "FUZZ_TMP_DIR=$fuzz_tmp_dir" >> "$GITHUB_ENV" + { + echo "## Fuzz nightly failed" + echo + echo "Exact drift is on (\`FUZZ_CHECK_DRIFT=1\`); a red nightly is a finding to triage." + echo "Use the failing variant's \`LOG.repro\`, or:" + echo + echo '```' + echo 'ENVFILTER=FUZZ_TARGET= FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_CHECK_DRIFT=1 ./task test-fuzz' + echo '```' + echo + echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + # Paste each failing variant's LOG.repro from the kept workdirs. + echo + while IFS= read -r repro; do + echo + echo "### \`${repro}\`" + echo + echo '```' + cat "$repro" + echo '```' + done < <(find "$fuzz_tmp_dir" -name LOG.repro 2>/dev/null | sort) + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload fuzz triage logs + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fuzz-triage-logs + # Same workdir root the summary step listed; path is not shell-expanded. + path: ${{ env.FUZZ_TMP_DIR }}/**/LOG.* + if-no-files-found: warn + retention-days: 14 + # This job groups the result of all the above test jobs. # It is a required check, so it blocks auto-merge and the merge queue. # @@ -417,6 +488,8 @@ jobs: # # The step checks `contains(needs.*.result, 'failure')` to fail if any dependency failed. # Reference: https://github.com/orgs/community/discussions/25970 + # + # test-fuzz is schedule-only (skipped on PRs); listed so nightlies gate test-result. test-result: needs: - test @@ -424,6 +497,7 @@ jobs: - test-exp-ssh - test-pipelines - test-sandbox + - test-fuzz if: ${{ always() }} name: test-result diff --git a/Taskfile.yml b/Taskfile.yml index 8ed24ad0f60..c48add03d73 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -733,6 +733,20 @@ tasks: --packages ./acceptance/... \ -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox" + test-fuzz: + desc: Run invariant fuzz tests (mutated configs, direct engine) + # No sources fingerprint: the window depends on FUZZ_* env vars Task can't see. + cmds: + - | + # Ceiling; run_fuzz.py stops at FUZZ_TIME_BUDGET. + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-10000}" + # Day-of-epoch start keeps consecutive nightly windows disjoint. + export FUZZ_SEED_START="${FUZZ_SEED_START:-$(( $(date -u +%s) / 86400 * FUZZ_SEED_COUNT ))}" + export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" + export FUZZ_TIME_BUDGET="${FUZZ_TIME_BUDGET:-900}" + # go test (not gotestsum): -keeptmp must follow the package list. + go test ./acceptance -count=1 -keeptmp -timeout=${LOCAL_TIMEOUT:-90m} -run "TestAccept/bundle/fuzz" + # --- Integration tests --- integration: diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6b11e812763..87ba551939a 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -346,6 +346,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { t.Setenv("CLI", execPath) repls.SetPath(execPath, "[CLI]") + // Fuzzer mutator is stdlib-only Python; yaml2json parses bases the way the bundle does. + yaml2jsonPath := BuildYaml2Json(t, buildDir, runtime.GOOS, runtime.GOARCH) + t.Setenv("YAML2JSON", yaml2jsonPath) + repls.SetPath(yaml2jsonPath, "[YAML2JSON]") + if !inprocessMode { cli293Path := DownloadCLI(t, buildDir, "0.293.0") t.Setenv("CLI_293", cli293Path) @@ -1300,6 +1305,24 @@ func BuildCLI(t *testing.T, buildDir, coverDir, osName, arch string) string { return execPath } +// BuildYaml2Json builds the acceptance-only yaml2json helper and returns its path. +func BuildYaml2Json(t *testing.T, buildDir, osName, arch string) string { + execPath := filepath.Join(buildDir, "yaml2json") + if osName == "windows" { + execPath += ".exe" + } + + args := []string{"go", "build", "-o", execPath} + if osName == "windows" { + // Same as BuildCLI: "error obtaining VCS status: exit status 128" without this. + args = append(args, "-buildvcs=false") + } + args = append(args, "./acceptance/cmd/yaml2json") + + RunCommand(t, args, "..", []string{"GOOS=" + osName, "GOARCH=" + arch}) + return execPath +} + // CreateReleaseArtifacts builds release artifacts for the given OS using amd64 and arm64 architectures, // archives them into zip files, and returns the directory containing the release artifacts. func CreateReleaseArtifacts(t *testing.T, cwd, coverDir, osName string) string { diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py new file mode 100755 index 00000000000..3ad09923ec1 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +""" +Mutate a curated, deploy-verified bundle config for the invariant fuzzer. + +Destructive: delete/replace a field (token, dangerous scalar, or empty container). +Additive: inject one optional from INJECT that the base omits. +Each seed picks exactly one mode so an additive finding maps to one catalog entry. + +Bases live in bundle/invariant/configs/ and are parsed via $YAML2JSON (stdlib Python +cannot read YAML). Emits JSON on stdout; the bundle reads it as YAML 1.2. +""" + +import json +import os +import random +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables + +# Prefer inject: that is how reconcile/drift bugs are reached. +ADD_PROB = 0.6 + +# Hostile free-form scalars; the CLI must reject or round-trip without panicking. +DANGEROUS_STRINGS = [ + "", + " ", + "a" * 300, + "line1\nline2", + "tab\there", + "\U0001f680-unicode-\u00e9", + "quote\"and'apostrophe", + "${resources.jobs.does_not_exist.id}", + "../../etc/passwd", +] +DANGEROUS_INTS = [ + 2**31 - 1, + 2**31, + -(2**31), + 2**63 - 1, + -(2**63), + -1, +] +DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS + +# Single-resource invariant configs; extend as more types prove mutable. +MUTATE_BASES = [ + "app", + "catalog", + "experiment", + "external_location", + "job", + "model", + "model_serving_endpoint", + "pipeline", + "registered_model", + "schema", + "secret_scope", + "sql_warehouse", + "volume", +] + +# Keep in sync with acceptance/bundle/invariant/migrate/test.toml EnvMatrixExclude: +# migrate seeds terraform first, so direct-only / known-drift bases cannot run there. +MIGRATE_SKIP_BASES = frozenset( + { + "catalog", + "external_location", + "sql_warehouse", + } +) + +CONFIGS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "invariant", "configs") + +# Schema-valid optionals from past drift/reconcile findings (may still fail to deploy). +INJECT = { + "apps": [ + ("description", "fuzz-app-description"), + ( + "config", + { + "command": ["python", "app.py"], + "env": [{"name": "FUZZ_ENV", "value": "1"}], + }, + ), + ("lifecycle", {"started": False}), + ], + "catalogs": [ + ("custom_max_retention_hours", 168), + ( + "managed_encryption_settings", + {"customer_managed_key_id": "00000000-0000-0000-0000-000000000000"}, + ), + ("properties", {"fuzz_key": "fuzz_val"}), + ], + "experiments": [ + ("artifact_location", "dbfs:/databricks/mlflow-tracking/fuzz"), + ("tags", [{"key": "fuzz", "value": "1"}]), + ], + "external_locations": [ + ("read_only", True), + ("skip_validation", True), + ], + "jobs": [ + ("description", "fuzz-job"), + ("max_concurrent_runs", 1), + ( + "webhook_notifications", + {"on_success": [{"id": "alpha"}, {"id": "beta"}]}, + ), + ("tags", {"fuzz": "1"}), + ], + "models": [ + ("description", "fuzz-model"), + ], + "model_serving_endpoints": [ + ("description", "fuzz-endpoint"), + ("route_optimized", True), + ( + "config", + { + "served_entities": [ + { + "name": "prod", + "burst_scaling_enabled": True, + "external_model": { + "name": "gpt-4o-mini", + "provider": "openai", + "task": "llm/v1/chat", + "openai_config": { + "openai_api_key_plaintext": "sk-test-plaintext-key", + }, + }, + } + ], + "traffic_config": { + "routes": [{"served_model_name": "prod", "traffic_percentage": 100}], + }, + }, + ), + ], + "pipelines": [ + ("allow_duplicate_names", True), + ("parameters", {"fuzz_param": "1"}), + ("development", True), + ("photon", False), + ], + "registered_models": [ + ("comment", "fuzz-registered-model"), + ], + "schemas": [ + ("comment", "fuzz-schema"), + ("properties", {"fuzz_key": "fuzz_val"}), + ], + "sql_warehouses": [ + ("enable_photon", True), + ("lifecycle", {"started": False}), + ("tags", {"fuzz": "1"}), + ], + "volumes": [ + ("comment", "fuzz-volume"), + ], +} + +# Audited-empty types (an absent INJECT key would look identical). +NO_INJECT = { + "secret_scopes": "only keyvault_metadata remains: Azure-only, conflicts with backend_type DATABRICKS", +} + + +def token(rng): + return "fuzz_" + "".join(rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def dump_config(config): + # ensure_ascii=False: default escapes become invalid YAML surrogates before the bundle sees them. + return json.dumps(config, indent=2, ensure_ascii=False) + "\n" + + +def load_base(name): + path = os.path.join(CONFIGS_DIR, name + ".yml.tmpl") + # Same loader as the bundle; substitute after parse so placeholders stay JSON strings. + result = subprocess.run([os.environ["YAML2JSON"], path], capture_output=True, check=True, text=True) + return json.loads(substitute_variables(result.stdout)) + + +def collect(node, out): + if isinstance(node, dict): + for k, v in node.items(): + out.append((node, k)) + collect(v, out) + elif isinstance(node, list): + for idx, v in enumerate(node): + out.append((node, idx)) + collect(v, out) + + +def mutate_once(rng, roots): + refs = [] + for root in roots: + collect(root, refs) + if not refs: + return + container, key = rng.choice(refs) + op = rng.choice(["delete", "scalar", "dangerous", "empty"]) + if op == "delete": + del container[key] + elif op == "scalar": + container[key] = token(rng) + elif op == "dangerous": + container[key] = rng.choice(DANGEROUS) + else: + container[key] = rng.choice([{}, [], None]) + + +def resource_instances(config): + for rtype, instances in config.get("resources", {}).items(): + if isinstance(instances, dict): + for instance in instances.values(): + if isinstance(instance, dict): + yield rtype, instance + + +def add_field(rng, config): + candidates = [] + for rtype, instance in resource_instances(config): + for name, value in INJECT.get(rtype, []): + if name not in instance: + candidates.append((instance, name, value)) + if not candidates: + return False + instance, name, value = rng.choice(candidates) + # Copy: INJECT values are shared across seeds. + instance[name] = json.loads(json.dumps(value)) + return True + + +def mutate(config, seed): + rng = random.Random(seed) + # Resource instances only: keep the bundle/resources skeleton intact. + roots = [instance for _, instance in resource_instances(config)] + + # Fall through when add has nothing to inject (e.g. NO_INJECT types). + if rng.random() < ADD_PROB and add_field(rng, config): + return config + for _ in range(rng.randint(1, 3)): + mutate_once(rng, roots) + return config + + +def bases_for_target(): + if os.environ.get("FUZZ_TARGET") == "migrate": + return [b for b in MUTATE_BASES if b not in MIGRATE_SKIP_BASES] + return MUTATE_BASES + + +def main(): + # Windows stdout is often ANSI; UTF-8 probes need an explicit encoding. + sys.stdout.reconfigure(encoding="utf-8") + + seed = int(os.environ["FUZZ_SEED"]) + bases = bases_for_target() + name = bases[seed % len(bases)] + sys.stdout.write(dump_config(mutate(load_base(name), seed))) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py new file mode 100755 index 00000000000..c3c648925fd --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +""" +Contract checks for mutate_fuzz_config. Failures go to stderr; stdout samples mutated +configs so an algorithm change shows up as an acceptance output diff. + +- every MUTATE_BASES entry has a YAML fixture with one non-empty resource instance +- every base type has INJECT entries or a NO_INJECT reason, never both or neither +- every INJECT field is a settable input in the committed reference schema +- mutate(seed) is deterministic for every base +- sample volume seeds stay pairwise distinct (so output.txt catches algorithm drift) +- INJECT eventually lands on a sparse base (registered_model) +""" + +import json +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from mutate_fuzz_config import ( + CONFIGS_DIR, + INJECT, + MIGRATE_SKIP_BASES, + MUTATE_BASES, + NO_INJECT, + dump_config, + load_base, + mutate, +) + +FIELDS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "refschema", "out.fields.txt") + + +def instance(config): + (instances,) = config["resources"].values() + (value,) = instances.values() + return value + + +def field_flags(): + """Map field path -> flags. A path can repeat (one Go type each), so union them.""" + flags = {} + with open(FIELDS) as f: + for line in f: + path, _, rest = line.rstrip("\n").partition("\t") + if path: + flags.setdefault(path, set()).update(rest.split("\t")[1:]) + return flags + + +def main(): + # Pin UNIQUE_NAME so printed configs are stable across harness runs. + os.environ["UNIQUE_NAME"] = "check" + os.environ.setdefault("CURRENT_USER_NAME", "check-user") + failed = False + + for name in MUTATE_BASES: + path = os.path.join(CONFIGS_DIR, name + ".yml.tmpl") + if not os.path.isfile(path): + sys.stderr.write(f"{name}: missing YAML fixture at {path}\n") + failed = True + continue + try: + parsed = load_base(name) + except (OSError, subprocess.CalledProcessError, json.JSONDecodeError) as e: + sys.stderr.write(f"{name}: could not load fixture: {e}\n") + failed = True + continue + if not isinstance(parsed, dict) or "resources" not in parsed: + sys.stderr.write(f"{name}: base did not parse to a config with resources\n") + failed = True + continue + resources = parsed["resources"] + if len(resources) != 1: + sys.stderr.write(f"{name}: expected one resource type, got {sorted(resources)}\n") + failed = True + continue + instances = next(iter(resources.values())) + if not isinstance(instances, dict) or len(instances) != 1: + sys.stderr.write(f"{name}: expected one resource instance\n") + failed = True + continue + if not next(iter(instances.values())): + sys.stderr.write(f"{name}: base parsed to an empty resource instance\n") + failed = True + rtype = next(iter(resources)) + if bool(INJECT.get(rtype)) == (rtype in NO_INJECT): + sys.stderr.write( + f"{name}: resources.{rtype} needs INJECT entries or a NO_INJECT reason, not both or neither\n" + ) + failed = True + + unknown_skip = sorted(MIGRATE_SKIP_BASES - set(MUTATE_BASES)) + if unknown_skip: + sys.stderr.write(f"MIGRATE_SKIP_BASES not in MUTATE_BASES: {unknown_skip}\n") + failed = True + if len(MUTATE_BASES) - len(MIGRATE_SKIP_BASES) < 1: + sys.stderr.write("MIGRATE_SKIP_BASES leaves no migrate bases\n") + failed = True + + # A typo would inject a field the bundle ignores, so the seed proves nothing. + flags = field_flags() + for rtype, fields in INJECT.items(): + for field, _ in fields: + path = f"resources.{rtype}.*.{field}" + if not flags.get(path, set()) & {"INPUT", "ALL"}: + sys.stderr.write(f"INJECT[{rtype}]: {path} is not a settable input field\n") + failed = True + + for name in MUTATE_BASES: + for seed in range(5): + a = dump_config(mutate(load_base(name), seed)) + b = dump_config(mutate(load_base(name), seed)) + if a != b: + sys.stderr.write(f"{name} seed {seed}: mutation is not deterministic\n") + failed = True + + # Distinct dumps: colliding samples hide algorithm changes in output.txt. + samples = [0, 1, 5] + dumps = [] + for seed in samples: + out = dump_config(mutate(load_base("volume"), seed)) + dumps.append(out) + sys.stdout.write(f"=== volume seed={seed} ===\n") + sys.stdout.write(out) + if len(set(dumps)) != len(dumps): + sys.stderr.write(f"sample seeds {samples} are not pairwise distinct\n") + failed = True + + # Sparse base: any new field must come from INJECT. + base_fields = set(instance(load_base("registered_model"))) + inject_names = {name for name, _ in INJECT["registered_models"]} + injected = False + for seed in range(30): + fields = set(instance(mutate(load_base("registered_model"), seed))) + added = fields - base_fields + if added: + injected = True + unexpected = added - inject_names + if unexpected: + sys.stderr.write(f"seed {seed}: injected non-catalog field(s): {sorted(unexpected)}\n") + failed = True + if not injected: + sys.stderr.write("mutation never injected a curated optional field\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/run_fuzz.py b/acceptance/bin/run_fuzz.py new file mode 100755 index 00000000000..85c34103069 --- /dev/null +++ b/acceptance/bin/run_fuzz.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Seed loop for the invariant fuzzer. Invokes fuzz/seed.sh per seed and classifies each: + + deployed - deployed and the invariant held + rejected - validate --strict refused the config before deploy + gap - needs a route the testserver does not model + hang - exceeded FUZZ_SEED_TIMEOUT + bug - panic, internal error, mutator failure, or a deploy that failed or drifted + +Writes LOG.summary per seed; on bug/hang writes LOG.repro and exits non-zero. +Stdout stays empty (the committed run asserts that). + +Knobs: FUZZ_TARGET (matrix), FUZZ_SEED_*, FUZZ_TIME_BUDGET, FUZZ_CHECK_DRIFT +(script.prepare acts on the latter). +""" + +import json +import os +import shutil +import signal +import subprocess +import sys +import time +from collections import Counter +from pathlib import Path + +# Stuck-seed cap; FUZZ_SEED_TIMEOUT=0 disables. +SEED_TIMEOUT = float(os.environ.get("FUZZ_SEED_TIMEOUT", "180")) + +# Nightly/task stop; seed count is only a ceiling. 0 disables. +BUDGET = float(os.environ.get("FUZZ_TIME_BUDGET", "900")) + +QUIT_GRACE = 10 # seconds between SIGQUIT and SIGKILL + +CLEANUP_LOG = "LOG.destroy" # EXIT-trap destroy; every target writes this + +TARGET = os.environ["FUZZ_TARGET"] + +# Repro knob: 0 = plan-determinism, 1 = exact no_drift (task test-fuzz default). +CHECK_DRIFT = os.environ.get("FUZZ_CHECK_DRIFT", "0") + +POSIX = os.name == "posix" + +# Resolved path: Windows CreateProcess otherwise picks System32\bash.exe (WSL stub). +BASH = shutil.which("bash") +if not BASH: + sys.exit("fuzz: bash not found on PATH") + + +def read(path): + """Bytes from a log; fuzzed configs can put arbitrary bytes there. Empty if absent.""" + return path.read_bytes() if path.exists() else b"" + + +def concat_logs(seed_dir, skip=()): + return b"".join(read(p) for p in sorted(seed_dir.glob("LOG.*")) if p.name not in skip) + + +def killpg(proc, sig): + try: + os.killpg(proc.pid, sig) + except ProcessLookupError: + # Exited between timeout and signal; still report as hang. + pass + + +def kill_seed(proc): + if not POSIX: + proc.kill() + return + # SIGQUIT for Go's goroutine dump, then SIGKILL. + killpg(proc, signal.SIGQUIT) + try: + proc.wait(timeout=QUIT_GRACE) + except subprocess.TimeoutExpired: + killpg(proc, signal.SIGKILL) + + +def run_seed(seed_dir, seed): + """Exit code and whether the seed was killed for timeout.""" + seed_sh = Path(os.environ["TESTDIR"]) / "seed.sh" + with open(seed_dir / "LOG.check", "wb") as log: + proc = subprocess.Popen( + [BASH, "-euo", "pipefail", str(seed_sh), str(seed_dir), str(seed)], + stdout=log, + stderr=subprocess.STDOUT, + # Own process group so a hung CLI dies with the seed. + start_new_session=POSIX, + ) + try: + return proc.wait(timeout=SEED_TIMEOUT or None), False + except subprocess.TimeoutExpired: + kill_seed(proc) + return proc.wait(), True + + +def oracle_verdict(seed_dir): + """Drift oracle wording if it fired; empty if it never ran or was happy.""" + # Checked before TESTSERVER_GAP: both can fire on one seed. + if b"Unexpected action=" in read(seed_dir / "LOG.check"): + return "planned a change after deploy" + if read(seed_dir / "LOG.plan.determinism.diff").strip(): + return "planned differently on two consecutive runs" + if read(seed_dir / "LOG.plan.failed").strip(): + return "could not be planned after deploy" + return "" + + +def classify(seed_dir): + """Kind and, for a failure, the reason.""" + gen_err = read(seed_dir / "LOG.gen.err").strip() + if gen_err: + last_line = gen_err.splitlines()[-1].decode(errors="replace") + return "bug", f"could not be mutated: {last_line}" + + # Mutated config text must not count as a CLI panic/gap. + skip_input = {"LOG.config"} + logs = concat_logs(seed_dir, skip=skip_input) + if b"panic:" in logs or b"internal error" in logs: + return "bug", "panicked or hit an internal error" + + verdict = oracle_verdict(seed_dir) + if verdict: + return "bug", verdict + + # Skip cleanup: destroy after failure must not mask the real cause. + if b"TESTSERVER_GAP" in concat_logs(seed_dir, skip=skip_input | {CLEANUP_LOG}): + return "gap", "" + + if b"INPUT_CONFIG_OK" in read(seed_dir / "LOG.check"): + return "bug", "failed after deploying; see the seed's LOG.* files" + + # LOG.deploy* exists only once validate --strict passed, so a seed here is a + # config the CLI accepted and then failed to deploy. + if any(seed_dir.glob("LOG.deploy*")): + return "bug", "deploy failed after validate --strict; see the seed's LOG.* files" + + return "rejected", "" + + +def resource_type(seed_dir): + # Empty only when the mutator failed; classify() already reports that as a bug. + raw = read(seed_dir / "LOG.config") + if not raw: + return "unknown" + (rtype,) = json.loads(raw)["resources"] + return rtype + + +def record(kind, seed, seed_dir): + with open("LOG.summary", "a") as f: + f.write(f"{kind} seed={seed} target={TARGET} type={resource_type(seed_dir)}\n") + + +def fail(seed, seed_dir, kind, reason, prefix=""): + record(kind, seed, seed_dir) + # Harness rewrites env values in stdout; ENVFILTER selects the matrix key. + Path("LOG.repro").write_text( + f"fuzz: seed {seed} {reason}, reproduce with: {prefix}" + f"ENVFILTER=FUZZ_TARGET={TARGET} FUZZ_SEED_START={seed} " + f"FUZZ_SEED_COUNT=1 FUZZ_CHECK_DRIFT={CHECK_DRIFT} ./task test-fuzz\n" + ) + sys.exit(1) + + +def totals(): + summary = Path("LOG.summary") + if not summary.exists(): + return Counter() + + kinds = Counter(line.split()[0] for line in summary.read_text().splitlines()) + with summary.open("a") as f: + f.write("--- totals ---\n") + for kind, n in sorted(kinds.items()): + f.write(f"{n} {kind}\n") + return kinds + + +def main(): + start = time.monotonic() + seed_start = int(os.environ.get("FUZZ_SEED_START", "0")) + # PR smoke default; nightly/task raise the ceiling and stop on FUZZ_TIME_BUDGET. + count = int(os.environ.get("FUZZ_SEED_COUNT", "25")) + + for offset in range(count): + if BUDGET and time.monotonic() - start >= BUDGET: + Path("LOG.budget").write_text( + f"fuzz: stopping after {offset}/{count} seeds; hit FUZZ_TIME_BUDGET={BUDGET:g}s\n" + ) + break + + seed = seed_start + offset + seed_dir = Path(f"seed-{seed}") + seed_dir.mkdir(exist_ok=True) + + returncode, killed = run_seed(seed_dir, seed) + if returncode == 0: + record("deployed", seed, seed_dir) + continue + + if killed: + fail(seed, seed_dir, "hang", f"hung (>{SEED_TIMEOUT:g}s)", "FUZZ_SEED_TIMEOUT=0 ") + + kind, reason = classify(seed_dir) + if reason: + fail(seed, seed_dir, kind, reason) + record(kind, seed, seed_dir) + + kinds = totals() + + # Zero deploys is fine only when every seed is a gap. Single-seed repro is exempt. + if count > 1 and not kinds: + sys.exit("fuzz: no seeds ran") + if count > 1 and not kinds["deployed"] and kinds["gap"] != sum(kinds.values()): + sys.exit("fuzz: no seed deployed; the mutator or fixtures are broken") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/fuzz/README.md b/acceptance/bundle/fuzz/README.md new file mode 100644 index 00000000000..ae7c1ee9262 --- /dev/null +++ b/acceptance/bundle/fuzz/README.md @@ -0,0 +1,21 @@ +Harness over ../invariant: mutates curated configs and runs a real target script. +`run_fuzz.py` owns the seed loop (`seed.sh` per seed) and classifies deployed / +rejected / gap / hang / bug. `FUZZ_TARGET` in test.toml picks the target. + +Each seed reads and mutates a deploy-verified YAML base from +`../invariant/configs/` and may inject a curated optional from INJECT. The mutator +is stdlib-only Python, so it parses bases through `$YAML2JSON` +(`acceptance/cmd/yaml2json`). + +Helpers come from ../invariant/script.prepare (sourced explicitly; prepare/test.toml +only merge along the directory chain). Server stubs are copied into test.toml; +script asserts every invariant `Pattern` is present. Unmodeled routes return +`TESTSERVER_GAP` (gaps). + +A failure is a CLI bug. `LOG.repro` prints e.g. +`ENVFILTER=FUZZ_TARGET=no_drift FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_CHECK_DRIFT=0 ./task test-fuzz` +(`ENVFILTER` because `FUZZ_TARGET` is a matrix key). + +`FUZZ_CHECK_DRIFT=0` (committed run) uses plan-determinism; `1` (`task test-fuzz` / +nightly) uses exact no_drift. Only the committed run is expected green; a red +nightly is a finding to triage (gates `test-result`; failure summary has the repro). diff --git a/acceptance/bundle/fuzz/out.test.toml b/acceptance/bundle/fuzz/out.test.toml new file mode 100644 index 00000000000..df2dbedbb74 --- /dev/null +++ b/acceptance/bundle/fuzz/out.test.toml @@ -0,0 +1,8 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_TARGET = [ + "no_drift", + "migrate", + "delete_idempotent", + "destroy_idempotent" +] diff --git a/acceptance/bundle/fuzz/output.txt b/acceptance/bundle/fuzz/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/fuzz/script b/acceptance/bundle/fuzz/script new file mode 100644 index 00000000000..2d87667247f --- /dev/null +++ b/acceptance/bundle/fuzz/script @@ -0,0 +1,11 @@ +# Mutate a curated config per seed and run ../invariant/$FUZZ_TARGET. Loop: run_fuzz.py. + +# Empty READPLAN: the saved-plan matrix is out of scope for fuzz. +export READPLAN="" + +# Fail if ../invariant/test.toml gained a stub that was not copied here. +grep '^Pattern = ' "$INVARIANT_DIR/test.toml" | while read -r stub; do + grep -qxF -- "$stub" "$TESTDIR/test.toml" || echo "stub missing from fuzz/test.toml: $stub" +done | contains.py '!stub missing' > /dev/null + +run_fuzz.py diff --git a/acceptance/bundle/fuzz/script.prepare b/acceptance/bundle/fuzz/script.prepare new file mode 100644 index 00000000000..ba95d333d0f --- /dev/null +++ b/acceptance/bundle/fuzz/script.prepare @@ -0,0 +1,53 @@ +# Fuzz overrides of invariant helpers. Source explicitly: we sit outside the invariant +# subtree (test.toml / script.prepare only merge along the directory chain). +export INVARIANT_DIR="$TESTDIR/../invariant" + +# Empty INPUT_CONFIG satisfies set -u; mutated configs are unnamed. +export INPUT_CONFIG="" + +source "$INVARIANT_DIR/script.prepare" + +# Mutator writes the config; validate runs on its own so a panic there is caught before deploy. +invariant_render() { + cp -r "$INVARIANT_DIR/data/." . &> LOG.cp + + mutate_fuzz_config.py > databricks.yml 2>LOG.gen.err + + cp databricks.yml LOG.config + + # --strict: type/required warnings must reject before deploy. + set +e + trace $CLI bundle validate --strict &> LOG.validate + local rc=$? + set -e + cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null + if [ "$rc" -ne 0 ]; then + return 1 + fi +} + +# Plan-determinism oracle: exact no_drift false-positives on fake-server gaps. +# Default 0 for the committed run; task test-fuzz and the nightly set 1. +if [ "${FUZZ_CHECK_DRIFT:-0}" = 0 ]; then + invariant_verify_no_drift() { + set +e + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err + local plan1_rc=$? + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err + local plan2_rc=$? + set -e + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + + if [ "$plan1_rc" -ne 0 ] || [ "$plan2_rc" -ne 0 ]; then + # Stderr is ignored (*.err); copy into LOG.plan.failed for classify(). + if ! grep -q TESTSERVER_GAP LOG.plan1.err LOG.plan2.err; then + echo "bundle plan exited $plan1_rc and $plan2_rc" > LOG.plan.failed + cat LOG.plan1.err LOG.plan2.err >> LOG.plan.failed + fi + return 1 + fi + + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + } +fi diff --git a/acceptance/bundle/fuzz/seed.sh b/acceptance/bundle/fuzz/seed.sh new file mode 100644 index 00000000000..e54d4a214cd --- /dev/null +++ b/acceptance/bundle/fuzz/seed.sh @@ -0,0 +1,9 @@ +# One seed: prepare chain + invariant target. Args: seed_dir seed. +cd "$1" +# Per-seed names: seeds share one workspace; leftover state looks like drift. +export UNIQUE_NAME="$UNIQUE_NAME-$2" +export FUZZ_SEED="$2" +# Same prepare chain the harness merged (root helpers like trace, then fuzz). +source "$TESTROOT/script.prepare" +source "$TESTDIR/script.prepare" +source "$INVARIANT_DIR/$FUZZ_TARGET/script" diff --git a/acceptance/bundle/fuzz/test.toml b/acceptance/bundle/fuzz/test.toml new file mode 100644 index 00000000000..be298eeb9f8 --- /dev/null +++ b/acceptance/bundle/fuzz/test.toml @@ -0,0 +1,60 @@ +# Local only: cloud round-trips are minutes per seed and trip SEED_TIMEOUT. +Cloud = false + +# Nightly FUZZ_TIME_BUDGET plus the last seed's tail. +Timeout = '20m' + +# Copied from ../invariant/test.toml (merge is directory-chain only); script asserts every +# Pattern there is present here. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + ".databricks", + ".venv", + "databricks.yml", + "plan.json", + "*.py", + "*.json", + "*.err", + "app", + # Idempotency targets' pre-delete snapshot; may linger if a seed fails. + ".databricks.backup", +] + +# continue_293 omitted: pinned v0.293.0 rejects most current fields/types first. +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "delete_idempotent", "destroy_idempotent"] + +# Local SQL stub used by some mutated configs. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{"status": "OK"}' + +# Unmodeled routes → TESTSERVER_GAP. HEAD shares the GET catch-all (ServeMux maps HEAD to GET). +[[Server]] +Pattern = "GET /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "POST /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "PUT /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "PATCH /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "DELETE /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' diff --git a/acceptance/bundle/invariant/script.prepare b/acceptance/bundle/invariant/script.prepare index 54f2a3dbf25..214e1da26c9 100644 --- a/acceptance/bundle/invariant/script.prepare +++ b/acceptance/bundle/invariant/script.prepare @@ -1,10 +1,13 @@ # Shared setup for the invariant targets; each script keeps only the invariant it asserts. +# Root of configs/ and data/. Callers outside this subtree set it before sourcing. +INVARIANT_DIR="${INVARIANT_DIR:-$TESTDIR/..}" + invariant_cleanup() { trace $CLI bundle destroy --auto-approve &> LOG.destroy cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + CLEANUP_SCRIPT="$INVARIANT_DIR/configs/$INPUT_CONFIG-cleanup.sh" if [ -f "$CLEANUP_SCRIPT" ]; then source "$CLEANUP_SCRIPT" &> LOG.cleanup fi @@ -13,14 +16,14 @@ invariant_cleanup() { # Separate from invariant_setup so a caller that generates its own config can override the # render alone; child script.prepare files are concatenated after this one. invariant_render() { - cp -r "$TESTDIR/../data/." . &> LOG.cp + cp -r "$INVARIANT_DIR/data/." . &> LOG.cp - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + INIT_SCRIPT="$INVARIANT_DIR/configs/$INPUT_CONFIG-init.sh" if [ -f "$INIT_SCRIPT" ]; then source "$INIT_SCRIPT" &> LOG.init fi - envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml + envsubst < "$INVARIANT_DIR/configs/$INPUT_CONFIG" > databricks.yml cp databricks.yml LOG.config } @@ -34,11 +37,18 @@ invariant_setup() { } # Goes through trace, so callers can prefix the command with VAR=val. +# set -e is off so a failed deploy is still scanned for panics; the exit code is re-raised after. invariant_deploy() { local logfile="$1" shift + set +e trace "$@" &> "$logfile" + local rc=$? + set -e cat "$logfile" | contains.py '!panic:' '!internal error' > /dev/null + if [ "$rc" -ne 0 ]; then + exit "$rc" + fi # Tells the fuzzer the generated config was valid; failures after this count as bugs. echo INPUT_CONFIG_OK diff --git a/acceptance/cmd/yaml2json/main.go b/acceptance/cmd/yaml2json/main.go new file mode 100644 index 00000000000..1b4da7c7ae5 --- /dev/null +++ b/acceptance/cmd/yaml2json/main.go @@ -0,0 +1,46 @@ +// Command yaml2json prints a YAML file as JSON, parsed the way the bundle parses it. +// +// Acceptance helpers are stdlib-only Python and cannot parse YAML; this lives under +// acceptance/ rather than the product CLI so it stays test-only. +package main + +import ( + "fmt" + "os" + + "github.com/databricks/cli/libs/dyn/jsonsaver" + "github.com/databricks/cli/libs/dyn/yamlloader" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: %s FILE\n", os.Args[0]) + os.Exit(1) + } + + if err := run(os.Args[1]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + v, err := yamlloader.LoadYAML(path, f) + if err != nil { + return err + } + + buf, err := jsonsaver.MarshalIndent(v, "", " ") + if err != nil { + return err + } + + _, err = os.Stdout.Write(buf) + return err +} diff --git a/acceptance/selftest/mutate_fuzz_config/out.test.toml b/acceptance/selftest/mutate_fuzz_config/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/selftest/mutate_fuzz_config/output.txt b/acceptance/selftest/mutate_fuzz_config/output.txt new file mode 100644 index 00000000000..d4448247920 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/output.txt @@ -0,0 +1,65 @@ +=== volume seed=0 === +{ + "bundle": { + "name": "test-bundle-check" + }, + "resources": { + "volumes": { + "foo": { + "name": "../../etc/passwd", + "catalog_name": "main", + "schema_name": "default", + "grants": [ + { + "principal": "account users", + "privileges": [ + [] + ] + } + ] + } + } + } +} +=== volume seed=1 === +{ + "bundle": { + "name": "test-bundle-check" + }, + "resources": { + "volumes": { + "foo": { + "name": "test-volume-check", + "catalog_name": "main", + "schema_name": "default", + "grants": [ + { + "principal": "account users", + "privileges": [ + "READ_VOLUME" + ] + } + ], + "comment": "fuzz-volume" + } + } + } +} +=== volume seed=5 === +{ + "bundle": { + "name": "test-bundle-check" + }, + "resources": { + "volumes": { + "foo": { + "name": "test-volume-check", + "catalog_name": "main", + "schema_name": "default", + "grants": [ + {} + ] + } + } + } +} diff --git a/acceptance/selftest/mutate_fuzz_config/script b/acceptance/selftest/mutate_fuzz_config/script new file mode 100644 index 00000000000..2d57c739261 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/script @@ -0,0 +1 @@ +mutate_fuzz_config_check.py diff --git a/acceptance/selftest/mutate_fuzz_config/test.toml b/acceptance/selftest/mutate_fuzz_config/test.toml new file mode 100644 index 00000000000..62ad996277f --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/test.toml @@ -0,0 +1,3 @@ +# Pure Python contract check; engine matrix would only double a no-op. +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]