diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 0000000000..95869a25cc --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 49418540ac..b84842ffdc 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -852,8 +852,11 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL @@ -876,6 +879,8 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.resolved_file_triggers map[string]string INPUT +resources.job_runs.*.resolved_file_triggers.* string INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 0000000000..0a6df75047 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 0000000000..57b0f61685 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 0000000000..d03307b90f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,226 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 0000000000..fd1e8800c4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,64 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "removing on_file_change rewrites state without a run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 0000000000..626799f0f8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 0000000000..7a950e9fd0 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 0000000000..44c5433e52 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 0000000000..79c53955ef --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 0000000000..c7d3a98e1c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,39 @@ + +=== a pattern outside the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 0000000000..39596f0c77 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,12 @@ +title "a pattern outside the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories is rejected" +# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the +# trigger would silently never fire. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace musterr $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "dirs/*" "missing/*.txt" +trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 0000000000..7b88d613d4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 0000000000..d47fcf4404 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore new file mode 100644 index 0000000000..57433d43f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 0000000000..5626abf0f7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 0000000000..f719efd430 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 0000000000..2bdf67abb1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 0000000000..ea10ec85c1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 0000000000..62e0af52c1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 0000000000..57b0f61685 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 0000000000..d424482b96 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,283 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 10 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing a file in a subdirectory re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" + } + } + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][5]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ** is not recursive: it matches the same files as * +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 0000000000..dd737f242b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,63 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. +changes() { + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "editing one matched file re-fires" +update_file.py migrations/001.txt "one" "one-edited" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt + +title "a new matching file re-fires" +printf 'three\n' > migrations/003.txt +changes +rm migrations/003.txt + +title "removing a matched file re-fires" +mv migrations/002.txt tmp.002.txt +changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +changes +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + +title "** is not recursive: it matches the same files as *" +update_file.py databricks.yml "migrations/*" "migrations/**" +changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 0000000000..98c98631e5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,13 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 0000000000..d74b9841c5 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,202 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "maps" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" +) + +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// content hashes on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + syncable, d := syncableRelPaths(ctx, b) + if d.HasError() { + return d + } + diags = diags.Extend(d) + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil || jr.Lifecycle == nil { + continue + } + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) + diags = diags.Extend(d) + maps.Copy(out, hashes) + } + if len(out) == 0 { + jr.ResolvedFileTriggers = nil + } else { + jr.ResolvedFileTriggers = out + } + } + return diags +} + +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + files, err := fl.Files(ctx) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, diags +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + localPattern := filepath.FromSlash(pattern) + // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + if err != nil { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) + } + if len(matches) == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + regularMatches := 0 + ignoredMatches := 0 + sawNonRegular := false + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(loc), + }) + continue + } + if !info.Mode().IsRegular() { + sawNonRegular = true + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), + Locations: b.Config.GetLocations(loc), + }) + continue + } + // Same membership as sync: .gitignore and sync.exclude drop a glob match. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + ignoredMatches++ + continue + } + regularMatches++ + hash, err := hashFile(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(loc), + }) + continue + } + out[filepath.ToSlash(rel)] = hash + } + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + if len(out) == 0 && ignoredMatches > 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + return out, diags +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 0000000000..1cc54eecf4 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,75 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) + }) + + t.Run("trims pattern whitespace", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) + pattern := " seed.txt " + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) + return &bundle.Bundle{ + SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, + Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func contentHash(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925..c185d3b578 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -26,30 +27,45 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c22545747..7e8772d8d8 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,13 +64,35 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 56f3c39a35..f4b1696345 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,18 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239e..a4e881d298 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,6 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index b391fd706d..63ed81cb78 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,10 +26,21 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// jobRunTriggerLocalPaths is shared by OverrideChangeDesc and DoUpdate so +// clearing a trigger stays a state-only update in both places. +var jobRunTriggerLocalPaths = []string{ + "lifecycle", + "lifecycle.triggers", + "lifecycle.triggers.on_bundle_deploy", + "lifecycle.triggers.on_file_change", +} + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Path → content hash from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. @@ -99,10 +110,18 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } + if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { + return state + } + onBundleDeploy := "" if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, - } + onBundleDeploy = uuid.NewString() + } + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: onBundleDeploy, + OnFileChange: input.ResolvedFileTriggers, + }, } return state } @@ -370,7 +389,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRunState, entry *PlanEntry) (*JobRunRemote, error) { // Clearing a trigger only drops its local-only fingerprint from state; wait on // the run only when some other field changed. - if !entry.Changes.HasChangeExcept("lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy") { + if !entry.Changes.HasChangeExcept(jobRunTriggerLocalPaths...) { config.ResultState = "" return nil, nil } @@ -386,8 +405,7 @@ func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRun // Clearing a trigger downgrades the recreate to a state-only update so the // fingerprint is dropped from state without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + if slices.Contains(jobRunTriggerLocalPaths, path.String()) { // A cleared trigger sets New empty; structdiff may report it at lifecycle, // lifecycle.triggers, or the leaf. DoUpdate treats these paths as no-ops. if change.New == nil || change.New == "" { @@ -395,6 +413,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } return nil + } + switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 27c1b93fac..72de912e9d 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -354,25 +354,46 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { - t.Run("unset", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) - assert.Nil(t, state.Lifecycle) - }) + on := true + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, + }, + } + first := (&ResourceJobRun{}).PrepareState(input) + require.NotNil(t, first.Lifecycle) + require.NotNil(t, first.Lifecycle.Triggers) + assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) + + second := (&ResourceJobRun{}).PrepareState(input) + assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) +} + +func TestJobRunPrepareStateOnFileChange(t *testing.T) { + hashes := map[string]string{"a.txt": "abc"} t.Run("armed", func(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ + ResolvedFileTriggers: hashes, + }) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) + }) + + t.Run("both triggers", func(t *testing.T) { on := true - input := &resources.JobRun{ + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ Lifecycle: &resources.JobRunLifecycle{ Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, }, - } - first := (&ResourceJobRun{}).PrepareState(input) - require.NotNil(t, first.Lifecycle) - require.NotNil(t, first.Lifecycle.Triggers) - assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) - - second := (&ResourceJobRun{}).PrepareState(input) - assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) + ResolvedFileTriggers: hashes, + }) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) }) } @@ -401,6 +422,17 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) + t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: map[string]string{"a.txt": "abc"}, + New: nil, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + assert.Equal(t, "trigger removed", change.Reason) + }) + t.Run("fresh fingerprint still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, @@ -410,6 +442,16 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) + + t.Run("changed on_file_change hash still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: map[string]string{"a.txt": "old"}, + New: map[string]string{"a.txt": "new"}, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) } // The planner diffs RemapState(remote) against PrepareState(config), so a run diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a47e815168..1fd6b45241 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -990,6 +990,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b15e1c30df..9eba6f466e 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,8 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), @@ -191,6 +192,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index e0c710428a..39d31e9216 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1300,6 +1300,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false