From 84608d678e3444127b21643dcb672b73c6ec92f5 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 28 Jul 2026 13:23:41 +0200 Subject: [PATCH 01/30] internal: Refactor immutable folder upload as an internal resource in direct engine --- .../deploy/immutable-no-artifacts/.gitignore | 2 + .../deploy/immutable-no-artifacts/output.txt | 87 ++++++++++++- .../deploy/immutable-no-artifacts/script | 20 ++- .../immutable-permissions-change/output.txt | 2 - acceptance/bundle/deploy/immutable/output.txt | 114 +++++++++++++++++- acceptance/bundle/deploy/immutable/script | 24 +++- .../config/mutator/default_workspace_paths.go | 12 +- .../process_static_resources.go | 2 +- .../resourcemutator/resource_mutator.go | 18 +-- bundle/config/mutator/translate_paths.go | 16 +-- bundle/deploy/snapshot/path.go | 26 +--- bundle/deploy/snapshot/path_test.go | 23 +--- bundle/deploy/snapshot/upload.go | 75 ++++++------ bundle/deploy/snapshot/upload_test.go | 5 +- bundle/deploy/snapshot/upload_warning_test.go | 9 +- bundle/direct/bundle_plan.go | 68 +++++++++-- bundle/direct/dresources/adapter.go | 19 +++ bundle/direct/dresources/config.go | 6 +- bundle/direct/dresources/resources.yml | 8 ++ bundle/direct/dresources/snapshot.go | 104 ++++++++++++++++ bundle/direct/pkg.go | 41 ++++++- bundle/libraries/filer.go | 6 + bundle/phases/build.go | 6 - bundle/phases/deploy.go | 33 ++--- bundle/phases/destroy.go | 7 ++ cmd/bundle/plan.go | 7 ++ {bundle/deploy => libs}/snapshot/client.go | 34 +++++- libs/snapshot/hash.go | 12 ++ libs/testserver/handlers.go | 7 ++ 29 files changed, 608 insertions(+), 185 deletions(-) create mode 100644 acceptance/bundle/deploy/immutable-no-artifacts/.gitignore create mode 100644 bundle/direct/dresources/snapshot.go rename {bundle/deploy => libs}/snapshot/client.go (74%) create mode 100644 libs/snapshot/hash.go diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore b/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore new file mode 100644 index 00000000000..78caa68e38e --- /dev/null +++ b/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore @@ -0,0 +1,2 @@ +*.txt +*.json diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 3fc999917ab..b6a34bac207 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -8,8 +8,47 @@ Workspace: Validation OK! +>>> [CLI] bundle plan -o json + +>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json +[ + { + "notebook_task": { + "base_parameters": { + "path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path" + }, + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" + }, + "task_key": "notebook_task" + }, + { + "environment_key": "env", + "spark_python_task": { + "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" + }, + "task_key": "spark_python_task" + } +] + +>>> jq .plan["internal.snapshot"] plan.json +{ + "action": "create", + "new_state": { + "value": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] + } + } +} + >>> [CLI] bundle deploy -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! @@ -23,6 +62,52 @@ Deployment complete! >>> [CLI] jobs get [NUMID] "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path" +>>> title Check snapshot plan after deploy + +=== Check snapshot plan after deploy +>>> [CLI] bundle plan -o json + +>>> jq .plan["internal.snapshot"] plan.json +{ + "action": "skip", + "remote_state": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "changes": { + "acl": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "new": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "remote": null + }, + "bundle_id": { + "action": "skip", + "reason": "missing_in_remote", + "old": "[UUID]", + "new": "[UUID]", + "remote": "" + } + } +} + +>>> [CLI] bundle deploy +Deploying resources... +Updating deployment state... +Deployment complete! +\nSnapshot did not change as expected + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable-no-artifacts/script index a7aad718750..3717f7f919f 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/script +++ b/acceptance/bundle/deploy/immutable-no-artifacts/script @@ -6,13 +6,31 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle deploy +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json +trace jq '.plan["internal.snapshot"]' plan.json +trace $CLI bundle deploy # Get a job and check that task paths point into the snapshot JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path' +PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') + +trace title "Check snapshot plan after deploy" +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["internal.snapshot"]' plan.json +rm plan.json + +trace $CLI bundle deploy +PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +if [ "$PYTHON_FILE_1" == "$PYTHON_FILE_2" ]; then + echo "\nSnapshot did not change as expected" +else + echo "\nSnapshot changed" + exit 1 +fi trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable-permissions-change/output.txt index 68b79efc831..3b4f5e73600 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable-permissions-change/output.txt @@ -1,7 +1,6 @@ === Deploy without permissions >>> [CLI] bundle deploy -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! @@ -21,7 +20,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:22:3 -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index dcc78a79103..139da38a088 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -10,10 +10,12 @@ Validation OK! >>> [CLI] bundle plan -o json Building python_artifact... + +>>> jq .plan["resources.jobs.my_job"].new_state.value.tasks plan.json [ { "notebook_task": { - "notebook_path": "${workspace.snapshot_path}/files/src/notebook" + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" }, "task_key": "notebook_task" }, @@ -28,15 +30,45 @@ Building python_artifact... { "environment_key": "env", "spark_python_task": { - "python_file": "${workspace.snapshot_path}/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" }, "task_key": "spark_python_task" } ] +>>> jq .plan["resources.jobs.my_job"].new_state.value.environments plan.json +[ + { + "environment_key": "env", + "spec": { + "dependencies": [ + "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } +] + +>>> jq .plan["internal.snapshot"] plan.json +{ + "action": "create", + "new_state": { + "value": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] + } + } +} + >>> [CLI] bundle deploy Building python_artifact... -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! @@ -52,6 +84,82 @@ Deployment complete! "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ] +>>> title Check snapshot plan after deploy + +=== Check snapshot plan after deploy +>>> [CLI] bundle plan -o json +Building python_artifact... + +>>> jq .plan["internal.snapshot"] plan.json +{ + "action": "recreate", + "new_state": { + "value": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] + } + }, + "remote_state": { + "relative_path": "[UUID]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "changes": { + "acl": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "new": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "remote": null + }, + "bundle_id": { + "action": "skip", + "reason": "missing_in_remote", + "old": "[UUID]", + "new": "[UUID]", + "remote": "" + }, + "full_path": { + "action": "recreate", + "reason": "immutable", + "old": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "new": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "remote": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "relative_path": { + "action": "recreate", + "reason": "immutable", + "old": "[UUID]/[SNAPSHOT_HASH]", + "new": "[UUID]/[SNAPSHOT_HASH]", + "remote": "[UUID]/[SNAPSHOT_HASH]" + } + } +} + +>>> [CLI] bundle deploy +Building python_artifact... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Check that job uses a new snapshot\nSnapshot changed as expected + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 85e9a908d1d..a0027758b7c 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,15 +5,35 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' -trace $CLI bundle deploy +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json +trace jq '.plan["resources.jobs.my_job"].new_state.value.environments' plan.json +trace jq '.plan["internal.snapshot"]' plan.json +trace $CLI bundle deploy # Get a job and check that task paths are immutable JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' trace $CLI jobs get $JOB_ID | jq '.settings.environments[0].spec.dependencies' +PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') + +trace title "Check snapshot plan after deploy" +trace $CLI bundle plan -o json > plan.json +trace jq '.plan["internal.snapshot"]' plan.json +rm plan.json + +trace $CLI bundle deploy + +title "Check that job uses a new snapshot" +PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +if [ "$PYTHON_FILE_1" != "$PYTHON_FILE_2" ]; then + echo "\nSnapshot changed as expected" +else + echo "\nSnapshot did not change" + exit 1 +fi # Redirect run output to a log file — the real workspace produces different output than the local test server. $CLI bundle run my_job &> LOG.run diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 02a1ddb3b11..38bf8c987c9 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -26,7 +26,11 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl } if b.Config.Workspace.FilePath == "" { - b.Config.Workspace.FilePath = path.Join(root, "files") + if b.IsImmutableFolder() { + b.Config.Workspace.FilePath = "${internal.snapshot.full_path}/files" + } else { + b.Config.Workspace.FilePath = path.Join(root, "files") + } } if b.Config.Workspace.ResourcePath == "" { @@ -34,7 +38,11 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl } if b.Config.Workspace.ArtifactPath == "" { - b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") + if b.IsImmutableFolder() { + b.Config.Workspace.ArtifactPath = "${internal.snapshot.full_path}/artifacts" + } else { + b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") + } } if b.Config.Workspace.StatePath == "" { diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 28a7cbefe78..b1f689808f6 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -45,7 +45,7 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes - immutableExcludingResolver(b), + mutator.ResolveVariableReferencesOnlyResources(), mutator.NormalizePaths(), // Translate dashboard paths into paths in the workspace file system diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e8c33f0c59b..32243687074 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -133,22 +133,6 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { ) } -// immutableExcludingResolver returns a variable reference resolver for the resources -// section. When experimental.immutable_folder is enabled it excludes -// workspace.file_path, workspace.artifact_path, and workspace.snapshot_path from -// resolution: those paths are set by snapshot.Upload() in the Deploy phase, so -// resolving them here would freeze them to the default bundle path instead. -// workspace.snapshot_path is also excluded so it stays as a literal ${...} template -// in the plan output (making the pre-upload intent visible). -func immutableExcludingResolver(b *bundle.Bundle) bundle.Mutator { - if b.IsImmutableFolder() { - return mutator.ResolveVariableReferencesOnlyResourcesExcluding( - []string{"workspace.file_path", "workspace.artifact_path", "workspace.snapshot_path"}, - ) - } - return mutator.ResolveVariableReferencesOnlyResources() -} - // Normalization is applied multiple times if resource is modified during initialization // // If bundle is modified outside of 'resources' section, these changes are discarded. @@ -162,7 +146,7 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Reads (dynamic): * (strings) (searches for variable references in string values) // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes - immutableExcludingResolver(b), + mutator.ResolveVariableReferencesOnlyResources(), // Reads (dynamic): resources.pipelines.*.libraries (checks for notebook.path and file.path fields) // Updates (dynamic): resources.pipelines.*.libraries (expands glob patterns in path fields to multiple library entries) diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index c44e91160b9..a1d11d79b4a 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -322,21 +322,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { switch { case b.IsImmutableFolder(): - // Reject an explicit workspace.file_path: immutable bundles control that path - // automatically (it is set to the content-addressed snapshot location after upload). - // A user-supplied value would be silently discarded, so we error early instead. - if loc := b.Config.GetLocation("workspace.file_path"); loc.File != "" { - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: "workspace.file_path cannot be configured when experimental.immutable_folder is true", - Locations: []dyn.Location{loc}, - }} - } - // Use a placeholder referencing workspace.snapshot_path so that paths are stored - // as ${workspace.snapshot_path}/files/ during validate. After - // snapshot.Upload() sets workspace.snapshot_path, a variable-resolution pass - // expands these references to the actual content-addressed paths. - t.remoteRoot = "${workspace.snapshot_path}/files" + t.remoteRoot = "${internal.snapshot.full_path}/files" case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment): t.remoteRoot = t.b.SyncRootPath default: diff --git a/bundle/deploy/snapshot/path.go b/bundle/deploy/snapshot/path.go index 62a4b9f91ab..b82cf42a336 100644 --- a/bundle/deploy/snapshot/path.go +++ b/bundle/deploy/snapshot/path.go @@ -4,8 +4,6 @@ import ( "archive/zip" "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "io" @@ -17,6 +15,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/snapshot" libsync "github.com/databricks/cli/libs/sync" ) @@ -71,12 +70,12 @@ func BundleZip(ctx context.Context, b *bundle.Bundle) ([]byte, int, error) { // addMetadataToZip writes the snapshot metadata file into the zip so that // any change to the ACL changes the snapshot hash and forces a new snapshot. -func addMetadataToZip(zw *zip.Writer, acl []ACLEntry) error { +func addMetadataToZip(zw *zip.Writer, acl []snapshot.ACLEntry) error { aclJSON, err := json.Marshal(acl) if err != nil { return fmt.Errorf("marshal ACL for permissions hash: %w", err) } - data, err := json.Marshal(snapshotMetadata{PermissionsHash: IDFromContent(aclJSON)}) + data, err := json.Marshal(snapshotMetadata{PermissionsHash: snapshot.HashFromContent(aclJSON)}) if err != nil { return fmt.Errorf("marshal snapshot metadata: %w", err) } @@ -93,23 +92,6 @@ func addMetadataToZip(zw *zip.Writer, acl []ACLEntry) error { return err } -// IDFromContent returns the SHA-256 hex digest of content. -func IDFromContent(content []byte) string { - h := sha256.Sum256(content) - return hex.EncodeToString(h[:]) -} - -// SnapshotID builds the bundle zip and returns its SHA-256 hex digest. -// Called after artifacts are built so that ApplyImmutableWorkspacePaths and -// snapshot.Upload both hash identical content. -func SnapshotID(ctx context.Context, b *bundle.Bundle) (string, error) { - content, _, err := BundleZip(ctx, b) - if err != nil { - return "", err - } - return IDFromContent(content), nil -} - // addSyncRootToZip returns the number of files added from the sync root. func addSyncRootToZip(ctx context.Context, zw *zip.Writer, b *bundle.Bundle) (int, error) { opts, err := files.GetSyncOptions(ctx, b) @@ -168,8 +150,6 @@ func addArtifactsToZip(zw *zip.Writer, b *bundle.Bundle) error { if af.Patched != "" { source = af.Patched } - // ".internal" matches libraries.InternalDirName so that ReplaceWithRemotePath - // produces library paths that resolve correctly inside the snapshot. if err := addLocalFileToZip(zw, source, "artifacts/.internal"); err != nil { return err } diff --git a/bundle/deploy/snapshot/path_test.go b/bundle/deploy/snapshot/path_test.go index 5f567d9f377..54407a98e20 100644 --- a/bundle/deploy/snapshot/path_test.go +++ b/bundle/deploy/snapshot/path_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/snapshot" + libsnapshot "github.com/databricks/cli/libs/snapshot" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" @@ -93,26 +94,6 @@ func TestBundleZipRespectsExcludes(t *testing.T) { assert.Less(t, len(zipExcl), len(zipAll)) } -func TestIDFromContent(t *testing.T) { - id := snapshot.IDFromContent([]byte("hello")) - // SHA-256 of "hello" - assert.Equal(t, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", id) - assert.Len(t, id, 64, "SHA-256 hex must be 64 characters") -} - -func TestSnapshotIDMatchesBundleZipHash(t *testing.T) { - b := makeBundleWithFiles(t, map[string]string{"task.py": "x = 1"}) - - zipContent, _, err := snapshot.BundleZip(t.Context(), b) - require.NoError(t, err) - expectedID := snapshot.IDFromContent(zipContent) - - id, err := snapshot.SnapshotID(t.Context(), b) - require.NoError(t, err) - - assert.Equal(t, expectedID, id) -} - func zipEntryNames(t *testing.T, zipContent []byte) []string { t.Helper() r, err := zip.NewReader(bytes.NewReader(zipContent), int64(len(zipContent))) @@ -166,6 +147,6 @@ func TestBundleZipChangesWithPermissions(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, zipNoPerms, zipWithPerms, "adding top-level permissions must produce a different snapshot zip") - assert.NotEqual(t, snapshot.IDFromContent(zipNoPerms), snapshot.IDFromContent(zipWithPerms), + assert.NotEqual(t, libsnapshot.HashFromContent(zipNoPerms), libsnapshot.HashFromContent(zipWithPerms), "snapshot IDs must differ when top-level permissions change") } diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 8ec7a215d03..19e647bf57a 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -3,27 +3,26 @@ package snapshot import ( "context" "fmt" - "path" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/snapshot" ) // fileLimitWarning is the file count above which immutable folder deployments may fail. const fileLimitWarning = 1000 type snapshotUpload struct { + skipZip bool // uploader allows test injection of a custom SnapshotUploader. - uploader SnapshotUploader + uploader snapshot.SnapshotUploader } -// Upload returns a mutator that builds the bundle zip, uploads it via -// /api/2.0/repos/snapshots, and updates workspace.file_path and -// workspace.artifact_path to the content-addressed location returned by the API. -func Upload() bundle.Mutator { - return &snapshotUpload{} +// PlanUpload returns a mutator that builds the bundle zip, uploads it via +// /api/2.0/repos/snapshots, and registers the snapshot as an internal resource. +func PlanUpload(skipZip bool) bundle.Mutator { + return &snapshotUpload{skipZip: skipZip} } func (m *snapshotUpload) Name() string { @@ -34,46 +33,40 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn uploader := m.uploader if uploader == nil { var err error - uploader, err = NewSnapshotUploader(b.WorkspaceClient(ctx)) + uploader, err = snapshot.NewSnapshotUploader(b.WorkspaceClient(ctx)) if err != nil { return diag.FromErr(err) } } - cmdio.LogString(ctx, "Uploading immutable bundle snapshot...") - - zipContent, fileCount, err := BundleZip(ctx, b) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to build snapshot zip: %w", err)) - } - var diags diag.Diagnostics - if fileCount > fileLimitWarning { - diags = append(diags, diag.Warningf( - "immutable folder deployment may not work correctly: bundle contains %d files (limit is %d)", - fileCount, fileLimitWarning, - )...) - } - snapshotID := IDFromContent(zipContent) - log.Debugf(ctx, "snapshot.Upload: snapshotID=%s zip=%d bytes", snapshotID, len(zipContent)) - - acl := BuildACL(b) - // Use the deployment lineage UUID as bundle_id so the snapshot directory is - // keyed to this specific deployment (not to the bundle name, which can be - // reused across unrelated deployments). bundleID := b.DeploymentBundle.StateDB.GetOrInitLineage() - info, err := uploader.Upload(ctx, bundleID, snapshotID, acl, zipContent) + input := &dresources.SnapshotConfig{ + BundleID: bundleID, + ACL: BuildACL(b), + } + snapshotRootPath, err := uploader.GetSnapshotRootPath(ctx) if err != nil { return diag.FromErr(err) } + var diags diag.Diagnostics + if !m.skipZip { + zipContent, fileCount, err := BundleZip(ctx, b) + if err != nil { + return diag.FromErr(fmt.Errorf("failed to build snapshot zip: %w", err)) + } - log.Infof(ctx, "Snapshot uploaded to %s", info.Path) + if fileCount > fileLimitWarning { + diags = append(diags, diag.Warningf( + "immutable folder deployment may not work correctly: bundle contains %d files (limit is %d)", + fileCount, fileLimitWarning, + )...) + } - b.Config.Workspace.SnapshotPath = info.Path - b.Config.Workspace.FilePath = path.Join(info.Path, "files") - // Only set artifact_path when artifacts are present; with no artifacts the - // zip has no "artifacts" directory and a get-status on it would 404. - if len(b.Config.Artifacts) > 0 { - b.Config.Workspace.ArtifactPath = path.Join(info.Path, "artifacts") + input.ZipContent = zipContent + } + instance := (*dresources.Snapshot)(nil).New(uploader, snapshotRootPath) + if err := b.DeploymentBundle.RegisterInternalResource("internal.snapshot", instance, input); err != nil { + return diag.FromErr(err) } return diags @@ -82,12 +75,12 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn // BuildACL constructs the access_control_list for the snapshot upload. // It grants CAN_READ to the current user and to every principal listed in the // top-level permissions section of the bundle config. -func BuildACL(b *bundle.Bundle) []ACLEntry { - acl := []ACLEntry{ +func BuildACL(b *bundle.Bundle) []snapshot.ACLEntry { + acl := []snapshot.ACLEntry{ {UserName: b.Config.Workspace.CurrentUser.UserName, PermissionLevel: "CAN_READ"}, } for _, p := range b.Config.Permissions { - acl = append(acl, ACLEntry{ + acl = append(acl, snapshot.ACLEntry{ UserName: p.UserName, GroupName: p.GroupName, ServicePrincipalName: p.ServicePrincipalName, diff --git a/bundle/deploy/snapshot/upload_test.go b/bundle/deploy/snapshot/upload_test.go index 3b53f31ae3c..5bace57e2be 100644 --- a/bundle/deploy/snapshot/upload_test.go +++ b/bundle/deploy/snapshot/upload_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/snapshot" + libsnapshot "github.com/databricks/cli/libs/snapshot" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) @@ -29,7 +30,7 @@ func TestBuildACLCurrentUserOnly(t *testing.T) { acl := snapshot.BuildACL(b) - assert.Equal(t, []snapshot.ACLEntry{ + assert.Equal(t, []libsnapshot.ACLEntry{ {UserName: "alice@example.com", PermissionLevel: "CAN_READ"}, }, acl) } @@ -44,7 +45,7 @@ func TestBuildACLWithTopLevelPermissions(t *testing.T) { acl := snapshot.BuildACL(b) - assert.Equal(t, []snapshot.ACLEntry{ + assert.Equal(t, []libsnapshot.ACLEntry{ {UserName: "alice@example.com", PermissionLevel: "CAN_READ"}, {UserName: "bob@example.com", PermissionLevel: "CAN_READ"}, {GroupName: "devs", PermissionLevel: "CAN_READ"}, diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index ae45c45a87d..6ce52cc4451 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/snapshot" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" @@ -20,8 +21,12 @@ import ( type mockUploader struct{ path string } -func (m *mockUploader) Upload(_ context.Context, _, _ string, _ []ACLEntry, _ []byte) (*SnapshotInfo, error) { - return &SnapshotInfo{Path: m.path}, nil +func (m *mockUploader) Upload(_ context.Context, _, _ string, _ []snapshot.ACLEntry, _ []byte) (*snapshot.SnapshotInfo, error) { + return &snapshot.SnapshotInfo{Path: m.path}, nil +} + +func (m *mockUploader) Get(_ context.Context, _ string) (*snapshot.SnapshotInfo, error) { + return &snapshot.SnapshotInfo{Path: m.path}, nil } func makeBundle(t *testing.T, nFiles int) *bundle.Bundle { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 139012423ea..467e2de06ef 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -670,6 +670,11 @@ func isEmptyStruct(rv reflect.Value) bool { // For regular resources like "resources.jobs.foo.name", returns ("resources.jobs.foo", "name"). // For sub-resources like "resources.jobs.foo.permissions[0].level", returns ("resources.jobs.foo.permissions", "[0].level"). func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) { + // Internal resources: "internal.." — key is first two components. + if key, ok := path.Prefix(1).StringKey(); ok && key == "internal" { + return path.Prefix(2).String(), path.SkipPrefix(2) + } + // Check if the 4th component is "permissions" or "grants" (sub-resource) if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) @@ -681,12 +686,6 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) } func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { - // ${workspace.snapshot_path} is resolved by the mutator pipeline after - // snapshot.Upload() — not by the direct engine. Return errDelayed so the - // template string is preserved in the plan output rather than causing an error. - if path.String() == "workspace.snapshot_path" { - return nil, errDelayed - } targetResourceKey, fieldPath := splitResourcePath(path) targetGroup := config.GetResourceTypeFromKey(targetResourceKey) @@ -744,7 +743,7 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s localConfig := sv.Value - adapter := b.Adapters[targetGroup] + adapter, err := b.getAdapterForKey(targetResourceKey) if adapter == nil { return nil, fmt.Errorf("internal error: %s: unknown resource type %q", targetResourceKey, targetGroup) } @@ -919,8 +918,29 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } } - slices.Sort(nodes) + // If configRoot is nil it means we are destroying the bundle, + // so we don't need to prepare internal resources same way as for regular resources. + if configRoot != nil { + // Register internal resources first so regular resources can depend on them. + for _, ir := range b.InternalResources { + delete(existingKeys, ir.Key) // prevent it being marked for deletion + stateVal, err := ir.Adapter.PrepareState(ir.InputConfig) + if err != nil { + return nil, fmt.Errorf("%s: preparing state: %w", ir.Key, err) + } + sv := &structvar.StructVar{Value: stateVal} + b.StateCache.Store(ir.Key, sv) + + newStateJSON, err := sv.ToJSON() + if err != nil { + return nil, fmt.Errorf("%s: serializing state: %w", ir.Key, err) + } + p.Plan[ir.Key] = &deployplan.PlanEntry{NewState: newStateJSON} + } + } + + slices.Sort(nodes) for _, node := range nodes { delete(existingKeys, node) @@ -994,11 +1014,8 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return nil, fmt.Errorf("parsing %q: %w", targetPath, err) } - targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) - targetNode := targetNodeDP.String() - // ${workspace.snapshot_path} is resolved by the mutator pipeline after - // snapshot.Upload(), not by the direct engine — skip it here. - if targetPath == "workspace.snapshot_path" { + targetNode := resourceNodeFromPath(targetPath, targetPathParsed) + if targetNode == "" { continue } @@ -1058,6 +1075,11 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root Action: deployplan.Delete, DependsOn: entry.DependsOn, } + + // If the node is an internal snapshot resource, we need to mark it as gone because it can't be deleted. + if n == "internal.snapshot" { + p.Plan[n].Gone = true + } } return p, nil @@ -1135,7 +1157,27 @@ func dynPathToStructPath(p dyn.Path) *structpath.PathNode { return node } +// resourceNodeFromPath extracts the plan node key from a reference target path. +// Returns "" for paths that aren't plan nodes (e.g. bundle.*, variables.*). +func resourceNodeFromPath(targetPath string, parsed dyn.Path) string { + if strings.HasPrefix(targetPath, "internal.") { + // "internal.." → "internal." + parts := strings.SplitN(targetPath, ".", 3) + if len(parts) >= 2 { + return parts[0] + "." + parts[1] + } + return "" + } + node, _ := config.GetNodeAndType(parsed) + return node.String() +} + func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Adapter, error) { + // Internal resources are looked up by exact key, not by parsed group. + if adapter := b.findInternalAdapter(resourceKey); adapter != nil { + return adapter, nil + } + group := config.GetResourceTypeFromKey(resourceKey) if group == "" { return nil, fmt.Errorf("internal error: bad node: %s", resourceKey) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 70a9f1f5e3d..cb0816e356b 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "strings" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/calladapt" @@ -153,6 +154,24 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC return adapter, nil } +// NewAdapterFromInstance creates an Adapter from an already-initialized resource +// instance. Use this for internal resources whose New() method doesn't take a +// workspace client. +func NewAdapterFromInstance(instance any, resourceType string) (*Adapter, error) { + resourceType = strings.TrimPrefix(resourceType, "internal.") + adapter := &Adapter{ + resourceConfig: GetResourceConfig(resourceType), + generatedResourceConfig: GetGeneratedResourceConfig(resourceType), + } + if err := adapter.initMethods(instance); err != nil { + return nil, err + } + if err := adapter.validate(); err != nil { + return nil, err + } + return adapter, nil +} + // loadKeyedSlices validates and calls KeyedSlices method, returning the resulting map. func loadKeyedSlices(call *calladapt.BoundCaller) (map[string]any, error) { outs, err := call.Call() diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 91175594a6a..e4d6c07795a 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -84,6 +84,7 @@ type ResourceLifecycleConfig struct { // Config is the root configuration structure for resource lifecycle behavior. type Config struct { Resources map[string]ResourceLifecycleConfig `yaml:"resources"` + Internal map[string]ResourceLifecycleConfig `yaml:"internal"` } //go:embed resources.yml @@ -104,7 +105,7 @@ var empty = ResourceLifecycleConfig{ func mustParseConfig(data []byte) func() *Config { return sync.OnceValue(func() *Config { - c := &Config{Resources: nil} + c := &Config{Resources: nil, Internal: nil} if err := yaml.Unmarshal(data, c); err != nil { panic(err) } @@ -133,6 +134,9 @@ func GetResourceConfig(resourceType string) *ResourceLifecycleConfig { if rc, ok := cfg.Resources[resourceType]; ok { return &rc } + if rc, ok := cfg.Internal[resourceType]; ok { + return &rc + } return &empty } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 580c3aa1558..c5e671bc557 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -795,3 +795,11 @@ resources: backend_defaults: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype + +internal: + snapshot: + recreate_on_changes: + - field: full_path + reason: immutable + - field: relative_path + reason: immutable diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go new file mode 100644 index 00000000000..ed44002921b --- /dev/null +++ b/bundle/direct/dresources/snapshot.go @@ -0,0 +1,104 @@ +package dresources + +import ( + "context" + "errors" + "path" + + "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/databricks-sdk-go/apierr" +) + +type Snapshot struct { + remoteRoot string + uploader snapshot.SnapshotUploader +} + +type SnapshotConfig struct { + BundleID string + ACL []snapshot.ACLEntry + ZipContent []byte +} + +type SnapshotState struct { + RelativePath string `json:"relative_path"` + FullPath string `json:"full_path"` + BundleID string `json:"bundle_id"` + ACL []snapshot.ACLEntry `json:"acl"` + ZipContent []byte `json:"-"` +} + +type SnapshotRemote struct { + RelativePath string `json:"relative_path"` + FullPath string `json:"full_path"` +} + +func (s *SnapshotConfig) RelativePath() string { + return path.Join(s.BundleID, snapshot.HashFromContent(s.ZipContent)) +} + +func (s *SnapshotConfig) FullPath(remoteRoot string) string { + return path.Join(remoteRoot, s.RelativePath()) +} + +func (s *Snapshot) New(uploader snapshot.SnapshotUploader, remoteRoot string) *Snapshot { + return &Snapshot{ + remoteRoot: remoteRoot, + uploader: uploader, + } +} + +func (s *Snapshot) PrepareState(input *SnapshotConfig) *SnapshotState { + return &SnapshotState{ + RelativePath: input.RelativePath(), + FullPath: input.FullPath(s.remoteRoot), + BundleID: input.BundleID, + ACL: input.ACL, + ZipContent: input.ZipContent, + } +} + +func (s *Snapshot) RemapState(remote *SnapshotRemote) *SnapshotState { + return &SnapshotState{ + RelativePath: remote.RelativePath, + FullPath: remote.FullPath, + BundleID: "", + ACL: nil, + ZipContent: nil, + } +} + +func (s *Snapshot) DoRead(ctx context.Context, id string) (*SnapshotRemote, error) { + fullPath := path.Join(s.remoteRoot, id) + _, err := s.uploader.Get(ctx, fullPath) + if err != nil { + if errors.Is(err, apierr.ErrNotFound) { + return &SnapshotRemote{ + RelativePath: id, + FullPath: "", + }, nil + } + return nil, err + } + return &SnapshotRemote{ + RelativePath: id, + FullPath: fullPath, + }, nil +} + +func (s *Snapshot) DoCreate(ctx context.Context, state *SnapshotState) (string, *SnapshotRemote, error) { + path := state.RelativePath + info, err := s.uploader.Upload(ctx, path, state.BundleID, state.ACL, state.ZipContent) + if err != nil { + return "", nil, err + } + return path, &SnapshotRemote{RelativePath: path, FullPath: info.Path}, nil +} + +func (s *Snapshot) DoUpdate(ctx context.Context, id string, newState *SnapshotState, entry *PlanEntry) (*SnapshotRemote, error) { + return nil, nil +} + +func (s *Snapshot) DoDelete(ctx context.Context, id string, state *SnapshotState) error { + return nil +} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..154a9415fc6 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -39,11 +39,42 @@ type DeploymentUnit struct { // DeploymentBundle holds everything needed to deploy a bundle type DeploymentBundle struct { - StateDB dstate.DeploymentState - Adapters map[string]*dresources.Adapter - Plan *deployplan.Plan - RemoteStateCache sync.Map - StateCache structvar.Cache + StateDB dstate.DeploymentState + Adapters map[string]*dresources.Adapter + InternalResources []InternalResource + Plan *deployplan.Plan + RemoteStateCache sync.Map + StateCache structvar.Cache +} + +type InternalResource struct { + Key string + Adapter *dresources.Adapter + InputConfig any // passed to adapter.PrepareState during makePlan +} + +// RegisterInternalResource creates an adapter from instance and queues the +// resource for planning and apply. Must be called before CalculatePlan. +func (b *DeploymentBundle) RegisterInternalResource(key string, instance any, inputConfig any) error { + adapter, err := dresources.NewAdapterFromInstance(instance, key) + if err != nil { + return fmt.Errorf("registering internal resource %s: %w", key, err) + } + b.InternalResources = append(b.InternalResources, InternalResource{ + Key: key, + Adapter: adapter, + InputConfig: inputConfig, + }) + return nil +} + +func (b *DeploymentBundle) findInternalAdapter(key string) *dresources.Adapter { + for i := range b.InternalResources { + if b.InternalResources[i].Key == key { + return b.InternalResources[i].Adapter + } + } + return nil } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index 762732262be..5280ea0915d 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -26,6 +26,9 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) uploadPath = ensureWorkspaceOrVolumesPrefix(uploadPath) + if b.IsImmutableFolder() { + uploadPath = path.Join("${internal.snapshot.full_path}", "artifacts", InternalDirName) + } switch { case IsVolumesPath(artifactPath): @@ -43,6 +46,9 @@ func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.F } artifactPath = ensureWorkspaceOrVolumesPrefix(artifactPath) + if b.IsImmutableFolder() { + artifactPath = path.Join("${internal.snapshot.full_path}", "artifacts") + } switch { case IsVolumesPath(artifactPath): diff --git a/bundle/phases/build.go b/bundle/phases/build.go index db376e07e28..f8069b73bcf 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -48,12 +48,6 @@ func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { return nil } - // For immutable bundles, library remote paths are set in the deploy phase - // after snapshot.Upload() provides the content-addressed workspace.artifact_path. - if b.IsImmutableFolder() { - return nil - } - libs, diags := libraries.ReplaceWithRemotePath(ctx, b) for _, d := range diags { logdiag.LogDiag(ctx, d) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f65e50a940e..61bd97acc06 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" @@ -171,29 +170,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - if immutable { - // Upload all source files and built artifacts as a single immutable snapshot. - // snapshot.Upload() sets workspace.snapshot_path; the variable-resolution - // pass expands ${workspace.snapshot_path} placeholders written by translate_paths. - bundle.ApplySeqContext(ctx, b, - snapshot.Upload(), - mutator.ResolveVariableReferencesOnlyResources("workspace"), - ) - if !logdiag.HasError(ctx) { - _, libDiags := libraries.ReplaceWithRemotePath(ctx, b) - for _, d := range libDiags { - logdiag.LogDiag(ctx, d) - } - } - } else { + if !immutable { uploadLibraries(ctx, b, libs) - } - - if logdiag.HasError(ctx) { - return - } + if logdiag.HasError(ctx) { + return + } - if !immutable { bundle.ApplySeqContext(ctx, b, files.Upload(outputHandler)) if logdiag.HasError(ctx) { return @@ -212,6 +194,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + if immutable { + bundle.ApplyContext(ctx, b, snapshot.PlanUpload(false)) + if logdiag.HasError(ctx) { + return + } + } + planFromFile := plan != nil if plan == nil { // State is already open for read by process.go (for direct engine) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..ad57e121cc1 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" + "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" @@ -155,6 +156,12 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { + if b.IsImmutableFolder() { + bundle.ApplyContext(ctx, b, snapshot.PlanUpload(true)) + if logdiag.HasError(ctx) { + return + } + } plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) if err != nil { logdiag.LogError(ctx, err) diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 20df8cb5f0f..5ce0464c0cc 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/bundle/utils" @@ -61,6 +62,12 @@ It is useful for previewing changes before running 'bundle deploy'.`, } ctx := cmd.Context() + if b.IsImmutableFolder() { + bundle.ApplyContext(ctx, b, snapshot.PlanUpload(false)) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + } plan := phases.RunPlan(ctx, b, stateDesc.Engine) if logdiag.HasError(ctx) { return root.ErrAlreadyPrinted diff --git a/bundle/deploy/snapshot/client.go b/libs/snapshot/client.go similarity index 74% rename from bundle/deploy/snapshot/client.go rename to libs/snapshot/client.go index 4e5df29e408..006042a38e1 100644 --- a/bundle/deploy/snapshot/client.go +++ b/libs/snapshot/client.go @@ -35,12 +35,15 @@ type ACLEntry struct { // the same workspace location. // This interface exists so the implementation can later be replaced with a Go SDK call. type SnapshotUploader interface { - Upload(ctx context.Context, bundleID, snapshotID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) + GetSnapshotRootPath(ctx context.Context) (string, error) + Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) + Get(ctx context.Context, path string) (*SnapshotInfo, error) } // snapshotAPIClient implements SnapshotUploader against /api/2.0/repos/snapshots. type snapshotAPIClient struct { - client *databricksclient.DatabricksClient + workspaceClient *databricks.WorkspaceClient + client *databricksclient.DatabricksClient } // snapshotUploadResponse mirrors the /api/2.0/repos/snapshots response body. @@ -50,23 +53,27 @@ type snapshotUploadResponse struct { } `json:"snapshot"` } +type snapshotRootPathResponse struct { + Path string `json:"path"` +} + // NewSnapshotUploader creates a SnapshotUploader backed by /api/2.0/repos/snapshots. func NewSnapshotUploader(w *databricks.WorkspaceClient) (SnapshotUploader, error) { c, err := databricksclient.New(w.Config) if err != nil { return nil, err } - return &snapshotAPIClient{client: c}, nil + return &snapshotAPIClient{workspaceClient: w, client: c}, nil } // Upload uploads zipContent as an immutable snapshot identified by snapshotID. // snapshotID is the SHA-256 of the zip and is used by the server as the // content-addressed path component. acl grants CAN_READ to each listed principal. -func (c *snapshotAPIClient) Upload(ctx context.Context, bundleID, snapshotID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { +func (c *snapshotAPIClient) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { var body bytes.Buffer mw := multipart.NewWriter(&body) - if err := mw.WriteField("snapshot_id", snapshotID); err != nil { + if err := mw.WriteField("snapshot_id", HashFromContent(zipContent)); err != nil { return nil, fmt.Errorf("failed to write snapshot_id: %w", err) } if err := mw.WriteField("bundle_id", bundleID); err != nil { @@ -112,3 +119,20 @@ func (c *snapshotAPIClient) Upload(ctx context.Context, bundleID, snapshotID str return &SnapshotInfo{Path: resp.Snapshot.Path}, nil } + +func (c *snapshotAPIClient) Get(ctx context.Context, path string) (*SnapshotInfo, error) { + resp, err := c.workspaceClient.Workspace.GetStatusByPath(ctx, path) + if err != nil { + return nil, fmt.Errorf("snapshot get: %w", err) + } + return &SnapshotInfo{Path: resp.Path}, nil +} + +func (c *snapshotAPIClient) GetSnapshotRootPath(ctx context.Context) (string, error) { + var resp snapshotRootPathResponse + err := c.client.Do(ctx, http.MethodGet, "/api/2.0/repos/snapshots/rootpath", auth.WorkspaceIDHeaders(c.client.Config), nil, nil, &resp) + if err != nil { + return "", fmt.Errorf("snapshot root path get: %w", err) + } + return resp.Path, nil +} diff --git a/libs/snapshot/hash.go b/libs/snapshot/hash.go new file mode 100644 index 00000000000..104858896c3 --- /dev/null +++ b/libs/snapshot/hash.go @@ -0,0 +1,12 @@ +package snapshot + +import ( + "crypto/sha256" + "encoding/hex" +) + +// HashFromContent returns the SHA-256 hex digest of content. +func HashFromContent(content []byte) string { + h := sha256.Sum256(content) + return hex.EncodeToString(h[:]) +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 1d534c47431..0efda60a867 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -603,6 +603,12 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ReposDelete(req) }) + server.Handle("GET", "/api/2.0/repos/snapshots/rootpath", func(req Request) any { + return map[string]any{ + "path": "/Workspace/Users/" + TestUserSP.UserName + "/.snapshots", + } + }) + server.Handle("POST", "/api/2.0/repos/snapshots", func(req Request) any { contentType := req.Headers.Get("Content-Type") mediaType, params, err := mime.ParseMediaType(contentType) @@ -635,6 +641,7 @@ func AddDefaultHandlers(server *Server) { // The real API uses the workspace user UUID (not email) in the snapshot path, // matching service-principal identities used in cloud acceptance tests. snapshotPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/%s", TestUserSP.UserName, bundleID, snapshotID) + req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: snapshotPath}) return map[string]any{ "snapshot": map[string]any{ "path": snapshotPath, From a5ab287a1a656ec96732b1a2f3217fb84ebb51ee Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 28 Jul 2026 14:00:20 +0200 Subject: [PATCH 02/30] fix test and tidy up --- bundle/config/workspace.go | 6 ------ bundle/deploy/snapshot/upload_warning_test.go | 5 ++++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index f68670b7e86..9e5d2522d0d 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -78,12 +78,6 @@ type Workspace struct { // Remote workspace path for deployment state. // This defaults to "${workspace.root}/state". StatePath string `json:"state_path,omitempty"` - - // SnapshotPath is the workspace path of the immutable snapshot uploaded during - // deployment. Set by snapshot.Upload() and used by the subsequent variable-resolution - // pass to expand ${workspace.snapshot_path} placeholders in resource configs. - // Only populated at runtime for bundles with experimental.immutable_folder = true. - SnapshotPath string `json:"snapshot_path,omitempty" bundle:"internal"` } type User struct { diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index 6ce52cc4451..01a543d94fc 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -29,6 +29,10 @@ func (m *mockUploader) Get(_ context.Context, _ string) (*snapshot.SnapshotInfo, return &snapshot.SnapshotInfo{Path: m.path}, nil } +func (m *mockUploader) GetSnapshotRootPath(_ context.Context) (string, error) { + return filepath.Join(m.path, "snapshots"), nil +} + func makeBundle(t *testing.T, nFiles int) *bundle.Bundle { t.Helper() dir := t.TempDir() @@ -71,7 +75,6 @@ func TestUploadWarnsAboveFileLimit(t *testing.T) { require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, fmt.Sprintf("%d files", fileLimitWarning+1)) - assert.Equal(t, "/snapshots/test", b.Config.Workspace.SnapshotPath) } func TestUploadNoWarningBelowFileLimit(t *testing.T) { From 4a0d4bdb3852b63389f71fb00e40507161bd23f7 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 28 Jul 2026 15:57:11 +0200 Subject: [PATCH 03/30] updated the output --- acceptance/bundle/resources/apps/immutable/output.txt | 3 +-- .../bundle/validate/immutable_workspace_paths/output.txt | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index e7ae9f1ba23..bed6b0fd19a 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -1,6 +1,5 @@ >>> [CLI] bundle deploy -Uploading immutable bundle snapshot... Deploying resources... Updating deployment state... Deployment complete! @@ -37,6 +36,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "${workspace.snapshot_path}/files/app" + "source_code_path": "${internal.snapshot.full_path}/files/app" } } diff --git a/acceptance/bundle/validate/immutable_workspace_paths/output.txt b/acceptance/bundle/validate/immutable_workspace_paths/output.txt index fef8083634f..9035835e2c7 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/output.txt +++ b/acceptance/bundle/validate/immutable_workspace_paths/output.txt @@ -6,14 +6,14 @@ Warning: Pattern user_repls.json does not match any files { "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/artifacts", + "artifact_path": "${internal.snapshot.full_path}/artifacts", "current_user": { "domain_friendly_name": "[USERNAME]", "id": "[USERID]", "short_name": "[USERNAME]", "userName": "[USERNAME]" }, - "file_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/files", + "file_path": "${internal.snapshot.full_path}/files", "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default", "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state" @@ -23,7 +23,7 @@ Warning: Pattern user_repls.json does not match any files "ai_runtime_task": { "deployments": [ { - "command_path": "${workspace.snapshot_path}/files/src/main.py", + "command_path": "${internal.snapshot.full_path}/files/src/main.py", "compute": { "accelerator_count": 1, "accelerator_type": "GPU_1xA10" @@ -37,7 +37,7 @@ Warning: Pattern user_repls.json does not match any files { "existing_cluster_id": "0101-120000-aaaaaaaa", "spark_python_task": { - "python_file": "${workspace.snapshot_path}/files/src/main.py" + "python_file": "${internal.snapshot.full_path}/files/src/main.py" }, "task_key": "my_task" } From aa73e340bd957dff2c067edf6237306a259d80de Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 28 Jul 2026 16:32:02 +0200 Subject: [PATCH 04/30] fix lint --- bundle/direct/bundle_plan.go | 3 ++- bundle/direct/dresources/adapter.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 53b817b49be..cae5d03d333 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -748,7 +748,8 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s localConfig := sv.Value - adapter, err := b.getAdapterForKey(targetResourceKey) + var adapter *dresources.Adapter + adapter, err = b.getAdapterForKey(targetResourceKey) if adapter == nil { return nil, fmt.Errorf("internal error: %s: unknown resource type %q", targetResourceKey, targetGroup) } diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 81d78471821..b3416c5c47a 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -169,6 +169,20 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC func NewAdapterFromInstance(instance any, resourceType string) (*Adapter, error) { resourceType = strings.TrimPrefix(resourceType, "internal.") adapter := &Adapter{ + prepareState: nil, + remapState: nil, + doRefresh: nil, + doDelete: nil, + doCreate: nil, + doUpdate: nil, + doUpdateWithID: nil, + doResize: nil, + waitAfterCreate: nil, + waitAfterUpdate: nil, + waitAfterDelete: nil, + overrideChangeDesc: nil, + keyedSlices: nil, + isGone: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), } From b4f272f3c739221126b4fdc2aa7ec2fbe6a99ded Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 31 Jul 2026 17:00:14 +0200 Subject: [PATCH 05/30] make snapshot a normal resource --- .../deploy/immutable-no-artifacts/output.txt | 4 +- .../deploy/immutable-no-artifacts/script | 4 +- acceptance/bundle/deploy/immutable/output.txt | 4 +- acceptance/bundle/deploy/immutable/script | 4 +- .../resources/apps/immutable/output.txt | 2 +- .../immutable_workspace_paths/output.txt | 8 +-- .../databricks.yml | 9 +++ .../out.test.toml | 3 + .../snapshot_resource_not_allowed/output.txt | 12 ++++ .../snapshot_resource_not_allowed/script | 1 + .../config/mutator/default_workspace_paths.go | 4 +- .../mutator/reject_internal_resources.go | 61 +++++++++++++++++++ bundle/config/mutator/translate_paths.go | 2 +- bundle/config/resources.go | 3 + bundle/config/resources/snapshot.go | 24 ++++++++ bundle/deploy/snapshot/upload.go | 23 +++---- bundle/direct/bundle_plan.go | 42 +------------ bundle/direct/dresources/adapter.go | 33 ---------- bundle/direct/dresources/all.go | 3 + bundle/direct/dresources/config.go | 6 +- bundle/direct/dresources/resources.yml | 1 - bundle/direct/dresources/snapshot.go | 53 ++++++++-------- bundle/direct/pkg.go | 41 ++----------- bundle/libraries/filer.go | 4 +- bundle/phases/initialize.go | 4 ++ 25 files changed, 183 insertions(+), 172 deletions(-) create mode 100644 acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml create mode 100644 acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml create mode 100644 acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt create mode 100644 acceptance/bundle/validate/snapshot_resource_not_allowed/script create mode 100644 bundle/config/mutator/reject_internal_resources.go create mode 100644 bundle/config/resources/snapshot.go diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index b6a34bac207..093b6eeccd4 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -30,7 +30,7 @@ Validation OK! } ] ->>> jq .plan["internal.snapshot"] plan.json +>>> jq .plan["resources.snapshot.immutable"] plan.json { "action": "create", "new_state": { @@ -67,7 +67,7 @@ Deployment complete! === Check snapshot plan after deploy >>> [CLI] bundle plan -o json ->>> jq .plan["internal.snapshot"] plan.json +>>> jq .plan["resources.snapshot.immutable"] plan.json { "action": "skip", "remote_state": { diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable-no-artifacts/script index 3717f7f919f..5e74d027b13 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/script +++ b/acceptance/bundle/deploy/immutable-no-artifacts/script @@ -8,7 +8,7 @@ trap cleanup EXIT trace $CLI bundle validate trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json -trace jq '.plan["internal.snapshot"]' plan.json +trace jq '.plan["resources.snapshot.immutable"]' plan.json trace $CLI bundle deploy @@ -21,7 +21,7 @@ PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select( trace title "Check snapshot plan after deploy" trace $CLI bundle plan -o json > plan.json -trace jq '.plan["internal.snapshot"]' plan.json +trace jq '.plan["resources.snapshot.immutable"]' plan.json rm plan.json trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 139da38a088..edeac4b5e18 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -49,7 +49,7 @@ Building python_artifact... } ] ->>> jq .plan["internal.snapshot"] plan.json +>>> jq .plan["resources.snapshot.immutable"] plan.json { "action": "create", "new_state": { @@ -90,7 +90,7 @@ Deployment complete! >>> [CLI] bundle plan -o json Building python_artifact... ->>> jq .plan["internal.snapshot"] plan.json +>>> jq .plan["resources.snapshot.immutable"] plan.json { "action": "recreate", "new_state": { diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index a0027758b7c..33f3b347da6 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -8,7 +8,7 @@ trace $CLI bundle validate trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.environments' plan.json -trace jq '.plan["internal.snapshot"]' plan.json +trace jq '.plan["resources.snapshot.immutable"]' plan.json trace $CLI bundle deploy @@ -21,7 +21,7 @@ PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select( trace title "Check snapshot plan after deploy" trace $CLI bundle plan -o json > plan.json -trace jq '.plan["internal.snapshot"]' plan.json +trace jq '.plan["resources.snapshot.immutable"]' plan.json rm plan.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index bed6b0fd19a..72dc9ebd61c 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -36,6 +36,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "${internal.snapshot.full_path}/files/app" + "source_code_path": "${resources.snapshot.immutable.full_path}/files/app" } } diff --git a/acceptance/bundle/validate/immutable_workspace_paths/output.txt b/acceptance/bundle/validate/immutable_workspace_paths/output.txt index 9035835e2c7..0d1d1ee6a5f 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/output.txt +++ b/acceptance/bundle/validate/immutable_workspace_paths/output.txt @@ -6,14 +6,14 @@ Warning: Pattern user_repls.json does not match any files { "workspace": { - "artifact_path": "${internal.snapshot.full_path}/artifacts", + "artifact_path": "${resources.snapshot.immutable.full_path}/artifacts", "current_user": { "domain_friendly_name": "[USERNAME]", "id": "[USERID]", "short_name": "[USERNAME]", "userName": "[USERNAME]" }, - "file_path": "${internal.snapshot.full_path}/files", + "file_path": "${resources.snapshot.immutable.full_path}/files", "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default", "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state" @@ -23,7 +23,7 @@ Warning: Pattern user_repls.json does not match any files "ai_runtime_task": { "deployments": [ { - "command_path": "${internal.snapshot.full_path}/files/src/main.py", + "command_path": "${resources.snapshot.immutable.full_path}/files/src/main.py", "compute": { "accelerator_count": 1, "accelerator_type": "GPU_1xA10" @@ -37,7 +37,7 @@ Warning: Pattern user_repls.json does not match any files { "existing_cluster_id": "0101-120000-aaaaaaaa", "spark_python_task": { - "python_file": "${internal.snapshot.full_path}/files/src/main.py" + "python_file": "${resources.snapshot.immutable.full_path}/files/src/main.py" }, "task_key": "my_task" } diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml new file mode 100644 index 00000000000..c738f20869c --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: snapshot_resource_not_allowed + +resources: + snapshot: + immutable: + bundle_id: 123 + zip_content: | + This is a test zip content. diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml b/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt new file mode 100644 index 00000000000..bae3072e941 --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle validate +Error: snapshot is an internal resource and cannot be set in bundle configuration + at resources.snapshot + in databricks.yml:6:5 + +Name: snapshot_resource_not_allowed +Target: default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/script b/acceptance/bundle/validate/snapshot_resource_not_allowed/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 38bf8c987c9..82ac6fd60b0 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -27,7 +27,7 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl if b.Config.Workspace.FilePath == "" { if b.IsImmutableFolder() { - b.Config.Workspace.FilePath = "${internal.snapshot.full_path}/files" + b.Config.Workspace.FilePath = "${resources.snapshot.immutable.full_path}/files" } else { b.Config.Workspace.FilePath = path.Join(root, "files") } @@ -39,7 +39,7 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl if b.Config.Workspace.ArtifactPath == "" { if b.IsImmutableFolder() { - b.Config.Workspace.ArtifactPath = "${internal.snapshot.full_path}/artifacts" + b.Config.Workspace.ArtifactPath = "${resources.snapshot.immutable.full_path}/artifacts" } else { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go new file mode 100644 index 00000000000..d548aafc10f --- /dev/null +++ b/bundle/config/mutator/reject_internal_resources.go @@ -0,0 +1,61 @@ +package mutator + +import ( + "context" + "reflect" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/structs/structwalk" +) + +type rejectInternalResources struct{} + +// RejectInternalResources returns a mutator that errors when a user has set +// any internal resource field in their bundle configuration. +func RejectInternalResources() bundle.Mutator { + return &rejectInternalResources{} +} + +func (m *rejectInternalResources) Name() string { + return "RejectInternalResources" +} + +func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + var internalResourceKeys []string + // collect all internal resource keys, only top level keys under "resources" + structwalk.WalkType(reflect.TypeFor[config.Resources](), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { + if path.Len() > 2 { + return false + } + if field == nil { + return true + } + tag := field.Tag.Get("bundle") + if structtag.BundleTag(tag).Internal() { + internalResourceKeys = append(internalResourceKeys, path.String()) + } + return true + }) + + for _, key := range internalResourceKeys { + v, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources."+key)) + if err != nil { + continue + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: key + " is an internal resource and cannot be set in bundle configuration", + Locations: v.Locations(), + Paths: []dyn.Path{dyn.MustPathFromString("resources." + key)}, + }) + } + + return diags +} diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a1d11d79b4a..1bd80dd74d2 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -322,7 +322,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { switch { case b.IsImmutableFolder(): - t.remoteRoot = "${internal.snapshot.full_path}/files" + t.remoteRoot = "${resources.snapshot.immutable.full_path}/files" case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment): t.remoteRoot = t.b.SyncRootPath default: diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 3ab06f49918..5628b3c1c1a 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -44,6 +44,9 @@ type Resources struct { VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` + + // Internal resources + Snapshot map[string]*resources.Snapshot `json:"snapshot,omitempty" bundle:"internal"` } type ConfigResource interface { diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go new file mode 100644 index 00000000000..b85b93621f1 --- /dev/null +++ b/bundle/config/resources/snapshot.go @@ -0,0 +1,24 @@ +package resources + +import ( + "path" + + "github.com/databricks/cli/libs/snapshot" +) + +// Snapshot is the configuration for the snapshot resource. +// This is an internal resource that is used to store the snapshot of the bundle. +// It is not meant to be used by the user. +type Snapshot struct { + BundleID string `json:"bundle_id"` + ACL []snapshot.ACLEntry `json:"acl"` + ZipContent string `json:"zip_content"` +} + +func (s *Snapshot) RelativePath() string { + return path.Join(s.BundleID, snapshot.HashFromContent([]byte(s.ZipContent))) +} + +func (s *Snapshot) FullPath(remoteRoot string) string { + return path.Join(remoteRoot, s.RelativePath()) +} diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 19e647bf57a..b0192445a4e 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/snapshot" ) @@ -39,15 +39,16 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } } - bundleID := b.DeploymentBundle.StateDB.GetOrInitLineage() - input := &dresources.SnapshotConfig{ - BundleID: bundleID, - ACL: BuildACL(b), + if b.Config.Resources.Snapshot == nil { + b.Config.Resources.Snapshot = make(map[string]*resources.Snapshot) } - snapshotRootPath, err := uploader.GetSnapshotRootPath(ctx) - if err != nil { - return diag.FromErr(err) + if _, ok := b.Config.Resources.Snapshot["immutable"]; !ok { + b.Config.Resources.Snapshot["immutable"] = &resources.Snapshot{ + BundleID: b.DeploymentBundle.StateDB.GetOrInitLineage(), + ACL: BuildACL(b), + } } + var diags diag.Diagnostics if !m.skipZip { zipContent, fileCount, err := BundleZip(ctx, b) @@ -62,11 +63,7 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn )...) } - input.ZipContent = zipContent - } - instance := (*dresources.Snapshot)(nil).New(uploader, snapshotRootPath) - if err := b.DeploymentBundle.RegisterInternalResource("internal.snapshot", instance, input); err != nil { - return diag.FromErr(err) + b.Config.Resources.Snapshot["immutable"].ZipContent = string(zipContent) } return diags diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index cae5d03d333..cccd9069f71 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -675,11 +675,6 @@ func isEmptyStruct(rv reflect.Value) bool { // For regular resources like "resources.jobs.foo.name", returns ("resources.jobs.foo", "name"). // For sub-resources like "resources.jobs.foo.permissions[0].level", returns ("resources.jobs.foo.permissions", "[0].level"). func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) { - // Internal resources: "internal.." — key is first two components. - if key, ok := path.Prefix(1).StringKey(); ok && key == "internal" { - return path.Prefix(2).String(), path.SkipPrefix(2) - } - // Check if the 4th component is "permissions" or "grants" (sub-resource) if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) @@ -924,28 +919,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } } - // If configRoot is nil it means we are destroying the bundle, - // so we don't need to prepare internal resources same way as for regular resources. - if configRoot != nil { - // Register internal resources first so regular resources can depend on them. - for _, ir := range b.InternalResources { - delete(existingKeys, ir.Key) // prevent it being marked for deletion - - stateVal, err := ir.Adapter.PrepareState(ir.InputConfig) - if err != nil { - return nil, fmt.Errorf("%s: preparing state: %w", ir.Key, err) - } - sv := &structvar.StructVar{Value: stateVal} - b.StateCache.Store(ir.Key, sv) - - newStateJSON, err := sv.ToJSON() - if err != nil { - return nil, fmt.Errorf("%s: serializing state: %w", ir.Key, err) - } - p.Plan[ir.Key] = &deployplan.PlanEntry{NewState: newStateJSON} - } - } - slices.Sort(nodes) for _, node := range nodes { delete(existingKeys, node) @@ -1083,7 +1056,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } // If the node is an internal snapshot resource, we need to mark it as gone because it can't be deleted. - if n == "internal.snapshot" { + if n == "resources.snapshot.immutable" { p.Plan[n].Gone = true } } @@ -1166,24 +1139,11 @@ func dynPathToStructPath(p dyn.Path) *structpath.PathNode { // resourceNodeFromPath extracts the plan node key from a reference target path. // Returns "" for paths that aren't plan nodes (e.g. bundle.*, variables.*). func resourceNodeFromPath(targetPath string, parsed dyn.Path) string { - if strings.HasPrefix(targetPath, "internal.") { - // "internal.." → "internal." - parts := strings.SplitN(targetPath, ".", 3) - if len(parts) >= 2 { - return parts[0] + "." + parts[1] - } - return "" - } node, _ := config.GetNodeAndType(parsed) return node.String() } func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Adapter, error) { - // Internal resources are looked up by exact key, not by parsed group. - if adapter := b.findInternalAdapter(resourceKey); adapter != nil { - return adapter, nil - } - group := config.GetResourceTypeFromKey(resourceKey) if group == "" { return nil, fmt.Errorf("internal error: bad node: %s", resourceKey) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index b3416c5c47a..fdaa15bfcea 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "reflect" - "strings" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/calladapt" @@ -163,38 +162,6 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC return adapter, nil } -// NewAdapterFromInstance creates an Adapter from an already-initialized resource -// instance. Use this for internal resources whose New() method doesn't take a -// workspace client. -func NewAdapterFromInstance(instance any, resourceType string) (*Adapter, error) { - resourceType = strings.TrimPrefix(resourceType, "internal.") - adapter := &Adapter{ - prepareState: nil, - remapState: nil, - doRefresh: nil, - doDelete: nil, - doCreate: nil, - doUpdate: nil, - doUpdateWithID: nil, - doResize: nil, - waitAfterCreate: nil, - waitAfterUpdate: nil, - waitAfterDelete: nil, - overrideChangeDesc: nil, - keyedSlices: nil, - isGone: nil, - resourceConfig: GetResourceConfig(resourceType), - generatedResourceConfig: GetGeneratedResourceConfig(resourceType), - } - if err := adapter.initMethods(instance); err != nil { - return nil, err - } - if err := adapter.validate(); err != nil { - return nil, err - } - return adapter, nil -} - // loadKeyedSlices validates and calls KeyedSlices method, returning the resulting map. func loadKeyedSlices(call *calladapt.BoundCaller) (map[string]any, error) { outs, err := call.Call() diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index d6bcd03f73c..abbb1cd979a 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -65,6 +65,9 @@ var SupportedResources = map[string]any{ "volumes.grants": (*ResourceGrants)(nil), "registered_models.grants": (*ResourceGrants)(nil), "vector_search_indexes.grants": (*ResourceGrants)(nil), + + // Internal resources + "snapshot": (*ResourceSnapshot)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index e4d6c07795a..91175594a6a 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -84,7 +84,6 @@ type ResourceLifecycleConfig struct { // Config is the root configuration structure for resource lifecycle behavior. type Config struct { Resources map[string]ResourceLifecycleConfig `yaml:"resources"` - Internal map[string]ResourceLifecycleConfig `yaml:"internal"` } //go:embed resources.yml @@ -105,7 +104,7 @@ var empty = ResourceLifecycleConfig{ func mustParseConfig(data []byte) func() *Config { return sync.OnceValue(func() *Config { - c := &Config{Resources: nil, Internal: nil} + c := &Config{Resources: nil} if err := yaml.Unmarshal(data, c); err != nil { panic(err) } @@ -134,9 +133,6 @@ func GetResourceConfig(resourceType string) *ResourceLifecycleConfig { if rc, ok := cfg.Resources[resourceType]; ok { return &rc } - if rc, ok := cfg.Internal[resourceType]; ok { - return &rc - } return &empty } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 229892921d2..b3282a5e426 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -798,7 +798,6 @@ resources: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype -internal: snapshot: recreate_on_changes: - field: full_path diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index ed44002921b..bcd7f007718 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -5,27 +5,23 @@ import ( "errors" "path" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" ) -type Snapshot struct { +type ResourceSnapshot struct { remoteRoot string uploader snapshot.SnapshotUploader } -type SnapshotConfig struct { - BundleID string - ACL []snapshot.ACLEntry - ZipContent []byte -} - type SnapshotState struct { RelativePath string `json:"relative_path"` FullPath string `json:"full_path"` BundleID string `json:"bundle_id"` ACL []snapshot.ACLEntry `json:"acl"` - ZipContent []byte `json:"-"` + ZipContent string `json:"-"` } type SnapshotRemote struct { @@ -33,22 +29,29 @@ type SnapshotRemote struct { FullPath string `json:"full_path"` } -func (s *SnapshotConfig) RelativePath() string { - return path.Join(s.BundleID, snapshot.HashFromContent(s.ZipContent)) -} +func (s *ResourceSnapshot) New(client *databricks.WorkspaceClient) *ResourceSnapshot { + // Return a zero-value instance when client is nil (e.g. refschema introspection). + if client == nil { + return &ResourceSnapshot{} + } -func (s *SnapshotConfig) FullPath(remoteRoot string) string { - return path.Join(remoteRoot, s.RelativePath()) -} + uploader, err := snapshot.NewSnapshotUploader(client) + if err != nil { + panic(err) + } + + snapshotRootPath, err := uploader.GetSnapshotRootPath(context.Background()) + if err != nil { + panic(err) + } -func (s *Snapshot) New(uploader snapshot.SnapshotUploader, remoteRoot string) *Snapshot { - return &Snapshot{ - remoteRoot: remoteRoot, + return &ResourceSnapshot{ + remoteRoot: snapshotRootPath, uploader: uploader, } } -func (s *Snapshot) PrepareState(input *SnapshotConfig) *SnapshotState { +func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotState { return &SnapshotState{ RelativePath: input.RelativePath(), FullPath: input.FullPath(s.remoteRoot), @@ -58,17 +61,17 @@ func (s *Snapshot) PrepareState(input *SnapshotConfig) *SnapshotState { } } -func (s *Snapshot) RemapState(remote *SnapshotRemote) *SnapshotState { +func (s *ResourceSnapshot) RemapState(remote *SnapshotRemote) *SnapshotState { return &SnapshotState{ RelativePath: remote.RelativePath, FullPath: remote.FullPath, BundleID: "", ACL: nil, - ZipContent: nil, + ZipContent: "", } } -func (s *Snapshot) DoRead(ctx context.Context, id string) (*SnapshotRemote, error) { +func (s *ResourceSnapshot) DoRead(ctx context.Context, id string) (*SnapshotRemote, error) { fullPath := path.Join(s.remoteRoot, id) _, err := s.uploader.Get(ctx, fullPath) if err != nil { @@ -86,19 +89,19 @@ func (s *Snapshot) DoRead(ctx context.Context, id string) (*SnapshotRemote, erro }, nil } -func (s *Snapshot) DoCreate(ctx context.Context, state *SnapshotState) (string, *SnapshotRemote, error) { +func (s *ResourceSnapshot) DoCreate(ctx context.Context, state *SnapshotState) (string, *SnapshotRemote, error) { path := state.RelativePath - info, err := s.uploader.Upload(ctx, path, state.BundleID, state.ACL, state.ZipContent) + info, err := s.uploader.Upload(ctx, path, state.BundleID, state.ACL, []byte(state.ZipContent)) if err != nil { return "", nil, err } return path, &SnapshotRemote{RelativePath: path, FullPath: info.Path}, nil } -func (s *Snapshot) DoUpdate(ctx context.Context, id string, newState *SnapshotState, entry *PlanEntry) (*SnapshotRemote, error) { +func (s *ResourceSnapshot) DoUpdate(ctx context.Context, id string, newState *SnapshotState, entry *PlanEntry) (*SnapshotRemote, error) { return nil, nil } -func (s *Snapshot) DoDelete(ctx context.Context, id string, state *SnapshotState) error { +func (s *ResourceSnapshot) DoDelete(ctx context.Context, id string, state *SnapshotState) error { return nil } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 154a9415fc6..48a9c5a2ff7 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -39,42 +39,11 @@ type DeploymentUnit struct { // DeploymentBundle holds everything needed to deploy a bundle type DeploymentBundle struct { - StateDB dstate.DeploymentState - Adapters map[string]*dresources.Adapter - InternalResources []InternalResource - Plan *deployplan.Plan - RemoteStateCache sync.Map - StateCache structvar.Cache -} - -type InternalResource struct { - Key string - Adapter *dresources.Adapter - InputConfig any // passed to adapter.PrepareState during makePlan -} - -// RegisterInternalResource creates an adapter from instance and queues the -// resource for planning and apply. Must be called before CalculatePlan. -func (b *DeploymentBundle) RegisterInternalResource(key string, instance any, inputConfig any) error { - adapter, err := dresources.NewAdapterFromInstance(instance, key) - if err != nil { - return fmt.Errorf("registering internal resource %s: %w", key, err) - } - b.InternalResources = append(b.InternalResources, InternalResource{ - Key: key, - Adapter: adapter, - InputConfig: inputConfig, - }) - return nil -} - -func (b *DeploymentBundle) findInternalAdapter(key string) *dresources.Adapter { - for i := range b.InternalResources { - if b.InternalResources[i].Key == key { - return b.InternalResources[i].Adapter - } - } - return nil + StateDB dstate.DeploymentState + Adapters map[string]*dresources.Adapter + Plan *deployplan.Plan + RemoteStateCache sync.Map + StateCache structvar.Cache } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index 5280ea0915d..c5884aea930 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -27,7 +27,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) uploadPath = ensureWorkspaceOrVolumesPrefix(uploadPath) if b.IsImmutableFolder() { - uploadPath = path.Join("${internal.snapshot.full_path}", "artifacts", InternalDirName) + uploadPath = path.Join("${resources.snapshot.immutable.full_path}", "artifacts", InternalDirName) } switch { @@ -47,7 +47,7 @@ func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.F artifactPath = ensureWorkspaceOrVolumesPrefix(artifactPath) if b.IsImmutableFolder() { - artifactPath = path.Join("${internal.snapshot.full_path}", "artifacts") + artifactPath = path.Join("${resources.snapshot.immutable.full_path}", "artifacts") } switch { diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..51b0d406eee 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,6 +26,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") bundle.ApplySeqContext(ctx, b, + // Reads (dynamic): resources.snapshot (and other internal keys) + // Warns and removes any internal resource fields set by the user. + mutator.RejectInternalResources(), + // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), From f31a466888639c03a59001c19638474e983a54a5 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 31 Jul 2026 17:11:27 +0200 Subject: [PATCH 06/30] clean up --- bundle/direct/bundle_plan.go | 15 ++------------- bundle/direct/dresources/snapshot.go | 5 +++++ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index cccd9069f71..83584207e9e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -993,7 +993,8 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return nil, fmt.Errorf("parsing %q: %w", targetPath, err) } - targetNode := resourceNodeFromPath(targetPath, targetPathParsed) + targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) + targetNode := targetNodeDP.String() if targetNode == "" { continue } @@ -1054,11 +1055,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root Action: deployplan.Delete, DependsOn: entry.DependsOn, } - - // If the node is an internal snapshot resource, we need to mark it as gone because it can't be deleted. - if n == "resources.snapshot.immutable" { - p.Plan[n].Gone = true - } } return p, nil @@ -1136,13 +1132,6 @@ func dynPathToStructPath(p dyn.Path) *structpath.PathNode { return node } -// resourceNodeFromPath extracts the plan node key from a reference target path. -// Returns "" for paths that aren't plan nodes (e.g. bundle.*, variables.*). -func resourceNodeFromPath(targetPath string, parsed dyn.Path) string { - node, _ := config.GetNodeAndType(parsed) - return node.String() -} - func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Adapter, error) { group := config.GetResourceTypeFromKey(resourceKey) if group == "" { diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index bcd7f007718..f520bb8aef4 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -105,3 +105,8 @@ func (s *ResourceSnapshot) DoUpdate(ctx context.Context, id string, newState *Sn func (s *ResourceSnapshot) DoDelete(ctx context.Context, id string, state *SnapshotState) error { return nil } + +// IsGone treats a snapshot as already-deleted. The snapshot is immutable, so it can't be deleted. +func (s *ResourceSnapshot) IsGone(remote *SnapshotRemote) bool { + return true +} From 5943979b133d471e4133e1b63ff3efb148e9a9cf Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 3 Aug 2026 13:00:10 +0200 Subject: [PATCH 07/30] fix test --- .../deploy/immutable-no-artifacts/output.txt | 39 ++-------------- .../deploy/immutable-no-artifacts/script | 4 +- acceptance/bundle/deploy/immutable/output.txt | 46 ++----------------- acceptance/bundle/deploy/immutable/script | 4 +- .../resources/apps/immutable/output.txt | 2 +- .../immutable_workspace_paths/output.txt | 8 ++-- .../databricks.yml | 2 +- .../snapshot_resource_not_allowed/output.txt | 4 +- .../config/mutator/default_workspace_paths.go | 4 +- bundle/config/mutator/translate_paths.go | 2 +- bundle/config/resources.go | 2 +- bundle/deploy/snapshot/upload.go | 10 ++-- bundle/direct/dresources/all.go | 2 +- bundle/direct/dresources/all_test.go | 34 +++++++++++--- bundle/direct/dresources/resources.yml | 2 +- bundle/direct/dresources/snapshot.go | 8 ---- bundle/direct/dresources/type_test.go | 7 +++ bundle/libraries/filer.go | 4 +- bundle/phases/initialize.go | 2 +- bundle/statemgmt/state_load_test.go | 5 ++ 20 files changed, 73 insertions(+), 118 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 093b6eeccd4..bc4d7278262 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -30,20 +30,13 @@ Validation OK! } ] ->>> jq .plan["resources.snapshot.immutable"] plan.json +>>> jq .plan["resources.snapshots.immutable"] plan.json { "action": "create", "new_state": { "value": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", - "bundle_id": "[UUID]", - "acl": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ] + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" } } } @@ -67,38 +60,12 @@ Deployment complete! === Check snapshot plan after deploy >>> [CLI] bundle plan -o json ->>> jq .plan["resources.snapshot.immutable"] plan.json +>>> jq .plan["resources.snapshots.immutable"] plan.json { "action": "skip", "remote_state": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" - }, - "changes": { - "acl": { - "action": "skip", - "reason": "missing_in_remote", - "old": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ], - "new": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ], - "remote": null - }, - "bundle_id": { - "action": "skip", - "reason": "missing_in_remote", - "old": "[UUID]", - "new": "[UUID]", - "remote": "" - } } } diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable-no-artifacts/script index 5e74d027b13..a944c4ad090 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/script +++ b/acceptance/bundle/deploy/immutable-no-artifacts/script @@ -8,7 +8,7 @@ trap cleanup EXIT trace $CLI bundle validate trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json -trace jq '.plan["resources.snapshot.immutable"]' plan.json +trace jq '.plan["resources.snapshots.immutable"]' plan.json trace $CLI bundle deploy @@ -21,7 +21,7 @@ PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select( trace title "Check snapshot plan after deploy" trace $CLI bundle plan -o json > plan.json -trace jq '.plan["resources.snapshot.immutable"]' plan.json +trace jq '.plan["resources.snapshots.immutable"]' plan.json rm plan.json trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index edeac4b5e18..8e280cddb45 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -49,20 +49,13 @@ Building python_artifact... } ] ->>> jq .plan["resources.snapshot.immutable"] plan.json +>>> jq .plan["resources.snapshots.immutable"] plan.json { "action": "create", "new_state": { "value": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", - "bundle_id": "[UUID]", - "acl": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ] + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" } } } @@ -90,20 +83,13 @@ Deployment complete! >>> [CLI] bundle plan -o json Building python_artifact... ->>> jq .plan["resources.snapshot.immutable"] plan.json +>>> jq .plan["resources.snapshots.immutable"] plan.json { "action": "recreate", "new_state": { "value": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", - "bundle_id": "[UUID]", - "acl": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ] + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" } }, "remote_state": { @@ -111,30 +97,6 @@ Building python_artifact... "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" }, "changes": { - "acl": { - "action": "skip", - "reason": "missing_in_remote", - "old": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ], - "new": [ - { - "user_name": "[USERNAME]", - "permission_level": "CAN_READ" - } - ], - "remote": null - }, - "bundle_id": { - "action": "skip", - "reason": "missing_in_remote", - "old": "[UUID]", - "new": "[UUID]", - "remote": "" - }, "full_path": { "action": "recreate", "reason": "immutable", diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 33f3b347da6..7e72e45efc1 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -8,7 +8,7 @@ trace $CLI bundle validate trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.environments' plan.json -trace jq '.plan["resources.snapshot.immutable"]' plan.json +trace jq '.plan["resources.snapshots.immutable"]' plan.json trace $CLI bundle deploy @@ -21,7 +21,7 @@ PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select( trace title "Check snapshot plan after deploy" trace $CLI bundle plan -o json > plan.json -trace jq '.plan["resources.snapshot.immutable"]' plan.json +trace jq '.plan["resources.snapshots.immutable"]' plan.json rm plan.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index 72dc9ebd61c..41491f053b8 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -36,6 +36,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "${resources.snapshot.immutable.full_path}/files/app" + "source_code_path": "${resources.snapshots.immutable.full_path}/files/app" } } diff --git a/acceptance/bundle/validate/immutable_workspace_paths/output.txt b/acceptance/bundle/validate/immutable_workspace_paths/output.txt index 0d1d1ee6a5f..c5d3b305605 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/output.txt +++ b/acceptance/bundle/validate/immutable_workspace_paths/output.txt @@ -6,14 +6,14 @@ Warning: Pattern user_repls.json does not match any files { "workspace": { - "artifact_path": "${resources.snapshot.immutable.full_path}/artifacts", + "artifact_path": "${resources.snapshots.immutable.full_path}/artifacts", "current_user": { "domain_friendly_name": "[USERNAME]", "id": "[USERID]", "short_name": "[USERNAME]", "userName": "[USERNAME]" }, - "file_path": "${resources.snapshot.immutable.full_path}/files", + "file_path": "${resources.snapshots.immutable.full_path}/files", "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default", "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state" @@ -23,7 +23,7 @@ Warning: Pattern user_repls.json does not match any files "ai_runtime_task": { "deployments": [ { - "command_path": "${resources.snapshot.immutable.full_path}/files/src/main.py", + "command_path": "${resources.snapshots.immutable.full_path}/files/src/main.py", "compute": { "accelerator_count": 1, "accelerator_type": "GPU_1xA10" @@ -37,7 +37,7 @@ Warning: Pattern user_repls.json does not match any files { "existing_cluster_id": "0101-120000-aaaaaaaa", "spark_python_task": { - "python_file": "${resources.snapshot.immutable.full_path}/files/src/main.py" + "python_file": "${resources.snapshots.immutable.full_path}/files/src/main.py" }, "task_key": "my_task" } diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml index c738f20869c..c8770db7c9f 100644 --- a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml @@ -2,7 +2,7 @@ bundle: name: snapshot_resource_not_allowed resources: - snapshot: + snapshots: immutable: bundle_id: 123 zip_content: | diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt index bae3072e941..686b016f05c 100644 --- a/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle validate -Error: snapshot is an internal resource and cannot be set in bundle configuration - at resources.snapshot +Error: snapshots is an internal resource and cannot be set in bundle configuration + at resources.snapshots in databricks.yml:6:5 Name: snapshot_resource_not_allowed diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 82ac6fd60b0..47c2dfc7202 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -27,7 +27,7 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl if b.Config.Workspace.FilePath == "" { if b.IsImmutableFolder() { - b.Config.Workspace.FilePath = "${resources.snapshot.immutable.full_path}/files" + b.Config.Workspace.FilePath = "${resources.snapshots.immutable.full_path}/files" } else { b.Config.Workspace.FilePath = path.Join(root, "files") } @@ -39,7 +39,7 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl if b.Config.Workspace.ArtifactPath == "" { if b.IsImmutableFolder() { - b.Config.Workspace.ArtifactPath = "${resources.snapshot.immutable.full_path}/artifacts" + b.Config.Workspace.ArtifactPath = "${resources.snapshots.immutable.full_path}/artifacts" } else { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 1bd80dd74d2..f6e43c067ac 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -322,7 +322,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { switch { case b.IsImmutableFolder(): - t.remoteRoot = "${resources.snapshot.immutable.full_path}/files" + t.remoteRoot = "${resources.snapshots.immutable.full_path}/files" case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment): t.remoteRoot = t.b.SyncRootPath default: diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 5628b3c1c1a..18ed05c4653 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -46,7 +46,7 @@ type Resources struct { InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` // Internal resources - Snapshot map[string]*resources.Snapshot `json:"snapshot,omitempty" bundle:"internal"` + Snapshots map[string]*resources.Snapshot `json:"snapshots,omitempty" bundle:"internal"` } type ConfigResource interface { diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index b0192445a4e..2203b829b6e 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -39,11 +39,11 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } } - if b.Config.Resources.Snapshot == nil { - b.Config.Resources.Snapshot = make(map[string]*resources.Snapshot) + if b.Config.Resources.Snapshots == nil { + b.Config.Resources.Snapshots = make(map[string]*resources.Snapshot) } - if _, ok := b.Config.Resources.Snapshot["immutable"]; !ok { - b.Config.Resources.Snapshot["immutable"] = &resources.Snapshot{ + if _, ok := b.Config.Resources.Snapshots["immutable"]; !ok { + b.Config.Resources.Snapshots["immutable"] = &resources.Snapshot{ BundleID: b.DeploymentBundle.StateDB.GetOrInitLineage(), ACL: BuildACL(b), } @@ -63,7 +63,7 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn )...) } - b.Config.Resources.Snapshot["immutable"].ZipContent = string(zipContent) + b.Config.Resources.Snapshots["immutable"].ZipContent = string(zipContent) } return diags diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index abbb1cd979a..e451253b393 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -67,7 +67,7 @@ var SupportedResources = map[string]any{ "vector_search_indexes.grants": (*ResourceGrants)(nil), // Internal resources - "snapshot": (*ResourceSnapshot)(nil), + "snapshots": (*ResourceSnapshot)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 54541d94ba9..6f8e83d0f96 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/snapshot" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" @@ -297,6 +298,11 @@ var testConfig map[string]any = map[string]any{ Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, }}, }, + + "snapshots": &resources.Snapshot{ + BundleID: "test-bundle-id", + ACL: []snapshot.ACLEntry{{UserName: "user@example.com", PermissionLevel: "CAN_READ"}}, + }, } type prepareWorkspace func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) @@ -890,6 +896,7 @@ func TestAll(t *testing.T) { // testIgnoreFilter encapsulates the logic for filtering fields based on ignore_remote_changes config. type testIgnoreFilter struct { ignoreFields map[string]bool + adapter *Adapter } // newTestIgnoreFilter creates a filter from the adapter's resource configs. @@ -903,7 +910,8 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { ignoreFields[p.Field.String()] = true } } - return &testIgnoreFilter{ignoreFields: ignoreFields} + + return &testIgnoreFilter{ignoreFields: ignoreFields, adapter: adapter} } // shouldIgnore returns true if the field at the given path should be ignored. @@ -916,7 +924,17 @@ func (f *testIgnoreFilter) shouldIgnore(path string) bool { if prefix, _, ok := strings.Cut(path, "."); ok { topLevelField = prefix } - return f.ignoreFields[topLevelField] + + parts := strings.Split(topLevelField, "[") + if len(parts) > 1 { + topLevelField = parts[0] + } + + if f.ignoreFields[topLevelField] { + return true + } + + return structaccess.ValidatePath(f.adapter.RemoteType(), structpath.MustParsePath(path)) != nil } // filterChanges returns only the changes that should not be ignored. @@ -1037,6 +1055,10 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W } require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any, field *reflect.StructField) { + // Skip fields configured in ignore_remote_changes. + if ignoreFilter.shouldIgnore(path.String()) { + return + } remoteValue, err := structaccess.Get(remappedState, path) if err != nil { t.Errorf("Failed to read %s from remapped remote state %#v", path.String(), remappedState) @@ -1051,10 +1073,6 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W // testserver can set field to backend-generated value return } - // Skip fields configured in ignore_remote_changes. - if ignoreFilter.shouldIgnore(path.String()) { - return - } // t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue) // We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today. // If not true for your resource, add exception here: @@ -1076,6 +1094,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W } deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + isImmutable := strings.HasSuffix(group, "snapshots") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver @@ -1086,6 +1105,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W remoteAfterDelete, err := adapter.DoRead(ctx, createdID) switch { + case isImmutable: + require.NoError(t, err) + assert.True(t, adapter.IsGone(remoteAfterDelete)) case deleteIsNoop: require.NoError(t, err) // The resource genuinely still exists, so it must not report as gone. diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b3282a5e426..fd2c87e1e04 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -798,7 +798,7 @@ resources: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype - snapshot: + snapshots: recreate_on_changes: - field: full_path reason: immutable diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index f520bb8aef4..7df109e8b83 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -2,13 +2,11 @@ package dresources import ( "context" - "errors" "path" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/snapshot" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" ) type ResourceSnapshot struct { @@ -75,12 +73,6 @@ func (s *ResourceSnapshot) DoRead(ctx context.Context, id string) (*SnapshotRemo fullPath := path.Join(s.remoteRoot, id) _, err := s.uploader.Get(ctx, fullPath) if err != nil { - if errors.Is(err, apierr.ErrNotFound) { - return &SnapshotRemote{ - RelativePath: id, - FullPath: "", - }, nil - } return nil, err } return &SnapshotRemote{ diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ac04be010b9..6edf80c0973 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -54,6 +54,10 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, + "snapshots": { + "bundle_id", + "acl", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. @@ -91,6 +95,9 @@ var knownMissingInStateType = map[string][]string{ "keyvault_metadata", "name", }, + "snapshots": { + "zip_content", + }, } // TestInputSubset validates that all fields in InputType exist in StateType. diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index c5884aea930..a9ee411cb73 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -27,7 +27,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) uploadPath = ensureWorkspaceOrVolumesPrefix(uploadPath) if b.IsImmutableFolder() { - uploadPath = path.Join("${resources.snapshot.immutable.full_path}", "artifacts", InternalDirName) + uploadPath = path.Join("${resources.snapshots.immutable.full_path}", "artifacts", InternalDirName) } switch { @@ -47,7 +47,7 @@ func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.F artifactPath = ensureWorkspaceOrVolumesPrefix(artifactPath) if b.IsImmutableFolder() { - artifactPath = path.Join("${resources.snapshot.immutable.full_path}", "artifacts") + artifactPath = path.Join("${resources.snapshots.immutable.full_path}", "artifacts") } switch { diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 51b0d406eee..af17a93f8b6 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") bundle.ApplySeqContext(ctx, b, - // Reads (dynamic): resources.snapshot (and other internal keys) + // Reads (dynamic): resources.snapshots (and other internal keys) // Warns and removes any internal resource fields set by the user. mutator.RejectInternalResources(), diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 1ec6dc82999..4167e45ec99 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -1150,6 +1150,11 @@ func AssertFullResourceCoverage(t *testing.T, config *config.Root) { resources := reflect.ValueOf(config.Resources) for i := range resources.NumField() { field := resources.Field(i) + // Get Tag field from reflect.Value + tag := resources.Type().Field(i).Tag.Get("bundle") + if tag == "internal" { + continue + } if field.Kind() == reflect.Map { assert.True( t, From 195974d2837e456f4fae93dc2d1ac26114127c69 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 3 Aug 2026 14:54:54 +0200 Subject: [PATCH 08/30] fixes --- .../deploy/immutable-no-artifacts/output.txt | 35 ++++++++++++++- acceptance/bundle/deploy/immutable/output.txt | 42 +++++++++++++++++- acceptance/bundle/refschema/out.fields.txt | 10 +++++ .../mutator/reject_internal_resources.go | 5 ++- .../resourcemutator/apply_target_mode_test.go | 6 +++ .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 2 + bundle/config/resources/snapshot.go | 44 ++++++++++++++++++- bundle/config/validate/files_to_sync.go | 5 +++ .../config/validate/validate_artifact_path.go | 2 +- bundle/deploy/snapshot/upload.go | 10 ++++- bundle/direct/bundle_plan.go | 5 +-- bundle/direct/dresources/snapshot.go | 23 ++++------ libs/snapshot/client.go | 10 ++++- 14 files changed, 172 insertions(+), 29 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index bc4d7278262..5d9b4d857b9 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -36,7 +36,14 @@ Validation OK! "new_state": { "value": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] } } } @@ -66,6 +73,32 @@ Deployment complete! "remote_state": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + }, + "changes": { + "acl": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "new": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "remote": null + }, + "bundle_id": { + "action": "skip", + "reason": "missing_in_remote", + "old": "[UUID]", + "new": "[UUID]", + "remote": "" + } } } diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 8e280cddb45..440c62b5cf8 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -55,7 +55,14 @@ Building python_artifact... "new_state": { "value": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] } } } @@ -89,7 +96,14 @@ Building python_artifact... "new_state": { "value": { "relative_path": "[UUID]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" + "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", + "bundle_id": "[UUID]", + "acl": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ] } }, "remote_state": { @@ -97,6 +111,30 @@ Building python_artifact... "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]" }, "changes": { + "acl": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "new": [ + { + "user_name": "[USERNAME]", + "permission_level": "CAN_READ" + } + ], + "remote": null + }, + "bundle_id": { + "action": "skip", + "reason": "missing_in_remote", + "old": "[UUID]", + "new": "[UUID]", + "remote": "" + }, "full_path": { "action": "recreate", "reason": "immutable", diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d35bfa350f0..1f212874404 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3473,6 +3473,16 @@ resources.secret_scopes.*.permissions.acls[*] workspace.AclItem ALL resources.secret_scopes.*.permissions.acls[*].permission workspace.AclPermission ALL resources.secret_scopes.*.permissions.acls[*].principal string ALL resources.secret_scopes.*.permissions.scope_name string ALL +resources.snapshots.*.acl []snapshot.ACLEntry INPUT STATE +resources.snapshots.*.acl[*] snapshot.ACLEntry INPUT STATE +resources.snapshots.*.acl[*].group_name string INPUT STATE +resources.snapshots.*.acl[*].permission_level string INPUT STATE +resources.snapshots.*.acl[*].service_principal_name string INPUT STATE +resources.snapshots.*.acl[*].user_name string INPUT STATE +resources.snapshots.*.bundle_id string INPUT STATE +resources.snapshots.*.full_path string REMOTE STATE +resources.snapshots.*.relative_path string REMOTE STATE +resources.snapshots.*.zip_content string INPUT resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL resources.sql_warehouses.*.channel.dbsql_version string ALL diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index d548aafc10f..742eb62a282 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -30,7 +30,7 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d var internalResourceKeys []string // collect all internal resource keys, only top level keys under "resources" - structwalk.WalkType(reflect.TypeFor[config.Resources](), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { + err := structwalk.WalkType(reflect.TypeFor[config.Resources](), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { if path.Len() > 2 { return false } @@ -43,6 +43,9 @@ func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) d } return true }) + if err != nil { + return diag.FromErr(err) + } for _, key := range internalResourceKeys { v, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources."+key)) diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index b97113028b8..b26edf2ce30 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -311,6 +311,12 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + Snapshots: map[string]*resources.Snapshot{ + "snapshot1": { + BundleID: "bundle1", + ACL: nil, + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 5faed5f7b1b..b574b7a2225 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -59,6 +59,7 @@ func allResourceTypes(t *testing.T) []string { "registered_models", "schemas", "secret_scopes", + "snapshots", "sql_warehouses", "synced_database_tables", "vector_search_endpoints", @@ -193,6 +194,7 @@ var allowList = []string{ "genie_spaces", "instance_pools", "job_runs", + "snapshots", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 18ed05c4653..67b36a95fa2 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -132,6 +132,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), + collectResourceMap(descriptions["snapshots"], r.Snapshots), } } @@ -195,5 +196,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), + "snapshots": (&resources.Snapshot{}).ResourceDescription(), } } diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index b85b93621f1..88a4da3a9e1 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -1,9 +1,12 @@ package resources import ( + "context" + "net/url" "path" "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/databricks-sdk-go" ) // Snapshot is the configuration for the snapshot resource. @@ -13,12 +16,49 @@ type Snapshot struct { BundleID string `json:"bundle_id"` ACL []snapshot.ACLEntry `json:"acl"` ZipContent string `json:"zip_content"` + RemoteRoot string `json:"remote_root"` + + Lifecycle Lifecycle `json:"-,omitempty"` } func (s *Snapshot) RelativePath() string { return path.Join(s.BundleID, snapshot.HashFromContent([]byte(s.ZipContent))) } -func (s *Snapshot) FullPath(remoteRoot string) string { - return path.Join(remoteRoot, s.RelativePath()) +func (s *Snapshot) FullPath() string { + return path.Join(s.RemoteRoot, s.RelativePath()) +} + +func (s Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Workspace.GetStatusByPath(ctx, s.FullPath()) + if err != nil { + return false, err + } + return true, nil +} + +func (s Snapshot) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "snapshot", + PluralName: "snapshots", + SingularTitle: "Snapshot", + PluralTitle: "Snapshots", + } +} + +func (s Snapshot) GetName() string { + return s.RelativePath() +} + +func (s Snapshot) GetURL() string { + // Skipping URL initialization for snapshots + return "" +} + +func (s Snapshot) InitializeURL(_ url.URL) { + // Secret scopes do not have a URL +} + +func (s Snapshot) GetLifecycle() LifecycleConfig { + return s.Lifecycle } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index aea78f7104b..0922d0fd19a 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -20,6 +20,11 @@ func (v *filesToSync) Name() string { } func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Immutable folder does not use the sync mechanism, so we don't need to validate it. + if b.IsImmutableFolder() { + return nil + } + // The user may be intentional about not synchronizing any files. // In this case, we should not show any warnings. if len(b.Config.Sync.Paths) == 0 { diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 4ea5c4308ad..e7070b20b02 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -76,7 +76,7 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // We only validate UC Volumes paths right now. - if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) { + if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) || b.IsImmutableFolder() { return nil } diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 2203b829b6e..c7e16b4f809 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -39,13 +39,19 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } } + remoteRoot, err := uploader.GetSnapshotRootPath(ctx) + if err != nil { + return diag.FromErr(err) + } + if b.Config.Resources.Snapshots == nil { b.Config.Resources.Snapshots = make(map[string]*resources.Snapshot) } if _, ok := b.Config.Resources.Snapshots["immutable"]; !ok { b.Config.Resources.Snapshots["immutable"] = &resources.Snapshot{ - BundleID: b.DeploymentBundle.StateDB.GetOrInitLineage(), - ACL: BuildACL(b), + BundleID: b.DeploymentBundle.StateDB.GetOrInitLineage(), + ACL: BuildACL(b), + RemoteRoot: remoteRoot, } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 83584207e9e..9931037b93e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -743,9 +743,8 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s localConfig := sv.Value - var adapter *dresources.Adapter - adapter, err = b.getAdapterForKey(targetResourceKey) - if adapter == nil { + adapter, err := b.getAdapterForKey(targetResourceKey) + if err != nil { return nil, fmt.Errorf("internal error: %s: unknown resource type %q", targetResourceKey, targetGroup) } diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index 7df109e8b83..43fff553c96 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -2,7 +2,6 @@ package dresources import ( "context" - "path" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/snapshot" @@ -10,8 +9,7 @@ import ( ) type ResourceSnapshot struct { - remoteRoot string - uploader snapshot.SnapshotUploader + uploader snapshot.SnapshotUploader } type SnapshotState struct { @@ -30,7 +28,9 @@ type SnapshotRemote struct { func (s *ResourceSnapshot) New(client *databricks.WorkspaceClient) *ResourceSnapshot { // Return a zero-value instance when client is nil (e.g. refschema introspection). if client == nil { - return &ResourceSnapshot{} + return &ResourceSnapshot{ + uploader: nil, + } } uploader, err := snapshot.NewSnapshotUploader(client) @@ -38,21 +38,15 @@ func (s *ResourceSnapshot) New(client *databricks.WorkspaceClient) *ResourceSnap panic(err) } - snapshotRootPath, err := uploader.GetSnapshotRootPath(context.Background()) - if err != nil { - panic(err) - } - return &ResourceSnapshot{ - remoteRoot: snapshotRootPath, - uploader: uploader, + uploader: uploader, } } func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotState { return &SnapshotState{ RelativePath: input.RelativePath(), - FullPath: input.FullPath(s.remoteRoot), + FullPath: input.FullPath(), BundleID: input.BundleID, ACL: input.ACL, ZipContent: input.ZipContent, @@ -70,14 +64,13 @@ func (s *ResourceSnapshot) RemapState(remote *SnapshotRemote) *SnapshotState { } func (s *ResourceSnapshot) DoRead(ctx context.Context, id string) (*SnapshotRemote, error) { - fullPath := path.Join(s.remoteRoot, id) - _, err := s.uploader.Get(ctx, fullPath) + info, err := s.uploader.Get(ctx, id) if err != nil { return nil, err } return &SnapshotRemote{ RelativePath: id, - FullPath: fullPath, + FullPath: info.Path, }, nil } diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index 006042a38e1..ac6ad674c8d 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -8,6 +8,7 @@ import ( "mime/multipart" "net/http" "net/textproto" + "path" "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go" @@ -120,8 +121,13 @@ func (c *snapshotAPIClient) Upload(ctx context.Context, path, bundleID string, a return &SnapshotInfo{Path: resp.Snapshot.Path}, nil } -func (c *snapshotAPIClient) Get(ctx context.Context, path string) (*SnapshotInfo, error) { - resp, err := c.workspaceClient.Workspace.GetStatusByPath(ctx, path) +func (c *snapshotAPIClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { + rootPath, err := c.GetSnapshotRootPath(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get snapshot root path: %w", err) + } + snapshotPath := path.Join(rootPath, snapshotRelativePath) + resp, err := c.workspaceClient.Workspace.GetStatusByPath(ctx, snapshotPath) if err != nil { return nil, fmt.Errorf("snapshot get: %w", err) } From b6bb79fc96ceadd003c545114e208a39e32ae29e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 3 Aug 2026 14:58:07 +0200 Subject: [PATCH 09/30] refschema --- acceptance/bundle/refschema/out.fields.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 1f212874404..23f335fe4c6 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3482,6 +3482,7 @@ resources.snapshots.*.acl[*].user_name string INPUT STATE resources.snapshots.*.bundle_id string INPUT STATE resources.snapshots.*.full_path string REMOTE STATE resources.snapshots.*.relative_path string REMOTE STATE +resources.snapshots.*.remote_root string INPUT resources.snapshots.*.zip_content string INPUT resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL From 2b7b48f04a657943135ea562a561be0788df47d3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 3 Aug 2026 14:59:51 +0200 Subject: [PATCH 10/30] lint fixes --- bundle/config/resources/snapshot.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index 88a4da3a9e1..c6ba3e3c562 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -18,7 +18,7 @@ type Snapshot struct { ZipContent string `json:"zip_content"` RemoteRoot string `json:"remote_root"` - Lifecycle Lifecycle `json:"-,omitempty"` + Lifecycle Lifecycle `json:"-"` } func (s *Snapshot) RelativePath() string { @@ -29,7 +29,7 @@ func (s *Snapshot) FullPath() string { return path.Join(s.RemoteRoot, s.RelativePath()) } -func (s Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { +func (s *Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { _, err := w.Workspace.GetStatusByPath(ctx, s.FullPath()) if err != nil { return false, err @@ -37,7 +37,7 @@ func (s Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, nam return true, nil } -func (s Snapshot) ResourceDescription() ResourceDescription { +func (s *Snapshot) ResourceDescription() ResourceDescription { return ResourceDescription{ SingularName: "snapshot", PluralName: "snapshots", @@ -46,19 +46,19 @@ func (s Snapshot) ResourceDescription() ResourceDescription { } } -func (s Snapshot) GetName() string { +func (s *Snapshot) GetName() string { return s.RelativePath() } -func (s Snapshot) GetURL() string { +func (s *Snapshot) GetURL() string { // Skipping URL initialization for snapshots return "" } -func (s Snapshot) InitializeURL(_ url.URL) { +func (s *Snapshot) InitializeURL(_ url.URL) { // Secret scopes do not have a URL } -func (s Snapshot) GetLifecycle() LifecycleConfig { +func (s *Snapshot) GetLifecycle() LifecycleConfig { return s.Lifecycle } From 54a7dc04e70e724cb27aea73e0db55fc88cb8ae6 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 3 Aug 2026 16:59:12 +0200 Subject: [PATCH 11/30] use HasInternalResources --- .../mutator/reject_internal_resources.go | 38 ++----------------- bundle/config/resources.go | 4 ++ bundle/config/resources_test.go | 35 +++++++++++++++++ 3 files changed, 43 insertions(+), 34 deletions(-) diff --git a/bundle/config/mutator/reject_internal_resources.go b/bundle/config/mutator/reject_internal_resources.go index 742eb62a282..c377ecb0aaa 100644 --- a/bundle/config/mutator/reject_internal_resources.go +++ b/bundle/config/mutator/reject_internal_resources.go @@ -2,15 +2,10 @@ package mutator import ( "context" - "reflect" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/structs/structtag" - "github.com/databricks/cli/libs/structs/structwalk" ) type rejectInternalResources struct{} @@ -27,36 +22,11 @@ func (m *rejectInternalResources) Name() string { func (m *rejectInternalResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - - var internalResourceKeys []string - // collect all internal resource keys, only top level keys under "resources" - err := structwalk.WalkType(reflect.TypeFor[config.Resources](), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { - if path.Len() > 2 { - return false - } - if field == nil { - return true - } - tag := field.Tag.Get("bundle") - if structtag.BundleTag(tag).Internal() { - internalResourceKeys = append(internalResourceKeys, path.String()) - } - return true - }) - if err != nil { - return diag.FromErr(err) - } - - for _, key := range internalResourceKeys { - v, err := dyn.GetByPath(b.Config.Value(), dyn.MustPathFromString("resources."+key)) - if err != nil { - continue - } + if b.Config.Resources.HasInternalResources() { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: key + " is an internal resource and cannot be set in bundle configuration", - Locations: v.Locations(), - Paths: []dyn.Path{dyn.MustPathFromString("resources." + key)}, + Severity: diag.Error, + Summary: "Internal resources cannot be set in bundle configuration", + Paths: []dyn.Path{dyn.MustPathFromString("resources")}, }) } diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 67b36a95fa2..007a35ff3fd 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -161,6 +161,10 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) return found[0], nil } +func (r *Resources) HasInternalResources() bool { + return len(r.Snapshots) > 0 +} + // SupportedResources returns a map which keys correspond to the resource key in the bundle configuration. func SupportedResources() map[string]resources.ResourceDescription { return map[string]resources.ResourceDescription{ diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index de9caa09dc2..9f20360c82b 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -15,6 +15,9 @@ import ( "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/apps" @@ -399,3 +402,35 @@ func TestResourcesBindSupport(t *testing.T) { } } } + +func TestAllInteralResourcesAreMarkedAsInternal(t *testing.T) { + internalResourceKeys := map[string]reflect.Type{} + err := structwalk.WalkType(reflect.TypeFor[Resources](), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { + if path.Len() > 2 { + return false + } + if field == nil { + return true + } + tag := field.Tag.Get("bundle") + if structtag.BundleTag(tag).Internal() { + internalResourceKeys[field.Name] = typ + } + return true + }) + assert.NoError(t, err) + + for key, typ := range internalResourceKeys { + r := reflect.MakeMap(typ) + r.SetMapIndex(reflect.ValueOf("my_resources"), reflect.New(typ.Elem()).Elem()) + + resources := &Resources{} + res := reflect.ValueOf(resources).Elem() + field := res.FieldByName(key) + if !field.IsValid() && !field.CanSet() { + t.Fatalf("Field %s is not valid", key) + } + field.Set(r) + assert.True(t, resources.HasInternalResources()) + } +} From fbd52299fcf441265d1d8a9e6a862af44273a4b2 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 4 Aug 2026 11:10:47 +0200 Subject: [PATCH 12/30] fix test and etc --- .../immutable-no-artifacts/out.test.toml | 1 + .../deploy/immutable-no-artifacts/output.txt | 8 +++---- .../deploy/immutable-no-artifacts/script | 22 ++++++++++--------- .../deploy/immutable-no-artifacts/test.toml | 1 + acceptance/bundle/deploy/immutable/output.txt | 8 +++---- acceptance/bundle/deploy/immutable/script | 16 ++++++++------ .../databricks.yml | 2 +- bundle/config/validate/files_to_sync.go | 5 ----- bundle/direct/dresources/all_test.go | 5 +++-- bundle/direct/dresources/snapshot.go | 3 +++ bundle/direct/dresources/type_test.go | 1 + libs/snapshot/client.go | 2 +- 12 files changed, 40 insertions(+), 34 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml index e90b6d5d1ba..71970b719d4 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 5d9b4d857b9..ef901a51c7f 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -53,13 +53,13 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path" >>> title Check snapshot plan after deploy @@ -106,7 +106,7 @@ Deployment complete! Deploying resources... Updating deployment state... Deployment complete! -\nSnapshot did not change as expected +Snapshot did not change as expected >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable-no-artifacts/script index a944c4ad090..1a68a7d06da 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/script +++ b/acceptance/bundle/deploy/immutable-no-artifacts/script @@ -10,27 +10,29 @@ trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json trace jq '.plan["resources.snapshots.immutable"]' plan.json -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg plan.json) # Get a job and check that task paths point into the snapshot JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path' -PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +$CLI jobs get $JOB_ID > job.json +trace jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' +trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' +trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.base_parameters.path' +PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') -trace title "Check snapshot plan after deploy" +trace title "Check snapshot plan after deploy: " trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.snapshots.immutable"]' plan.json -rm plan.json -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg plan.json) +rm plan.json +rm job.json PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') if [ "$PYTHON_FILE_1" == "$PYTHON_FILE_2" ]; then - echo "\nSnapshot did not change as expected" + echo "Snapshot did not change as expected" else - echo "\nSnapshot changed" + echo "Snapshot changed" exit 1 fi trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml index 9b345bc3395..ea7bdeaf04a 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml @@ -1,6 +1,7 @@ Local = true Cloud = false # Temporary disable cloud tests until the API is fully available RecordRequests = true +EnvMatrix.READPLAN = ["", "1"] # immutable_folder only works with the direct engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 440c62b5cf8..2625f718098 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -73,13 +73,13 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook" ->>> [CLI] jobs get [NUMID] +>>> jq .settings.environments[0].spec.dependencies job.json [ "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ] @@ -158,7 +158,7 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Check that job uses a new snapshot\nSnapshot changed as expected +=== Check that job uses a new snapshotSnapshot changed as expected >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 7e72e45efc1..2ca5554e5a1 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -14,24 +14,26 @@ trace $CLI bundle deploy # Get a job and check that task paths are immutable JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' -trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' -trace $CLI jobs get $JOB_ID | jq '.settings.environments[0].spec.dependencies' -PYTHON_FILE_1=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') +$CLI jobs get $JOB_ID > job.json +trace jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file' +trace jq '.settings.tasks' job.json | jq '.[] | select(.notebook_task != null) | .notebook_task.notebook_path' +trace jq '.settings.environments[0].spec.dependencies' job.json +PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') trace title "Check snapshot plan after deploy" trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.snapshots.immutable"]' plan.json rm plan.json +rm job.json trace $CLI bundle deploy -title "Check that job uses a new snapshot" +title "Check that job uses a new snapshot: " PYTHON_FILE_2=$($CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.spark_python_task != null) | .spark_python_task.python_file') if [ "$PYTHON_FILE_1" != "$PYTHON_FILE_2" ]; then - echo "\nSnapshot changed as expected" + echo "Snapshot changed as expected" else - echo "\nSnapshot did not change" + echo "Snapshot did not change" exit 1 fi diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml index c8770db7c9f..a36fc1ba93d 100644 --- a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml @@ -5,5 +5,5 @@ resources: snapshots: immutable: bundle_id: 123 - zip_content: | + zip_content: |- This is a test zip content. diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index 0922d0fd19a..aea78f7104b 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -20,11 +20,6 @@ func (v *filesToSync) Name() string { } func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // Immutable folder does not use the sync mechanism, so we don't need to validate it. - if b.IsImmutableFolder() { - return nil - } - // The user may be intentional about not synchronizing any files. // In this case, we should not show any warnings. if len(b.Config.Sync.Paths) == 0 { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 6f8e83d0f96..915e4f997c5 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -300,8 +300,9 @@ var testConfig map[string]any = map[string]any{ }, "snapshots": &resources.Snapshot{ - BundleID: "test-bundle-id", - ACL: []snapshot.ACLEntry{{UserName: "user@example.com", PermissionLevel: "CAN_READ"}}, + RemoteRoot: "/Workspace/Users/" + testserver.TestUserSP.UserName + "/.snapshots", + BundleID: "test-bundle-id", + ACL: []snapshot.ACLEntry{{UserName: "user@example.com", PermissionLevel: "CAN_READ"}}, }, } diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index 43fff553c96..b585730d92b 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -13,6 +13,7 @@ type ResourceSnapshot struct { } type SnapshotState struct { + RemoteRoot string `json:"remote_root"` RelativePath string `json:"relative_path"` FullPath string `json:"full_path"` BundleID string `json:"bundle_id"` @@ -45,6 +46,7 @@ func (s *ResourceSnapshot) New(client *databricks.WorkspaceClient) *ResourceSnap func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotState { return &SnapshotState{ + RemoteRoot: input.RemoteRoot, RelativePath: input.RelativePath(), FullPath: input.FullPath(), BundleID: input.BundleID, @@ -55,6 +57,7 @@ func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotStat func (s *ResourceSnapshot) RemapState(remote *SnapshotRemote) *SnapshotState { return &SnapshotState{ + RemoteRoot: "", RelativePath: remote.RelativePath, FullPath: remote.FullPath, BundleID: "", diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 6edf80c0973..a99bece3ba5 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -57,6 +57,7 @@ var knownMissingInRemoteType = map[string][]string{ "snapshots": { "bundle_id", "acl", + "remote_root", }, } diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index ac6ad674c8d..95d6afd33e0 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -34,7 +34,7 @@ type ACLEntry struct { // snapshotID is the content-addressed key supplied by the caller; the API uses // it as the final path component so that identical content always resolves to // the same workspace location. -// This interface exists so the implementation can later be replaced with a Go SDK call. +// The interface exists so we can replace the implementation in tests with a mock. type SnapshotUploader interface { GetSnapshotRootPath(ctx context.Context) (string, error) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) From 78257227f52ec602d59773bb68e935c20e07eca4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 4 Aug 2026 19:20:27 +0200 Subject: [PATCH 13/30] stable bundle id --- .../deploy/immutable-no-artifacts/output.txt | 12 +++++-- .../deploy/immutable-no-artifacts/test.toml | 6 ++++ acceptance/bundle/deploy/immutable/output.txt | 11 +++++- bundle/deploy/snapshot/upload.go | 36 ++++++++++++++++++- bundle/phases/deploy.go | 21 ++++++++--- libs/snapshot/client.go | 2 +- libs/sync/path.go | 6 ++++ libs/testserver/handlers.go | 2 +- 8 files changed, 85 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index ef901a51c7f..9ae7e4ef2cd 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -35,6 +35,7 @@ Validation OK! "action": "create", "new_state": { "value": { + "remote_root": "/Workspace/Users/[UUID]/.snapshots", "relative_path": "[UUID]/[SNAPSHOT_HASH]", "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", "bundle_id": "[UUID]", @@ -62,9 +63,9 @@ Deployment complete! >>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path" ->>> title Check snapshot plan after deploy +>>> title Check snapshot plan after deploy: -=== Check snapshot plan after deploy +=== Check snapshot plan after deploy: >>> [CLI] bundle plan -o json >>> jq .plan["resources.snapshots.immutable"] plan.json @@ -98,6 +99,13 @@ Deployment complete! "old": "[UUID]", "new": "[UUID]", "remote": "" + }, + "remote_root": { + "action": "skip", + "reason": "missing_in_remote", + "old": "/Workspace/Users/[UUID]/.snapshots", + "new": "/Workspace/Users/[UUID]/.snapshots", + "remote": "" } } } diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml index ea7bdeaf04a..4bfd1bb19de 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml @@ -19,3 +19,9 @@ Ignore = [ [[Repls]] Old = '[0-9a-f]{64}' New = '[SNAPSHOT_HASH]' + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan plan.json' +New = '' diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 2625f718098..224eca0b517 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -54,6 +54,7 @@ Building python_artifact... "action": "create", "new_state": { "value": { + "remote_root": "/Workspace/Users/[UUID]/.snapshots", "relative_path": "[UUID]/[SNAPSHOT_HASH]", "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", "bundle_id": "[UUID]", @@ -95,6 +96,7 @@ Building python_artifact... "action": "recreate", "new_state": { "value": { + "remote_root": "/Workspace/Users/[UUID]/.snapshots", "relative_path": "[UUID]/[SNAPSHOT_HASH]", "full_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]", "bundle_id": "[UUID]", @@ -148,6 +150,13 @@ Building python_artifact... "old": "[UUID]/[SNAPSHOT_HASH]", "new": "[UUID]/[SNAPSHOT_HASH]", "remote": "[UUID]/[SNAPSHOT_HASH]" + }, + "remote_root": { + "action": "skip", + "reason": "missing_in_remote", + "old": "/Workspace/Users/[UUID]/.snapshots", + "new": "/Workspace/Users/[UUID]/.snapshots", + "remote": "" } } } @@ -158,7 +167,7 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Check that job uses a new snapshotSnapshot changed as expected +=== Check that job uses a new snapshot: Snapshot changed as expected >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index c7e16b4f809..d11ecd55422 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -6,8 +6,10 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/snapshot" + "github.com/google/uuid" ) // fileLimitWarning is the file count above which immutable folder deployments may fail. @@ -49,7 +51,7 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } if _, ok := b.Config.Resources.Snapshots["immutable"]; !ok { b.Config.Resources.Snapshots["immutable"] = &resources.Snapshot{ - BundleID: b.DeploymentBundle.StateDB.GetOrInitLineage(), + BundleID: BundleID(b), ACL: BuildACL(b), RemoteRoot: remoteRoot, } @@ -75,6 +77,38 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return diags } +// SyncZipContent copies the zip content from b.Config.Resources.Snapshots["immutable"] +// into the in-memory state cache entry for the snapshot resource. This is needed when +// deploying from a plan file: the plan JSON omits ZipContent (json:"-"), so InitForApply +// leaves it empty, causing DoCreate to upload an empty zip and derive a wrong snapshot ID. +func SyncZipContent(b *bundle.Bundle) { + snap := b.Config.Resources.Snapshots["immutable"] + if snap == nil || snap.ZipContent == "" { + return + } + sv, ok := b.DeploymentBundle.StateCache.Load("resources.snapshots.immutable") + if !ok { + return + } + state, ok := sv.Value.(*dresources.SnapshotState) + if !ok { + return + } + state.ZipContent = snap.ZipContent +} + +// bundleIDNamespace is the UUID namespace used to derive the bundle ID. +var bundleIDNamespace = uuid.MustParse("4b4e4b5a-3c3d-4e4f-8b8c-9d9e9f0a0b0c") + +// BundleID returns a stable UUID that identifies the bundle deployment. +// It is derived deterministically from the bundle name, target, and workspace host +// so that every CLI invocation for the same deployment produces the same value. +// This is used as the path prefix for immutable snapshots in the workspace. +func BundleID(b *bundle.Bundle) string { + key := b.Config.Bundle.Name + "/" + b.Config.Bundle.Target + "/" + b.Config.Workspace.Host + return uuid.NewSHA1(bundleIDNamespace, []byte(key)).String() +} + // BuildACL constructs the access_control_list for the snapshot upload. // It grants CAN_READ to the current user and to every principal listed in the // top-level permissions section of the bundle config. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 61bd97acc06..88c2138fd1d 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -108,7 +108,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st return } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, statemgmt.Load(state), metadata.Compute(), metadata.Upload(), @@ -133,7 +134,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st // It also cleans up the artifacts directory and transforms wheel tasks. // It is called by only "bundle deploy". func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, artifacts.CleanUp(), libraries.Upload(libs), ) @@ -149,7 +151,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, scripts.Execute(config.ScriptPreDeploy), lock.Acquire(), ) @@ -182,7 +185,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), @@ -229,6 +233,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } + if immutable { + // The plan JSON omits ZipContent (json:"-"), so InitForApply leaves it + // empty in the state cache. Transfer the zip content built by PlanUpload + // above so DoCreate uploads the correct content and derives the right ID. + snapshot.SyncZipContent(b) + } } // InitForApply receives ctx and could log a diagnostic without returning an @@ -273,7 +283,8 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is // never set here. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("deploy")), diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index 95d6afd33e0..d75859b556a 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -140,5 +140,5 @@ func (c *snapshotAPIClient) GetSnapshotRootPath(ctx context.Context) (string, er if err != nil { return "", fmt.Errorf("snapshot root path get: %w", err) } - return resp.Path, nil + return path.Clean(resp.Path), nil } diff --git a/libs/sync/path.go b/libs/sync/path.go index 6b60ef682de..760695acfe8 100644 --- a/libs/sync/path.go +++ b/libs/sync/path.go @@ -26,6 +26,12 @@ func repoPathForPath(me *iam.User, remotePath string) string { // expected base paths and if it is a directory or repository. // If dryRun is set, a missing remote directory is not created. func EnsureRemotePathIsUsable(ctx context.Context, wsc *databricks.WorkspaceClient, remotePath string, me *iam.User, dryRun bool) error { + // If the remote path is a variable, we don't need to check if it's usable. + // This happens for immutable folder paths. Their paths are fully resolved at deployment stage and can't be validated at this point. + if strings.HasPrefix(remotePath, "${") { + return nil + } + var err error // TODO: we should cache CurrentUser.Me at the SDK level diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 0efda60a867..87d91bed810 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -605,7 +605,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/repos/snapshots/rootpath", func(req Request) any { return map[string]any{ - "path": "/Workspace/Users/" + TestUserSP.UserName + "/.snapshots", + "path": "/Workspace/Users/" + TestUserSP.UserName + "/.snapshots/", } }) From b1e0be181ba0352503b00a298507e1a23ceb3018 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 4 Aug 2026 19:33:27 +0200 Subject: [PATCH 14/30] fix tests --- .../mutator/resourcemutator/apply_bundle_permissions_test.go | 1 + bundle/config/resources_test.go | 4 +++- bundle/deploy/terraform/lifecycle_test.go | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 85f30c952f2..2f7df79908c 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -35,6 +35,7 @@ var unsupportedResources = []string{ "postgres_synced_tables", "vector_search_indexes", "job_runs", + "snapshots", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 9f20360c82b..c64c2672419 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -138,6 +138,7 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { "postgres_projects": true, "postgres_roles": true, "secret_scopes": true, + "snapshots": true, } supported := SupportedResources() @@ -345,7 +346,8 @@ func TestResourcesBindSupport(t *testing.T) { }, } unbindableResources := map[string]bool{ - "model": true, + "model": true, + "snapshots": true, } ctx := t.Context() diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 9fb59329ffd..4f5b51b7a46 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -22,6 +22,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "job_runs", "vector_search_endpoints", "vector_search_indexes", + "snapshots", } for resourceType := range supportedResources { From b8cdcec38b59a00ff89d64436ceb5224f5b260ed Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Aug 2026 10:07:45 +0200 Subject: [PATCH 15/30] fix typo --- bundle/config/resources_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index c64c2672419..cec127828e9 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -346,8 +346,8 @@ func TestResourcesBindSupport(t *testing.T) { }, } unbindableResources := map[string]bool{ - "model": true, - "snapshots": true, + "model": true, + "snapshot": true, } ctx := t.Context() From f32d7fcd125dbbe0ca0b3a011c3a45e1d7de5b2d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Aug 2026 10:39:51 +0200 Subject: [PATCH 16/30] update refschema --- acceptance/bundle/refschema/out.fields.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 23f335fe4c6..85c7ef3ac00 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3482,7 +3482,7 @@ resources.snapshots.*.acl[*].user_name string INPUT STATE resources.snapshots.*.bundle_id string INPUT STATE resources.snapshots.*.full_path string REMOTE STATE resources.snapshots.*.relative_path string REMOTE STATE -resources.snapshots.*.remote_root string INPUT +resources.snapshots.*.remote_root string INPUT STATE resources.snapshots.*.zip_content string INPUT resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL From 94eefc76d9129fc3ff44dd851bd5896c80363d1a Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Aug 2026 10:52:09 +0200 Subject: [PATCH 17/30] remove deadcode + update test output --- .../snapshot_resource_not_allowed/output.txt | 5 ++--- .../mutator/resolve_variable_references.go | 18 ------------------ 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt index 686b016f05c..b390e82ce64 100644 --- a/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/output.txt @@ -1,8 +1,7 @@ >>> [CLI] bundle validate -Error: snapshots is an internal resource and cannot be set in bundle configuration - at resources.snapshots - in databricks.yml:6:5 +Error: Internal resources cannot be set in bundle configuration + at resources Name: snapshot_resource_not_allowed Target: default diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 9a03c85d362..8e98bc52092 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -80,24 +80,6 @@ func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator { } } -// ResolveVariableReferencesOnlyResourcesExcluding is like ResolveVariableReferencesOnlyResources -// but leaves the listed variable reference paths unresolved. Use this when a workspace path will -// be updated by a later mutator (e.g. snapshot.Upload sets workspace.file_path to the snapshot -// location) and the final value should be substituted at that later point. -func ResolveVariableReferencesOnlyResourcesExcluding(excluded []string, prefixes ...string) bundle.Mutator { - if len(prefixes) == 0 { - prefixes = defaultPrefixes - } - return &resolveVariableReferences{ - prefixes: prefixes, - lookupFn: lookup, - extraRounds: maxResolutionRounds - 1, - pattern: dyn.NewPattern(dyn.Key("resources")), - includeResources: true, - excludePaths: excluded, - } -} - func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutator { if len(prefixes) == 0 { prefixes = defaultPrefixes From 769c0c314d43ba52d348082b3742fa2ed94f0c4d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Aug 2026 13:15:01 +0200 Subject: [PATCH 18/30] adr feedback --- .../deploy/immutable-no-artifacts/output.txt | 4 +-- .../deploy/immutable-no-artifacts/script | 4 +-- acceptance/bundle/deploy/immutable/output.txt | 4 +-- acceptance/bundle/deploy/immutable/script | 4 +-- acceptance/bundle/refschema/out.fields.txt | 22 ++++++------ .../resources/apps/immutable/output.txt | 2 +- .../immutable_workspace_paths/output.txt | 8 ++--- .../databricks.yml | 2 +- .../config/mutator/default_workspace_paths.go | 4 +-- .../apply_bundle_permissions_test.go | 2 +- .../mutator/resourcemutator/run_as_test.go | 4 +-- bundle/config/mutator/translate_paths.go | 2 +- bundle/config/resources.go | 6 ++-- bundle/config/resources/snapshot.go | 36 +++++++++++++------ bundle/config/resources_test.go | 6 ++-- bundle/deploy/snapshot/upload.go | 14 +++----- bundle/deploy/snapshot/upload_warning_test.go | 33 +++++++++-------- bundle/deploy/terraform/lifecycle_test.go | 2 +- bundle/direct/dresources/all.go | 2 +- bundle/direct/dresources/all_test.go | 4 +-- bundle/direct/dresources/resources.yml | 7 ++-- bundle/direct/dresources/snapshot.go | 4 +-- bundle/direct/dresources/type_test.go | 4 +-- bundle/libraries/filer.go | 4 +-- bundle/phases/initialize.go | 2 +- libs/snapshot/client.go | 27 +++++--------- 26 files changed, 105 insertions(+), 108 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 9ae7e4ef2cd..3eaf0316527 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -30,7 +30,7 @@ Validation OK! } ] ->>> jq .plan["resources.snapshots.immutable"] plan.json +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json { "action": "create", "new_state": { @@ -68,7 +68,7 @@ Deployment complete! === Check snapshot plan after deploy: >>> [CLI] bundle plan -o json ->>> jq .plan["resources.snapshots.immutable"] plan.json +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json { "action": "skip", "remote_state": { diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable-no-artifacts/script index 1a68a7d06da..6f8f5f97bd6 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/script +++ b/acceptance/bundle/deploy/immutable-no-artifacts/script @@ -8,7 +8,7 @@ trap cleanup EXIT trace $CLI bundle validate trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json -trace jq '.plan["resources.snapshots.immutable"]' plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json trace $CLI bundle deploy $(readplanarg plan.json) @@ -22,7 +22,7 @@ PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_t trace title "Check snapshot plan after deploy: " trace $CLI bundle plan -o json > plan.json -trace jq '.plan["resources.snapshots.immutable"]' plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json trace $CLI bundle deploy $(readplanarg plan.json) rm plan.json diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 224eca0b517..bc97046230b 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -49,7 +49,7 @@ Building python_artifact... } ] ->>> jq .plan["resources.snapshots.immutable"] plan.json +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json { "action": "create", "new_state": { @@ -91,7 +91,7 @@ Deployment complete! >>> [CLI] bundle plan -o json Building python_artifact... ->>> jq .plan["resources.snapshots.immutable"] plan.json +>>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json { "action": "recreate", "new_state": { diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 2ca5554e5a1..5babe793a9a 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -8,7 +8,7 @@ trace $CLI bundle validate trace $CLI bundle plan -o json > plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.tasks' plan.json trace jq '.plan["resources.jobs.my_job"].new_state.value.environments' plan.json -trace jq '.plan["resources.snapshots.immutable"]' plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json trace $CLI bundle deploy @@ -22,7 +22,7 @@ PYTHON_FILE_1=$(jq '.settings.tasks' job.json | jq '.[] | select(.spark_python_t trace title "Check snapshot plan after deploy" trace $CLI bundle plan -o json > plan.json -trace jq '.plan["resources.snapshots.immutable"]' plan.json +trace jq '.plan["resources.internal_immutable_snapshots.immutable"]' plan.json rm plan.json rm job.json diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 85c7ef3ac00..6f3e7ccc3fc 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3473,17 +3473,17 @@ resources.secret_scopes.*.permissions.acls[*] workspace.AclItem ALL resources.secret_scopes.*.permissions.acls[*].permission workspace.AclPermission ALL resources.secret_scopes.*.permissions.acls[*].principal string ALL resources.secret_scopes.*.permissions.scope_name string ALL -resources.snapshots.*.acl []snapshot.ACLEntry INPUT STATE -resources.snapshots.*.acl[*] snapshot.ACLEntry INPUT STATE -resources.snapshots.*.acl[*].group_name string INPUT STATE -resources.snapshots.*.acl[*].permission_level string INPUT STATE -resources.snapshots.*.acl[*].service_principal_name string INPUT STATE -resources.snapshots.*.acl[*].user_name string INPUT STATE -resources.snapshots.*.bundle_id string INPUT STATE -resources.snapshots.*.full_path string REMOTE STATE -resources.snapshots.*.relative_path string REMOTE STATE -resources.snapshots.*.remote_root string INPUT STATE -resources.snapshots.*.zip_content string INPUT +resources.internal_immutable_snapshots.*.acl []snapshot.ACLEntry INPUT STATE +resources.internal_immutable_snapshots.*.acl[*] snapshot.ACLEntry INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].group_name string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].permission_level string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].service_principal_name string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].user_name string INPUT STATE +resources.internal_immutable_snapshots.*.bundle_id string INPUT STATE +resources.internal_immutable_snapshots.*.full_path string REMOTE STATE +resources.internal_immutable_snapshots.*.relative_path string REMOTE STATE +resources.internal_immutable_snapshots.*.remote_root string INPUT STATE +resources.internal_immutable_snapshots.*.zip_content string INPUT resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL resources.sql_warehouses.*.channel.dbsql_version string ALL diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index 41491f053b8..d9df97347e0 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -36,6 +36,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "${resources.snapshots.immutable.full_path}/files/app" + "source_code_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files/app" } } diff --git a/acceptance/bundle/validate/immutable_workspace_paths/output.txt b/acceptance/bundle/validate/immutable_workspace_paths/output.txt index c5d3b305605..a2f776d66d4 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/output.txt +++ b/acceptance/bundle/validate/immutable_workspace_paths/output.txt @@ -6,14 +6,14 @@ Warning: Pattern user_repls.json does not match any files { "workspace": { - "artifact_path": "${resources.snapshots.immutable.full_path}/artifacts", + "artifact_path": "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts", "current_user": { "domain_friendly_name": "[USERNAME]", "id": "[USERID]", "short_name": "[USERNAME]", "userName": "[USERNAME]" }, - "file_path": "${resources.snapshots.immutable.full_path}/files", + "file_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files", "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources", "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default", "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state" @@ -23,7 +23,7 @@ Warning: Pattern user_repls.json does not match any files "ai_runtime_task": { "deployments": [ { - "command_path": "${resources.snapshots.immutable.full_path}/files/src/main.py", + "command_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py", "compute": { "accelerator_count": 1, "accelerator_type": "GPU_1xA10" @@ -37,7 +37,7 @@ Warning: Pattern user_repls.json does not match any files { "existing_cluster_id": "0101-120000-aaaaaaaa", "spark_python_task": { - "python_file": "${resources.snapshots.immutable.full_path}/files/src/main.py" + "python_file": "${resources.internal_immutable_snapshots.immutable.full_path}/files/src/main.py" }, "task_key": "my_task" } diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml index a36fc1ba93d..a715ab3b03e 100644 --- a/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/databricks.yml @@ -2,7 +2,7 @@ bundle: name: snapshot_resource_not_allowed resources: - snapshots: + internal_immutable_snapshots: immutable: bundle_id: 123 zip_content: |- diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 47c2dfc7202..33e1531fdd6 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -27,7 +27,7 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl if b.Config.Workspace.FilePath == "" { if b.IsImmutableFolder() { - b.Config.Workspace.FilePath = "${resources.snapshots.immutable.full_path}/files" + b.Config.Workspace.FilePath = "${resources.internal_immutable_snapshots.immutable.full_path}/files" } else { b.Config.Workspace.FilePath = path.Join(root, "files") } @@ -39,7 +39,7 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl if b.Config.Workspace.ArtifactPath == "" { if b.IsImmutableFolder() { - b.Config.Workspace.ArtifactPath = "${resources.snapshots.immutable.full_path}/artifacts" + b.Config.Workspace.ArtifactPath = "${resources.internal_immutable_snapshots.immutable.full_path}/artifacts" } else { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 2f7df79908c..e186387882e 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -35,7 +35,7 @@ var unsupportedResources = []string{ "postgres_synced_tables", "vector_search_indexes", "job_runs", - "snapshots", + "internal_immutable_snapshots", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index b574b7a2225..e49169801b7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -43,6 +43,7 @@ func allResourceTypes(t *testing.T) []string { "external_locations", "genie_spaces", "instance_pools", + "internal_immutable_snapshots", "job_runs", "jobs", "model_serving_endpoints", @@ -59,7 +60,6 @@ func allResourceTypes(t *testing.T) []string { "registered_models", "schemas", "secret_scopes", - "snapshots", "sql_warehouses", "synced_database_tables", "vector_search_endpoints", @@ -194,7 +194,7 @@ var allowList = []string{ "genie_spaces", "instance_pools", "job_runs", - "snapshots", + "internal_immutable_snapshots", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index f6e43c067ac..18fbafc43cc 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -322,7 +322,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { switch { case b.IsImmutableFolder(): - t.remoteRoot = "${resources.snapshots.immutable.full_path}/files" + t.remoteRoot = "${resources.internal_immutable_snapshots.immutable.full_path}/files" case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment): t.remoteRoot = t.b.SyncRootPath default: diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 007a35ff3fd..bd0c20b8189 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -46,7 +46,7 @@ type Resources struct { InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` // Internal resources - Snapshots map[string]*resources.Snapshot `json:"snapshots,omitempty" bundle:"internal"` + Snapshots map[string]*resources.Snapshot `json:"internal_immutable_snapshots,omitempty" bundle:"internal"` } type ConfigResource interface { @@ -132,7 +132,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), - collectResourceMap(descriptions["snapshots"], r.Snapshots), + collectResourceMap(descriptions["internal_immutable_snapshots"], r.Snapshots), } } @@ -200,6 +200,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), - "snapshots": (&resources.Snapshot{}).ResourceDescription(), + "internal_immutable_snapshots": (&resources.Snapshot{}).ResourceDescription(), } } diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index c6ba3e3c562..5147609bc8d 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -9,14 +9,28 @@ import ( "github.com/databricks/databricks-sdk-go" ) -// Snapshot is the configuration for the snapshot resource. -// This is an internal resource that is used to store the snapshot of the bundle. -// It is not meant to be used by the user. +// Snapshot is an internal resource that stores the bundle zip as an immutable +// workspace object. It is created by the deploy pipeline and is not intended +// to be declared in user-authored databricks.yml files. +// +// JSON tags are present because the direct-deploy engine serialises the in-memory +// state to a JSON plan file (resources.internal_immutable_snapshots.*). Fields +// that must not leak into the plan file use json:"-". type Snapshot struct { - BundleID string `json:"bundle_id"` - ACL []snapshot.ACLEntry `json:"acl"` - ZipContent string `json:"zip_content"` - RemoteRoot string `json:"remote_root"` + // BundleID is the stable UUID that identifies the bundle deployment, used + // as the first path component of the snapshot workspace path. + BundleID string `json:"bundle_id"` + // ACL is the access control list applied to the uploaded snapshot, granting + // CAN_READ to the deploying user and to every principal in bundle.permissions. + ACL []snapshot.ACLEntry `json:"acl"` + // ZipContent holds the raw zip bytes of the bundle source tree. It is + // populated just before upload. The counterpart SnapshotState.ZipContent + // carries json:"-" so the zip bytes never reach the plan file; SyncZipContent + // re-injects them from here when deploying from a plan. + ZipContent string `json:"zip_content"` + // RemoteRoot is the workspace root path returned by the snapshot rootpath + // API (e.g. /Workspace/Users//.snapshots). + RemoteRoot string `json:"remote_root"` Lifecycle Lifecycle `json:"-"` } @@ -39,10 +53,10 @@ func (s *Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, na func (s *Snapshot) ResourceDescription() ResourceDescription { return ResourceDescription{ - SingularName: "snapshot", - PluralName: "snapshots", - SingularTitle: "Snapshot", - PluralTitle: "Snapshots", + SingularName: "internal_immutable_snapshot", + PluralName: "internal_immutable_snapshots", + SingularTitle: "Internal Immutable Snapshot", + PluralTitle: "Internal Immutable Snapshots", } } diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index cec127828e9..f6b0369f6ab 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -138,7 +138,7 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { "postgres_projects": true, "postgres_roles": true, "secret_scopes": true, - "snapshots": true, + "internal_immutable_snapshots": true, } supported := SupportedResources() @@ -346,8 +346,8 @@ func TestResourcesBindSupport(t *testing.T) { }, } unbindableResources := map[string]bool{ - "model": true, - "snapshot": true, + "model": true, + "internal_immutable_snapshot": true, } ctx := t.Context() diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index d11ecd55422..a0dfb1157b9 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -17,8 +17,6 @@ const fileLimitWarning = 1000 type snapshotUpload struct { skipZip bool - // uploader allows test injection of a custom SnapshotUploader. - uploader snapshot.SnapshotUploader } // PlanUpload returns a mutator that builds the bundle zip, uploads it via @@ -32,13 +30,9 @@ func (m *snapshotUpload) Name() string { } func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - uploader := m.uploader - if uploader == nil { - var err error - uploader, err = snapshot.NewSnapshotUploader(b.WorkspaceClient(ctx)) - if err != nil { - return diag.FromErr(err) - } + uploader, err := snapshot.NewSnapshotClient(b.WorkspaceClient(ctx)) + if err != nil { + return diag.FromErr(err) } remoteRoot, err := uploader.GetSnapshotRootPath(ctx) @@ -86,7 +80,7 @@ func SyncZipContent(b *bundle.Bundle) { if snap == nil || snap.ZipContent == "" { return } - sv, ok := b.DeploymentBundle.StateCache.Load("resources.snapshots.immutable") + sv, ok := b.DeploymentBundle.StateCache.Load("resources.internal_immutable_snapshots.immutable") if !ok { return } diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index 01a543d94fc..778c6b4795c 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -3,6 +3,7 @@ package snapshot import ( "context" "fmt" + "math" "os" "path/filepath" "testing" @@ -12,25 +13,25 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/cli/libs/testserver" "github.com/databricks/cli/libs/vfs" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type mockUploader struct{ path string } - -func (m *mockUploader) Upload(_ context.Context, _, _ string, _ []snapshot.ACLEntry, _ []byte) (*snapshot.SnapshotInfo, error) { - return &snapshot.SnapshotInfo{Path: m.path}, nil -} - -func (m *mockUploader) Get(_ context.Context, _ string) (*snapshot.SnapshotInfo, error) { - return &snapshot.SnapshotInfo{Path: m.path}, nil -} - -func (m *mockUploader) GetSnapshotRootPath(_ context.Context) (string, error) { - return filepath.Join(m.path, "snapshots"), nil +func setupTestClient(t *testing.T) *databricks.WorkspaceClient { + t.Helper() + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + RateLimitPerSecond: math.MaxInt, + }) + require.NoError(t, err) + return client } func makeBundle(t *testing.T, nFiles int) *bundle.Bundle { @@ -68,7 +69,8 @@ func testContext(t *testing.T) context.Context { func TestUploadWarnsAboveFileLimit(t *testing.T) { b := makeBundle(t, fileLimitWarning+1) - m := &snapshotUpload{uploader: &mockUploader{path: "/snapshots/test"}} + b.SetWorkpaceClient(setupTestClient(t)) + m := &snapshotUpload{} diags := m.Apply(testContext(t), b) @@ -79,7 +81,8 @@ func TestUploadWarnsAboveFileLimit(t *testing.T) { func TestUploadNoWarningBelowFileLimit(t *testing.T) { b := makeBundle(t, 5) - m := &snapshotUpload{uploader: &mockUploader{path: "/snapshots/test"}} + b.SetWorkpaceClient(setupTestClient(t)) + m := &snapshotUpload{} diags := m.Apply(testContext(t), b) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 4f5b51b7a46..26aa79ae1f3 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -22,7 +22,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "job_runs", "vector_search_endpoints", "vector_search_indexes", - "snapshots", + "internal_immutable_snapshots", } for resourceType := range supportedResources { diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index e451253b393..2a992326425 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -67,7 +67,7 @@ var SupportedResources = map[string]any{ "vector_search_indexes.grants": (*ResourceGrants)(nil), // Internal resources - "snapshots": (*ResourceSnapshot)(nil), + "internal_immutable_snapshots": (*ResourceSnapshot)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 915e4f997c5..77061a3eaa2 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -299,7 +299,7 @@ var testConfig map[string]any = map[string]any{ }}, }, - "snapshots": &resources.Snapshot{ + "internal_immutable_snapshots": &resources.Snapshot{ RemoteRoot: "/Workspace/Users/" + testserver.TestUserSP.UserName + "/.snapshots", BundleID: "test-bundle-id", ACL: []snapshot.ACLEntry{{UserName: "user@example.com", PermissionLevel: "CAN_READ"}}, @@ -1095,7 +1095,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W } deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") - isImmutable := strings.HasSuffix(group, "snapshots") + isImmutable := strings.HasSuffix(group, "internal_immutable_snapshots") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index fd2c87e1e04..53e7987d2c9 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -798,9 +798,6 @@ resources: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype - snapshots: + internal_immutable_snapshots: recreate_on_changes: - - field: full_path - reason: immutable - - field: relative_path - reason: immutable + - reason: immutable diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index b585730d92b..872aa424f62 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -9,7 +9,7 @@ import ( ) type ResourceSnapshot struct { - uploader snapshot.SnapshotUploader + uploader *snapshot.SnapshotClient } type SnapshotState struct { @@ -34,7 +34,7 @@ func (s *ResourceSnapshot) New(client *databricks.WorkspaceClient) *ResourceSnap } } - uploader, err := snapshot.NewSnapshotUploader(client) + uploader, err := snapshot.NewSnapshotClient(client) if err != nil { panic(err) } diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index a99bece3ba5..b31f3dbc791 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -54,7 +54,7 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "snapshots": { + "internal_immutable_snapshots": { "bundle_id", "acl", "remote_root", @@ -96,7 +96,7 @@ var knownMissingInStateType = map[string][]string{ "keyvault_metadata", "name", }, - "snapshots": { + "internal_immutable_snapshots": { "zip_content", }, } diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index a9ee411cb73..ddd178ae948 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -27,7 +27,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s uploadPath := path.Join(b.Config.Workspace.ArtifactPath, InternalDirName) uploadPath = ensureWorkspaceOrVolumesPrefix(uploadPath) if b.IsImmutableFolder() { - uploadPath = path.Join("${resources.snapshots.immutable.full_path}", "artifacts", InternalDirName) + uploadPath = path.Join("${resources.internal_immutable_snapshots.immutable.full_path}", "artifacts", InternalDirName) } switch { @@ -47,7 +47,7 @@ func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.F artifactPath = ensureWorkspaceOrVolumesPrefix(artifactPath) if b.IsImmutableFolder() { - artifactPath = path.Join("${resources.snapshots.immutable.full_path}", "artifacts") + artifactPath = path.Join("${resources.internal_immutable_snapshots.immutable.full_path}", "artifacts") } switch { diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index af17a93f8b6..3771392ce8f 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") bundle.ApplySeqContext(ctx, b, - // Reads (dynamic): resources.snapshots (and other internal keys) + // Reads (dynamic): resources.internal_immutable_snapshots (and other internal keys) // Warns and removes any internal resource fields set by the user. mutator.RejectInternalResources(), diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index d75859b556a..10ebd58b8b1 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -30,19 +30,8 @@ type ACLEntry struct { PermissionLevel string `json:"permission_level"` } -// SnapshotUploader abstracts the /api/2.0/repos/snapshots endpoint. -// snapshotID is the content-addressed key supplied by the caller; the API uses -// it as the final path component so that identical content always resolves to -// the same workspace location. -// The interface exists so we can replace the implementation in tests with a mock. -type SnapshotUploader interface { - GetSnapshotRootPath(ctx context.Context) (string, error) - Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) - Get(ctx context.Context, path string) (*SnapshotInfo, error) -} - -// snapshotAPIClient implements SnapshotUploader against /api/2.0/repos/snapshots. -type snapshotAPIClient struct { +// SnapshotClient implements the /api/2.0/repos/snapshots endpoint. +type SnapshotClient struct { workspaceClient *databricks.WorkspaceClient client *databricksclient.DatabricksClient } @@ -58,19 +47,19 @@ type snapshotRootPathResponse struct { Path string `json:"path"` } -// NewSnapshotUploader creates a SnapshotUploader backed by /api/2.0/repos/snapshots. -func NewSnapshotUploader(w *databricks.WorkspaceClient) (SnapshotUploader, error) { +// NewSnapshotClient creates a SnapshotClient backed by /api/2.0/repos/snapshots. +func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { c, err := databricksclient.New(w.Config) if err != nil { return nil, err } - return &snapshotAPIClient{workspaceClient: w, client: c}, nil + return &SnapshotClient{workspaceClient: w, client: c}, nil } // Upload uploads zipContent as an immutable snapshot identified by snapshotID. // snapshotID is the SHA-256 of the zip and is used by the server as the // content-addressed path component. acl grants CAN_READ to each listed principal. -func (c *snapshotAPIClient) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { +func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { var body bytes.Buffer mw := multipart.NewWriter(&body) @@ -121,7 +110,7 @@ func (c *snapshotAPIClient) Upload(ctx context.Context, path, bundleID string, a return &SnapshotInfo{Path: resp.Snapshot.Path}, nil } -func (c *snapshotAPIClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { +func (c *SnapshotClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { rootPath, err := c.GetSnapshotRootPath(ctx) if err != nil { return nil, fmt.Errorf("failed to get snapshot root path: %w", err) @@ -134,7 +123,7 @@ func (c *snapshotAPIClient) Get(ctx context.Context, snapshotRelativePath string return &SnapshotInfo{Path: resp.Path}, nil } -func (c *snapshotAPIClient) GetSnapshotRootPath(ctx context.Context) (string, error) { +func (c *SnapshotClient) GetSnapshotRootPath(ctx context.Context) (string, error) { var resp snapshotRootPathResponse err := c.client.Do(ctx, http.MethodGet, "/api/2.0/repos/snapshots/rootpath", auth.WorkspaceIDHeaders(c.client.Config), nil, nil, &resp) if err != nil { From c5cbf31b5b5f03f9be3a973207a1dc31b2d3f4f0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 5 Aug 2026 13:57:10 +0200 Subject: [PATCH 19/30] refschema --- acceptance/bundle/refschema/out.fields.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6f3e7ccc3fc..4dca290689d 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -828,6 +828,17 @@ resources.instance_pools.*.permissions[*].group_name string ALL resources.instance_pools.*.permissions[*].level iam.PermissionLevel ALL resources.instance_pools.*.permissions[*].service_principal_name string ALL resources.instance_pools.*.permissions[*].user_name string ALL +resources.internal_immutable_snapshots.*.acl []snapshot.ACLEntry INPUT STATE +resources.internal_immutable_snapshots.*.acl[*] snapshot.ACLEntry INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].group_name string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].permission_level string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].service_principal_name string INPUT STATE +resources.internal_immutable_snapshots.*.acl[*].user_name string INPUT STATE +resources.internal_immutable_snapshots.*.bundle_id string INPUT STATE +resources.internal_immutable_snapshots.*.full_path string REMOTE STATE +resources.internal_immutable_snapshots.*.relative_path string REMOTE STATE +resources.internal_immutable_snapshots.*.remote_root string INPUT STATE +resources.internal_immutable_snapshots.*.zip_content string INPUT resources.job_runs.*.dbt_commands []string ALL resources.job_runs.*.dbt_commands[*] string ALL resources.job_runs.*.id string INPUT @@ -3473,17 +3484,6 @@ resources.secret_scopes.*.permissions.acls[*] workspace.AclItem ALL resources.secret_scopes.*.permissions.acls[*].permission workspace.AclPermission ALL resources.secret_scopes.*.permissions.acls[*].principal string ALL resources.secret_scopes.*.permissions.scope_name string ALL -resources.internal_immutable_snapshots.*.acl []snapshot.ACLEntry INPUT STATE -resources.internal_immutable_snapshots.*.acl[*] snapshot.ACLEntry INPUT STATE -resources.internal_immutable_snapshots.*.acl[*].group_name string INPUT STATE -resources.internal_immutable_snapshots.*.acl[*].permission_level string INPUT STATE -resources.internal_immutable_snapshots.*.acl[*].service_principal_name string INPUT STATE -resources.internal_immutable_snapshots.*.acl[*].user_name string INPUT STATE -resources.internal_immutable_snapshots.*.bundle_id string INPUT STATE -resources.internal_immutable_snapshots.*.full_path string REMOTE STATE -resources.internal_immutable_snapshots.*.relative_path string REMOTE STATE -resources.internal_immutable_snapshots.*.remote_root string INPUT STATE -resources.internal_immutable_snapshots.*.zip_content string INPUT resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL resources.sql_warehouses.*.channel.dbsql_version string ALL From af7f3ab4645c13f9851a92707242115c247922b4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 6 Aug 2026 11:08:03 +0200 Subject: [PATCH 20/30] fix invariant coverage test --- acceptance/invariant_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index 1c8204055ed..ad8e1d14022 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -22,7 +22,8 @@ const invariantConfigsDir = "bundle/invariant/configs" // for permissions/grants coverage. Add a config and remove the entry to close a gap; // the test fails if an entry here is actually covered, so the list only shrinks. var LackingInvariantTest = map[string]bool{ - "quality_monitors": true, + "quality_monitors": true, + "internal_immutable_snapshots": true, } // TestInvariantConfigsCoverage ensures that the invariant test configs in From 5a050415e41c854faed43142d9a2564a2eb31c22 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 6 Aug 2026 11:12:29 +0200 Subject: [PATCH 21/30] fix fmt --- bundle/config/resources.go | 64 ++++++++++++++++----------------- bundle/config/resources_test.go | 18 +++++----- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/bundle/config/resources.go b/bundle/config/resources.go index bd0c20b8189..f060b074b23 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -168,38 +168,38 @@ func (r *Resources) HasInternalResources() bool { // SupportedResources returns a map which keys correspond to the resource key in the bundle configuration. func SupportedResources() map[string]resources.ResourceDescription { return map[string]resources.ResourceDescription{ - "jobs": (&resources.Job{}).ResourceDescription(), - "job_runs": (&resources.JobRun{}).ResourceDescription(), - "pipelines": (&resources.Pipeline{}).ResourceDescription(), - "models": (&resources.MlflowModel{}).ResourceDescription(), - "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), - "instance_pools": (&resources.InstancePool{}).ResourceDescription(), - "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), - "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), - "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), - "catalogs": (&resources.Catalog{}).ResourceDescription(), - "schemas": (&resources.Schema{}).ResourceDescription(), - "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), - "clusters": (&resources.Cluster{}).ResourceDescription(), - "dashboards": (&resources.Dashboard{}).ResourceDescription(), - "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), - "volumes": (&resources.Volume{}).ResourceDescription(), - "apps": (&resources.App{}).ResourceDescription(), - "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), - "alerts": (&resources.Alert{}).ResourceDescription(), - "sql_warehouses": (&resources.SqlWarehouse{}).ResourceDescription(), - "database_instances": (&resources.DatabaseInstance{}).ResourceDescription(), - "database_catalogs": (&resources.DatabaseCatalog{}).ResourceDescription(), - "synced_database_tables": (&resources.SyncedDatabaseTable{}).ResourceDescription(), - "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), - "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), - "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), - "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), - "postgres_databases": (&resources.PostgresDatabase{}).ResourceDescription(), - "postgres_roles": (&resources.PostgresRole{}).ResourceDescription(), - "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), - "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), - "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), + "jobs": (&resources.Job{}).ResourceDescription(), + "job_runs": (&resources.JobRun{}).ResourceDescription(), + "pipelines": (&resources.Pipeline{}).ResourceDescription(), + "models": (&resources.MlflowModel{}).ResourceDescription(), + "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), + "instance_pools": (&resources.InstancePool{}).ResourceDescription(), + "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), + "registered_models": (&resources.RegisteredModel{}).ResourceDescription(), + "quality_monitors": (&resources.QualityMonitor{}).ResourceDescription(), + "catalogs": (&resources.Catalog{}).ResourceDescription(), + "schemas": (&resources.Schema{}).ResourceDescription(), + "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), + "clusters": (&resources.Cluster{}).ResourceDescription(), + "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), + "volumes": (&resources.Volume{}).ResourceDescription(), + "apps": (&resources.App{}).ResourceDescription(), + "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), + "alerts": (&resources.Alert{}).ResourceDescription(), + "sql_warehouses": (&resources.SqlWarehouse{}).ResourceDescription(), + "database_instances": (&resources.DatabaseInstance{}).ResourceDescription(), + "database_catalogs": (&resources.DatabaseCatalog{}).ResourceDescription(), + "synced_database_tables": (&resources.SyncedDatabaseTable{}).ResourceDescription(), + "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), + "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), + "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), + "postgres_databases": (&resources.PostgresDatabase{}).ResourceDescription(), + "postgres_roles": (&resources.PostgresRole{}).ResourceDescription(), + "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), + "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), + "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "internal_immutable_snapshots": (&resources.Snapshot{}).ResourceDescription(), } } diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index f6b0369f6ab..2a76b9d44b5 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -131,13 +131,13 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // A job run does have a workspace URL, but it's addressed by two IDs // (job + run) so it can't be expressed as a single-ID pattern here; it's // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. - "job_runs": true, - "postgres_branches": true, - "postgres_databases": true, - "postgres_endpoints": true, - "postgres_projects": true, - "postgres_roles": true, - "secret_scopes": true, + "job_runs": true, + "postgres_branches": true, + "postgres_databases": true, + "postgres_endpoints": true, + "postgres_projects": true, + "postgres_roles": true, + "secret_scopes": true, "internal_immutable_snapshots": true, } @@ -346,8 +346,8 @@ func TestResourcesBindSupport(t *testing.T) { }, } unbindableResources := map[string]bool{ - "model": true, - "internal_immutable_snapshot": true, + "model": true, + "internal_immutable_snapshot": true, } ctx := t.Context() From e816436b6f4b05ea377c29f12e218237bf18e163 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 10 Aug 2026 10:56:01 +0200 Subject: [PATCH 22/30] fmt --- bundle/config/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 88d2844c9cc..891a72333ed 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -203,6 +203,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "internal_immutable_snapshots": (&resources.Snapshot{}).ResourceDescription(), - "secrets": (&resources.Secret{}).ResourceDescription(), + "secrets": (&resources.Secret{}).ResourceDescription(), } } From 03d79bd69fbcbab05a105f030ba07d5f0bd37f1c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 13 Aug 2026 11:41:05 +0200 Subject: [PATCH 23/30] update out.test.toml --- .../bundle/validate/snapshot_resource_not_allowed/out.test.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml b/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml index f784a183258..98ea5040486 100644 --- a/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml +++ b/acceptance/bundle/validate/snapshot_resource_not_allowed/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] From ba0d5b8b899367a0195c3dfd09bdcf490e9a88ff Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 18 Aug 2026 17:38:12 +0200 Subject: [PATCH 24/30] fix build --- bundle/deploy/snapshot/upload.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 461580f9f79..a491a60bda6 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -37,11 +37,6 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn remoteRoot, err := uploader.GetSnapshotRootPath(ctx) - zipContent, fileCount, err := BundleZip(ctx, b) - if err != nil { - return diag.FromErr(err) - } - if b.Config.Resources.Snapshots == nil { b.Config.Resources.Snapshots = make(map[string]*resources.Snapshot) } From 4ac3aed2419af6ee56165ee9128ea58aadcfa8f0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 18 Aug 2026 17:42:10 +0200 Subject: [PATCH 25/30] fix lint --- bundle/deploy/snapshot/upload.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index a491a60bda6..a0dfb1157b9 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -36,6 +36,9 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } remoteRoot, err := uploader.GetSnapshotRootPath(ctx) + if err != nil { + return diag.FromErr(err) + } if b.Config.Resources.Snapshots == nil { b.Config.Resources.Snapshots = make(map[string]*resources.Snapshot) From d07f4f8726fb4fa769f9680bf38d103ed3899adc Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 18 Aug 2026 17:53:56 +0200 Subject: [PATCH 26/30] fix output --- .../deploy/immutable-no-artifacts/output.txt | 13 +++++-------- .../immutable-permissions-change/output.txt | 14 +++++--------- acceptance/bundle/deploy/immutable/output.txt | 15 +++++++-------- .../bundle/resources/apps/immutable/output.txt | 6 ++---- 4 files changed, 19 insertions(+), 29 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 1781f9fb07e..a484d1e5dc1 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -50,12 +50,10 @@ Validation OK! } >>> [CLI] bundle deploy -Deploying resources... -Updating deployment state... -Deployment complete! +Created internal_immutable_snapshots.immutable Created jobs.my_job Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" @@ -114,9 +112,8 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } >>> [CLI] bundle deploy -Deploying resources... -Updating deployment state... -Deployment complete! +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged Snapshot did not change as expected >>> [CLI] bundle destroy --auto-approve @@ -125,4 +122,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-no-artifacts-[UNIQUE_NAME]/default -Destroy: 1 deleted +Destroy: 2 deleted diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable-permissions-change/output.txt index 563dc5dc4c6..9f5e66dbfeb 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable-permissions-change/output.txt @@ -1,12 +1,10 @@ === Deploy without permissions >>> [CLI] bundle deploy -Deploying resources... -Updating deployment state... -Deployment complete! +Created internal_immutable_snapshots.immutable Created jobs.my_job Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged First snapshot path: /Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py === Add permissions and redeploy @@ -23,13 +21,11 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:22:3 -Deploying resources... -Updating deployment state... -Deployment complete! +Recreated internal_immutable_snapshots.immutable Updated jobs.my_job Created jobs.my_job.permissions Files: 0 uploaded, 0 deleted -Resources: 1 created, 1 changed, 0 deleted, 0 unchanged +Resources: 2 created, 1 changed, 1 deleted, 0 unchanged Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py === Verify snapshot path changed after permissions change @@ -53,4 +49,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-perms-[UNIQUE_NAME]/default -Destroy: 1 deleted +Destroy: 2 deleted diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 0cedd352cd8..fdcfafcea47 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -70,12 +70,10 @@ Building python_artifact... >>> [CLI] bundle deploy Building python_artifact... -Deploying resources... -Updating deployment state... -Deployment complete! +Created internal_immutable_snapshots.immutable Created jobs.my_job Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .settings.tasks job.json "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py" @@ -166,9 +164,10 @@ Building python_artifact... >>> [CLI] bundle deploy Building python_artifact... -Deploying resources... -Updating deployment state... -Deployment complete! +Recreated internal_immutable_snapshots.immutable +Updated jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged === Check that job uses a new snapshot: Snapshot changed as expected @@ -178,4 +177,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-[UNIQUE_NAME]/default -Destroy: 1 deleted +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index b6e16b99d04..6c827e90906 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -1,11 +1,9 @@ >>> [CLI] bundle deploy -Deploying resources... -Updating deployment state... -Deployment complete! Created apps.my_app +Created internal_immutable_snapshots.immutable Files: 0 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle run my_app ✓ Getting the status of the app my-immutable-app From 2408d9a3eb621fd4983c312ab0faa74705535e95 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 18 Aug 2026 18:19:19 +0200 Subject: [PATCH 27/30] fix delete count --- .../bundle/deploy/immutable-no-artifacts/output.txt | 2 +- .../deploy/immutable-permissions-change/output.txt | 2 +- acceptance/bundle/deploy/immutable/output.txt | 2 +- bundle/phases/destroy.go | 11 ++++++----- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index a484d1e5dc1..55dde26d833 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -122,4 +122,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-no-artifacts-[UNIQUE_NAME]/default -Destroy: 2 deleted +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable-permissions-change/output.txt index 9f5e66dbfeb..680d9579aae 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable-permissions-change/output.txt @@ -49,4 +49,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-perms-[UNIQUE_NAME]/default -Destroy: 2 deleted +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index fdcfafcea47..765e73a51bd 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -177,4 +177,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-[UNIQUE_NAME]/default -Destroy: 2 deleted +Destroy: 1 deleted diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 97c97c0df71..6e97d9bd4b7 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -163,13 +163,14 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e if !logdiag.HasError(ctx) && b.Quiet < bundle.QuietAll { // Count top-level resources only, matching the approval list above (which - // skips children); this also keeps the count stable across engines. Gone - // resources are included: they are excluded from the approval prompt because - // they are not destructive, but destroying them does remove them from state, - // and "bundle deploy" likewise counts them as deleted. + // skips children and gone resources). A gone delete does not destroy + // anything: the resource is already absent remotely, or, like an immutable + // snapshot, cannot be deleted at all, so the delete only cleans up state. + // Counting it would overstate what was destroyed and disagree with the list + // of resources shown above. deleted := 0 for _, a := range plan.GetActions() { - if a.ActionType == deployplan.Delete && !a.IsChildResource() { + if a.ActionType == deployplan.Delete && !a.IsChildResource() && !a.Gone { deleted++ } } From 8f2b500318071fedfb563cc423d5593a5f056c73 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 19 Aug 2026 16:58:16 +0200 Subject: [PATCH 28/30] fixes --- .../resources/apps/immutable/output.txt | 2 +- bundle/config/mutator/translate_paths.go | 10 +++ bundle/config/resources/snapshot.go | 7 +- bundle/deploy/snapshot/upload.go | 15 ++-- bundle/direct/bundle_plan.go | 2 +- bundle/phases/destroy.go | 7 -- bundle/run/app.go | 77 ++++++++++++++++++- libs/snapshot/client.go | 7 +- 8 files changed, 105 insertions(+), 22 deletions(-) diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index 6c827e90906..14c87a9eb2b 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -37,6 +37,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "${resources.internal_immutable_snapshots.immutable.full_path}/files/app" + "source_code_path": "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/app" } } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 18fbafc43cc..8e2d65d1af0 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -322,6 +322,16 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { switch { case b.IsImmutableFolder(): + // Reject an explicit workspace.file_path: immutable bundles set it + // automatically to the content-addressed snapshot location. A user-supplied + // value would be silently discarded during path translation, so we error early. + if loc := b.Config.GetLocation("workspace.file_path"); loc.File != "" { + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "workspace.file_path cannot be configured when experimental.immutable_folder is enabled", + Locations: []dyn.Location{loc}, + }} + } t.remoteRoot = "${resources.internal_immutable_snapshots.immutable.full_path}/files" case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment): t.remoteRoot = t.b.SyncRootPath diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index 5147609bc8d..7a53eb575fc 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -65,12 +65,15 @@ func (s *Snapshot) GetName() string { } func (s *Snapshot) GetURL() string { - // Skipping URL initialization for snapshots + // A snapshot is a workspace folder owned by the project's service principal, so + // a browser URL is constructible from its path. We don't surface one yet: + // workspaceurls has no folder-path helper, and the path is only known during + // deploy (its content hash depends on ZipContent, which is empty otherwise). return "" } func (s *Snapshot) InitializeURL(_ url.URL) { - // Secret scopes do not have a URL + // See GetURL: no browser URL is surfaced for the snapshot folder yet. } func (s *Snapshot) GetLifecycle() LifecycleConfig { diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index a0dfb1157b9..8b20474b4c5 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -19,8 +19,9 @@ type snapshotUpload struct { skipZip bool } -// PlanUpload returns a mutator that builds the bundle zip, uploads it via -// /api/2.0/repos/snapshots, and registers the snapshot as an internal resource. +// PlanUpload returns a mutator that registers the immutable snapshot as an internal +// resource. Unless skipZip is set, it also builds the bundle zip and stages it in +// memory on the resource; the zip is uploaded when the resource is created on apply. func PlanUpload(skipZip bool) bundle.Mutator { return &snapshotUpload{skipZip: skipZip} } @@ -95,12 +96,12 @@ func SyncZipContent(b *bundle.Bundle) { var bundleIDNamespace = uuid.MustParse("4b4e4b5a-3c3d-4e4f-8b8c-9d9e9f0a0b0c") // BundleID returns a stable UUID that identifies the bundle deployment. -// It is derived deterministically from the bundle name, target, and workspace host -// so that every CLI invocation for the same deployment produces the same value. -// This is used as the path prefix for immutable snapshots in the workspace. +// It is derived deterministically from workspace.state_path, which is the +// canonical unique identifier for a deployment (name, target, and workspace root +// are all encoded in it). Two bundles with the same name and target but different +// workspace.state_path values get distinct IDs. func BundleID(b *bundle.Bundle) string { - key := b.Config.Bundle.Name + "/" + b.Config.Bundle.Target + "/" + b.Config.Workspace.Host - return uuid.NewSHA1(bundleIDNamespace, []byte(key)).String() + return uuid.NewSHA1(bundleIDNamespace, []byte(b.Config.Workspace.StatePath)).String() } // BuildACL constructs the access_control_list for the snapshot upload. diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 078a451af37..1982f9b2bb1 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -781,7 +781,7 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s adapter, err := b.getAdapterForKey(targetResourceKey) if err != nil { - return nil, fmt.Errorf("internal error: %s: unknown resource type %q", targetResourceKey, targetGroup) + return nil, fmt.Errorf("internal error: %s: %w", targetResourceKey, err) } configValidErr := structaccess.ValidatePath(reflect.TypeOf(localConfig), fieldPath) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 6e97d9bd4b7..5d608744b6b 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" - "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" @@ -222,12 +221,6 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { - if b.IsImmutableFolder() { - bundle.ApplyContext(ctx, b, snapshot.PlanUpload(true)) - if logdiag.HasError(ctx) { - return - } - } plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) if err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/run/app.go b/bundle/run/app.go index 8c0f135cc5d..b2d261d65fc 100644 --- a/bundle/run/app.go +++ b/bundle/run/app.go @@ -2,6 +2,7 @@ package run import ( "context" + "encoding/json" "errors" "fmt" "time" @@ -142,10 +143,84 @@ func (a *appRunner) deploy(ctx context.Context) error { if err != nil { return err } - deployment := appdeploy.BuildDeployment(a.app.SourceCodePath, config, a.app.GitSource) + sourceCodePath, err := a.resolvedSourceCodePath() + if err != nil { + return err + } + deployment := appdeploy.BuildDeployment(sourceCodePath, config, a.app.GitSource) return appdeploy.Deploy(ctx, w, a.app.Name, deployment) } +// resolvedSourceCodePath returns source_code_path with any ${resources.*} variable +// references resolved against the current bundle state. This is needed for immutable +// folder bundles where source_code_path contains a reference to the snapshot's +// full_path, which is only known after deploy. +func (a *appRunner) resolvedSourceCodePath() (string, error) { + if !dynvar.ContainsVariableReference(a.app.SourceCodePath) { + return a.app.SourceCodePath, nil + } + + root := a.bundle.Config.Value() + + // Build a lookup for ${resources.*} references. For most fields, the + // normalized config value is enough. For compute-only fields like the + // snapshot's full_path (a method, not a stored field), we supplement with + // the deployed state from the direct engine's state DB. + stateOverrides := a.snapshotStateOverrides() + normalized, _ := convert.Normalize(a.bundle.Config, root, convert.IncludeMissingFields) + + sourceCodePathKey := dyn.MustPathFromString("resources." + a.Key() + ".source_code_path") + pathV, err := dyn.GetByPath(root, sourceCodePathKey) + if err != nil || !pathV.IsValid() { + return a.app.SourceCodePath, nil //nolint:nilerr + } + + resourcesPrefix := dyn.MustPathFromString("resources") + resolved, err := dynvar.Resolve(pathV, func(path dyn.Path) (dyn.Value, error) { + if !path.HasPrefix(resourcesPrefix) { + return dyn.InvalidValue, dynvar.ErrSkipResolution + } + // Prefer state-derived overrides (e.g. snapshot.full_path) over the + // config-computed value, which would be wrong when ZipContent is empty. + if v, ok := stateOverrides[path.String()]; ok { + return dyn.V(v), nil + } + return dyn.GetByPath(normalized, path) + }) + if err != nil { + return "", err + } + + s, ok := resolved.AsString() + if !ok { + return a.app.SourceCodePath, nil + } + return s, nil +} + +// snapshotStateOverrides returns a map of resource path → deployed value for +// fields that are only correct in the persisted state (not computable from the +// bundle config alone). Currently this covers the snapshot's full_path and +// relative_path, whose content hash depends on ZipContent (json:"-"). +func (a *appRunner) snapshotStateOverrides() map[string]string { + const snapshotKey = "resources.internal_immutable_snapshots.immutable" + entry, ok := a.bundle.DeploymentBundle.StateDB.GetResourceEntry(snapshotKey) + if !ok { + return nil + } + var s struct { + RelativePath string `json:"relative_path"` + FullPath string `json:"full_path"` + } + if err := json.Unmarshal(entry.State, &s); err != nil || s.FullPath == "" { + return nil + } + return map[string]string{ + snapshotKey + ".full_path": s.FullPath, + snapshotKey + ".relative_path": s.RelativePath, + } +} + // resolvedConfig returns the app config with any ${resources.*} variable references // resolved against the current bundle state. This is needed because the app runtime // configuration (env vars, command) can reference other bundle resources whose diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index 10ebd58b8b1..8539cf0bda5 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -56,9 +56,10 @@ func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { return &SnapshotClient{workspaceClient: w, client: c}, nil } -// Upload uploads zipContent as an immutable snapshot identified by snapshotID. -// snapshotID is the SHA-256 of the zip and is used by the server as the -// content-addressed path component. acl grants CAN_READ to each listed principal. +// Upload uploads zipContent as an immutable snapshot. The server derives the +// content-addressed path from the SHA-256 of the zip, and acl grants CAN_READ to +// each listed principal. path is reserved for forthcoming API support for a +// caller-specified upload location; it is not sent yet. func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { var body bytes.Buffer mw := multipart.NewWriter(&body) From 82f271401aab9da05d86154bbab101bfc99d3f9e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 20 Aug 2026 13:11:55 +0200 Subject: [PATCH 29/30] updated output --- .../bundle/resources/apps/lifecycle-started-toggle/output.txt | 2 +- .../bundle/resources/genie_spaces/recreate_when_gone/output.txt | 2 +- .../resources/jobs/remote_delete/destroy/out.destroy.direct.txt | 2 +- .../permissions/genie_spaces/out_of_band_deletion/output.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt index ef03a4541d9..37fd111532d 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... Created apps.mykey -Files: 7 uploaded, 0 deleted +Files: 8 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_app_requests diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt index e5621c9677d..d7fdd89d8a1 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt @@ -16,4 +16,4 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default -Destroy: 1 deleted +Destroy: 0 deleted diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.destroy.direct.txt b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.destroy.direct.txt index cbbde0249f1..ea08bea2059 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.destroy.direct.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.destroy.direct.txt @@ -2,4 +2,4 @@ >>> errcode [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Destroy: 1 deleted +Destroy: 0 deleted diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt index ec85eecf2b4..e2a8af52883 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt @@ -75,4 +75,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Destroy: 1 deleted +Destroy: 0 deleted From 92ec65ce65789cec3ed14a9b9a58faa497b4043e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 20 Aug 2026 13:16:10 +0200 Subject: [PATCH 30/30] fix output --- .../bundle/resources/apps/lifecycle-started-toggle/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt index 37fd111532d..ef03a4541d9 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... Created apps.mykey -Files: 8 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_app_requests