From 2042b3a0389230b9e3c6cd934e8873e7a9eb46f7 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Tue, 9 Dec 2025 23:15:22 +0530 Subject: [PATCH 1/9] Added DependsOn to State --- .../job_id/out.plan_delete.direct.json | 5 ++++ .../bundle/resource_deps/job_id/output.txt | 6 ++-- acceptance/bundle/resource_deps/job_id/script | 3 +- .../resource_deps/jobs_update/output.txt | 6 ++-- .../bundle/resource_deps/jobs_update/script | 3 +- .../jobs_update_remote/output.txt | 30 +++++++++---------- .../resource_deps/jobs_update_remote/script | 3 +- .../pipelines_recreate/output.txt | 10 +++---- .../resource_deps/pipelines_recreate/script | 3 +- bundle/direct/apply.go | 10 +++---- bundle/direct/bundle_apply.go | 8 ++++- bundle/direct/bundle_plan.go | 19 +++++++++++- bundle/direct/dstate/state.go | 12 ++++---- bundle/direct/pkg.go | 3 ++ 14 files changed, 75 insertions(+), 46 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index 5e92e2c9a5..dbc13a42bd 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,6 +1,11 @@ { "plan": { "resources.jobs.bar": { + "depends_on": [ + { + "node": "resources.jobs.foo" + } + ], "action": "delete", "remote_state": { "created_time": [UNIX_TIME_MILLIS][0], diff --git a/acceptance/bundle/resource_deps/job_id/output.txt b/acceptance/bundle/resource_deps/job_id/output.txt index 5d4350509d..89f0656f90 100644 --- a/acceptance/bundle/resource_deps/job_id/output.txt +++ b/acceptance/bundle/resource_deps/job_id/output.txt @@ -65,19 +65,19 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [BAR_ID] + "job_id": [FOO_ID] } } { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [FOO_ID] + "job_id": [BAR_ID] } } diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index daa89e8f66..4f5481f8c3 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -13,8 +13,7 @@ echo "$bar_id:BAR_ID" >> ACC_REPLS cp empty.yml databricks.yml trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -# TODO sorting requests should not be needed one we persist depends_on in state -trace print_requests.py --sort //jobs +trace print_requests.py //jobs trace $CLI bundle destroy --auto-approve trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/output.txt b/acceptance/bundle/resource_deps/jobs_update/output.txt index 0629b5d465..2096504d5e 100644 --- a/acceptance/bundle/resource_deps/jobs_update/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update/output.txt @@ -73,19 +73,19 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [FOO_ID] + "job_id": [BAR_ID] } } { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [BAR_ID] + "job_id": [FOO_ID] } } diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index deabdbb529..dbd8363dd9 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -28,8 +28,7 @@ trace $CLI jobs get $bar_id rm out.requests.txt trace $CLI bundle destroy --auto-approve -# Sort, because the order of requests is different. TODO: remember deps in order to delete in reverse deployment order. -trace print_requests.py --sort //jobs +trace print_requests.py //jobs trace musterr $CLI jobs get $foo_id trace musterr $CLI jobs get $bar_id diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt index 1ec1c91943..8cd27e9c35 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt @@ -76,21 +76,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort //jobs -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [FOO_ID] - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [BAR_ID] - } -} +>>> print_requests.py //jobs { "method": "POST", "path": "/api/2.2/jobs/reset", @@ -126,3 +112,17 @@ Destroy complete! } } } +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index 42b7b9751e..da734a46e1 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -18,5 +18,4 @@ trace $CLI jobs reset --json @job_update.json $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve -# Sort, because the order of requests is different. TODO: remember deps in order to delete in reverse deployment order. -trace print_requests.py --sort //jobs +trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index 646250706a..50cc2526b2 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -115,11 +115,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort //jobs //pipelines -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[UUID]" -} +>>> print_requests.py //jobs //pipelines { "method": "POST", "path": "/api/2.2/jobs/delete", @@ -127,6 +123,10 @@ Destroy complete! "job_id": [BAR_ID] } } +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]" +} >>> musterr [CLI] pipelines get [FOO_ID] Error: The specified pipeline [FOO_ID] was not found. diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 978f43e42b..c11e9993ed 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -37,8 +37,7 @@ trace $CLI bundle deploy trace print_requests.py //jobs //pipelines trace $CLI bundle destroy --auto-approve -# Sort, because the order of requests is different. TODO: remember deps in order to delete in reverse deployment order. -trace print_requests.py --sort //jobs //pipelines +trace print_requests.py //jobs //pipelines trace musterr $CLI pipelines get $foo_id trace musterr $CLI pipelines get $foo_id_2 diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 99cbd66c1b..5b4ef20adc 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -71,7 +71,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, newID, newState) + err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -96,7 +96,7 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("deleting old id=%s: %w", oldID, err) } - err = db.SaveState(d.ResourceKey, "", nil) + err = db.SaveState(d.ResourceKey, "", nil, nil) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -119,7 +119,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, id, newState) + err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -155,7 +155,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(d.ResourceKey, newID, newState) + err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -202,7 +202,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(d.ResourceKey, id, newState) + err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index ec695558ec..3e84ca16c4 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -75,9 +75,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + var dependsOnNodes []string + for _, dep := range entry.DependsOn { + dependsOnNodes = append(dependsOnNodes, dep.Node) + } + d := &DeploymentUnit{ ResourceKey: resourceKey, Adapter: adapter, + DependsOn: dependsOnNodes, } if at == deployplan.ActionTypeDelete { @@ -112,7 +118,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey)) return false } - err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.Value) + err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.Value, dependsOnNodes) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, entry.NewState.Value, at, entry.Changes) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 605ced24bd..82e60ceea0 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -591,12 +591,29 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root p.Plan[node] = &e } + // Build reverse dependency map: reverseDeps[B] = resources that depend on B. + reverseDeps := make(map[string][]string) + for n, entry := range existingKeys { + for _, dep := range entry.DependsOn { + reverseDeps[dep] = append(reverseDeps[dep], n) + } + } + for n := range existingKeys { if p.Plan[n] != nil { panic("unexpected node " + n) } + + var dependsOn []deployplan.DependsOnEntry + for _, dep := range reverseDeps[n] { + dependsOn = append(dependsOn, deployplan.DependsOnEntry{ + Node: dep, + }) + } + p.Plan[n] = &deployplan.PlanEntry{ - Action: deployplan.ActionTypeDelete.String(), + Action: deployplan.ActionTypeDelete.String(), + DependsOn: dependsOn, } } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e2fda94001..52fb551908 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -25,11 +25,12 @@ type Database struct { } type ResourceEntry struct { - ID string `json:"__id__"` - State any `json:"state"` + ID string `json:"__id__"` + State any `json:"state"` + DependsOn []string `json:"depends_on,omitempty"` } -func (db *DeploymentState) SaveState(key, newID string, state any) error { +func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []string) error { db.AssertOpened() db.mu.Lock() defer db.mu.Unlock() @@ -39,8 +40,9 @@ func (db *DeploymentState) SaveState(key, newID string, state any) error { } db.Data.State[key] = ResourceEntry{ - ID: newID, - State: state, + ID: newID, + State: state, + DependsOn: dependsOn, } return nil diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 6930464bb0..12e6ff3b66 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -31,6 +31,9 @@ type DeploymentUnit struct { // If the resource does not implement withRefresh variants of those methods, remoteState remains nil and // will be populated lazily by calling DoRead(). RemoteState any + + // DependsOn lists resource keys this resource depends on (persisted in state). + DependsOn []string } // DeploymentBundle holds everything needed to deploy a bundle From cab7a92a3959269394b290b9e8ca7a93fe1ef5f6 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Thu, 11 Dec 2025 08:52:05 +0530 Subject: [PATCH 2/9] Changed the state to have the DependsOnEntry and updated the tests --- .../bundle/migrate/basic/out.new_state.json | 28 +- .../out.state_after_migration.json | 8 +- .../bundle/migrate/grants/out.new_state.json | 40 +- .../migrate/permissions/out.new_state.json | 16 +- .../bundle/migrate/runas/out.new_state.json | 8 +- .../job_id/out.plan_delete.direct.json | 11 +- .../job_id_big_graph/databricks.yml | 78 ++++ .../resource_deps/job_id_big_graph/empty.yml | 4 + .../out.plan_delete.direct.json | 408 ++++++++++++++++++ .../out.plan_delete.terraform.json | 40 ++ .../job_id_big_graph/out.test.toml | 5 + .../resource_deps/job_id_big_graph/output.txt | 287 ++++++++++++ .../resource_deps/job_id_big_graph/script | 18 + .../job_id_delete_bar/databricks.yml | 13 + .../job_id_delete_bar/only_foo.yml | 7 + .../out.deploy2.requests.direct.json | 27 ++ .../out.deploy2.requests.terraform.json | 32 ++ .../out.plan_delete.direct.json | 111 +++++ .../out.plan_delete.terraform.json | 10 + .../job_id_delete_bar/out.test.toml | 5 + .../job_id_delete_bar/output.txt | 79 ++++ .../resource_deps/job_id_delete_bar/script | 18 + .../job_id_delete_foo/databricks.yml | 13 + .../job_id_delete_foo/only_bar.yml | 7 + .../out.plan_delete.direct.json | 84 ++++ .../out.plan_delete.terraform.json | 10 + .../job_id_delete_foo/out.test.toml | 5 + .../job_id_delete_foo/output.txt | 86 ++++ .../resource_deps/job_id_delete_foo/script | 18 + acceptance/bundle/resource_deps/test.toml | 1 - bundle/direct/bundle_apply.go | 9 +- bundle/direct/bundle_plan.go | 17 +- bundle/direct/dstate/state.go | 9 +- bundle/direct/graph.go | 14 +- bundle/direct/pkg.go | 4 +- 35 files changed, 1481 insertions(+), 49 deletions(-) create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/empty.yml create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/out.test.toml create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/output.txt create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/script create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/output.txt create mode 100644 acceptance/bundle/resource_deps/job_id_delete_bar/script create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/output.txt create mode 100644 acceptance/bundle/resource_deps/job_id_delete_foo/script diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index a93d0cab70..73dd158cd7 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -51,7 +51,33 @@ "volume_catalog_name": "mycat", "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } - } + }, + "depends_on": [ + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.id}" + }, + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.name}" + }, + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.timeout_seconds}" + }, + { + "node": "resources.volumes.test_volume", + "label": "${resources.volumes.test_volume.catalog_name}" + }, + { + "node": "resources.volumes.test_volume", + "label": "${resources.volumes.test_volume.id}" + }, + { + "node": "resources.volumes.test_volume", + "label": "${resources.volumes.test_volume.storage_location}" + } + ] }, "resources.volumes.test_volume": { "__id__": "mycat.myschema.myvol", diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index fc12770f74..267aca779d 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -99,7 +99,13 @@ "unit": "DAYS" } } - } + }, + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ] }, "resources.pipelines.my_default_python_etl": { "__id__": "[UUID]", diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index cf2dc8bc60..febf0c5f36 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -9,7 +9,13 @@ "comment": "mycomment", "name": "mymodel", "schema_name": "schema_grants" - } + }, + "depends_on": [ + { + "node": "resources.schemas.my_schema", + "label": "${resources.schemas.my_schema.name}" + } + ] }, "resources.registered_models.my_registered_model.grants": { "__id__": "function/main.schema_grants.mymodel", @@ -24,7 +30,13 @@ ] } ] - } + }, + "depends_on": [ + { + "node": "resources.registered_models.my_registered_model", + "label": "${resources.registered_models.my_registered_model.id}" + } + ] }, "resources.schemas.my_schema": { "__id__": "main.schema_grants", @@ -47,7 +59,13 @@ ] } ] - } + }, + "depends_on": [ + { + "node": "resources.schemas.my_schema", + "label": "${resources.schemas.my_schema.id}" + } + ] }, "resources.volumes.my_volume": { "__id__": "main.schema_grants.volume_name", @@ -56,7 +74,13 @@ "name": "volume_name", "schema_name": "schema_grants", "volume_type": "MANAGED" - } + }, + "depends_on": [ + { + "node": "resources.schemas.my_schema", + "label": "${resources.schemas.my_schema.name}" + } + ] }, "resources.volumes.my_volume.grants": { "__id__": "volume/main.schema_grants.volume_name", @@ -72,7 +96,13 @@ ] } ] - } + }, + "depends_on": [ + { + "node": "resources.volumes.my_volume", + "label": "${resources.volumes.my_volume.id}" + } + ] } } } diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index dca45c1ac0..7a3f1a8050 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -40,7 +40,13 @@ "user_name": "[USERNAME]" } ] - } + }, + "depends_on": [ + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.id}" + } + ] }, "resources.pipelines.test_pipeline": { "__id__": "[UUID]", @@ -75,7 +81,13 @@ "user_name": "[USERNAME]" } ] - } + }, + "depends_on": [ + { + "node": "resources.pipelines.test_pipeline", + "label": "${resources.pipelines.test_pipeline.id}" + } + ] } } } diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index 76bfaf8899..87b49b3729 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -41,7 +41,13 @@ "user_name": "[USERNAME]" } ] - } + }, + "depends_on": [ + { + "node": "resources.pipelines.foo", + "label": "${resources.pipelines.foo.id}" + } + ] } } } diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index dbc13a42bd..e54d39c9c3 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,11 +1,6 @@ { "plan": { "resources.jobs.bar": { - "depends_on": [ - { - "node": "resources.jobs.foo" - } - ], "action": "delete", "remote_state": { "created_time": [UNIX_TIME_MILLIS][0], @@ -31,6 +26,12 @@ } }, "resources.jobs.foo": { + "depends_on": [ + { + "node": "resources.jobs.bar", + "label": "${resources.jobs.bar.id}" + } + ], "action": "delete", "remote_state": { "created_time": [UNIX_TIME_MILLIS][1], diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml b/acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml new file mode 100644 index 0000000000..424a1f55f3 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml @@ -0,0 +1,78 @@ +bundle: + name: test-bundle + +# Complex graph combining: chain, diamond, independent, multi-deps +# +# independent1 independent2 +# +# chain_top -> chain_mid -> chain_bottom +# +# diamond_top -> diamond_left -> diamond_bottom +# diamond_right -> +# +# multi_child -> multi_parent1 +# -> multi_parent2 + +resources: + jobs: + # Independent nodes (no dependencies) + independent1: + name: job independent1 + independent2: + name: job independent2 + + # Chain: chain_top -> chain_mid -> chain_bottom + chain_bottom: + name: job chain_bottom + chain_mid: + name: job chain_mid + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.chain_bottom.id} + chain_top: + name: job chain_top + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.chain_mid.id} + + # Diamond: diamond_top -> diamond_left/right -> diamond_bottom + diamond_bottom: + name: job diamond_bottom + diamond_left: + name: job diamond_left + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.diamond_bottom.id} + diamond_right: + name: job diamond_right + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.diamond_bottom.id} + diamond_top: + name: job diamond_top + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.diamond_left.id} + - task_key: t2 + run_job_task: + job_id: ${resources.jobs.diamond_right.id} + + # Multi-deps: multi_child -> multi_parent1, multi_parent2 + multi_parent1: + name: job multi_parent1 + multi_parent2: + name: job multi_parent2 + multi_child: + name: job multi_child + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.multi_parent1.id} + - task_key: t2 + run_job_task: + job_id: ${resources.jobs.multi_parent2.id} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/empty.yml b/acceptance/bundle/resource_deps/job_id_big_graph/empty.yml new file mode 100644 index 0000000000..f12a895592 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/empty.yml @@ -0,0 +1,4 @@ +bundle: + name: test-bundle + +resources: {} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json new file mode 100644 index 0000000000..601ad10324 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json @@ -0,0 +1,408 @@ +{ + "plan": { + "resources.jobs.chain_bottom": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [CHAIN_BOTTOM_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_bottom", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.chain_mid": { + "depends_on": [ + { + "node": "resources.jobs.chain_bottom", + "label": "${resources.jobs.chain_bottom.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "job_id": [CHAIN_MID_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_mid", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_BOTTOM_ID] + }, + "task_key": "t1" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.chain_top": { + "depends_on": [ + { + "node": "resources.jobs.chain_mid", + "label": "${resources.jobs.chain_mid.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][2], + "creator_user_name": "[USERNAME]", + "job_id": [CHAIN_TOP_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_MID_ID] + }, + "task_key": "t1" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.diamond_bottom": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][3], + "creator_user_name": "[USERNAME]", + "job_id": [DIAMOND_BOTTOM_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_bottom", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.diamond_left": { + "depends_on": [ + { + "node": "resources.jobs.diamond_bottom", + "label": "${resources.jobs.diamond_bottom.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][4], + "creator_user_name": "[USERNAME]", + "job_id": [DIAMOND_LEFT_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_left", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.diamond_right": { + "depends_on": [ + { + "node": "resources.jobs.diamond_bottom", + "label": "${resources.jobs.diamond_bottom.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][5], + "creator_user_name": "[USERNAME]", + "job_id": [DIAMOND_RIGHT_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_right", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.diamond_top": { + "depends_on": [ + { + "node": "resources.jobs.diamond_left", + "label": "${resources.jobs.diamond_left.id}" + }, + { + "node": "resources.jobs.diamond_right", + "label": "${resources.jobs.diamond_right.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][6], + "creator_user_name": "[USERNAME]", + "job_id": [DIAMOND_TOP_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_LEFT_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [DIAMOND_RIGHT_ID] + }, + "task_key": "t2" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.independent1": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][7], + "creator_user_name": "[USERNAME]", + "job_id": [INDEPENDENT1_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent1", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.independent2": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][8], + "creator_user_name": "[USERNAME]", + "job_id": [INDEPENDENT2_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent2", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.multi_child": { + "depends_on": [ + { + "node": "resources.jobs.multi_parent1", + "label": "${resources.jobs.multi_parent1.id}" + }, + { + "node": "resources.jobs.multi_parent2", + "label": "${resources.jobs.multi_parent2.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][9], + "creator_user_name": "[USERNAME]", + "job_id": [MULTI_CHILD_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_child", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MULTI_PARENT1_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [MULTI_PARENT2_ID] + }, + "task_key": "t2" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.multi_parent1": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][10], + "creator_user_name": "[USERNAME]", + "job_id": [MULTI_PARENT1_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent1", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.multi_parent2": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][11], + "creator_user_name": "[USERNAME]", + "job_id": [MULTI_PARENT2_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent2", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json new file mode 100644 index 0000000000..04ac0e78ae --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json @@ -0,0 +1,40 @@ +{ + "plan": { + "resources.jobs.chain_bottom": { + "action": "delete" + }, + "resources.jobs.chain_mid": { + "action": "delete" + }, + "resources.jobs.chain_top": { + "action": "delete" + }, + "resources.jobs.diamond_bottom": { + "action": "delete" + }, + "resources.jobs.diamond_left": { + "action": "delete" + }, + "resources.jobs.diamond_right": { + "action": "delete" + }, + "resources.jobs.diamond_top": { + "action": "delete" + }, + "resources.jobs.independent1": { + "action": "delete" + }, + "resources.jobs.independent2": { + "action": "delete" + }, + "resources.jobs.multi_child": { + "action": "delete" + }, + "resources.jobs.multi_parent1": { + "action": "delete" + }, + "resources.jobs.multi_parent2": { + "action": "delete" + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/output.txt new file mode 100644 index 0000000000..84912f3168 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/output.txt @@ -0,0 +1,287 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_mid", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_MID_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_left", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_right", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_LEFT_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [DIAMOND_RIGHT_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent2", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_child", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MULTI_PARENT1_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [MULTI_PARENT2_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent2", + "queue": { + "enabled": true + } + } +} + +=== Delete all - complex graph with chain, diamond, independent, multi-deps +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/script b/acceptance/bundle/resource_deps/job_id_big_graph/script new file mode 100644 index 0000000000..2a03c03320 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore +trace $CLI bundle deploy +trace print_requests.py --sort //jobs + +# Capture all job IDs +for name in independent1 independent2 chain_bottom chain_mid chain_top diamond_bottom diamond_left diamond_right diamond_top multi_parent1 multi_parent2 multi_child; do + id=`read_id.py jobs $name` + upper=$(echo $name | tr '[:lower:]' '[:upper:]') + echo "$id:${upper}_ID" >> ACC_REPLS +done + +title "Delete all - complex graph with chain, diamond, independent, multi-deps" +cp empty.yml databricks.yml +trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml b/acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml new file mode 100644 index 0000000000..3b06625845 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar + foo: + name: job foo + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.bar.id} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml b/acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml new file mode 100644 index 0000000000..6c423728e1 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: job foo diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json new file mode 100644 index 0000000000..0f2d4370cc --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json @@ -0,0 +1,27 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [FOO_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json new file mode 100644 index 0000000000..7b9df15328 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json @@ -0,0 +1,32 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [FOO_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "webhook_notifications": {} + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json new file mode 100644 index 0000000000..a64d4a30b3 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -0,0 +1,111 @@ +{ + "plan": { + "resources.jobs.bar": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [BAR_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.foo": { + "action": "update", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + } + } + }, + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "job_id": [FOO_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + }, + "changes": { + "local": { + "tasks": { + "action": "update", + "old": [ + { + "run_job_task": { + "job_id": [NUMID] + }, + "task_key": "job_task" + } + ] + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='job_task'].run_job_task.job_id": { + "action": "update", + "old": [NUMID], + "new": [BAR_ID] + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json new file mode 100644 index 0000000000..abb6234cd9 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json @@ -0,0 +1,10 @@ +{ + "plan": { + "resources.jobs.bar": { + "action": "delete" + }, + "resources.jobs.foo": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt new file mode 100644 index 0000000000..850328ea49 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt @@ -0,0 +1,79 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ] + } +} + +=== Delete bar, keep foo (foo config updated to not depend on bar) +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script new file mode 100644 index 0000000000..149668335e --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore +trace $CLI bundle deploy +trace print_requests.py //jobs + +foo_id=`read_id.py jobs foo` +echo "$foo_id:FOO_ID" >> ACC_REPLS + +bar_id=`read_id.py jobs bar` +echo "$bar_id:BAR_ID" >> ACC_REPLS + +title "Delete bar, keep foo (foo config updated to not depend on bar)" +cp only_foo.yml databricks.yml +trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy +trace print_requests.py //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml b/acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml new file mode 100644 index 0000000000..3b06625845 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar + foo: + name: job foo + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.bar.id} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml b/acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml new file mode 100644 index 0000000000..703748a7d5 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json new file mode 100644 index 0000000000..fde4361d19 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -0,0 +1,84 @@ +{ + "plan": { + "resources.jobs.bar": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [BAR_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + }, + "changes": { + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.jobs.foo": { + "depends_on": [ + { + "node": "resources.jobs.bar", + "label": "${resources.jobs.bar.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "job_id": [FOO_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json new file mode 100644 index 0000000000..54b6d1acdb --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json @@ -0,0 +1,10 @@ +{ + "plan": { + "resources.jobs.bar": { + "action": "skip" + }, + "resources.jobs.foo": { + "action": "delete" + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt new file mode 100644 index 0000000000..dd604c5148 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt @@ -0,0 +1,86 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ] + } +} + +=== Delete foo, keep bar +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.bar + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script new file mode 100644 index 0000000000..d30b8d36ef --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore +trace $CLI bundle deploy +trace print_requests.py //jobs + +foo_id=`read_id.py jobs foo` +echo "$foo_id:FOO_ID" >> ACC_REPLS + +bar_id=`read_id.py jobs bar` +echo "$bar_id:BAR_ID" >> ACC_REPLS + +title "Delete foo, keep bar" +cp only_bar.yml databricks.yml +trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy +trace print_requests.py //jobs + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index a2b2d9fc33..dc29b70c32 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -1,4 +1,3 @@ -Badness = "Delete order is not respected by direct" RecordRequests = true Ignore = [ diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 3e84ca16c4..670a9e8783 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -75,15 +75,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - var dependsOnNodes []string - for _, dep := range entry.DependsOn { - dependsOnNodes = append(dependsOnNodes, dep.Node) - } - d := &DeploymentUnit{ ResourceKey: resourceKey, Adapter: adapter, - DependsOn: dependsOnNodes, + DependsOn: entry.DependsOn, } if at == deployplan.ActionTypeDelete { @@ -118,7 +113,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey)) return false } - err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.Value, dependsOnNodes) + err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.Value, entry.DependsOn) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, entry.NewState.Value, at, entry.Changes) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 82e60ceea0..298f526fe8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -591,29 +591,14 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root p.Plan[node] = &e } - // Build reverse dependency map: reverseDeps[B] = resources that depend on B. - reverseDeps := make(map[string][]string) for n, entry := range existingKeys { - for _, dep := range entry.DependsOn { - reverseDeps[dep] = append(reverseDeps[dep], n) - } - } - - for n := range existingKeys { if p.Plan[n] != nil { panic("unexpected node " + n) } - var dependsOn []deployplan.DependsOnEntry - for _, dep := range reverseDeps[n] { - dependsOn = append(dependsOn, deployplan.DependsOnEntry{ - Node: dep, - }) - } - p.Plan[n] = &deployplan.PlanEntry{ Action: deployplan.ActionTypeDelete.String(), - DependsOn: dependsOn, + DependsOn: entry.DependsOn, } } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 52fb551908..a5a6c5fa75 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/google/uuid" ) @@ -25,12 +26,12 @@ type Database struct { } type ResourceEntry struct { - ID string `json:"__id__"` - State any `json:"state"` - DependsOn []string `json:"depends_on,omitempty"` + ID string `json:"__id__"` + State any `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []string) error { +func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpened() db.mu.Lock() defer db.mu.Unlock() diff --git a/bundle/direct/graph.go b/bundle/direct/graph.go index fc24f70388..5770377c90 100644 --- a/bundle/direct/graph.go +++ b/bundle/direct/graph.go @@ -16,17 +16,23 @@ func makeGraph(plan *deployplan.Plan) (*dagrun.Graph, error) { g.AddNode(resourceKey) } - // Add edges based on depends_on field exclusively + // Add edges based on depends_on field. + // For deletions, reverse direction so children are deleted before parents. for resourceKey, entry := range plan.Plan { if entry.DependsOn == nil { continue } + isDelete := entry.Action == deployplan.ActionTypeDelete.String() + for _, dep := range entry.DependsOn { - // Only add edge if target node exists in the plan if _, exists := plan.Plan[dep.Node]; exists { - g.AddDirectedEdge(dep.Node, resourceKey, dep.Label) - } else { + if isDelete { + g.AddDirectedEdge(resourceKey, dep.Node, dep.Label) + } else { + g.AddDirectedEdge(dep.Node, resourceKey, dep.Label) + } + } else if !isDelete { return nil, fmt.Errorf("invalid dependency %q, no such node %q", dep.Label, dep.Node) } } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 12e6ff3b66..28d1e0d1b1 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -32,8 +32,8 @@ type DeploymentUnit struct { // will be populated lazily by calling DoRead(). RemoteState any - // DependsOn lists resource keys this resource depends on (persisted in state). - DependsOn []string + // DependsOn lists resources this resource depends on (persisted in state). + DependsOn []deployplan.DependsOnEntry } // DeploymentBundle holds everything needed to deploy a bundle From 39bc6f51028cad1d4328ffcd0a18eaf8ef3a236e Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Thu, 11 Dec 2025 23:34:33 +0530 Subject: [PATCH 3/9] Tests updated plus change marshal to decoder --- .../delete_all/out.plan_delete.direct.txt | 14 + .../delete_all/out.plan_delete.terraform.txt | 14 + .../delete_all/out.plan_sanity.direct.txt | 1 + .../delete_all/out.plan_sanity.terraform.txt | 1 + .../{ => delete_all}/out.test.toml | 0 .../{ => delete_all}/output.txt | 6 +- .../job_id_big_graph/delete_all/script | 24 ++ .../destroy/out.plan_sanity.direct.txt | 1 + .../destroy/out.plan_sanity.terraform.txt | 1 + .../job_id_big_graph/destroy/out.test.toml | 5 + .../job_id_big_graph/destroy/output.txt | 295 +++++++++++++ .../job_id_big_graph/{ => destroy}/script | 9 +- .../out.plan_delete.direct.json | 408 ------------------ .../out.plan_delete.terraform.json | 40 -- .../resource_deps/job_id_big_graph/test.toml | 3 + .../resource_deps/job_id_chain/databricks.yml | 19 + .../out.deploy2.requests.direct.json | 14 +- .../out.plan_delete.direct.json | 7 +- .../update/out.plan_delete_all.direct.json | 6 + .../update/out.plan_delete_all.direct.json | 6 + acceptance/test.toml | 4 +- bundle/direct/dstate/state.go | 8 +- 22 files changed, 416 insertions(+), 470 deletions(-) create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt rename acceptance/bundle/resource_deps/job_id_big_graph/{ => delete_all}/out.test.toml (100%) rename acceptance/bundle/resource_deps/job_id_big_graph/{ => delete_all}/output.txt (98%) create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt rename acceptance/bundle/resource_deps/job_id_big_graph/{ => destroy}/script (68%) delete mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json delete mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json create mode 100644 acceptance/bundle/resource_deps/job_id_big_graph/test.toml create mode 100644 acceptance/bundle/resource_deps/job_id_chain/databricks.yml diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt new file mode 100644 index 0000000000..c2e0cdc7d9 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt @@ -0,0 +1,14 @@ +delete jobs.chain_bottom +delete jobs.chain_mid +delete jobs.chain_top +delete jobs.diamond_bottom +delete jobs.diamond_left +delete jobs.diamond_right +delete jobs.diamond_top +delete jobs.independent1 +delete jobs.independent2 +delete jobs.multi_child +delete jobs.multi_parent1 +delete jobs.multi_parent2 + +Plan: 0 to add, 0 to change, 12 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt new file mode 100644 index 0000000000..c2e0cdc7d9 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt @@ -0,0 +1,14 @@ +delete jobs.chain_bottom +delete jobs.chain_mid +delete jobs.chain_top +delete jobs.diamond_bottom +delete jobs.diamond_left +delete jobs.diamond_right +delete jobs.diamond_top +delete jobs.independent1 +delete jobs.independent2 +delete jobs.multi_child +delete jobs.multi_parent1 +delete jobs.multi_parent2 + +Plan: 0 to add, 0 to change, 12 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml similarity index 100% rename from acceptance/bundle/resource_deps/job_id_big_graph/out.test.toml rename to acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt similarity index 98% rename from acceptance/bundle/resource_deps/job_id_big_graph/output.txt rename to acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt index 84912f3168..b6c9814242 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt @@ -271,8 +271,10 @@ Deployment complete! } } -=== Delete all - complex graph with chain, diamond, independent, multi-deps ->>> [CLI] bundle plan -o json +>>> [CLI] bundle plan + +=== Delete all via empty.yml +>>> [CLI] bundle plan >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script new file mode 100644 index 0000000000..2ddb42983b --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script @@ -0,0 +1,24 @@ +cp $TESTDIR/../databricks.yml . +cp $TESTDIR/../empty.yml . +echo "*" > .gitignore + +trace $CLI bundle deploy +trace print_requests.py --sort //jobs + +# Capture all job IDs +for name in independent1 independent2 chain_bottom chain_mid chain_top diamond_bottom diamond_left diamond_right diamond_top multi_parent1 multi_parent2 multi_child; do + id=`read_id.py jobs $name` + upper=$(echo $name | tr '[:lower:]' '[:upper:]') + echo "$id:${upper}_ID" >> ACC_REPLS +done + +# Sanity check plan after deploy +trace $CLI bundle plan > out.plan_sanity.$DATABRICKS_BUNDLE_ENGINE.txt + +title "Delete all via empty.yml" +cp empty.yml databricks.yml +trace $CLI bundle plan > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt new file mode 100644 index 0000000000..72b7277ec7 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt @@ -0,0 +1,295 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_mid", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_MID_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_left", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_right", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_LEFT_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [DIAMOND_RIGHT_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent2", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_child", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MULTI_PARENT1_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [MULTI_PARENT2_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent2", + "queue": { + "enabled": true + } + } +} + +>>> [CLI] bundle plan + +=== Destroy all via bundle destroy +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.chain_bottom + delete resources.jobs.chain_mid + delete resources.jobs.chain_top + delete resources.jobs.diamond_bottom + delete resources.jobs.diamond_left + delete resources.jobs.diamond_right + delete resources.jobs.diamond_top + delete resources.jobs.independent1 + delete resources.jobs.independent2 + delete resources.jobs.multi_child + delete resources.jobs.multi_parent1 + delete resources.jobs.multi_parent2 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/script b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script similarity index 68% rename from acceptance/bundle/resource_deps/job_id_big_graph/script rename to acceptance/bundle/resource_deps/job_id_big_graph/destroy/script index 2a03c03320..db68e1c827 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/script +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script @@ -1,4 +1,6 @@ +cp $TESTDIR/../databricks.yml . echo "*" > .gitignore + trace $CLI bundle deploy trace print_requests.py --sort //jobs @@ -9,10 +11,9 @@ for name in independent1 independent2 chain_bottom chain_mid chain_top diamond_b echo "$id:${upper}_ID" >> ACC_REPLS done -title "Delete all - complex graph with chain, diamond, independent, multi-deps" -cp empty.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json -trace $CLI bundle deploy +# Sanity check plan after deploy +trace $CLI bundle plan > out.plan_sanity.$DATABRICKS_BUNDLE_ENGINE.txt +title "Destroy all via bundle destroy" trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json deleted file mode 100644 index 601ad10324..0000000000 --- a/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.direct.json +++ /dev/null @@ -1,408 +0,0 @@ -{ - "plan": { - "resources.jobs.chain_bottom": { - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "job_id": [CHAIN_BOTTOM_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job chain_bottom", - "queue": { - "enabled": true - }, - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.chain_mid": { - "depends_on": [ - { - "node": "resources.jobs.chain_bottom", - "label": "${resources.jobs.chain_bottom.id}" - } - ], - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "job_id": [CHAIN_MID_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job chain_mid", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [CHAIN_BOTTOM_ID] - }, - "task_key": "t1" - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.chain_top": { - "depends_on": [ - { - "node": "resources.jobs.chain_mid", - "label": "${resources.jobs.chain_mid.id}" - } - ], - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][2], - "creator_user_name": "[USERNAME]", - "job_id": [CHAIN_TOP_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job chain_top", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [CHAIN_MID_ID] - }, - "task_key": "t1" - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.diamond_bottom": { - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][3], - "creator_user_name": "[USERNAME]", - "job_id": [DIAMOND_BOTTOM_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job diamond_bottom", - "queue": { - "enabled": true - }, - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.diamond_left": { - "depends_on": [ - { - "node": "resources.jobs.diamond_bottom", - "label": "${resources.jobs.diamond_bottom.id}" - } - ], - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][4], - "creator_user_name": "[USERNAME]", - "job_id": [DIAMOND_LEFT_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job diamond_left", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [DIAMOND_BOTTOM_ID] - }, - "task_key": "t1" - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.diamond_right": { - "depends_on": [ - { - "node": "resources.jobs.diamond_bottom", - "label": "${resources.jobs.diamond_bottom.id}" - } - ], - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][5], - "creator_user_name": "[USERNAME]", - "job_id": [DIAMOND_RIGHT_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job diamond_right", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [DIAMOND_BOTTOM_ID] - }, - "task_key": "t1" - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.diamond_top": { - "depends_on": [ - { - "node": "resources.jobs.diamond_left", - "label": "${resources.jobs.diamond_left.id}" - }, - { - "node": "resources.jobs.diamond_right", - "label": "${resources.jobs.diamond_right.id}" - } - ], - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][6], - "creator_user_name": "[USERNAME]", - "job_id": [DIAMOND_TOP_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job diamond_top", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [DIAMOND_LEFT_ID] - }, - "task_key": "t1" - }, - { - "run_job_task": { - "job_id": [DIAMOND_RIGHT_ID] - }, - "task_key": "t2" - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.independent1": { - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][7], - "creator_user_name": "[USERNAME]", - "job_id": [INDEPENDENT1_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job independent1", - "queue": { - "enabled": true - }, - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.independent2": { - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][8], - "creator_user_name": "[USERNAME]", - "job_id": [INDEPENDENT2_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job independent2", - "queue": { - "enabled": true - }, - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.multi_child": { - "depends_on": [ - { - "node": "resources.jobs.multi_parent1", - "label": "${resources.jobs.multi_parent1.id}" - }, - { - "node": "resources.jobs.multi_parent2", - "label": "${resources.jobs.multi_parent2.id}" - } - ], - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][9], - "creator_user_name": "[USERNAME]", - "job_id": [MULTI_CHILD_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job multi_child", - "queue": { - "enabled": true - }, - "tasks": [ - { - "run_job_task": { - "job_id": [MULTI_PARENT1_ID] - }, - "task_key": "t1" - }, - { - "run_job_task": { - "job_id": [MULTI_PARENT2_ID] - }, - "task_key": "t2" - } - ], - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.multi_parent1": { - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][10], - "creator_user_name": "[USERNAME]", - "job_id": [MULTI_PARENT1_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job multi_parent1", - "queue": { - "enabled": true - }, - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - }, - "resources.jobs.multi_parent2": { - "action": "delete", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][11], - "creator_user_name": "[USERNAME]", - "job_id": [MULTI_PARENT2_ID], - "run_as_user_name": "[USERNAME]", - "settings": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "job multi_parent2", - "queue": { - "enabled": true - }, - "timeout_seconds": 0, - "webhook_notifications": {} - } - } - } - } -} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json deleted file mode 100644 index 04ac0e78ae..0000000000 --- a/acceptance/bundle/resource_deps/job_id_big_graph/out.plan_delete.terraform.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "plan": { - "resources.jobs.chain_bottom": { - "action": "delete" - }, - "resources.jobs.chain_mid": { - "action": "delete" - }, - "resources.jobs.chain_top": { - "action": "delete" - }, - "resources.jobs.diamond_bottom": { - "action": "delete" - }, - "resources.jobs.diamond_left": { - "action": "delete" - }, - "resources.jobs.diamond_right": { - "action": "delete" - }, - "resources.jobs.diamond_top": { - "action": "delete" - }, - "resources.jobs.independent1": { - "action": "delete" - }, - "resources.jobs.independent2": { - "action": "delete" - }, - "resources.jobs.multi_child": { - "action": "delete" - }, - "resources.jobs.multi_parent1": { - "action": "delete" - }, - "resources.jobs.multi_parent2": { - "action": "delete" - } - } -} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/test.toml new file mode 100644 index 0000000000..4abc760b0c --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/test.toml @@ -0,0 +1,3 @@ +Ignore = [ + "empty.yml", +] diff --git a/acceptance/bundle/resource_deps/job_id_chain/databricks.yml b/acceptance/bundle/resource_deps/job_id_chain/databricks.yml new file mode 100644 index 0000000000..59b69d0b93 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_chain/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + jobs: + baz: + name: job baz + bar: + name: job bar + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.baz.id} + foo: + name: job foo + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.bar.id} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json index 0f2d4370cc..39625bcb71 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json @@ -1,10 +1,3 @@ -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [BAR_ID] - } -} { "method": "POST", "path": "/api/2.2/jobs/reset", @@ -25,3 +18,10 @@ } } } +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index a64d4a30b3..da2cfc6418 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -79,7 +79,7 @@ "old": [ { "run_job_task": { - "job_id": [NUMID] + "job_id": [BAR_ID] }, "task_key": "job_task" } @@ -91,11 +91,6 @@ "action": "skip", "reason": "server_side_default" }, - "tasks[task_key='job_task'].run_job_task.job_id": { - "action": "update", - "old": [NUMID], - "new": [BAR_ID] - }, "timeout_seconds": { "action": "skip", "reason": "server_side_default" diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index 4ea12c64d4..3c00ccc6f2 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -53,6 +53,12 @@ } }, "resources.jobs.job_with_permissions.permissions": { + "depends_on": [ + { + "node": "resources.jobs.job_with_permissions", + "label": "${resources.jobs.job_with_permissions.id}" + } + ], "action": "delete", "remote_state": { "object_id": "/jobs/[JOB_ID]", diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index ec5bc66f02..ffe36e6c0d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -33,6 +33,12 @@ } }, "resources.pipelines.foo.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.foo", + "label": "${resources.pipelines.foo.id}" + } + ], "action": "delete", "remote_state": { "object_id": "/pipelines/[PIPELINE_ID]", diff --git a/acceptance/test.toml b/acceptance/test.toml index d2f3e3864a..e9cacb082a 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -3,8 +3,8 @@ Local = true Cloud = false # default timeouts -Timeout = '30s' -TimeoutWindows = '60s' +Timeout = '120s' +TimeoutWindows = '120s' # Slowest test I saw: # github.com/databricks/cli/acceptance TestAccept/bundle/integration_whl/interactive_single_user 18m8.69s diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 64c39f7d45..53ff64c7f0 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -1,6 +1,7 @@ package dstate import ( + "bytes" "context" "encoding/json" "fmt" @@ -116,7 +117,12 @@ func (db *DeploymentState) Open(path string) error { return err } - err = json.Unmarshal(data, &db.Data) + // Use json.Decoder with UseNumber() to preserve precision of large integers. + // Without this, numbers are decoded as float64 when the destination is interface{}/any, + // which loses precision for int64 values larger than 2^53. + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + err = dec.Decode(&db.Data) if err != nil { return err } From 767bae53b57909892b9b1c3e9ac72de524861c12 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Fri, 12 Dec 2025 00:16:49 +0530 Subject: [PATCH 4/9] updated tests after rebase --- .../out.deploy2.requests.direct.json | 14 +++++++------- .../job_id_delete_bar/out.plan_delete.direct.json | 2 ++ .../out.plan_delete.terraform.json | 2 ++ .../job_id_delete_foo/out.plan_delete.direct.json | 2 ++ .../out.plan_delete.terraform.json | 2 ++ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json index 39625bcb71..0f2d4370cc 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json @@ -1,3 +1,10 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} { "method": "POST", "path": "/api/2.2/jobs/reset", @@ -18,10 +25,3 @@ } } } -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [BAR_ID] - } -} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index da2cfc6418..a002f0744f 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -1,4 +1,6 @@ { + "plan_version": 1, + "cli_version": "[DEV_VERSION]", "plan": { "resources.jobs.bar": { "action": "delete", diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json index abb6234cd9..755e0199b3 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json @@ -1,4 +1,6 @@ { + "plan_version": 1, + "cli_version": "[DEV_VERSION]", "plan": { "resources.jobs.bar": { "action": "delete" diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index fde4361d19..b2e02c2505 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -1,4 +1,6 @@ { + "plan_version": 1, + "cli_version": "[DEV_VERSION]", "plan": { "resources.jobs.bar": { "action": "skip", diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json index 54b6d1acdb..7d214b09c9 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json @@ -1,4 +1,6 @@ { + "plan_version": 1, + "cli_version": "[DEV_VERSION]", "plan": { "resources.jobs.bar": { "action": "skip" From e936999f2bec8f1bf00b529071c1e260a690f775 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Fri, 12 Dec 2025 00:38:45 +0530 Subject: [PATCH 5/9] updated tests --- acceptance/bundle/resource_deps/job_id_delete_bar/output.txt | 2 +- acceptance/bundle/resource_deps/job_id_delete_bar/script | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt index 850328ea49..fdba7e0c42 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt @@ -58,7 +58,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --sort //jobs >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script index 149668335e..bf99dfc0f0 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/script +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -12,7 +12,9 @@ title "Delete bar, keep foo (foo config updated to not depend on bar)" cp only_foo.yml databricks.yml trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json +# Sort requests because foo's update and bar's delete have no dependency edge +# (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. +trace print_requests.py --sort //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve trace print_requests.py //jobs From 01f454dc2fdc31e52c987819c0f496f0f3ad0fd6 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Fri, 12 Dec 2025 18:07:46 +0530 Subject: [PATCH 6/9] Updated tests --- .../delete_all/out.plan_sanity.direct.txt | 9 ++++++++- .../destroy/out.plan_sanity.direct.txt | 9 ++++++++- .../job_id_delete_bar/out.plan_delete.direct.json | 7 ++++++- acceptance/test.toml | 4 ++-- bundle/direct/dstate/state.go | 10 ++-------- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt index 28f35f7966..7f1a4a3a5c 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt @@ -1 +1,8 @@ -Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged +update jobs.chain_mid +update jobs.chain_top +update jobs.diamond_left +update jobs.diamond_right +update jobs.diamond_top +update jobs.multi_child + +Plan: 0 to add, 6 to change, 0 to delete, 6 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt index 28f35f7966..7f1a4a3a5c 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt @@ -1 +1,8 @@ -Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged +update jobs.chain_mid +update jobs.chain_top +update jobs.diamond_left +update jobs.diamond_right +update jobs.diamond_top +update jobs.multi_child + +Plan: 0 to add, 6 to change, 0 to delete, 6 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index a002f0744f..ffc20c21bf 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -81,7 +81,7 @@ "old": [ { "run_job_task": { - "job_id": [BAR_ID] + "job_id": [NUMID] }, "task_key": "job_task" } @@ -93,6 +93,11 @@ "action": "skip", "reason": "server_side_default" }, + "tasks[task_key='job_task'].run_job_task.job_id": { + "action": "update", + "old": [NUMID], + "new": [BAR_ID] + }, "timeout_seconds": { "action": "skip", "reason": "server_side_default" diff --git a/acceptance/test.toml b/acceptance/test.toml index e9cacb082a..d2f3e3864a 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -3,8 +3,8 @@ Local = true Cloud = false # default timeouts -Timeout = '120s' -TimeoutWindows = '120s' +Timeout = '30s' +TimeoutWindows = '60s' # Slowest test I saw: # github.com/databricks/cli/acceptance TestAccept/bundle/integration_whl/interactive_single_user 18m8.69s diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 53ff64c7f0..b30eac3b66 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -1,7 +1,6 @@ package dstate import ( - "bytes" "context" "encoding/json" "fmt" @@ -116,13 +115,8 @@ func (db *DeploymentState) Open(path string) error { } return err } - - // Use json.Decoder with UseNumber() to preserve precision of large integers. - // Without this, numbers are decoded as float64 when the destination is interface{}/any, - // which loses precision for int64 values larger than 2^53. - dec := json.NewDecoder(bytes.NewReader(data)) - dec.UseNumber() - err = dec.Decode(&db.Data) + // TODO: Use json.Decoder with UseNumber() to preserve precision of large integers. + err = json.Unmarshal(data, &db.Data) if err != nil { return err } From dada9af2a28d977018c7c5f0be6d7a4b1f0befd1 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Fri, 12 Dec 2025 19:23:28 +0530 Subject: [PATCH 7/9] changelog updates --- NEXT_CHANGELOG.md | 1 + .../resource_deps/job_id_chain/databricks.yml | 19 ------------------- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 acceptance/bundle/resource_deps/job_id_chain/databricks.yml diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 7f34ce2a5e..0e86b064f2 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* engine/direct: Fix dependency-ordered deletion by persisting depends_on in state ([#4105](https://github.com/databricks/cli/pull/4105)) ### Dependency updates diff --git a/acceptance/bundle/resource_deps/job_id_chain/databricks.yml b/acceptance/bundle/resource_deps/job_id_chain/databricks.yml deleted file mode 100644 index 59b69d0b93..0000000000 --- a/acceptance/bundle/resource_deps/job_id_chain/databricks.yml +++ /dev/null @@ -1,19 +0,0 @@ -bundle: - name: test-bundle - -resources: - jobs: - baz: - name: job baz - bar: - name: job bar - tasks: - - task_key: job_task - run_job_task: - job_id: ${resources.jobs.baz.id} - foo: - name: job foo - tasks: - - task_key: job_task - run_job_task: - job_id: ${resources.jobs.bar.id} From a555e98a1bdbf5226fe4cb7af924f5769248be72 Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Fri, 12 Dec 2025 22:15:09 +0530 Subject: [PATCH 8/9] fixed tests --- .../delete_all/out.plan_sanity.direct.txt | 9 +-------- .../job_id_big_graph/destroy/out.plan_sanity.direct.txt | 9 +-------- .../job_id_delete_bar/out.plan_delete.direct.json | 7 +------ bundle/direct/dstate/state.go | 1 - 4 files changed, 3 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt index 7f1a4a3a5c..28f35f7966 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt @@ -1,8 +1 @@ -update jobs.chain_mid -update jobs.chain_top -update jobs.diamond_left -update jobs.diamond_right -update jobs.diamond_top -update jobs.multi_child - -Plan: 0 to add, 6 to change, 0 to delete, 6 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt index 7f1a4a3a5c..28f35f7966 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt @@ -1,8 +1 @@ -update jobs.chain_mid -update jobs.chain_top -update jobs.diamond_left -update jobs.diamond_right -update jobs.diamond_top -update jobs.multi_child - -Plan: 0 to add, 6 to change, 0 to delete, 6 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index ffc20c21bf..a002f0744f 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -81,7 +81,7 @@ "old": [ { "run_job_task": { - "job_id": [NUMID] + "job_id": [BAR_ID] }, "task_key": "job_task" } @@ -93,11 +93,6 @@ "action": "skip", "reason": "server_side_default" }, - "tasks[task_key='job_task'].run_job_task.job_id": { - "action": "update", - "old": [NUMID], - "new": [BAR_ID] - }, "timeout_seconds": { "action": "skip", "reason": "server_side_default" diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index d0cd33cad1..f57708dedc 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" From ab9c1b10081dc3263aa109426f6b29513100b74f Mon Sep 17 00:00:00 2001 From: Varun Deep Saini Date: Sat, 13 Dec 2025 01:37:17 +0530 Subject: [PATCH 9/9] lint fixes --- bundle/direct/dstate/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f57708dedc..2556553044 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -32,7 +32,7 @@ type Database struct { type ResourceEntry struct { ID string `json:"__id__"` - State json.RawMessage `json:"state"` + State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` }