diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 368e327e66..491e5b29dc 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,7 +7,8 @@ ### CLI ### Bundles -* Pass SYSTEM_ACCESSTOKEN from env to the Terraform provider ([#4135](https://github.com/databricks/cli/pull/4135)) +* engine/direct: Fix dependency-ordered deletion by persisting depends_on in state ([#4105](https://github.com/databricks/cli/pull/4105)) +* Pass SYSTEM_ACCESSTOKEN from env to the Terraform provider ([#4135](https://github.com/databricks/cli/pull/4135) ### Dependency updates diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index b0af3e2cb7..86116b0bd0 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -53,7 +53,33 @@ "volume_catalog_name": "mycat", "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } - } + }, + "depends_on": [ + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.id}" + }, + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.name}" + }, + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.timeout_seconds}" + }, + { + "node": "resources.volumes.test_volume", + "label": "${resources.volumes.test_volume.catalog_name}" + }, + { + "node": "resources.volumes.test_volume", + "label": "${resources.volumes.test_volume.id}" + }, + { + "node": "resources.volumes.test_volume", + "label": "${resources.volumes.test_volume.storage_location}" + } + ] }, "resources.volumes.test_volume": { "__id__": "mycat.myschema.myvol", diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index 16b0a354ca..c29e8fbd7c 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -101,7 +101,13 @@ "unit": "DAYS" } } - } + }, + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ] }, "resources.pipelines.my_default_python_etl": { "__id__": "[UUID]", diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 93a6e0fc5f..8a0116f88a 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -11,7 +11,13 @@ "comment": "mycomment", "name": "mymodel", "schema_name": "schema_grants" - } + }, + "depends_on": [ + { + "node": "resources.schemas.my_schema", + "label": "${resources.schemas.my_schema.name}" + } + ] }, "resources.registered_models.my_registered_model.grants": { "__id__": "function/main.schema_grants.mymodel", @@ -26,7 +32,13 @@ ] } ] - } + }, + "depends_on": [ + { + "node": "resources.registered_models.my_registered_model", + "label": "${resources.registered_models.my_registered_model.id}" + } + ] }, "resources.schemas.my_schema": { "__id__": "main.schema_grants", @@ -49,7 +61,13 @@ ] } ] - } + }, + "depends_on": [ + { + "node": "resources.schemas.my_schema", + "label": "${resources.schemas.my_schema.id}" + } + ] }, "resources.volumes.my_volume": { "__id__": "main.schema_grants.volume_name", @@ -58,7 +76,13 @@ "name": "volume_name", "schema_name": "schema_grants", "volume_type": "MANAGED" - } + }, + "depends_on": [ + { + "node": "resources.schemas.my_schema", + "label": "${resources.schemas.my_schema.name}" + } + ] }, "resources.volumes.my_volume.grants": { "__id__": "volume/main.schema_grants.volume_name", @@ -74,7 +98,13 @@ ] } ] - } + }, + "depends_on": [ + { + "node": "resources.volumes.my_volume", + "label": "${resources.volumes.my_volume.id}" + } + ] } } } diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index 8867ec289d..9bb913044f 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -42,7 +42,13 @@ "user_name": "[USERNAME]" } ] - } + }, + "depends_on": [ + { + "node": "resources.jobs.test_job", + "label": "${resources.jobs.test_job.id}" + } + ] }, "resources.pipelines.test_pipeline": { "__id__": "[UUID]", @@ -77,7 +83,13 @@ "user_name": "[USERNAME]" } ] - } + }, + "depends_on": [ + { + "node": "resources.pipelines.test_pipeline", + "label": "${resources.pipelines.test_pipeline.id}" + } + ] } } } diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index f27ab58e73..017f90d468 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -43,7 +43,13 @@ "user_name": "[USERNAME]" } ] - } + }, + "depends_on": [ + { + "node": "resources.pipelines.foo", + "label": "${resources.pipelines.foo.id}" + } + ] } } } diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index 78f1a5d075..0a9f77c288 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -28,6 +28,12 @@ } }, "resources.jobs.foo": { + "depends_on": [ + { + "node": "resources.jobs.bar", + "label": "${resources.jobs.bar.id}" + } + ], "action": "delete", "remote_state": { "created_time": [UNIX_TIME_MILLIS][1], diff --git a/acceptance/bundle/resource_deps/job_id/output.txt b/acceptance/bundle/resource_deps/job_id/output.txt index 5d4350509d..89f0656f90 100644 --- a/acceptance/bundle/resource_deps/job_id/output.txt +++ b/acceptance/bundle/resource_deps/job_id/output.txt @@ -65,19 +65,19 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [BAR_ID] + "job_id": [FOO_ID] } } { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [FOO_ID] + "job_id": [BAR_ID] } } diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index daa89e8f66..4f5481f8c3 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -13,8 +13,7 @@ echo "$bar_id:BAR_ID" >> ACC_REPLS cp empty.yml databricks.yml trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -# TODO sorting requests should not be needed one we persist depends_on in state -trace print_requests.py --sort //jobs +trace print_requests.py //jobs trace $CLI bundle destroy --auto-approve trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml b/acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml new file mode 100644 index 0000000000..424a1f55f3 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/databricks.yml @@ -0,0 +1,78 @@ +bundle: + name: test-bundle + +# Complex graph combining: chain, diamond, independent, multi-deps +# +# independent1 independent2 +# +# chain_top -> chain_mid -> chain_bottom +# +# diamond_top -> diamond_left -> diamond_bottom +# diamond_right -> +# +# multi_child -> multi_parent1 +# -> multi_parent2 + +resources: + jobs: + # Independent nodes (no dependencies) + independent1: + name: job independent1 + independent2: + name: job independent2 + + # Chain: chain_top -> chain_mid -> chain_bottom + chain_bottom: + name: job chain_bottom + chain_mid: + name: job chain_mid + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.chain_bottom.id} + chain_top: + name: job chain_top + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.chain_mid.id} + + # Diamond: diamond_top -> diamond_left/right -> diamond_bottom + diamond_bottom: + name: job diamond_bottom + diamond_left: + name: job diamond_left + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.diamond_bottom.id} + diamond_right: + name: job diamond_right + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.diamond_bottom.id} + diamond_top: + name: job diamond_top + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.diamond_left.id} + - task_key: t2 + run_job_task: + job_id: ${resources.jobs.diamond_right.id} + + # Multi-deps: multi_child -> multi_parent1, multi_parent2 + multi_parent1: + name: job multi_parent1 + multi_parent2: + name: job multi_parent2 + multi_child: + name: job multi_child + tasks: + - task_key: t1 + run_job_task: + job_id: ${resources.jobs.multi_parent1.id} + - task_key: t2 + run_job_task: + job_id: ${resources.jobs.multi_parent2.id} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt new file mode 100644 index 0000000000..c2e0cdc7d9 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.direct.txt @@ -0,0 +1,14 @@ +delete jobs.chain_bottom +delete jobs.chain_mid +delete jobs.chain_top +delete jobs.diamond_bottom +delete jobs.diamond_left +delete jobs.diamond_right +delete jobs.diamond_top +delete jobs.independent1 +delete jobs.independent2 +delete jobs.multi_child +delete jobs.multi_parent1 +delete jobs.multi_parent2 + +Plan: 0 to add, 0 to change, 12 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt new file mode 100644 index 0000000000..c2e0cdc7d9 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_delete.terraform.txt @@ -0,0 +1,14 @@ +delete jobs.chain_bottom +delete jobs.chain_mid +delete jobs.chain_top +delete jobs.diamond_bottom +delete jobs.diamond_left +delete jobs.diamond_right +delete jobs.diamond_top +delete jobs.independent1 +delete jobs.independent2 +delete jobs.multi_child +delete jobs.multi_parent1 +delete jobs.multi_parent2 + +Plan: 0 to add, 0 to change, 12 to delete, 0 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.direct.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.plan_sanity.terraform.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt new file mode 100644 index 0000000000..b6c9814242 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt @@ -0,0 +1,289 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_mid", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_MID_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_left", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_right", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_LEFT_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [DIAMOND_RIGHT_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent2", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_child", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MULTI_PARENT1_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [MULTI_PARENT2_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent2", + "queue": { + "enabled": true + } + } +} + +>>> [CLI] bundle plan + +=== Delete all via empty.yml +>>> [CLI] bundle plan + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script new file mode 100644 index 0000000000..2ddb42983b --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script @@ -0,0 +1,24 @@ +cp $TESTDIR/../databricks.yml . +cp $TESTDIR/../empty.yml . +echo "*" > .gitignore + +trace $CLI bundle deploy +trace print_requests.py --sort //jobs + +# Capture all job IDs +for name in independent1 independent2 chain_bottom chain_mid chain_top diamond_bottom diamond_left diamond_right diamond_top multi_parent1 multi_parent2 multi_child; do + id=`read_id.py jobs $name` + upper=$(echo $name | tr '[:lower:]' '[:upper:]') + echo "$id:${upper}_ID" >> ACC_REPLS +done + +# Sanity check plan after deploy +trace $CLI bundle plan > out.plan_sanity.$DATABRICKS_BUNDLE_ENGINE.txt + +title "Delete all via empty.yml" +cp empty.yml databricks.yml +trace $CLI bundle plan > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.direct.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt new file mode 100644 index 0000000000..28f35f7966 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.plan_sanity.terraform.txt @@ -0,0 +1 @@ +Plan: 0 to add, 0 to change, 0 to delete, 12 unchanged diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt new file mode 100644 index 0000000000..72b7277ec7 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt @@ -0,0 +1,295 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_mid", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job chain_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [CHAIN_MID_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_bottom", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_left", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_right", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_BOTTOM_ID] + }, + "task_key": "t1" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job diamond_top", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [DIAMOND_LEFT_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [DIAMOND_RIGHT_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job independent2", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_child", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MULTI_PARENT1_ID] + }, + "task_key": "t1" + }, + { + "run_job_task": { + "job_id": [MULTI_PARENT2_ID] + }, + "task_key": "t2" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent1", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job multi_parent2", + "queue": { + "enabled": true + } + } +} + +>>> [CLI] bundle plan + +=== Destroy all via bundle destroy +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.chain_bottom + delete resources.jobs.chain_mid + delete resources.jobs.chain_top + delete resources.jobs.diamond_bottom + delete resources.jobs.diamond_left + delete resources.jobs.diamond_right + delete resources.jobs.diamond_top + delete resources.jobs.independent1 + delete resources.jobs.independent2 + delete resources.jobs.multi_child + delete resources.jobs.multi_parent1 + delete resources.jobs.multi_parent2 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script new file mode 100644 index 0000000000..db68e1c827 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script @@ -0,0 +1,19 @@ +cp $TESTDIR/../databricks.yml . +echo "*" > .gitignore + +trace $CLI bundle deploy +trace print_requests.py --sort //jobs + +# Capture all job IDs +for name in independent1 independent2 chain_bottom chain_mid chain_top diamond_bottom diamond_left diamond_right diamond_top multi_parent1 multi_parent2 multi_child; do + id=`read_id.py jobs $name` + upper=$(echo $name | tr '[:lower:]' '[:upper:]') + echo "$id:${upper}_ID" >> ACC_REPLS +done + +# Sanity check plan after deploy +trace $CLI bundle plan > out.plan_sanity.$DATABRICKS_BUNDLE_ENGINE.txt + +title "Destroy all via bundle destroy" +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/empty.yml b/acceptance/bundle/resource_deps/job_id_big_graph/empty.yml new file mode 100644 index 0000000000..f12a895592 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/empty.yml @@ -0,0 +1,4 @@ +bundle: + name: test-bundle + +resources: {} diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/test.toml new file mode 100644 index 0000000000..4abc760b0c --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_big_graph/test.toml @@ -0,0 +1,3 @@ +Ignore = [ + "empty.yml", +] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml b/acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml new file mode 100644 index 0000000000..3b06625845 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar + foo: + name: job foo + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.bar.id} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml b/acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml new file mode 100644 index 0000000000..6c423728e1 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/only_foo.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + foo: + name: job foo diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json new file mode 100644 index 0000000000..0f2d4370cc --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.direct.json @@ -0,0 +1,27 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [FOO_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json new file mode 100644 index 0000000000..7b9df15328 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.deploy2.requests.terraform.json @@ -0,0 +1,32 @@ +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [FOO_ID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "run_as": { + "user_name": "[USERNAME]" + }, + "webhook_notifications": {} + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json new file mode 100644 index 0000000000..a002f0744f --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -0,0 +1,108 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.bar": { + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [BAR_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + }, + "resources.jobs.foo": { + "action": "update", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + } + } + }, + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "job_id": [FOO_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + }, + "changes": { + "local": { + "tasks": { + "action": "update", + "old": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ] + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json new file mode 100644 index 0000000000..755e0199b3 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.terraform.json @@ -0,0 +1,12 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.bar": { + "action": "delete" + }, + "resources.jobs.foo": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt new file mode 100644 index 0000000000..fdba7e0c42 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt @@ -0,0 +1,79 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ] + } +} + +=== Delete bar, keep foo (foo config updated to not depend on bar) +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --sort //jobs + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script new file mode 100644 index 0000000000..bf99dfc0f0 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -0,0 +1,20 @@ +echo "*" > .gitignore +trace $CLI bundle deploy +trace print_requests.py //jobs + +foo_id=`read_id.py jobs foo` +echo "$foo_id:FOO_ID" >> ACC_REPLS + +bar_id=`read_id.py jobs bar` +echo "$bar_id:BAR_ID" >> ACC_REPLS + +title "Delete bar, keep foo (foo config updated to not depend on bar)" +cp only_foo.yml databricks.yml +trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy +# Sort requests because foo's update and bar's delete have no dependency edge +# (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. +trace print_requests.py --sort //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml b/acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml new file mode 100644 index 0000000000..3b06625845 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar + foo: + name: job foo + tasks: + - task_key: job_task + run_job_task: + job_id: ${resources.jobs.bar.id} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml b/acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml new file mode 100644 index 0000000000..703748a7d5 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/only_bar.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + jobs: + bar: + name: job bar diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json new file mode 100644 index 0000000000..b2e02c2505 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -0,0 +1,86 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.bar": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [BAR_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + }, + "timeout_seconds": 0, + "webhook_notifications": {} + } + }, + "changes": { + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.jobs.foo": { + "depends_on": [ + { + "node": "resources.jobs.bar", + "label": "${resources.jobs.bar.id}" + } + ], + "action": "delete", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "job_id": [FOO_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json new file mode 100644 index 0000000000..7d214b09c9 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.terraform.json @@ -0,0 +1,12 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.bar": { + "action": "skip" + }, + "resources.jobs.foo": { + "action": "delete" + } + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt new file mode 100644 index 0000000000..dd604c5148 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt @@ -0,0 +1,86 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job bar", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "job foo", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ] + } +} + +=== Delete foo, keep bar +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.bar + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script new file mode 100644 index 0000000000..d30b8d36ef --- /dev/null +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -0,0 +1,18 @@ +echo "*" > .gitignore +trace $CLI bundle deploy +trace print_requests.py //jobs + +foo_id=`read_id.py jobs foo` +echo "$foo_id:FOO_ID" >> ACC_REPLS + +bar_id=`read_id.py jobs bar` +echo "$bar_id:BAR_ID" >> ACC_REPLS + +title "Delete foo, keep bar" +cp only_bar.yml databricks.yml +trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy +trace print_requests.py //jobs + +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/output.txt b/acceptance/bundle/resource_deps/jobs_update/output.txt index 0629b5d465..2096504d5e 100644 --- a/acceptance/bundle/resource_deps/jobs_update/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update/output.txt @@ -73,19 +73,19 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [FOO_ID] + "job_id": [BAR_ID] } } { "method": "POST", "path": "/api/2.2/jobs/delete", "body": { - "job_id": [BAR_ID] + "job_id": [FOO_ID] } } diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index deabdbb529..dbd8363dd9 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -28,8 +28,7 @@ trace $CLI jobs get $bar_id rm out.requests.txt trace $CLI bundle destroy --auto-approve -# Sort, because the order of requests is different. TODO: remember deps in order to delete in reverse deployment order. -trace print_requests.py --sort //jobs +trace print_requests.py //jobs trace musterr $CLI jobs get $foo_id trace musterr $CLI jobs get $bar_id diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt index 1ec1c91943..8cd27e9c35 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt @@ -76,21 +76,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort //jobs -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [FOO_ID] - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [BAR_ID] - } -} +>>> print_requests.py //jobs { "method": "POST", "path": "/api/2.2/jobs/reset", @@ -126,3 +112,17 @@ Destroy complete! } } } +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [BAR_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [FOO_ID] + } +} diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index 42b7b9751e..da734a46e1 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -18,5 +18,4 @@ trace $CLI jobs reset --json @job_update.json $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve -# Sort, because the order of requests is different. TODO: remember deps in order to delete in reverse deployment order. -trace print_requests.py --sort //jobs +trace print_requests.py //jobs diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index 646250706a..50cc2526b2 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -115,11 +115,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort //jobs //pipelines -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[UUID]" -} +>>> print_requests.py //jobs //pipelines { "method": "POST", "path": "/api/2.2/jobs/delete", @@ -127,6 +123,10 @@ Destroy complete! "job_id": [BAR_ID] } } +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]" +} >>> musterr [CLI] pipelines get [FOO_ID] Error: The specified pipeline [FOO_ID] was not found. diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 978f43e42b..c11e9993ed 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -37,8 +37,7 @@ trace $CLI bundle deploy trace print_requests.py //jobs //pipelines trace $CLI bundle destroy --auto-approve -# Sort, because the order of requests is different. TODO: remember deps in order to delete in reverse deployment order. -trace print_requests.py --sort //jobs //pipelines +trace print_requests.py //jobs //pipelines trace musterr $CLI pipelines get $foo_id trace musterr $CLI pipelines get $foo_id_2 diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index a2b2d9fc33..dc29b70c32 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -1,4 +1,3 @@ -Badness = "Delete order is not respected by direct" RecordRequests = true Ignore = [ diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index 4ea12c64d4..3c00ccc6f2 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -53,6 +53,12 @@ } }, "resources.jobs.job_with_permissions.permissions": { + "depends_on": [ + { + "node": "resources.jobs.job_with_permissions", + "label": "${resources.jobs.job_with_permissions.id}" + } + ], "action": "delete", "remote_state": { "object_id": "/jobs/[JOB_ID]", diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index ec5bc66f02..ffe36e6c0d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -33,6 +33,12 @@ } }, "resources.pipelines.foo.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.foo", + "label": "${resources.pipelines.foo.id}" + } + ], "action": "delete", "remote_state": { "object_id": "/pipelines/[PIPELINE_ID]", diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index f8eb5c9fc2..92625c82e8 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -70,7 +70,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, newID, newState) + err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -95,7 +95,7 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("deleting old id=%s: %w", oldID, err) } - err = db.SaveState(d.ResourceKey, "", nil) + err = db.SaveState(d.ResourceKey, "", nil, nil) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -118,7 +118,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, id, newState) + err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -154,7 +154,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(d.ResourceKey, newID, newState) + err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -201,7 +201,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(d.ResourceKey, id, newState) + err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index ec695558ec..670a9e8783 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -78,6 +78,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa d := &DeploymentUnit{ ResourceKey: resourceKey, Adapter: adapter, + DependsOn: entry.DependsOn, } if at == deployplan.ActionTypeDelete { @@ -112,7 +113,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey)) return false } - err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.Value) + err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.Value, entry.DependsOn) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, entry.NewState.Value, at, entry.Changes) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 3daca7de2a..2d7f7ed657 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -604,12 +604,14 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root p.Plan[node] = &e } - for n := range existingKeys { + for n, entry := range existingKeys { if p.Plan[n] != nil { panic("unexpected node " + n) } + p.Plan[n] = &deployplan.PlanEntry{ - Action: deployplan.ActionTypeDelete.String(), + Action: deployplan.ActionTypeDelete.String(), + DependsOn: entry.DependsOn, } } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ce8b6adbec..2556553044 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" "github.com/google/uuid" @@ -30,8 +31,9 @@ type Database struct { } type ResourceEntry struct { - ID string `json:"__id__"` - State json.RawMessage `json:"state"` + ID string `json:"__id__"` + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } func NewDatabase() Database { @@ -48,7 +50,7 @@ func NewMigratedDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any) error { +func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpened() db.mu.Lock() defer db.mu.Unlock() @@ -63,8 +65,9 @@ func (db *DeploymentState) SaveState(key, newID string, state any) error { } db.Data.State[key] = ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, } return nil @@ -117,7 +120,7 @@ func (db *DeploymentState) Open(path string) error { } return err } - + // TODO: Use json.Decoder with UseNumber() to preserve precision of large integers. err = json.Unmarshal(data, &db.Data) if err != nil { return err diff --git a/bundle/direct/graph.go b/bundle/direct/graph.go index fc24f70388..5770377c90 100644 --- a/bundle/direct/graph.go +++ b/bundle/direct/graph.go @@ -16,17 +16,23 @@ func makeGraph(plan *deployplan.Plan) (*dagrun.Graph, error) { g.AddNode(resourceKey) } - // Add edges based on depends_on field exclusively + // Add edges based on depends_on field. + // For deletions, reverse direction so children are deleted before parents. for resourceKey, entry := range plan.Plan { if entry.DependsOn == nil { continue } + isDelete := entry.Action == deployplan.ActionTypeDelete.String() + for _, dep := range entry.DependsOn { - // Only add edge if target node exists in the plan if _, exists := plan.Plan[dep.Node]; exists { - g.AddDirectedEdge(dep.Node, resourceKey, dep.Label) - } else { + if isDelete { + g.AddDirectedEdge(resourceKey, dep.Node, dep.Label) + } else { + g.AddDirectedEdge(dep.Node, resourceKey, dep.Label) + } + } else if !isDelete { return nil, fmt.Errorf("invalid dependency %q, no such node %q", dep.Label, dep.Node) } } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 6930464bb0..28d1e0d1b1 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -31,6 +31,9 @@ type DeploymentUnit struct { // If the resource does not implement withRefresh variants of those methods, remoteState remains nil and // will be populated lazily by calling DoRead(). RemoteState any + + // DependsOn lists resources this resource depends on (persisted in state). + DependsOn []deployplan.DependsOnEntry } // DeploymentBundle holds everything needed to deploy a bundle