Skip to content

Commit

Permalink
Don't fail while parsing outdated terraform state (#1404)
Browse files Browse the repository at this point in the history 
`terraform show -json` (`terraform.Show()`) fails if the state file
contains resources with fields that non longer conform to the provider
schemas.

This can happen when you deploy a bundle with one version of the CLI,
then updated the CLI to a version that uses different databricks
terraform provider, and try to run `bundle run` or `bundle summary`.
Those commands don't recreate local terraform state (only `terraform
apply` or `plan` do) and terraform itself fails while parsing it.
[Terraform
docs](https://developer.hashicorp.com/terraform/language/state#format)
point out that it's best to use `terraform show` after successful
`apply` or `plan`.

Here we parse the state ourselves. The state file format is internal to
terraform, but it's more stable than our resource schemas. We only parse
a subset of fields from the state, and only update ID and ModifiedStatus
of bundle resources in the `terraform.Load` mutator.
  • Loading branch information
@ilia-db
ilia-db authored May 1, 2024
1 parent 781688c commit 153141d
Show file tree
Hide file tree
Showing 8 changed files with 423 additions and 285 deletions.
92 changes: 32 additions & 60 deletions bundle/deploy/check_running_resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@ import (
"strconv"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
"golang.org/x/sync/errgroup"
)

Expand All @@ -35,82 +34,55 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia
if !b.Config.Bundle.Deployment.FailOnActiveRuns {
return nil
}

tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}

err := tf.Init(ctx, tfexec.Upgrade(true))
if err != nil {
return diag.Errorf("terraform init: %v", err)
}

state, err := b.Terraform.Show(ctx)
w := b.WorkspaceClient()
err := checkAnyResourceRunning(ctx, w, &b.Config.Resources)
if err != nil {
return diag.FromErr(err)
}

err = checkAnyResourceRunning(ctx, b.WorkspaceClient(), state)
if err != nil {
return diag.Errorf("deployment aborted, err: %v", err)
}

return nil
}

func CheckRunningResource() *checkRunningResources {
return &checkRunningResources{}
}

func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *tfjson.State) error {
if state.Values == nil || state.Values.RootModule == nil {
return nil
}

func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, resources *config.Resources) error {
errs, errCtx := errgroup.WithContext(ctx)

for _, resource := range state.Values.RootModule.Resources {
// Limit to resources.
if resource.Mode != tfjson.ManagedResourceMode {
for _, job := range resources.Jobs {
id := job.ID
if id == "" {
continue
}
errs.Go(func() error {
isRunning, err := IsJobRunning(errCtx, w, id)
// If there's an error retrieving the job, we assume it's not running
if err != nil {
return err
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
}
return nil
})
}

value, ok := resource.AttributeValues["id"]
if !ok {
for _, pipeline := range resources.Pipelines {
id := pipeline.ID
if id == "" {
continue
}
id, ok := value.(string)
if !ok {
continue
}

switch resource.Type {
case "databricks_job":
errs.Go(func() error {
isRunning, err := IsJobRunning(errCtx, w, id)
// If there's an error retrieving the job, we assume it's not running
if err != nil {
return err
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
}
errs.Go(func() error {
isRunning, err := IsPipelineRunning(errCtx, w, id)
// If there's an error retrieving the pipeline, we assume it's not running
if err != nil {
return nil
})
case "databricks_pipeline":
errs.Go(func() error {
isRunning, err := IsPipelineRunning(errCtx, w, id)
// If there's an error retrieving the pipeline, we assume it's not running
if err != nil {
return nil
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
}
return nil
})
}
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
}
return nil
})
}

return errs.Wait()
Expand Down
64 changes: 17 additions & 47 deletions bundle/deploy/check_running_resources_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,26 @@ import (
"errors"
"testing"

"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)

func TestIsAnyResourceRunningWithEmptyState(t *testing.T) {
mock := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{}
err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &config.Resources{})
require.NoError(t, err)
}

func TestIsAnyResourceRunningWithJob(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
},
},
resources := &config.Resources{
Jobs: map[string]*resources.Job{
"job1": {ID: "123"},
},
}

Expand All @@ -46,33 +36,23 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
{RunId: 1234},
}, nil).Once()

err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.ErrorContains(t, err, "job 123 is running")

jobsApi.EXPECT().ListRunsAll(mock.Anything, jobs.ListRunsRequest{
JobId: 123,
ActiveOnly: true,
}).Return([]jobs.BaseRun{}, nil).Once()

err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}

func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_pipeline",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
},
},
resources := &config.Resources{
Pipelines: map[string]*resources.Pipeline{
"pipeline1": {ID: "123"},
},
}

Expand All @@ -84,7 +64,7 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
State: pipelines.PipelineStateRunning,
}, nil).Once()

err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.ErrorContains(t, err, "pipeline 123 is running")

pipelineApi.EXPECT().Get(mock.Anything, pipelines.GetPipelineRequest{
Expand All @@ -93,25 +73,15 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
PipelineId: "123",
State: pipelines.PipelineStateIdle,
}, nil).Once()
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}

func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_pipeline",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
},
},
resources := &config.Resources{
Pipelines: map[string]*resources.Pipeline{
"pipeline1": {ID: "123"},
},
}

Expand All @@ -120,6 +90,6 @@ func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
PipelineId: "123",
}).Return(nil, errors.New("API failure")).Once()

err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}
Loading

0 comments on commit 153141d

Please sign in to comment.