From 4ca6a0b3d98c8930e128423d057051a7a17ba62f Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 26 Jul 2023 17:12:12 +0300 Subject: [PATCH 01/60] Add the NoForkConnector to reconcile MRs without process forks - Switch to Resource.RefreshWithoutUpgrade & Resource.Apply Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 26 +--- pkg/controller/external_nofork.go | 220 ++++++++++++++++++++++++++++++ pkg/terraform/store.go | 2 + 3 files changed, 227 insertions(+), 21 deletions(-) create mode 100644 pkg/controller/external_nofork.go diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 8c6e1328..c5f489da 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -8,10 +8,10 @@ import ( "fmt" "regexp" - "github.com/crossplane/upjet/pkg/registry" - conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" - tfjson "github.com/hashicorp/terraform-json" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/registry" ) // ResourceConfiguratorFn is a function that implements the ResourceConfigurator @@ -199,24 +199,8 @@ func WithMainTemplate(template string) ProviderOption { } } -// NewProvider builds and returns a new Provider from provider -// tfjson schema, that is generated using Terraform CLI with: -// `terraform providers schema --json` -func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { //nolint:gocyclo - ps := tfjson.ProviderSchemas{} - if err := ps.UnmarshalJSON(schema); err != nil { - panic(err) - } - if len(ps.Schemas) != 1 { - panic(fmt.Sprintf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas))) - } - var rs map[string]*tfjson.Schema - for _, v := range ps.Schemas { - rs = v.ResourceSchemas - break - } - - resourceMap := conversiontfjson.GetV2ResourceMap(rs) +// NewProvider builds and returns a new Provider from provider native schema. +func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo providerMetadata, err := registry.NewProviderMetadataFromFile(metadata) if err != nil { panic(errors.Wrap(err, "cannot load provider metadata")) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go new file mode 100644 index 00000000..9a87ff10 --- /dev/null +++ b/pkg/controller/external_nofork.go @@ -0,0 +1,220 @@ +// Copyright 2023 Upbound Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "fmt" + "strings" + + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/meta" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/hashicorp/go-cty/cty" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/pkg/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/upbound/upjet/pkg/config" + "github.com/upbound/upjet/pkg/resource" + "github.com/upbound/upjet/pkg/terraform" +) + +func convertMapToCty(data map[string]any) cty.Value { + transformedData := make(map[string]cty.Value) + + for key, value := range data { + switch v := value.(type) { + case int: + transformedData[key] = cty.NumberIntVal(int64(v)) + case string: + transformedData[key] = cty.StringVal(v) + case bool: + transformedData[key] = cty.BoolVal(v) + case float64: + transformedData[key] = cty.NumberFloatVal(v) + case map[string]any: + transformedData[key] = convertMapToCty(v) + // more cases... + default: + // handle unknown types, for now we will ignore them + continue + } + } + + return cty.ObjectVal(transformedData) +} + +func fromFlatmap(m map[string]string) map[string]any { + result := make(map[string]any, len(m)) + for k, v := range m { + // we need to handle name hierarchies + if strings.Contains(k, ".") { + continue + } + result[k] = v + } + return result +} + +type NoForkConnector struct { + getTerraformSetup terraform.SetupFn + kube client.Client + config *config.Resource +} + +func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource) *NoForkConnector { + return &NoForkConnector{ + kube: kube, + getTerraformSetup: sf, + config: cfg, + } +} + +func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { + /* tr, ok := mg.(resource.Terraformed) + if !ok { + return nil, errors.New(errUnexpectedObject) + } + */ + ts, err := c.getTerraformSetup(ctx, c.kube, mg) + if err != nil { + return nil, errors.Wrap(err, errGetTerraformSetup) + } + + // To Compute the ResourceDiff: n.resourceSchema.Diff(...) + tr := mg.(resource.Terraformed) + params, err := tr.GetParameters() + if err != nil { + return nil, errors.Wrap(err, "cannot get parameters") + } + if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { + return nil, errors.Wrap(err, "cannot get sensitive parameters") + } + c.config.ExternalName.SetIdentifierArgumentFn(params, meta.GetExternalName(tr)) + + tfID, err := c.config.ExternalName.GetIDFn(ctx, meta.GetExternalName(mg), params, ts.Map()) + if err != nil { + return nil, errors.Wrap(err, "cannot get ID") + } + + tfState, err := tr.GetObservation() + if err != nil { + return nil, errors.Wrap(err, "failed to get the observation") + } + tfState["id"] = tfID + s, err := c.config.TerraformResource.ShimInstanceStateFromValue(convertMapToCty(tfState)) + if err != nil { + return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") + } + + params["id"] = tfID + instanceDiff, err := schema.InternalMap(c.config.TerraformResource.Schema).Diff(ctx, s, &tf.ResourceConfig{ + Raw: params, + Config: params, + }, nil, ts.Meta, false) + if err != nil { + return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") + } + + resourceData, err := schema.InternalMap(c.config.TerraformResource.Schema).Data(s, instanceDiff) + if err != nil { + return nil, errors.Wrap(err, "failed to get *schema.ResourceData") + } + + return &noForkExternal{ + ts: ts, + resourceSchema: c.config.TerraformResource, + config: c.config, + kube: c.kube, + resourceData: resourceData, + instanceState: s, + instanceDiff: instanceDiff, + }, nil +} + +type noForkExternal struct { + ts terraform.Setup + resourceSchema *schema.Resource + config *config.Resource + kube client.Client + resourceData *schema.ResourceData + instanceState *tf.InstanceState + instanceDiff *tf.InstanceDiff +} + +func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { + s, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.instanceState, n.ts.Meta) + fmt.Println(diag) + if diag != nil && diag.HasError() { + return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) + } + resourceExists := n.resourceData.Id() != "" + if resourceExists { + mg.SetConditions(xpv1.Available()) + if s != nil { + tr := mg.(resource.Terraformed) + tr.SetObservation(fromFlatmap(s.Attributes)) + } + } + noDiff := n.instanceDiff.Empty() + return managed.ExternalObservation{ + ResourceExists: resourceExists, + ResourceUpToDate: noDiff, + }, nil +} + +func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { + newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + // diag := n.resourceSchema.CreateWithoutTimeout(ctx, n.resourceData, n.ts.Meta) + fmt.Println(diag) + if diag != nil && diag.HasError() { + return managed.ExternalCreation{}, errors.Errorf("failed to create the resource: %v", diag) + } + + if newState == nil || newState.ID == "" { + return managed.ExternalCreation{}, errors.New("failed to read the ID of the new resource") + } + + en, err := n.config.ExternalName.GetExternalNameFn(map[string]any{ + "id": newState.ID, + }) + if err != nil { + return managed.ExternalCreation{}, errors.Wrapf(err, "failed to get the external-name from ID: %s", newState.ID) + } + // we have to make sure the newly set externa-name is recorded + meta.SetExternalName(mg, en) + + return managed.ExternalCreation{}, nil +} + +func (n *noForkExternal) Update(ctx context.Context, _ xpresource.Managed) (managed.ExternalUpdate, error) { + _, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + fmt.Println(diag) + if diag != nil && diag.HasError() { + return managed.ExternalUpdate{}, errors.Errorf("failed to update the resource: %v", diag) + } + return managed.ExternalUpdate{}, nil +} + +func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error { + diag := n.resourceSchema.DeleteWithoutTimeout(ctx, n.resourceData, n.ts.Meta) + fmt.Println(diag) + + return nil +} diff --git a/pkg/terraform/store.go b/pkg/terraform/store.go index 2f9b7f57..2e65ad06 100644 --- a/pkg/terraform/store.go +++ b/pkg/terraform/store.go @@ -120,6 +120,8 @@ type Setup struct { // the lifecycle of Terraform provider processes will be managed by // the Terraform CLI. Scheduler ProviderScheduler + + Meta any } // Map returns the Setup object in map form. The initial reason was so that From 54e5dd9be7b3d0556da71593196490dccaa4c766 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 29 Aug 2023 07:23:03 +0300 Subject: [PATCH 02/60] Disable updates Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 9a87ff10..bb425dcd 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -104,7 +104,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, errors.Wrap(err, "cannot get parameters") } if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { - return nil, errors.Wrap(err, "cannot get sensitive parameters") + return nil, errors.Wrap(err, "cannot store sensitive parameters into params") } c.config.ExternalName.SetIdentifierArgumentFn(params, meta.GetExternalName(tr)) @@ -112,18 +112,22 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m if err != nil { return nil, errors.Wrap(err, "cannot get ID") } + params["id"] = tfID tfState, err := tr.GetObservation() if err != nil { return nil, errors.Wrap(err, "failed to get the observation") } + if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, tfState, tr.GetConnectionDetailsMapping()); err != nil { + return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") + } + //c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) tfState["id"] = tfID s, err := c.config.TerraformResource.ShimInstanceStateFromValue(convertMapToCty(tfState)) if err != nil { return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } - params["id"] = tfID instanceDiff, err := schema.InternalMap(c.config.TerraformResource.Schema).Diff(ctx, s, &tf.ResourceConfig{ Raw: params, Config: params, @@ -164,7 +168,10 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if diag != nil && diag.HasError() { return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } - resourceExists := n.resourceData.Id() != "" + resourceExists := s != nil && s.ID != "" + if !resourceExists { + n.instanceState = s + } if resourceExists { mg.SetConditions(xpv1.Available()) if s != nil { @@ -172,10 +179,10 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma tr.SetObservation(fromFlatmap(s.Attributes)) } } - noDiff := n.instanceDiff.Empty() + //noDiff := n.instanceDiff.Empty() return managed.ExternalObservation{ ResourceExists: resourceExists, - ResourceUpToDate: noDiff, + ResourceUpToDate: true, }, nil } @@ -213,8 +220,11 @@ func (n *noForkExternal) Update(ctx context.Context, _ xpresource.Managed) (mana } func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error { - diag := n.resourceSchema.DeleteWithoutTimeout(ctx, n.resourceData, n.ts.Meta) + n.instanceDiff.Destroy = true + _, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) fmt.Println(diag) - + if diag != nil && diag.HasError() { + return errors.Errorf("failed to delete the resource: %v", diag) + } return nil } From 97849ad0d3c652bae447173e7e3296fc47bce389 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 29 Aug 2023 08:04:02 +0300 Subject: [PATCH 03/60] Fix updates Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 107 +++++++++++++++++++----------- 1 file changed, 69 insertions(+), 38 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index bb425dcd..2e8da5d6 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -86,12 +86,18 @@ func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Re } } +func copy(tfState, params map[string]any) map[string]any { + targetState := make(map[string]any, len(params)) + for k, v := range params { + targetState[k] = v + } + for k, v := range tfState { + targetState[k] = v + } + return targetState +} + func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { - /* tr, ok := mg.(resource.Terraformed) - if !ok { - return nil, errors.New(errUnexpectedObject) - } - */ ts, err := c.getTerraformSetup(ctx, c.kube, mg) if err != nil { return nil, errors.Wrap(err, errGetTerraformSetup) @@ -113,32 +119,27 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, errors.Wrap(err, "cannot get ID") } params["id"] = tfID + // we need to parameterize the following for a provider + // not all providers may have this attribute + params["tags_all"] = params["tags"] tfState, err := tr.GetObservation() if err != nil { return nil, errors.Wrap(err, "failed to get the observation") } + copyParams := len(tfState) == 0 if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, tfState, tr.GetConnectionDetailsMapping()); err != nil { return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") } - //c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) + c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) tfState["id"] = tfID - s, err := c.config.TerraformResource.ShimInstanceStateFromValue(convertMapToCty(tfState)) - if err != nil { - return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") + if copyParams { + tfState = copy(tfState, params) } - instanceDiff, err := schema.InternalMap(c.config.TerraformResource.Schema).Diff(ctx, s, &tf.ResourceConfig{ - Raw: params, - Config: params, - }, nil, ts.Meta, false) - if err != nil { - return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") - } - - resourceData, err := schema.InternalMap(c.config.TerraformResource.Schema).Data(s, instanceDiff) + s, err := c.config.TerraformResource.ShimInstanceStateFromValue(convertMapToCty(tfState)) if err != nil { - return nil, errors.Wrap(err, "failed to get *schema.ResourceData") + return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } return &noForkExternal{ @@ -146,9 +147,8 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m resourceSchema: c.config.TerraformResource, config: c.config, kube: c.kube, - resourceData: resourceData, instanceState: s, - instanceDiff: instanceDiff, + params: params, }, nil } @@ -157,37 +157,54 @@ type noForkExternal struct { resourceSchema *schema.Resource config *config.Resource kube client.Client - resourceData *schema.ResourceData instanceState *tf.InstanceState - instanceDiff *tf.InstanceDiff + params map[string]any +} + +func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState) (*tf.InstanceDiff, error) { + instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, &tf.ResourceConfig{ + Raw: n.params, + Config: n.params, + }, nil, n.ts.Meta, false) + if err != nil { + return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") + } + + return instanceDiff, nil } func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { - s, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.instanceState, n.ts.Meta) + newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.instanceState, n.ts.Meta) fmt.Println(diag) if diag != nil && diag.HasError() { return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } - resourceExists := s != nil && s.ID != "" - if !resourceExists { - n.instanceState = s - } + n.instanceState = newState + noDiff := false + resourceExists := newState != nil && newState.ID != "" if resourceExists { mg.SetConditions(xpv1.Available()) - if s != nil { - tr := mg.(resource.Terraformed) - tr.SetObservation(fromFlatmap(s.Attributes)) + mg.(resource.Terraformed).SetObservation(fromFlatmap(newState.Attributes)) + + instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + if err != nil { + return managed.ExternalObservation{}, err } + noDiff = instanceDiff.Empty() } - //noDiff := n.instanceDiff.Empty() + return managed.ExternalObservation{ ResourceExists: resourceExists, - ResourceUpToDate: true, + ResourceUpToDate: noDiff, }, nil } func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { - newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + if err != nil { + return managed.ExternalCreation{}, err + } + newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) // diag := n.resourceSchema.CreateWithoutTimeout(ctx, n.resourceData, n.ts.Meta) fmt.Println(diag) if diag != nil && diag.HasError() { @@ -206,22 +223,36 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man } // we have to make sure the newly set externa-name is recorded meta.SetExternalName(mg, en) + mg.(resource.Terraformed).SetObservation(fromFlatmap(newState.Attributes)) return managed.ExternalCreation{}, nil } -func (n *noForkExternal) Update(ctx context.Context, _ xpresource.Managed) (managed.ExternalUpdate, error) { - _, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) +func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { + instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + if err != nil { + return managed.ExternalUpdate{}, err + } + newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) fmt.Println(diag) if diag != nil && diag.HasError() { return managed.ExternalUpdate{}, errors.Errorf("failed to update the resource: %v", diag) } + mg.(resource.Terraformed).SetObservation(fromFlatmap(newState.Attributes)) return managed.ExternalUpdate{}, nil } func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error { - n.instanceDiff.Destroy = true - _, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + if err != nil { + return err + } + if instanceDiff == nil { + instanceDiff = tf.NewInstanceDiff() + } + + instanceDiff.Destroy = true + _, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) fmt.Println(diag) if diag != nil && diag.HasError() { return errors.Errorf("failed to delete the resource: %v", diag) From f4ad2a52eb39fb9cb8cd6c692dd0f25d75b92662 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Mon, 18 Sep 2023 15:39:51 +0300 Subject: [PATCH 04/60] tf instance state converters Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 108 +++++++++++++++++------------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 2e8da5d6..4076d579 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -17,15 +17,13 @@ package controller import ( "context" "fmt" - "strings" - tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" - "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,55 +33,33 @@ import ( "github.com/upbound/upjet/pkg/terraform" ) -func convertMapToCty(data map[string]any) cty.Value { - transformedData := make(map[string]cty.Value) - - for key, value := range data { - switch v := value.(type) { - case int: - transformedData[key] = cty.NumberIntVal(int64(v)) - case string: - transformedData[key] = cty.StringVal(v) - case bool: - transformedData[key] = cty.BoolVal(v) - case float64: - transformedData[key] = cty.NumberFloatVal(v) - case map[string]any: - transformedData[key] = convertMapToCty(v) - // more cases... - default: - // handle unknown types, for now we will ignore them - continue - } - } - - return cty.ObjectVal(transformedData) -} - -func fromFlatmap(m map[string]string) map[string]any { - result := make(map[string]any, len(m)) - for k, v := range m { - // we need to handle name hierarchies - if strings.Contains(k, ".") { - continue - } - result[k] = v - } - return result -} - type NoForkConnector struct { getTerraformSetup terraform.SetupFn kube client.Client config *config.Resource + logger logging.Logger } -func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource) *NoForkConnector { - return &NoForkConnector{ +// NoForkOption allows you to configure NoForkConnector. +type NoForkOption func(connector *NoForkConnector) + +// WithNoForkLogger configures a logger for the NoForkConnector. +func WithNoForkLogger(l logging.Logger) NoForkOption { + return func(c *NoForkConnector) { + c.logger = l + } +} + +func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkOption) *NoForkConnector { + nfc := &NoForkConnector{ kube: kube, getTerraformSetup: sf, config: cfg, } + for _, f := range opts { + f(nfc) + } + return nfc } func copy(tfState, params map[string]any) map[string]any { @@ -121,7 +97,11 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m params["id"] = tfID // we need to parameterize the following for a provider // not all providers may have this attribute - params["tags_all"] = params["tags"] + // TODO: tags_all handling + // attrs := c.config.TerraformResource.CoreConfigSchema().Attributes + // if _, ok := attrs["tags_all"]; ok { + // params["tags_all"] = params["tags"] + // } tfState, err := tr.GetObservation() if err != nil { @@ -137,7 +117,12 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m tfState = copy(tfState, params) } - s, err := c.config.TerraformResource.ShimInstanceStateFromValue(convertMapToCty(tfState)) + tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, c.config.TerraformResource.CoreConfigSchema()) + if err != nil { + return nil, err + } + s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) + if err != nil { return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } @@ -149,6 +134,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m kube: c.kube, instanceState: s, params: params, + logger: c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()), }, nil } @@ -159,6 +145,7 @@ type noForkExternal struct { kube client.Client instanceState *tf.InstanceState params map[string]any + logger logging.Logger } func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState) (*tf.InstanceDiff, error) { @@ -184,8 +171,11 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma resourceExists := newState != nil && newState.ID != "" if resourceExists { mg.SetConditions(xpv1.Available()) - mg.(resource.Terraformed).SetObservation(fromFlatmap(newState.Attributes)) - + stateValueMap, err := n.fromInstanceStateToJSONMap(newState) + if err != nil { + return managed.ExternalObservation{}, err + } + mg.(resource.Terraformed).SetObservation(stateValueMap) instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) if err != nil { return managed.ExternalObservation{}, err @@ -223,7 +213,11 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man } // we have to make sure the newly set externa-name is recorded meta.SetExternalName(mg, en) - mg.(resource.Terraformed).SetObservation(fromFlatmap(newState.Attributes)) + stateValueMap, err := n.fromInstanceStateToJSONMap(newState) + if err != nil { + return managed.ExternalCreation{}, err + } + mg.(resource.Terraformed).SetObservation(stateValueMap) return managed.ExternalCreation{}, nil } @@ -238,7 +232,12 @@ func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (man if diag != nil && diag.HasError() { return managed.ExternalUpdate{}, errors.Errorf("failed to update the resource: %v", diag) } - mg.(resource.Terraformed).SetObservation(fromFlatmap(newState.Attributes)) + + stateValueMap, err := n.fromInstanceStateToJSONMap(newState) + if err != nil { + return managed.ExternalUpdate{}, err + } + mg.(resource.Terraformed).SetObservation(stateValueMap) return managed.ExternalUpdate{}, nil } @@ -259,3 +258,16 @@ func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error } return nil } + +func (n *noForkExternal) fromInstanceStateToJSONMap(newState *tf.InstanceState) (map[string]interface{}, error) { + impliedType := n.resourceSchema.CoreConfigSchema().ImpliedType() + attrsAsCtyValue, err := newState.AttrsAsObjectValue(impliedType) + if err != nil { + return nil, errors.Wrap(err, "could not convert attrs to cty value") + } + stateValueMap, err := schema.StateValueToJSONMap(attrsAsCtyValue, impliedType) + if err != nil { + return nil, errors.Wrap(err, "could not convert instance state value to JSON") + } + return stateValueMap, nil +} From 921bce71d64663d16f09ddf2237dc7a82a6babbb Mon Sep 17 00:00:00 2001 From: Cem Mergenci Date: Mon, 18 Sep 2023 17:55:20 +0300 Subject: [PATCH 05/60] Configure no-fork external client for resources using a bool flag. Signed-off-by: Cem Mergenci Signed-off-by: Alper Rifat Ulucinar --- pkg/config/resource.go | 4 ++++ pkg/pipeline/controller.go | 1 + pkg/pipeline/templates/controller.go.tmpl | 18 ++++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 7e5aa036..34582b50 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -293,6 +293,10 @@ type Resource struct { // databases. UseAsync bool + // UseNoForkClient indicates that a no-fork external client should + // be generated instead of the Terraform CLI-forking client. + UseNoForkClient bool + InitializerFns []NewInitializerFn // OperationTimeouts allows configuring resource operation timeouts. diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index e34f05aa..8305e951 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -49,6 +49,7 @@ func (cg *ControllerGenerator) Generate(cfg *config.Resource, typesPkgPath strin "DisableNameInitializer": cfg.ExternalName.DisableNameInitializer, "TypePackageAlias": ctrlFile.Imports.UsePackage(typesPkgPath), "UseAsync": cfg.UseAsync, + "UseNoForkClient": cfg.UseNoForkClient, "ResourceType": cfg.Name, "Initializers": cfg.InitializerFns, } diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index be17d390..fc3d6522 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -41,15 +41,21 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { cps = append(cps, connection.NewDetailsManager(mgr.GetClient(), *o.SecretStoreConfigGVK, connection.WithTLSConfig(o.ESSOptions.TLSConfig))) } eventHandler := handler.NewEventHandler(handler.WithLogger(o.Logger.WithValues("gvk", {{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind))) - {{- if .UseAsync }} + {{- if and .UseAsync (not .UseNoForkClient) }} ac := tjcontroller.NewAPICallbacks(mgr, xpresource.ManagedKind({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind), tjcontroller.WithEventHandler(eventHandler)) {{- end}} opts := []managed.ReconcilerOption{ - managed.WithExternalConnecter(tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), - {{- if .UseAsync }} - tjcontroller.WithCallbackProvider(ac), - {{- end}} - )), + managed.WithExternalConnecter( + {{- if .UseNoForkClient -}} + tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"]) + {{- else -}} + tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), + {{- if .UseAsync }} + tjcontroller.WithCallbackProvider(ac), + {{- end}} + ) + {{- end -}} + ), managed.WithLogger(o.Logger.WithValues("controller", name)), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), managed.WithFinalizer(terraform.NewWorkspaceFinalizer(o.WorkspaceStore, xpresource.NewAPIFinalizer(mgr.GetClient(), managed.FinalizerName))), From 3a1991d6ed82872379d608833b9c44731891586e Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Mon, 18 Sep 2023 18:41:58 +0300 Subject: [PATCH 06/60] Add no-fork external client's log option to controller template Signed-off-by: Alper Rifat Ulucinar --- pkg/pipeline/templates/controller.go.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index fc3d6522..005e724e 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -47,7 +47,7 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { opts := []managed.ReconcilerOption{ managed.WithExternalConnecter( {{- if .UseNoForkClient -}} - tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"]) + tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger)) {{- else -}} tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), {{- if .UseAsync }} From 728532253277a3f28aaf6e2867a006d38c63b808 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Tue, 19 Sep 2023 12:01:13 +0300 Subject: [PATCH 07/60] handle SetObservation errors & rename copy func Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 4076d579..9c5c3ca1 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -62,7 +62,7 @@ func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Re return nfc } -func copy(tfState, params map[string]any) map[string]any { +func copyParameters(tfState, params map[string]any) map[string]any { targetState := make(map[string]any, len(params)) for k, v := range params { targetState[k] = v @@ -114,7 +114,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) tfState["id"] = tfID if copyParams { - tfState = copy(tfState, params) + tfState = copyParameters(tfState, params) } tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, c.config.TerraformResource.CoreConfigSchema()) @@ -175,7 +175,10 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if err != nil { return managed.ExternalObservation{}, err } - mg.(resource.Terraformed).SetObservation(stateValueMap) + err = mg.(resource.Terraformed).SetObservation(stateValueMap) + if err != nil { + return managed.ExternalObservation{}, errors.Errorf("could not set observation: %v", err) + } instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) if err != nil { return managed.ExternalObservation{}, err @@ -217,7 +220,10 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man if err != nil { return managed.ExternalCreation{}, err } - mg.(resource.Terraformed).SetObservation(stateValueMap) + err = mg.(resource.Terraformed).SetObservation(stateValueMap) + if err != nil { + return managed.ExternalCreation{}, errors.Errorf("could not set observation: %v", err) + } return managed.ExternalCreation{}, nil } @@ -237,7 +243,11 @@ func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (man if err != nil { return managed.ExternalUpdate{}, err } - mg.(resource.Terraformed).SetObservation(stateValueMap) + + err = mg.(resource.Terraformed).SetObservation(stateValueMap) + if err != nil { + return managed.ExternalUpdate{}, errors.Errorf("failed to set observation: %v", err) + } return managed.ExternalUpdate{}, nil } From ca6d80fbd75ba1902ce380f219e49f8b8fa431a5 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 20 Sep 2023 12:59:12 +0300 Subject: [PATCH 08/60] Add upjet_resource_ext_api_duration histogram metric Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 18 +++++++++++++----- pkg/metrics/metrics.go | 13 +++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 9c5c3ca1..1b681fd0 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -16,7 +16,8 @@ package controller import ( "context" - "fmt" + "time" + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" @@ -29,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/upbound/upjet/pkg/config" + "github.com/upbound/upjet/pkg/metrics" "github.com/upbound/upjet/pkg/resource" "github.com/upbound/upjet/pkg/terraform" ) @@ -74,7 +76,9 @@ func copyParameters(tfState, params map[string]any) map[string]any { } func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { + start := time.Now() ts, err := c.getTerraformSetup(ctx, c.kube, mg) + metrics.ExternalAPITime.WithLabelValues("connect").Observe(time.Since(start).Seconds()) if err != nil { return nil, errors.Wrap(err, errGetTerraformSetup) } @@ -161,8 +165,9 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { + start := time.Now() newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.instanceState, n.ts.Meta) - fmt.Println(diag) + metrics.ExternalAPITime.WithLabelValues("read").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } @@ -197,9 +202,10 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man if err != nil { return managed.ExternalCreation{}, err } + start := time.Now() newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) + metrics.ExternalAPITime.WithLabelValues("create").Observe(time.Since(start).Seconds()) // diag := n.resourceSchema.CreateWithoutTimeout(ctx, n.resourceData, n.ts.Meta) - fmt.Println(diag) if diag != nil && diag.HasError() { return managed.ExternalCreation{}, errors.Errorf("failed to create the resource: %v", diag) } @@ -233,8 +239,9 @@ func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (man if err != nil { return managed.ExternalUpdate{}, err } + start := time.Now() newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) - fmt.Println(diag) + metrics.ExternalAPITime.WithLabelValues("update").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return managed.ExternalUpdate{}, errors.Errorf("failed to update the resource: %v", diag) } @@ -261,8 +268,9 @@ func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error } instanceDiff.Destroy = true + start := time.Now() _, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) - fmt.Println(diag) + metrics.ExternalAPITime.WithLabelValues("delete").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return errors.Errorf("failed to delete the resource: %v", diag) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4e71dc7d..345085fd 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -25,6 +25,15 @@ var ( Buckets: []float64{1.0, 3, 5, 10, 15, 30, 60, 120, 300}, }, []string{"subcommand", "mode"}) + // ExternalAPITime is the SDK processing times histogram. + ExternalAPITime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: promNSUpjet, + Subsystem: promSysResource, + Name: "ext_api_duration", + Help: "Measures in seconds how long it takes a Cloud SDK call to complete", + Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600}, + }, []string{"operation"}) + // CLIExecutions are the active number of terraform CLI invocations. CLIExecutions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: promNSUpjet, @@ -49,10 +58,10 @@ var ( Subsystem: promSysResource, Name: "ttr", Help: "Measures in seconds the time-to-readiness (TTR) for managed resources", - Buckets: []float64{10, 15, 30, 60, 120, 300, 600, 1800, 3600}, + Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600}, }, []string{"group", "version", "kind"}) ) func init() { - metrics.Registry.MustRegister(CLITime, CLIExecutions, TFProcesses, TTRMeasurements) + metrics.Registry.MustRegister(CLITime, CLIExecutions, TFProcesses, TTRMeasurements, ExternalAPITime) } From 59cc664e01b2b200c84d49dc44959e263cd3c174 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 20 Sep 2023 21:29:27 +0300 Subject: [PATCH 09/60] Add TTR metric for the forkless client Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 1b681fd0..f5336c34 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -19,6 +19,7 @@ import ( "time" tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + corev1 "k8s.io/api/core/v1" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/logging" @@ -175,6 +176,9 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma noDiff := false resourceExists := newState != nil && newState.ID != "" if resourceExists { + if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown { + addTTR(mg) + } mg.SetConditions(xpv1.Available()) stateValueMap, err := n.fromInstanceStateToJSONMap(newState) if err != nil { From 1cc1a824cdd110d13b539705d00cb2dc8ede2187 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 21 Sep 2023 17:45:24 +0300 Subject: [PATCH 10/60] Add upjet_resource_deletion_seconds & upjet_resource_reconcile_delay_seconds metrics Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 31 +++++- pkg/metrics/metrics.go | 113 +++++++++++++++++++++- pkg/pipeline/templates/controller.go.tmpl | 3 +- 3 files changed, 140 insertions(+), 7 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index f5336c34..aefb09ae 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -41,6 +41,7 @@ type NoForkConnector struct { kube client.Client config *config.Resource logger logging.Logger + metricRecorder *metrics.MetricRecorder } // NoForkOption allows you to configure NoForkConnector. @@ -53,6 +54,14 @@ func WithNoForkLogger(l logging.Logger) NoForkOption { } } +// WithNoForkMetricRecorder configures a metrics.MetricRecorder for the +// NoForkConnector. +func WithNoForkMetricRecorder(r *metrics.MetricRecorder) NoForkOption { + return func(c *NoForkConnector) { + c.metricRecorder = r + } +} + func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkOption) *NoForkConnector { nfc := &NoForkConnector{ kube: kube, @@ -77,6 +86,7 @@ func copyParameters(tfState, params map[string]any) map[string]any { } func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { + c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) start := time.Now() ts, err := c.getTerraformSetup(ctx, c.kube, mg) metrics.ExternalAPITime.WithLabelValues("connect").Observe(time.Since(start).Seconds()) @@ -103,10 +113,10 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m // we need to parameterize the following for a provider // not all providers may have this attribute // TODO: tags_all handling - // attrs := c.config.TerraformResource.CoreConfigSchema().Attributes - // if _, ok := attrs["tags_all"]; ok { - // params["tags_all"] = params["tags"] - // } + attrs := c.config.TerraformResource.CoreConfigSchema().Attributes + if _, ok := attrs["tags_all"]; ok { + params["tags_all"] = params["tags"] + } tfState, err := tr.GetObservation() if err != nil { @@ -140,6 +150,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m instanceState: s, params: params, logger: c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()), + metricRecorder: c.metricRecorder, }, nil } @@ -151,6 +162,7 @@ type noForkExternal struct { instanceState *tf.InstanceState params map[string]any logger logging.Logger + metricRecorder *metrics.MetricRecorder } func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState) (*tf.InstanceDiff, error) { @@ -175,8 +187,13 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma n.instanceState = newState noDiff := false resourceExists := newState != nil && newState.ID != "" + if !resourceExists && mg.GetDeletionTimestamp() != nil { + gvk := mg.GetObjectKind().GroupVersionKind() + metrics.DeletionTime.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(time.Since(mg.GetDeletionTimestamp().Time).Seconds()) + } if resourceExists { - if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown { + if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown || + mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionFalse { addTTR(mg) } mg.SetConditions(xpv1.Available()) @@ -193,6 +210,10 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma return managed.ExternalObservation{}, err } noDiff = instanceDiff.Empty() + + if noDiff { + n.metricRecorder.SetReconcileTime(mg.GetName()) + } } return managed.ExternalObservation{ diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 345085fd..a06d41aa 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -5,7 +5,17 @@ package metrics import ( + "context" + "sync" + "time" + + "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -34,6 +44,29 @@ var ( Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600}, }, []string{"operation"}) + // DeletionTime is the histogram metric for collecting statistics on the + // intervals between the deletion timestamp and the moment when + // the resource is observed to be missing (actually deleted). + DeletionTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: promNSUpjet, + Subsystem: promSysResource, + Name: "deletion_seconds", + Help: "Measures in seconds how long it takes for a resource to be deleted", + Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600}, + }, []string{"group", "version", "kind"}) + + // ReconcileDelay is the histogram metric for collecting statistics on the + // delays between when the expected reconciles of an up-to-date resource + // should happen and when the resource is actually reconciled. Only + // delays from the expected reconcile times are considered. + ReconcileDelay = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: promNSUpjet, + Subsystem: promSysResource, + Name: "reconcile_delay_seconds", + Help: "Measures in seconds how long the reconciles for a resource have been delayed from the configured poll periods", + Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600}, + }, []string{"group", "version", "kind"}) + // CLIExecutions are the active number of terraform CLI invocations. CLIExecutions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: promNSUpjet, @@ -62,6 +95,84 @@ var ( }, []string{"group", "version", "kind"}) ) +var _ manager.Runnable = &MetricRecorder{} + +type MetricRecorder struct { + observations sync.Map + gvk schema.GroupVersionKind + cluster cluster.Cluster + + pollInterval time.Duration +} + +type Observations struct { + expectedReconcileTime *time.Time + observeReconcileDelay bool +} + +func NewMetricRecorder(gvk schema.GroupVersionKind, c cluster.Cluster, pollInterval time.Duration) *MetricRecorder { + return &MetricRecorder{ + gvk: gvk, + cluster: c, + pollInterval: pollInterval, + } +} + +func (r *MetricRecorder) SetReconcileTime(name string) { + if r == nil { + return + } + o, ok := r.observations.Load(name) + if !ok { + o = &Observations{} + r.observations.Store(name, o) + } + t := time.Now().Add(r.pollInterval) + o.(*Observations).expectedReconcileTime = &t + o.(*Observations).observeReconcileDelay = true +} + +func (r *MetricRecorder) ObserveReconcileDelay(gvk schema.GroupVersionKind, name string) { + if r == nil { + return + } + o, _ := r.observations.Load(name) + if o == nil || !o.(*Observations).observeReconcileDelay || o.(*Observations).expectedReconcileTime == nil { + return + } + d := time.Now().Sub(*o.(*Observations).expectedReconcileTime) + if d < 0 { + d = 0 + } + ReconcileDelay.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(d.Seconds()) + o.(*Observations).observeReconcileDelay = false +} + +func (r *MetricRecorder) Start(ctx context.Context) error { + inf, err := r.cluster.GetCache().GetInformerForKind(ctx, r.gvk) + if err != nil { + return errors.Wrapf(err, "cannot get informer for metric recorder for resource %s", r.gvk) + } + + registered, err := inf.AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: func(obj interface{}) { + if final, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = final.Obj + } + managed := obj.(resource.Managed) + r.observations.Delete(managed.GetName()) + }, + }) + if err != nil { + return errors.Wrap(err, "cannot add delete event handler to informer for metric recorder") + } + defer inf.RemoveEventHandler(registered) //nolint:errcheck // this happens on destruction. We cannot do anything anyway. + + <-ctx.Done() + + return nil +} + func init() { - metrics.Registry.MustRegister(CLITime, CLIExecutions, TFProcesses, TTRMeasurements, ExternalAPITime) + metrics.Registry.MustRegister(CLITime, CLIExecutions, TFProcesses, TTRMeasurements, ExternalAPITime, DeletionTime, ReconcileDelay) } diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index 005e724e..d28cd3b2 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -47,7 +47,8 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { opts := []managed.ReconcilerOption{ managed.WithExternalConnecter( {{- if .UseNoForkClient -}} - tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger)) + tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger), + tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) {{- else -}} tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), {{- if .UseAsync }} From d46bde43dcc656a369ebce53a5493dbc680c4f1b Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Mon, 25 Sep 2023 10:42:59 +0300 Subject: [PATCH 11/60] publish connection details Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index aefb09ae..4eeb86c0 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -186,6 +186,7 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma } n.instanceState = newState noDiff := false + var connDetails managed.ConnectionDetails resourceExists := newState != nil && newState.ID != "" if !resourceExists && mg.GetDeletionTimestamp() != nil { gvk := mg.GetObjectKind().GroupVersionKind() @@ -205,6 +206,10 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if err != nil { return managed.ExternalObservation{}, errors.Errorf("could not set observation: %v", err) } + connDetails, err = resource.GetConnectionDetails(stateValueMap, mg.(resource.Terraformed), n.config) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot get connection details") + } instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) if err != nil { return managed.ExternalObservation{}, err @@ -217,8 +222,9 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma } return managed.ExternalObservation{ - ResourceExists: resourceExists, - ResourceUpToDate: noDiff, + ResourceExists: resourceExists, + ResourceUpToDate: noDiff, + ConnectionDetails: connDetails, }, nil } @@ -255,8 +261,12 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man if err != nil { return managed.ExternalCreation{}, errors.Errorf("could not set observation: %v", err) } + conn, err := resource.GetConnectionDetails(stateValueMap, mg.(resource.Terraformed), n.config) + if err != nil { + return managed.ExternalCreation{}, errors.Wrap(err, "cannot get connection details") + } - return managed.ExternalCreation{}, nil + return managed.ExternalCreation{ConnectionDetails: conn}, nil } func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { From f07a6e14f1cbaa50b206b6f9f882b0264ac852f6 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Mon, 25 Sep 2023 10:47:57 +0300 Subject: [PATCH 12/60] register eventHandler Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 10 ++++++++++ pkg/pipeline/templates/controller.go.tmpl | 1 + 2 files changed, 11 insertions(+) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 4eeb86c0..9bcdbe52 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -16,6 +16,7 @@ package controller import ( "context" + "github.com/upbound/upjet/pkg/controller/handler" "time" tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" @@ -41,6 +42,7 @@ type NoForkConnector struct { kube client.Client config *config.Resource logger logging.Logger + eventHandler *handler.EventHandler metricRecorder *metrics.MetricRecorder } @@ -62,6 +64,14 @@ func WithNoForkMetricRecorder(r *metrics.MetricRecorder) NoForkOption { } } +// WithNoForkConnectorEventHandler configures the EventHandler so that +// the no-fork external clients can requeue reconciliation requests. +func WithNoForkConnectorEventHandler(e *handler.EventHandler) NoForkOption { + return func(c *NoForkConnector) { + c.eventHandler = e + } +} + func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkOption) *NoForkConnector { nfc := &NoForkConnector{ kube: kube, diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index d28cd3b2..3c40f93a 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -48,6 +48,7 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { managed.WithExternalConnecter( {{- if .UseNoForkClient -}} tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger), + tjcontroller.WithNoForkConnectorEventHandler(eventHandler), tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) {{- else -}} tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), From 3c664304f26f0aa61e1d43328e3f3f1d1ca3dd78 Mon Sep 17 00:00:00 2001 From: Cem Mergenci Date: Tue, 26 Sep 2023 16:29:50 +0300 Subject: [PATCH 13/60] Add Terraform provider schema to config.Provider. Signed-off-by: Cem Mergenci Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 3 +++ pkg/controller/external_nofork.go | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index c5f489da..6e406bb5 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -116,6 +116,9 @@ type Provider struct { // resource name. Resources map[string]*Resource + // TerraformProvider is the Terraform schema of the provider. + TerraformProvider *schema.Provider + // refInjectors is an ordered list of `ReferenceInjector`s for // injecting references across this Provider's resources. refInjectors []ReferenceInjector diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 9bcdbe52..54fd6907 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -16,9 +16,10 @@ package controller import ( "context" - "github.com/upbound/upjet/pkg/controller/handler" "time" + "github.com/upbound/upjet/pkg/controller/handler" + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" corev1 "k8s.io/api/core/v1" From b3f2bac5c4abf552ba743b64b168cba9f29a8ee8 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 26 Sep 2023 17:41:44 +0300 Subject: [PATCH 14/60] Update crossplane-runtime to commit 4c4b0b47b6ed Signed-off-by: Alper Rifat Ulucinar --- pkg/terraform/files.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/terraform/files.go b/pkg/terraform/files.go index a8483012..404488aa 100644 --- a/pkg/terraform/files.go +++ b/pkg/terraform/files.go @@ -12,11 +12,12 @@ import ( "strings" "dario.cat/mergo" + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/resource" "github.com/crossplane/upjet/pkg/resource/json" - "github.com/pkg/errors" - "github.com/spf13/afero" "github.com/crossplane/crossplane-runtime/pkg/feature" "github.com/crossplane/crossplane-runtime/pkg/meta" @@ -184,7 +185,7 @@ func (fp *FileProducer) WriteMainTF() (ProviderHandle, error) { // EnsureTFState writes the Terraform state that should exist in the filesystem // to start any Terraform operation. -func (fp *FileProducer) EnsureTFState(ctx context.Context, tfID string) error { +func (fp *FileProducer) EnsureTFState(_ context.Context, tfID string) error { // TODO(muvaf): Reduce the cyclomatic complexity by separating the attributes // generation into its own function/interface. empty, err := fp.isStateEmpty() From 4be6a72a2a129fa61788e46cf84d303e5b16f0c4 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 27 Sep 2023 11:31:14 +0300 Subject: [PATCH 15/60] Compute Instance{State,Diff}.RawPlan to be able to handle Terraform AWS v5.x tags Signed-off-by: Alper Rifat Ulucinar --- go.mod | 3 ++- go.sum | 2 ++ pkg/controller/external_nofork.go | 20 +++++++++++++------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f524c161..ed4d2938 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/fatih/camelcase v1.0.0 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 + github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 github.com/hashicorp/hcl/v2 v2.14.1 github.com/hashicorp/terraform-json v0.14.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.24.0 @@ -25,6 +26,7 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/spf13/afero v1.10.0 github.com/tmccombs/hcl2json v0.3.3 + github.com/upbound/upjet v0.10.0 github.com/yuin/goldmark v1.4.13 github.com/zclconf/go-cty v1.11.0 golang.org/x/net v0.15.0 @@ -70,7 +72,6 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect github.com/hashicorp/go-hclog v1.2.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect diff --git a/go.sum b/go.sum index c8cf1cdf..3ce11661 100644 --- a/go.sum +++ b/go.sum @@ -339,6 +339,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= +github.com/upbound/upjet v0.10.0 h1:6nxc0GUBcL4BDHxQUUZWjw4ROXu5KRK9jOpb7LeJ+NQ= +github.com/upbound/upjet v0.10.0/go.mod h1:2RXHgpIugCL/S/Use1QJAeVaev901RBeUByQh5gUtGk= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 54fd6907..b7eab25b 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -18,21 +18,20 @@ import ( "context" "time" - "github.com/upbound/upjet/pkg/controller/handler" - - tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" - corev1 "k8s.io/api/core/v1" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/upbound/upjet/pkg/config" + "github.com/upbound/upjet/pkg/controller/handler" "github.com/upbound/upjet/pkg/metrics" "github.com/upbound/upjet/pkg/resource" "github.com/upbound/upjet/pkg/terraform" @@ -148,10 +147,10 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, err } s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) - if err != nil { return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } + s.RawPlan = tfStateCtyValue return &noForkExternal{ ts: ts, @@ -184,7 +183,14 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") } - + if instanceDiff != nil { + v := cty.EmptyObjectVal + v, err = instanceDiff.ApplyToValue(v, n.resourceSchema.CoreConfigSchema()) + if err != nil { + return nil, errors.Wrap(err, "cannot apply Terraform instance diff to an empty value") + } + instanceDiff.RawPlan = v + } return instanceDiff, nil } From bdbd1d2bdafd8c24163057d4d77c286152d32a6d Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 6 Oct 2023 14:23:48 +0300 Subject: [PATCH 16/60] Add late-initialization logic Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index b7eab25b..df7be4e4 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -34,6 +34,7 @@ import ( "github.com/upbound/upjet/pkg/controller/handler" "github.com/upbound/upjet/pkg/metrics" "github.com/upbound/upjet/pkg/resource" + "github.com/upbound/upjet/pkg/resource/json" "github.com/upbound/upjet/pkg/terraform" ) @@ -144,7 +145,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, c.config.TerraformResource.CoreConfigSchema()) if err != nil { - return nil, err + return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") } s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) if err != nil { @@ -209,6 +210,7 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma gvk := mg.GetObjectKind().GroupVersionKind() metrics.DeletionTime.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(time.Since(mg.GetDeletionTimestamp().Time).Seconds()) } + lateInitialized := false if resourceExists { if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown || mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionFalse { @@ -217,8 +219,18 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma mg.SetConditions(xpv1.Available()) stateValueMap, err := n.fromInstanceStateToJSONMap(newState) if err != nil { - return managed.ExternalObservation{}, err + return managed.ExternalObservation{}, errors.Wrap(err, "cannot convert instance state to JSON map") + } + + buff, err := json.TFParser.Marshal(stateValueMap) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot marshal the attributes of the new state for late-initialization") } + lateInitialized, err = mg.(resource.Terraformed).LateInitialize(buff) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot late-initialize the managed resource") + } + err = mg.(resource.Terraformed).SetObservation(stateValueMap) if err != nil { return managed.ExternalObservation{}, errors.Errorf("could not set observation: %v", err) @@ -236,12 +248,16 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if noDiff { n.metricRecorder.SetReconcileTime(mg.GetName()) } + if !lateInitialized { + resource.SetUpToDateCondition(mg, noDiff) + } } return managed.ExternalObservation{ - ResourceExists: resourceExists, - ResourceUpToDate: noDiff, - ConnectionDetails: connDetails, + ResourceExists: resourceExists, + ResourceUpToDate: noDiff, + ConnectionDetails: connDetails, + ResourceLateInitialized: lateInitialized, }, nil } From 0709592c8e5d16a8e9040d011f82556f1abca900 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 12 Oct 2023 01:26:29 +0300 Subject: [PATCH 17/60] Add config.Resource.SchemaElementOptions to configure options for resource schema elements - Add support for explicitly configuring fields to be added to the Observation types. Signed-off-by: Alper Rifat Ulucinar --- pkg/config/common.go | 21 ++++++++++---------- pkg/config/resource.go | 30 ++++++++++++++++++++++++++++ pkg/controller/external_nofork.go | 5 +---- pkg/types/builder.go | 5 +++-- pkg/types/comments/comment.go | 10 ++++++++++ pkg/types/field.go | 33 ++++++++++++++++++++----------- 6 files changed, 77 insertions(+), 27 deletions(-) diff --git a/pkg/config/common.go b/pkg/config/common.go index ac21fd66..7bf2226a 100644 --- a/pkg/config/common.go +++ b/pkg/config/common.go @@ -76,16 +76,17 @@ func DefaultResource(name string, terraformSchema *schema.Resource, terraformReg } r := &Resource{ - Name: name, - TerraformResource: terraformSchema, - MetaResource: terraformRegistry, - ShortGroup: group, - Kind: kind, - Version: "v1alpha1", - ExternalName: NameAsIdentifier, - References: map[string]Reference{}, - Sensitive: NopSensitive, - UseAsync: true, + Name: name, + TerraformResource: terraformSchema, + MetaResource: terraformRegistry, + ShortGroup: group, + Kind: kind, + Version: "v1alpha1", + ExternalName: NameAsIdentifier, + References: map[string]Reference{}, + Sensitive: NopSensitive, + UseAsync: true, + SchemaElementOptions: make(map[string]*SchemaElementOption), } for _, f := range opts { f(r) diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 34582b50..2d682b8d 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -322,4 +322,34 @@ type Resource struct { // the plural name of the generated CRD. Overriding this sets both the // path and the plural name for the generated CRD. Path string + + // SchemaElementOptions is a map from the schema element paths to + // SchemaElementOption for configuring options for schema elements. + SchemaElementOptions SchemaElementOptions +} + +// SchemaElementOptions represents schema element options for the +// schema elements of a Resource. +type SchemaElementOptions map[string]*SchemaElementOption + +// SetAddToObservation sets the AddToObservation for the specified key. +func (m SchemaElementOptions) SetAddToObservation(el string) { + if m[el] == nil { + m[el] = &SchemaElementOption{} + } + m[el].AddToObservation = true +} + +// AddToObservation returns true if the schema element at the specified path +// should be added to the CRD type's Observation type. +func (m SchemaElementOptions) AddToObservation(el string) bool { + return m[el] != nil && m[el].AddToObservation +} + +// SchemaElementOption represents configuration options on a schema element. +type SchemaElementOption struct { + // AddToObservation is set to true if the field represented by + // a schema element is to be added to the generated CRD type's + // Observation type. + AddToObservation bool } diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index df7be4e4..11767a17 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -177,10 +177,7 @@ type noForkExternal struct { } func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState) (*tf.InstanceDiff, error) { - instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, &tf.ResourceConfig{ - Raw: n.params, - Config: n.params, - }, nil, n.ts.Meta, false) + instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, tf.NewResourceConfigRaw(n.params), nil, n.ts.Meta, false) if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") } diff --git a/pkg/types/builder.go b/pkg/types/builder.go index 88b91039..cae32821 100644 --- a/pkg/types/builder.go +++ b/pkg/types/builder.go @@ -93,7 +93,8 @@ func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPa r := &resource{} for _, snakeFieldName := range keys { var reference *config.Reference - ref, ok := cfg.References[fieldPath(append(tfPath, snakeFieldName))] + cPath := fieldPath(append(tfPath, snakeFieldName)) + ref, ok := cfg.References[cPath] // if a reference is configured and the field does not belong to status if ok && !IsObservation(res.Schema[snakeFieldName]) { reference = &ref @@ -121,7 +122,7 @@ func (g *Builder) buildResource(res *schema.Resource, cfg *config.Resource, tfPa return nil, nil, nil, err } } - f.AddToResource(g, r, typeNames) + f.AddToResource(g, r, typeNames, cfg.SchemaElementOptions.AddToObservation(cPath)) } paramType, obsType, initType := g.AddToBuilder(typeNames, r) diff --git a/pkg/types/comments/comment.go b/pkg/types/comments/comment.go index baaec46e..53f583b2 100644 --- a/pkg/types/comments/comment.go +++ b/pkg/types/comments/comment.go @@ -90,3 +90,13 @@ func (c *Comment) Build() string { all := strings.ReplaceAll("// "+c.String(), "\n", "\n// ") return strings.TrimSuffix(all, "// ") } + +// CommentWithoutOptions returns a new Comment without the Options. +func (c *Comment) CommentWithoutOptions() *Comment { + if c == nil { + return nil + } + return &Comment{ + Text: c.Text, + } +} diff --git a/pkg/types/field.go b/pkg/types/field.go index 838d9073..c127affa 100644 --- a/pkg/types/field.go +++ b/pkg/types/field.go @@ -213,15 +213,21 @@ func NewReferenceField(g *Builder, cfg *config.Resource, r *resource, sch *schem } // AddToResource adds built field to the resource. -func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames) { - if f.Comment.UpjetOptions.FieldTFTag != nil { - f.TFTag = *f.Comment.UpjetOptions.FieldTFTag - } +func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames, addToObservation bool) { if f.Comment.UpjetOptions.FieldJSONTag != nil { f.JSONTag = *f.Comment.UpjetOptions.FieldJSONTag } field := types.NewField(token.NoPos, g.Package, f.FieldNameCamel, f.FieldType, false) + // if the field is explicitly configured to be added to + // the Observation type + if addToObservation { + r.addObservationField(f, field) + } + + if f.Comment.UpjetOptions.FieldTFTag != nil { + f.TFTag = *f.Comment.UpjetOptions.FieldTFTag + } // Note(turkenh): We want atProvider to be a superset of forProvider, so // we always add the field as an observation field and then add it as a @@ -230,9 +236,10 @@ func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames) { // We do this only if tf tag is not set to "-" because otherwise it won't // be populated from the tfstate. We typically set tf tag to "-" for // sensitive fields which were replaced with secretKeyRefs. - if f.TFTag != "-" { + if f.TFTag != "-" && !addToObservation { r.addObservationField(f, field) } + if !IsObservation(f.Schema) { if f.AsBlocksMode { f.TFTag = strings.TrimSuffix(f.TFTag, ",omitempty") @@ -259,12 +266,16 @@ func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames) { f.Comment.Required = nil g.comments.AddFieldComment(typeNames.InitTypeName, f.FieldNameCamel, f.Comment.Build()) - // Note(turkenh): We don't want reference resolver to be generated for - // fields under status.atProvider. So, we don't want reference comments to - // be added, hence we are unsetting reference on the field comment just - // before adding it as an observation field. - f.Comment.Reference = config.Reference{} - g.comments.AddFieldComment(typeNames.ObservationTypeName, f.FieldNameCamel, f.Comment.Build()) + if addToObservation { + g.comments.AddFieldComment(typeNames.ObservationTypeName, f.FieldNameCamel, f.Comment.CommentWithoutOptions().Build()) + } else { + // Note(turkenh): We don't want reference resolver to be generated for + // fields under status.atProvider. So, we don't want reference comments to + // be added, hence we are unsetting reference on the field comment just + // before adding it as an observation field. + f.Comment.Reference = config.Reference{} + g.comments.AddFieldComment(typeNames.ObservationTypeName, f.FieldNameCamel, f.Comment.Build()) + } } // isInit returns true if the field should be added to initProvider. From c19b5d017eec3b9c1cd852acf1876c3dbe882fd2 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 12 Oct 2023 01:59:00 +0300 Subject: [PATCH 18/60] Add config.Resource.TerraformConfigurationInjector to allow injecting Terraform configuration parameters Signed-off-by: Alper Rifat Ulucinar --- pkg/config/resource.go | 13 +++++++++++++ pkg/controller/external_nofork.go | 3 +++ 2 files changed, 16 insertions(+) diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 2d682b8d..c4de2d37 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -326,8 +326,21 @@ type Resource struct { // SchemaElementOptions is a map from the schema element paths to // SchemaElementOption for configuring options for schema elements. SchemaElementOptions SchemaElementOptions + + // TerraformConfigurationInjector allows a managed resource to inject + // configuration values in the Terraform configuration map obtained by + // deserializing its `spec.forProvider` value. Managed resources can + // use this resource configuration option to inject Terraform + // configuration parameters into their deserialized configuration maps, + // if the deserialization skips certain fields. + TerraformConfigurationInjector ConfigurationInjector } +// ConfigurationInjector is a function that injects Terraform configuration +// values from the specified managed resource into the specified configuration +// map. +type ConfigurationInjector func(xpresource.Managed, map[string]any) + // SchemaElementOptions represents schema element options for the // schema elements of a Resource. type SchemaElementOptions map[string]*SchemaElementOption diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 11767a17..7388b156 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -115,6 +115,9 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, errors.Wrap(err, "cannot store sensitive parameters into params") } c.config.ExternalName.SetIdentifierArgumentFn(params, meta.GetExternalName(tr)) + if c.config.TerraformConfigurationInjector != nil { + c.config.TerraformConfigurationInjector(mg, params) + } tfID, err := c.config.ExternalName.GetIDFn(ctx, meta.GetExternalName(mg), params, ts.Map()) if err != nil { From c61363fd9e5f89e2482f3471a4b0454a2e3baa50 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 12 Oct 2023 03:28:20 +0300 Subject: [PATCH 19/60] Confine no-fork client's instance diff computation to Observe Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 42 +++++++++++++------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 7388b156..60a0eab4 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -174,6 +174,7 @@ type noForkExternal struct { config *config.Resource kube client.Client instanceState *tf.InstanceState + instanceDiff *tf.InstanceDiff params map[string]any logger logging.Logger metricRecorder *metrics.MetricRecorder @@ -192,6 +193,9 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } instanceDiff.RawPlan = v } + if instanceDiff != nil { + n.logger.Debug("Diff detected", "instanceDiff", instanceDiff.GoString()) + } return instanceDiff, nil } @@ -203,7 +207,14 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } n.instanceState = newState - noDiff := false + // compute the instance diff + instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot compute the instance diff") + } + n.instanceDiff = instanceDiff + noDiff := instanceDiff.Empty() + var connDetails managed.ConnectionDetails resourceExists := newState != nil && newState.ID != "" if !resourceExists && mg.GetDeletionTimestamp() != nil { @@ -239,11 +250,6 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot get connection details") } - instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) - if err != nil { - return managed.ExternalObservation{}, err - } - noDiff = instanceDiff.Empty() if noDiff { n.metricRecorder.SetReconcileTime(mg.GetName()) @@ -262,12 +268,8 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma } func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { - instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) - if err != nil { - return managed.ExternalCreation{}, err - } start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("create").Observe(time.Since(start).Seconds()) // diag := n.resourceSchema.CreateWithoutTimeout(ctx, n.resourceData, n.ts.Meta) if diag != nil && diag.HasError() { @@ -303,12 +305,8 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man } func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { - instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) - if err != nil { - return managed.ExternalUpdate{}, err - } start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("update").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return managed.ExternalUpdate{}, errors.Errorf("failed to update the resource: %v", diag) @@ -327,17 +325,13 @@ func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (man } func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error { - instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) - if err != nil { - return err - } - if instanceDiff == nil { - instanceDiff = tf.NewInstanceDiff() + if n.instanceDiff == nil { + n.instanceDiff = tf.NewInstanceDiff() } - instanceDiff.Destroy = true + n.instanceDiff.Destroy = true start := time.Now() - _, diag := n.resourceSchema.Apply(ctx, n.instanceState, instanceDiff, n.ts.Meta) + _, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("delete").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return errors.Errorf("failed to delete the resource: %v", diag) From 1f01d7065cc699e40ea1c088d4e08eb793881984 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 12 Oct 2023 04:19:07 +0300 Subject: [PATCH 20/60] Do not add the resource.WithZeroValueJSONOmitEmptyFilter late-initialization option by default - May result in unnecessary drift if some fields are not late-initialized. Signed-off-by: Alper Rifat Ulucinar --- pkg/config/resource.go | 5 +++-- pkg/controller/external_nofork.go | 19 ++++++++++++++++++- pkg/pipeline/templates/terraformed.go.tmpl | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/config/resource.go b/pkg/config/resource.go index c4de2d37..395d7cac 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -338,8 +338,9 @@ type Resource struct { // ConfigurationInjector is a function that injects Terraform configuration // values from the specified managed resource into the specified configuration -// map. -type ConfigurationInjector func(xpresource.Managed, map[string]any) +// map. jsonMap is the map obtained by converting the `spec.forProvider` using +// the JSON tags and tfMap is obtained by using the TF tags. +type ConfigurationInjector func(jsonMap map[string]any, tfMap map[string]any) // SchemaElementOptions represents schema element options for the // schema elements of a Resource. diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 60a0eab4..9bb7049e 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -19,6 +19,7 @@ import ( "time" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" @@ -96,6 +97,18 @@ func copyParameters(tfState, params map[string]any) map[string]any { return targetState } +func getJSONMap(mg xpresource.Managed) (map[string]any, error) { + pv, err := fieldpath.PaveObject(mg) + if err != nil { + return nil, errors.Wrap(err, "cannot pave the managed resource") + } + v, err := pv.GetValue("spec.forProvider") + if err != nil { + return nil, errors.Wrap(err, "cannot get spec.forProvider value from paved object") + } + return v.(map[string]any), nil +} + func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) start := time.Now() @@ -116,7 +129,11 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m } c.config.ExternalName.SetIdentifierArgumentFn(params, meta.GetExternalName(tr)) if c.config.TerraformConfigurationInjector != nil { - c.config.TerraformConfigurationInjector(mg, params) + m, err := getJSONMap(mg) + if err != nil { + return nil, errors.Wrap(err, "cannot get JSON map for the managed resource's spec.forProvider value") + } + c.config.TerraformConfigurationInjector(m, params) } tfID, err := c.config.ExternalName.GetIDFn(ctx, meta.GetExternalName(mg), params, ts.Map()) diff --git a/pkg/pipeline/templates/terraformed.go.tmpl b/pkg/pipeline/templates/terraformed.go.tmpl index ee503fc4..89e71258 100644 --- a/pkg/pipeline/templates/terraformed.go.tmpl +++ b/pkg/pipeline/templates/terraformed.go.tmpl @@ -93,7 +93,7 @@ import ( if err := json.TFParser.Unmarshal(attrs, params); err != nil { return false, errors.Wrap(err, "failed to unmarshal Terraform state parameters for late-initialization") } - opts := []resource.GenericLateInitializerOption{resource.WithZeroValueJSONOmitEmptyFilter(resource.CNameWildcard)} + var opts []resource.GenericLateInitializerOption {{ range .LateInitializer.IgnoredFields -}} opts = append(opts, resource.WithNameFilter("{{ . }}")) {{ end }} From eecf3bea8df27bf46681a6d68df039b0396d0048 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 12 Oct 2023 05:04:00 +0300 Subject: [PATCH 21/60] Add resource.WithZeroValueJSONOmitEmptyFilter late-initialization option back - Late-initialization of nil values from corresponding zero-values results in a late-initialization loop and prevents the resource from becoming ready. Signed-off-by: Alper Rifat Ulucinar --- pkg/pipeline/templates/terraformed.go.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/templates/terraformed.go.tmpl b/pkg/pipeline/templates/terraformed.go.tmpl index 89e71258..ee503fc4 100644 --- a/pkg/pipeline/templates/terraformed.go.tmpl +++ b/pkg/pipeline/templates/terraformed.go.tmpl @@ -93,7 +93,7 @@ import ( if err := json.TFParser.Unmarshal(attrs, params); err != nil { return false, errors.Wrap(err, "failed to unmarshal Terraform state parameters for late-initialization") } - var opts []resource.GenericLateInitializerOption + opts := []resource.GenericLateInitializerOption{resource.WithZeroValueJSONOmitEmptyFilter(resource.CNameWildcard)} {{ range .LateInitializer.IgnoredFields -}} opts = append(opts, resource.WithNameFilter("{{ . }}")) {{ end }} From 5ac3a183ca08092f3b80ddc93880383638325dfb Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 13 Oct 2023 15:31:57 +0300 Subject: [PATCH 22/60] Set RawConfig of InstanceState and InstanceDiff Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 37 +++++++++++++++++++------------ 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 9bb7049e..607a293c 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -109,6 +109,19 @@ func getJSONMap(mg xpresource.Managed) (map[string]any, error) { return v.(map[string]any), nil } +type noForkExternal struct { + ts terraform.Setup + resourceSchema *schema.Resource + config *config.Resource + kube client.Client + instanceState *tf.InstanceState + instanceDiff *tf.InstanceDiff + params map[string]any + rawConfig cty.Value + logger logging.Logger + metricRecorder *metrics.MetricRecorder +} + func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) start := time.Now() @@ -144,7 +157,8 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m // we need to parameterize the following for a provider // not all providers may have this attribute // TODO: tags_all handling - attrs := c.config.TerraformResource.CoreConfigSchema().Attributes + schemaBlock := c.config.TerraformResource.CoreConfigSchema() + attrs := schemaBlock.Attributes if _, ok := attrs["tags_all"]; ok { params["tags_all"] = params["tags"] } @@ -163,7 +177,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m tfState = copyParameters(tfState, params) } - tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, c.config.TerraformResource.CoreConfigSchema()) + tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, schemaBlock) if err != nil { return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") } @@ -172,6 +186,11 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } s.RawPlan = tfStateCtyValue + rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) + if err != nil { + return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") + } + s.RawConfig = rawConfig return &noForkExternal{ ts: ts, @@ -180,23 +199,12 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m kube: c.kube, instanceState: s, params: params, + rawConfig: rawConfig, logger: c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()), metricRecorder: c.metricRecorder, }, nil } -type noForkExternal struct { - ts terraform.Setup - resourceSchema *schema.Resource - config *config.Resource - kube client.Client - instanceState *tf.InstanceState - instanceDiff *tf.InstanceDiff - params map[string]any - logger logging.Logger - metricRecorder *metrics.MetricRecorder -} - func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState) (*tf.InstanceDiff, error) { instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, tf.NewResourceConfigRaw(n.params), nil, n.ts.Meta, false) if err != nil { @@ -212,6 +220,7 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } if instanceDiff != nil { n.logger.Debug("Diff detected", "instanceDiff", instanceDiff.GoString()) + instanceDiff.RawConfig = n.rawConfig } return instanceDiff, nil } From 716daa25b32329658a58eae02da696c1e84f96c3 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 17 Oct 2023 21:27:56 +0300 Subject: [PATCH 23/60] Add config.Resource.TerraformCustomDiff to customize Terraform InstanceDiffs Signed-off-by: Alper Rifat Ulucinar --- pkg/config/resource.go | 10 ++++++++++ pkg/controller/external_nofork.go | 8 +++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 395d7cac..dd5e5aeb 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -11,6 +11,7 @@ import ( "github.com/crossplane/upjet/pkg/registry" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" @@ -334,8 +335,17 @@ type Resource struct { // configuration parameters into their deserialized configuration maps, // if the deserialization skips certain fields. TerraformConfigurationInjector ConfigurationInjector + + // TerraformCustomDiff allows a resource.Terraformed to customize how its + // Terraform InstanceDiff is computed during reconciliation. + TerraformCustomDiff CustomDiff } +// CustomDiff customizes the computed Terraform InstanceDiff. This can be used +// in cases where, for example, changes in a certain argument should just be +// dismissed. The new InstanceDiff is returned along with any errors. +type CustomDiff func(diff *terraform.InstanceDiff) (*terraform.InstanceDiff, error) + // ConfigurationInjector is a function that injects Terraform configuration // values from the specified managed resource into the specified configuration // map. jsonMap is the map obtained by converting the `spec.forProvider` using diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 607a293c..c05bae64 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -210,6 +210,12 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") } + if n.config.TerraformCustomDiff != nil { + instanceDiff, err = n.config.TerraformCustomDiff(instanceDiff) + if err != nil { + return nil, errors.Wrap(err, "failed to compute the customized terraform.InstanceDiff") + } + } if instanceDiff != nil { v := cty.EmptyObjectVal v, err = instanceDiff.ApplyToValue(v, n.resourceSchema.CoreConfigSchema()) @@ -218,7 +224,7 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } instanceDiff.RawPlan = v } - if instanceDiff != nil { + if instanceDiff != nil && len(instanceDiff.Attributes) > 0 { n.logger.Debug("Diff detected", "instanceDiff", instanceDiff.GoString()) instanceDiff.RawConfig = n.rawConfig } From a3e1935a177a4d2a70da2ec306327ab094652749 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Mon, 9 Oct 2023 19:22:06 +0300 Subject: [PATCH 24/60] no-fork async connector & client Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 377 ++++++++++++++++++++++ pkg/controller/external_nofork.go | 10 - pkg/controller/nofork_finalizer.go | 46 +++ pkg/controller/nofork_store.go | 101 ++++++ pkg/controller/options.go | 2 + pkg/pipeline/templates/controller.go.tmpl | 22 +- pkg/terraform/operation.go | 16 + 7 files changed, 560 insertions(+), 14 deletions(-) create mode 100644 pkg/controller/external_async_nofork.go create mode 100644 pkg/controller/nofork_finalizer.go create mode 100644 pkg/controller/nofork_store.go diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go new file mode 100644 index 00000000..ada84a71 --- /dev/null +++ b/pkg/controller/external_async_nofork.go @@ -0,0 +1,377 @@ +// Copyright 2023 Upbound Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/upbound/upjet/pkg/config" + "github.com/upbound/upjet/pkg/controller/handler" + "github.com/upbound/upjet/pkg/resource/json" + "github.com/upbound/upjet/pkg/terraform" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/meta" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "github.com/pkg/errors" + "github.com/upbound/upjet/pkg/metrics" + "github.com/upbound/upjet/pkg/resource" + corev1 "k8s.io/api/core/v1" +) + +var defaultAsyncTimeout = 1 * time.Hour + +type NoForkAsyncConnector struct { + *NoForkConnector + operationTrackerStore *OperationTrackerStore + callback CallbackProvider + eventHandler *handler.EventHandler +} + +type NoForkAsyncOption func(connector *NoForkAsyncConnector) + +func NewNoForkAsyncConnector(kube client.Client, ots *OperationTrackerStore, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkAsyncOption) *NoForkAsyncConnector { + nfac := &NoForkAsyncConnector{ + NoForkConnector: &NoForkConnector{ + kube: kube, + getTerraformSetup: sf, + config: cfg, + }, + operationTrackerStore: ots, + } + for _, f := range opts { + f(nfac) + } + return nfac +} + +func (c *NoForkAsyncConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { + asyncTracker := c.operationTrackerStore.Tracker(mg.(resource.Terraformed)) + c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) + start := time.Now() + ts, err := c.getTerraformSetup(ctx, c.kube, mg) + metrics.ExternalAPITime.WithLabelValues("connect").Observe(time.Since(start).Seconds()) + if err != nil { + return nil, errors.Wrap(err, errGetTerraformSetup) + } + + // To Compute the ResourceDiff: n.resourceSchema.Diff(...) + tr := mg.(resource.Terraformed) + params, err := tr.GetParameters() + if err != nil { + return nil, errors.Wrap(err, "cannot get parameters") + } + if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { + return nil, errors.Wrap(err, "cannot store sensitive parameters into params") + } + + externalName := meta.GetExternalName(tr) + if externalName == "" { + externalName = asyncTracker.GetTfID() + } + + c.config.ExternalName.SetIdentifierArgumentFn(params, externalName) + + tfID, err := c.config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) + if err != nil { + return nil, errors.Wrap(err, "cannot get ID") + } + params["id"] = tfID + // we need to parameterize the following for a provider + // not all providers may have this attribute + // TODO: tags_all handling + attrs := c.config.TerraformResource.CoreConfigSchema().Attributes + if _, ok := attrs["tags_all"]; ok { + params["tags_all"] = params["tags"] + } + // construct TF + if asyncTracker.GetTfState() == nil || asyncTracker.GetTfState().Attributes == nil { + tfState, err := tr.GetObservation() + if err != nil { + return nil, errors.Wrap(err, "failed to get the observation") + } + copyParams := len(tfState) == 0 + if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, tfState, tr.GetConnectionDetailsMapping()); err != nil { + return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") + } + c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) + tfState["id"] = tfID + if copyParams { + tfState = copyParameters(tfState, params) + } + + tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, c.config.TerraformResource.CoreConfigSchema()) + if err != nil { + return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") + } + s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) + if err != nil { + return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") + } + s.RawPlan = tfStateCtyValue + asyncTracker.SetTfState(s) + } + + return &noForkAsyncExternal{ + noForkExternal: &noForkExternal{ + ts: ts, + resourceSchema: c.config.TerraformResource, + config: c.config, + kube: c.kube, + params: params, + logger: c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()), + metricRecorder: c.metricRecorder, + }, + opTracker: asyncTracker, + callback: c.callback, + eventHandler: c.eventHandler, + }, nil +} + +// WithNoForkAsyncConnectorEventHandler configures the EventHandler so that +// the no-fork external clients can requeue reconciliation requests. +func WithNoForkAsyncConnectorEventHandler(e *handler.EventHandler) NoForkAsyncOption { + return func(c *NoForkAsyncConnector) { + c.eventHandler = e + } +} + +// WithNoForkAsyncCallbackProvider configures the controller to use async variant of the functions +// of the Terraform client and run given callbacks once those operations are +// completed. +func WithNoForkAsyncCallbackProvider(ac CallbackProvider) NoForkAsyncOption { + return func(c *NoForkAsyncConnector) { + c.callback = ac + } +} + +// WithNoForkAsyncLogger configures a logger for the NoForkAsyncConnector. +func WithNoForkAsyncLogger(l logging.Logger) NoForkAsyncOption { + return func(c *NoForkAsyncConnector) { + c.logger = l + } +} + +// WithNoForkAsyncMetricRecorder configures a metrics.MetricRecorder for the +// NoForkAsyncConnector. +func WithNoForkAsyncMetricRecorder(r *metrics.MetricRecorder) NoForkAsyncOption { + return func(c *NoForkAsyncConnector) { + c.metricRecorder = r + } +} + +type noForkAsyncExternal struct { + *noForkExternal + callback CallbackProvider + eventHandler *handler.EventHandler + opTracker *AsyncTracker +} + +type CallbackFn func(error, context.Context) error + +func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { + if n.opTracker.LastOperation.IsRunning() { + n.logger.WithValues("opType", n.opTracker.LastOperation.Type).Debug("ongoing async operation") + mg.SetConditions(resource.AsyncOperationOngoingCondition()) + return managed.ExternalObservation{ + ResourceExists: true, + ResourceUpToDate: true, + }, nil + } + n.opTracker.LastOperation.Flush() + + start := time.Now() + newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.opTracker.GetTfState(), n.ts.Meta) + metrics.ExternalAPITime.WithLabelValues("read").Observe(time.Since(start).Seconds()) + if diag != nil && diag.HasError() { + return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) + } + n.opTracker.SetTfState(newState) + noDiff := false + var connDetails managed.ConnectionDetails + resourceExists := newState != nil && newState.ID != "" + if !resourceExists && mg.GetDeletionTimestamp() != nil { + gvk := mg.GetObjectKind().GroupVersionKind() + metrics.DeletionTime.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(time.Since(mg.GetDeletionTimestamp().Time).Seconds()) + } + lateInitialized := false + if resourceExists { + if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown || + mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionFalse { + addTTR(mg) + } + // Set external name + en, err := n.config.ExternalName.GetExternalNameFn(map[string]any{ + "id": n.opTracker.GetTfID(), + }) + if err != nil { + return managed.ExternalObservation{}, errors.Wrapf(err, "failed to get the external-name from ID: %s", n.opTracker.GetTfID()) + } + // if external name is set for the first time or if it has changed, this is a spec update + // therefore managed reconciler needs to be informed to trigger a spec update + externalNameChanged := en != "" && mg.GetAnnotations()[meta.AnnotationKeyExternalName] != en + meta.SetExternalName(mg, en) + mg.SetConditions(xpv1.Available()) + stateValueMap, err := n.fromInstanceStateToJSONMap(newState) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot convert instance state to JSON map") + } + buff, err := json.TFParser.Marshal(stateValueMap) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot marshal the attributes of the new state for late-initialization") + } + lateInitialized, err = mg.(resource.Terraformed).LateInitialize(buff) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot late-initialize the managed resource") + } + // external name updates are considered as lateInitialized + lateInitialized = lateInitialized || externalNameChanged + err = mg.(resource.Terraformed).SetObservation(stateValueMap) + if err != nil { + return managed.ExternalObservation{}, errors.Errorf("could not set observation: %v", err) + } + connDetails, err = resource.GetConnectionDetails(stateValueMap, mg.(resource.Terraformed), n.config) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot get connection details") + } + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return managed.ExternalObservation{}, err + } + noDiff = instanceDiff.Empty() + + if noDiff { + n.metricRecorder.SetReconcileTime(mg.GetName()) + } + if !lateInitialized { + resource.SetUpToDateCondition(mg, noDiff) + } + } + + return managed.ExternalObservation{ + ResourceExists: resourceExists, + ResourceUpToDate: noDiff, + ConnectionDetails: connDetails, + ResourceLateInitialized: lateInitialized, + }, nil + +} + +func (n *noForkAsyncExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { + if !n.opTracker.LastOperation.MarkStart("create") { + return managed.ExternalCreation{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) + } + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return managed.ExternalCreation{}, err + } + + ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) + go func() { + defer cancel() + defer n.opTracker.LastOperation.MarkEnd() + start := time.Now() + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) + metrics.ExternalAPITime.WithLabelValues("create_async").Observe(time.Since(start).Seconds()) + var tfErr error + if diag != nil && diag.HasError() { + tfErr = errors.Errorf("failed to create the resource: %v", diag) + n.opTracker.LastOperation.SetError(tfErr) + } + n.opTracker.SetTfState(newState) + n.opTracker.logger.Debug("create async ended", "resource", n.opTracker.GetTfID()) + + defer func() { + if cErr := n.callback.Create(mg.GetName())(tfErr, ctx); cErr != nil { + n.opTracker.logger.Info("create callback failed", "error", cErr.Error()) + } + }() + }() + return managed.ExternalCreation{}, nil +} + +func (n *noForkAsyncExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { + if !n.opTracker.LastOperation.MarkStart("update") { + return managed.ExternalUpdate{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) + } + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return managed.ExternalUpdate{}, err + } + ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) + go func() { + defer cancel() + defer n.opTracker.LastOperation.MarkEnd() + start := time.Now() + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) + metrics.ExternalAPITime.WithLabelValues("update_async").Observe(time.Since(start).Seconds()) + var tfErr error + if diag != nil && diag.HasError() { + tfErr = errors.Errorf("failed to update the resource: %v", diag) + n.opTracker.LastOperation.SetError(tfErr) + } + n.opTracker.SetTfState(newState) + n.opTracker.logger.Debug("update async ended", "resource", n.opTracker.GetTfID()) + + defer func() { + if cErr := n.callback.Update(mg.GetName())(tfErr, ctx); cErr != nil { + n.opTracker.logger.Info("update callback failed", "error", cErr.Error()) + } + }() + }() + + return managed.ExternalUpdate{}, nil +} + +func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) error { + if !n.opTracker.LastOperation.MarkStart("destroy") { + return errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) + } + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return err + } + if instanceDiff == nil { + instanceDiff = tf.NewInstanceDiff() + } + + instanceDiff.Destroy = true + ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) + go func() { + defer cancel() + defer n.opTracker.LastOperation.MarkEnd() + start := time.Now() + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) + metrics.ExternalAPITime.WithLabelValues("destroy_async").Observe(time.Since(start).Seconds()) + var tfErr error + if diag != nil && diag.HasError() { + tfErr = errors.Errorf("failed to destroy the resource: %v", diag) + n.opTracker.LastOperation.SetError(tfErr) + } + n.opTracker.SetTfState(newState) + n.opTracker.logger.Debug("destroy async ended", "resource", n.opTracker.GetTfID()) + defer func() { + if cErr := n.callback.Destroy(mg.GetName())(tfErr, ctx); cErr != nil { + n.opTracker.logger.Info("destroy callback failed", "error", cErr.Error()) + } + }() + }() + return nil +} diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index c05bae64..f4668df2 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -32,7 +32,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/upbound/upjet/pkg/config" - "github.com/upbound/upjet/pkg/controller/handler" "github.com/upbound/upjet/pkg/metrics" "github.com/upbound/upjet/pkg/resource" "github.com/upbound/upjet/pkg/resource/json" @@ -44,7 +43,6 @@ type NoForkConnector struct { kube client.Client config *config.Resource logger logging.Logger - eventHandler *handler.EventHandler metricRecorder *metrics.MetricRecorder } @@ -66,14 +64,6 @@ func WithNoForkMetricRecorder(r *metrics.MetricRecorder) NoForkOption { } } -// WithNoForkConnectorEventHandler configures the EventHandler so that -// the no-fork external clients can requeue reconciliation requests. -func WithNoForkConnectorEventHandler(e *handler.EventHandler) NoForkOption { - return func(c *NoForkConnector) { - c.eventHandler = e - } -} - func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkOption) *NoForkConnector { nfc := &NoForkConnector{ kube: kube, diff --git a/pkg/controller/nofork_finalizer.go b/pkg/controller/nofork_finalizer.go new file mode 100644 index 00000000..2da9a655 --- /dev/null +++ b/pkg/controller/nofork_finalizer.go @@ -0,0 +1,46 @@ +package controller + +import ( + "context" + + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/pkg/errors" +) + +const ( + errRemoveTracker = "cannot remove tracker from the store" +) + +// TrackerCleaner is the interface that the no-fork finalizer needs to work with. +type TrackerCleaner interface { + RemoveTracker(obj xpresource.Object) error +} + +// NewNoForkFinalizer returns a new NoForkFinalizer. +func NewNoForkFinalizer(tc TrackerCleaner, af xpresource.Finalizer) *NoForkFinalizer { + return &NoForkFinalizer{ + Finalizer: af, + OperationStore: tc, + } +} + +// NoForkFinalizer removes the operation tracker from the workspace store and only +// then calls RemoveFinalizer of the underlying Finalizer. +type NoForkFinalizer struct { + xpresource.Finalizer + OperationStore TrackerCleaner +} + +// AddFinalizer to the supplied Managed resource. +func (nf *NoForkFinalizer) AddFinalizer(ctx context.Context, obj xpresource.Object) error { + return nf.Finalizer.AddFinalizer(ctx, obj) +} + +// RemoveFinalizer removes the workspace from workspace store before removing +// the finalizer. +func (nf *NoForkFinalizer) RemoveFinalizer(ctx context.Context, obj xpresource.Object) error { + if err := nf.OperationStore.RemoveTracker(obj); err != nil { + return errors.Wrap(err, errRemoveTracker) + } + return nf.Finalizer.RemoveFinalizer(ctx, obj) +} diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go new file mode 100644 index 00000000..b2cba672 --- /dev/null +++ b/pkg/controller/nofork_store.go @@ -0,0 +1,101 @@ +package controller + +import ( + "github.com/crossplane/crossplane-runtime/pkg/logging" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + tfsdk "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "github.com/upbound/upjet/pkg/resource" + "github.com/upbound/upjet/pkg/terraform" + "k8s.io/apimachinery/pkg/types" + "sync" +) + +type AsyncTracker struct { + LastOperation *terraform.Operation + logger logging.Logger + mu *sync.Mutex + tfID string + tfState *tfsdk.InstanceState +} + +type AsyncTrackerOption func(manager *AsyncTracker) + +// WithAsyncTrackerLogger sets the logger of AsyncTracker. +func WithAsyncTrackerLogger(l logging.Logger) AsyncTrackerOption { + return func(w *AsyncTracker) { + w.logger = l + } +} +func NewAsyncTracker(opts ...AsyncTrackerOption) *AsyncTracker { + w := &AsyncTracker{ + LastOperation: &terraform.Operation{}, + logger: logging.NewNopLogger(), + mu: &sync.Mutex{}, + } + for _, f := range opts { + f(w) + } + return w +} + +func (a *AsyncTracker) GetTfState() *tfsdk.InstanceState { + a.mu.Lock() + defer a.mu.Unlock() + return a.tfState +} + +func (a *AsyncTracker) SetTfState(state *tfsdk.InstanceState) { + a.mu.Lock() + defer a.mu.Unlock() + a.tfState = state +} + +func (a *AsyncTracker) GetTfID() string { + a.mu.Lock() + defer a.mu.Unlock() + if a.tfState == nil { + return "" + } + return a.tfState.ID +} + +func (a *AsyncTracker) SetTfID(tfId string) { + //a.mu.Lock() + //defer a.mu.Unlock() + a.tfID = tfId +} + +type OperationTrackerStore struct { + store map[types.UID]*AsyncTracker + logger logging.Logger + mu *sync.Mutex +} + +func NewOperationStore(l logging.Logger) *OperationTrackerStore { + ops := &OperationTrackerStore{ + store: map[types.UID]*AsyncTracker{}, + logger: l, + mu: &sync.Mutex{}, + } + + return ops +} + +func (ops *OperationTrackerStore) Tracker(tr resource.Terraformed) *AsyncTracker { + ops.mu.Lock() + defer ops.mu.Unlock() + tracker, ok := ops.store[tr.GetUID()] + if !ok { + l := ops.logger.WithValues("trackerUID", tr.GetUID()) + ops.store[tr.GetUID()] = NewAsyncTracker(WithAsyncTrackerLogger(l)) + tracker = ops.store[tr.GetUID()] + } + return tracker +} + +func (ops *OperationTrackerStore) RemoveTracker(obj xpresource.Object) error { + ops.mu.Lock() + defer ops.mu.Unlock() + delete(ops.store, obj.GetUID()) + return nil +} diff --git a/pkg/controller/options.go b/pkg/controller/options.go index e9e54a95..491e12e2 100644 --- a/pkg/controller/options.go +++ b/pkg/controller/options.go @@ -28,6 +28,8 @@ type Options struct { // instance should use. WorkspaceStore *terraform.WorkspaceStore + OperationTrackerStore *OperationTrackerStore + // SetupFn contains the provider-specific initialization logic, such as // preparing the auth token for Terraform CLI. SetupFn terraform.SetupFn diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index 3c40f93a..a3adba80 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -41,15 +41,22 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { cps = append(cps, connection.NewDetailsManager(mgr.GetClient(), *o.SecretStoreConfigGVK, connection.WithTLSConfig(o.ESSOptions.TLSConfig))) } eventHandler := handler.NewEventHandler(handler.WithLogger(o.Logger.WithValues("gvk", {{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind))) - {{- if and .UseAsync (not .UseNoForkClient) }} + {{- if .UseAsync }} ac := tjcontroller.NewAPICallbacks(mgr, xpresource.ManagedKind({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind), tjcontroller.WithEventHandler(eventHandler)) {{- end}} opts := []managed.ReconcilerOption{ managed.WithExternalConnecter( {{- if .UseNoForkClient -}} - tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger), - tjcontroller.WithNoForkConnectorEventHandler(eventHandler), + {{- if .UseAsync }} + tjcontroller.NewNoForkAsyncConnector(mgr.GetClient(), o.OperationTrackerStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], + tjcontroller.WithNoForkAsyncLogger(o.Logger), + tjcontroller.WithNoForkAsyncConnectorEventHandler(eventHandler), + tjcontroller.WithNoForkAsyncCallbackProvider(ac), + tjcontroller.WithNoForkAsyncMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) + {{- else -}} + tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger), tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) + {{- end -}} {{- else -}} tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), {{- if .UseAsync }} @@ -60,7 +67,14 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { ), managed.WithLogger(o.Logger.WithValues("controller", name)), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), - managed.WithFinalizer(terraform.NewWorkspaceFinalizer(o.WorkspaceStore, xpresource.NewAPIFinalizer(mgr.GetClient(), managed.FinalizerName))), + {{- if .UseNoForkClient }} + {{- if .UseAsync }} + managed.WithFinalizer(tjcontroller.NewNoForkFinalizer(o.OperationTrackerStore, xpresource.NewAPIFinalizer(mgr.GetClient(), managed.FinalizerName))), + {{- end }} + {{- else }} + managed.WithFinalizer(terraform.NewWorkspaceFinalizer(o.WorkspaceStore, xpresource.NewAPIFinalizer(mgr.GetClient(), managed.FinalizerName))), + {{- end }} + managed.WithTimeout(3*time.Minute), managed.WithInitializers(initializers), managed.WithConnectionPublishers(cps...), diff --git a/pkg/terraform/operation.go b/pkg/terraform/operation.go index 8f71ddb0..3229d2fa 100644 --- a/pkg/terraform/operation.go +++ b/pkg/terraform/operation.go @@ -15,6 +15,7 @@ type Operation struct { startTime *time.Time endTime *time.Time + err error mu sync.RWMutex } @@ -49,6 +50,7 @@ func (o *Operation) Flush() { o.Type = "" o.startTime = nil o.endTime = nil + o.err = nil } // IsEnded returns whether the operation has ended, regardless of its result. @@ -78,3 +80,17 @@ func (o *Operation) EndTime() time.Time { defer o.mu.RUnlock() return *o.endTime } + +// SetError records the given error on the current operation. +func (o *Operation) SetError(err error) { + o.mu.Lock() + defer o.mu.Unlock() + o.err = err +} + +// Error returns the recorded error of the current operation. +func (o *Operation) Error() error { + o.mu.RLock() + defer o.mu.RUnlock() + return o.err +} From a78ccac118856b3a8a562bdaa9da38abe8e7c019 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Tue, 17 Oct 2023 18:08:14 +0300 Subject: [PATCH 25/60] minor logging changes Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 7 ++++--- pkg/controller/nofork_store.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index ada84a71..1bb27f1e 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -296,7 +296,7 @@ func (n *noForkAsyncExternal) Create(ctx context.Context, mg xpresource.Managed) n.opTracker.LastOperation.SetError(tfErr) } n.opTracker.SetTfState(newState) - n.opTracker.logger.Debug("create async ended", "resource", n.opTracker.GetTfID()) + n.opTracker.logger.Debug("create async ended", "tfID", n.opTracker.GetTfID()) defer func() { if cErr := n.callback.Create(mg.GetName())(tfErr, ctx); cErr != nil { @@ -328,7 +328,7 @@ func (n *noForkAsyncExternal) Update(ctx context.Context, mg xpresource.Managed) n.opTracker.LastOperation.SetError(tfErr) } n.opTracker.SetTfState(newState) - n.opTracker.logger.Debug("update async ended", "resource", n.opTracker.GetTfID()) + n.opTracker.logger.Debug("update async ended", "tfID", n.opTracker.GetTfID()) defer func() { if cErr := n.callback.Update(mg.GetName())(tfErr, ctx); cErr != nil { @@ -358,6 +358,7 @@ func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() + tfID := n.opTracker.GetTfID() newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("destroy_async").Observe(time.Since(start).Seconds()) var tfErr error @@ -366,7 +367,7 @@ func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) n.opTracker.LastOperation.SetError(tfErr) } n.opTracker.SetTfState(newState) - n.opTracker.logger.Debug("destroy async ended", "resource", n.opTracker.GetTfID()) + n.opTracker.logger.Debug("destroy async ended", "tfID", tfID) defer func() { if cErr := n.callback.Destroy(mg.GetName())(tfErr, ctx); cErr != nil { n.opTracker.logger.Info("destroy callback failed", "error", cErr.Error()) diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index b2cba672..fba31db1 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -86,7 +86,7 @@ func (ops *OperationTrackerStore) Tracker(tr resource.Terraformed) *AsyncTracker defer ops.mu.Unlock() tracker, ok := ops.store[tr.GetUID()] if !ok { - l := ops.logger.WithValues("trackerUID", tr.GetUID()) + l := ops.logger.WithValues("trackerUID", tr.GetUID(), "resourceName", tr.GetName()) ops.store[tr.GetUID()] = NewAsyncTracker(WithAsyncTrackerLogger(l)) tracker = ops.store[tr.GetUID()] } From f2f4180e7bb06067b98defdf6fba93312d202ff5 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Wed, 18 Oct 2023 14:47:48 +0300 Subject: [PATCH 26/60] port instance diff fixes to async client Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 58 ++++++++++++++----------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index 1bb27f1e..980b063e 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -88,6 +88,13 @@ func (c *NoForkAsyncConnector) Connect(ctx context.Context, mg xpresource.Manage } c.config.ExternalName.SetIdentifierArgumentFn(params, externalName) + if c.config.TerraformConfigurationInjector != nil { + m, err := getJSONMap(mg) + if err != nil { + return nil, errors.Wrap(err, "cannot get JSON map for the managed resource's spec.forProvider value") + } + c.config.TerraformConfigurationInjector(m, params) + } tfID, err := c.config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) if err != nil { @@ -97,7 +104,8 @@ func (c *NoForkAsyncConnector) Connect(ctx context.Context, mg xpresource.Manage // we need to parameterize the following for a provider // not all providers may have this attribute // TODO: tags_all handling - attrs := c.config.TerraformResource.CoreConfigSchema().Attributes + schemaBlock := c.config.TerraformResource.CoreConfigSchema() + attrs := schemaBlock.Attributes if _, ok := attrs["tags_all"]; ok { params["tags_all"] = params["tags"] } @@ -117,7 +125,7 @@ func (c *NoForkAsyncConnector) Connect(ctx context.Context, mg xpresource.Manage tfState = copyParameters(tfState, params) } - tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, c.config.TerraformResource.CoreConfigSchema()) + tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, schemaBlock) if err != nil { return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") } @@ -126,6 +134,11 @@ func (c *NoForkAsyncConnector) Connect(ctx context.Context, mg xpresource.Manage return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } s.RawPlan = tfStateCtyValue + rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) + if err != nil { + return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") + } + s.RawConfig = rawConfig asyncTracker.SetTfState(s) } @@ -204,7 +217,14 @@ func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } n.opTracker.SetTfState(newState) - noDiff := false + // compute the instance diff + instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + if err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, "cannot compute the instance diff") + } + n.instanceDiff = instanceDiff + noDiff := instanceDiff.Empty() + var connDetails managed.ConnectionDetails resourceExists := newState != nil && newState.ID != "" if !resourceExists && mg.GetDeletionTimestamp() != nil { @@ -251,11 +271,6 @@ func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot get connection details") } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return managed.ExternalObservation{}, err - } - noDiff = instanceDiff.Empty() if noDiff { n.metricRecorder.SetReconcileTime(mg.GetName()) @@ -278,17 +293,13 @@ func (n *noForkAsyncExternal) Create(ctx context.Context, mg xpresource.Managed) if !n.opTracker.LastOperation.MarkStart("create") { return managed.ExternalCreation{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return managed.ExternalCreation{}, err - } ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("create_async").Observe(time.Since(start).Seconds()) var tfErr error if diag != nil && diag.HasError() { @@ -311,16 +322,13 @@ func (n *noForkAsyncExternal) Update(ctx context.Context, mg xpresource.Managed) if !n.opTracker.LastOperation.MarkStart("update") { return managed.ExternalUpdate{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return managed.ExternalUpdate{}, err - } + ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("update_async").Observe(time.Since(start).Seconds()) var tfErr error if diag != nil && diag.HasError() { @@ -344,22 +352,20 @@ func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) if !n.opTracker.LastOperation.MarkStart("destroy") { return errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return err - } - if instanceDiff == nil { - instanceDiff = tf.NewInstanceDiff() + + if n.instanceDiff == nil { + n.instanceDiff = tf.NewInstanceDiff() } - instanceDiff.Destroy = true + // TODO: sync + n.instanceDiff.Destroy = true ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() tfID := n.opTracker.GetTfID() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("destroy_async").Observe(time.Since(start).Seconds()) var tfErr error if diag != nil && diag.HasError() { From 0707799502a913a1be1e848f4929749f019a78f2 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 18 Oct 2023 16:49:41 +0300 Subject: [PATCH 27/60] Refactor commons between async & sync no-fork external clients - Terraformed resources which are reconciled using the synchronous no-fork external client also use the in-memory Terraform state cache. This was needed because there are resources who put certain attributes into the TF state only during the Create call. Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/api.go | 25 ++- pkg/controller/external_async_nofork.go | 229 ++++------------------ pkg/controller/external_nofork.go | 154 +++++++++------ pkg/controller/nofork_common.go | 15 ++ pkg/controller/nofork_store.go | 12 +- pkg/pipeline/templates/controller.go.tmpl | 6 +- 6 files changed, 179 insertions(+), 262 deletions(-) create mode 100644 pkg/controller/nofork_common.go diff --git a/pkg/controller/api.go b/pkg/controller/api.go index f5a5c5f5..e9946bee 100644 --- a/pkg/controller/api.go +++ b/pkg/controller/api.go @@ -68,6 +68,17 @@ func WithEventHandler(e *handler.EventHandler) APICallbacksOption { } } +// WithStatusUpdates sets whether the LastAsyncOperation status condition +// is enabled. If set to false, APICallbacks will not use the +// LastAsyncOperation status condition for reporting ongoing async +// operations or errors. Error conditions will still be reported +// as usual in the `Synced` status condition. +func WithStatusUpdates(enabled bool) APICallbacksOption { + return func(callbacks *APICallbacks) { + callbacks.enableStatusUpdates = enabled + } +} + // NewAPICallbacks returns a new APICallbacks. func NewAPICallbacks(m ctrl.Manager, of xpresource.ManagedKind, opts ...APICallbacksOption) *APICallbacks { nt := func() resource.Terraformed { @@ -76,6 +87,9 @@ func NewAPICallbacks(m ctrl.Manager, of xpresource.ManagedKind, opts ...APICallb cb := &APICallbacks{ kube: m.GetClient(), newTerraformed: nt, + // the default behavior is to use the LastAsyncOperation + // status condition for backwards compatibility. + enableStatusUpdates: true, } for _, o := range opts { o(cb) @@ -87,8 +101,9 @@ func NewAPICallbacks(m ctrl.Manager, of xpresource.ManagedKind, opts ...APICallb type APICallbacks struct { eventHandler *handler.EventHandler - kube client.Client - newTerraformed func() resource.Terraformed + kube client.Client + newTerraformed func() resource.Terraformed + enableStatusUpdates bool } func (ac *APICallbacks) callbackFn(name, op string, requeue bool) terraform.CallbackFn { @@ -98,8 +113,10 @@ func (ac *APICallbacks) callbackFn(name, op string, requeue bool) terraform.Call if kErr := ac.kube.Get(ctx, nn, tr); kErr != nil { return errors.Wrapf(kErr, errGetFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) } - tr.SetConditions(resource.LastAsyncOperationCondition(err)) - tr.SetConditions(resource.AsyncOperationFinishedCondition()) + if ac.enableStatusUpdates { + tr.SetConditions(resource.LastAsyncOperationCondition(err)) + tr.SetConditions(resource.AsyncOperationFinishedCondition()) + } uErr := errors.Wrapf(ac.kube.Status().Update(ctx, tr), errUpdateStatusFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) if ac.eventHandler != nil && requeue { switch { diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index 980b063e..23695dd7 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -16,45 +16,34 @@ package controller import ( "context" - "github.com/crossplane/crossplane-runtime/pkg/logging" - "github.com/upbound/upjet/pkg/config" - "github.com/upbound/upjet/pkg/controller/handler" - "github.com/upbound/upjet/pkg/resource/json" - "github.com/upbound/upjet/pkg/terraform" - "sigs.k8s.io/controller-runtime/pkg/client" "time" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - "github.com/crossplane/crossplane-runtime/pkg/meta" + "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/upbound/upjet/pkg/config" + "github.com/upbound/upjet/pkg/controller/handler" "github.com/upbound/upjet/pkg/metrics" - "github.com/upbound/upjet/pkg/resource" - corev1 "k8s.io/api/core/v1" + "github.com/upbound/upjet/pkg/terraform" ) var defaultAsyncTimeout = 1 * time.Hour type NoForkAsyncConnector struct { *NoForkConnector - operationTrackerStore *OperationTrackerStore - callback CallbackProvider - eventHandler *handler.EventHandler + callback CallbackProvider + eventHandler *handler.EventHandler } type NoForkAsyncOption func(connector *NoForkAsyncConnector) func NewNoForkAsyncConnector(kube client.Client, ots *OperationTrackerStore, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkAsyncOption) *NoForkAsyncConnector { nfac := &NoForkAsyncConnector{ - NoForkConnector: &NoForkConnector{ - kube: kube, - getTerraformSetup: sf, - config: cfg, - }, - operationTrackerStore: ots, + NoForkConnector: NewNoForkConnector(kube, sf, cfg, ots), } for _, f := range opts { f(nfac) @@ -63,98 +52,15 @@ func NewNoForkAsyncConnector(kube client.Client, ots *OperationTrackerStore, sf } func (c *NoForkAsyncConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { - asyncTracker := c.operationTrackerStore.Tracker(mg.(resource.Terraformed)) - c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) - start := time.Now() - ts, err := c.getTerraformSetup(ctx, c.kube, mg) - metrics.ExternalAPITime.WithLabelValues("connect").Observe(time.Since(start).Seconds()) + ec, err := c.NoForkConnector.Connect(ctx, mg) if err != nil { - return nil, errors.Wrap(err, errGetTerraformSetup) - } - - // To Compute the ResourceDiff: n.resourceSchema.Diff(...) - tr := mg.(resource.Terraformed) - params, err := tr.GetParameters() - if err != nil { - return nil, errors.Wrap(err, "cannot get parameters") - } - if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { - return nil, errors.Wrap(err, "cannot store sensitive parameters into params") - } - - externalName := meta.GetExternalName(tr) - if externalName == "" { - externalName = asyncTracker.GetTfID() - } - - c.config.ExternalName.SetIdentifierArgumentFn(params, externalName) - if c.config.TerraformConfigurationInjector != nil { - m, err := getJSONMap(mg) - if err != nil { - return nil, errors.Wrap(err, "cannot get JSON map for the managed resource's spec.forProvider value") - } - c.config.TerraformConfigurationInjector(m, params) - } - - tfID, err := c.config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) - if err != nil { - return nil, errors.Wrap(err, "cannot get ID") - } - params["id"] = tfID - // we need to parameterize the following for a provider - // not all providers may have this attribute - // TODO: tags_all handling - schemaBlock := c.config.TerraformResource.CoreConfigSchema() - attrs := schemaBlock.Attributes - if _, ok := attrs["tags_all"]; ok { - params["tags_all"] = params["tags"] - } - // construct TF - if asyncTracker.GetTfState() == nil || asyncTracker.GetTfState().Attributes == nil { - tfState, err := tr.GetObservation() - if err != nil { - return nil, errors.Wrap(err, "failed to get the observation") - } - copyParams := len(tfState) == 0 - if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, tfState, tr.GetConnectionDetailsMapping()); err != nil { - return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") - } - c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) - tfState["id"] = tfID - if copyParams { - tfState = copyParameters(tfState, params) - } - - tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, schemaBlock) - if err != nil { - return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") - } - s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) - if err != nil { - return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") - } - s.RawPlan = tfStateCtyValue - rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) - if err != nil { - return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") - } - s.RawConfig = rawConfig - asyncTracker.SetTfState(s) + return nil, errors.Wrap(err, "cannot initialize the no-fork async external client") } return &noForkAsyncExternal{ - noForkExternal: &noForkExternal{ - ts: ts, - resourceSchema: c.config.TerraformResource, - config: c.config, - kube: c.kube, - params: params, - logger: c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()), - metricRecorder: c.metricRecorder, - }, - opTracker: asyncTracker, - callback: c.callback, - eventHandler: c.eventHandler, + noForkExternal: ec.(*noForkExternal), + callback: c.callback, + eventHandler: c.eventHandler, }, nil } @@ -194,7 +100,6 @@ type noForkAsyncExternal struct { *noForkExternal callback CallbackProvider eventHandler *handler.EventHandler - opTracker *AsyncTracker } type CallbackFn func(error, context.Context) error @@ -202,7 +107,6 @@ type CallbackFn func(error, context.Context) error func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { if n.opTracker.LastOperation.IsRunning() { n.logger.WithValues("opType", n.opTracker.LastOperation.Type).Debug("ongoing async operation") - mg.SetConditions(resource.AsyncOperationOngoingCondition()) return managed.ExternalObservation{ ResourceExists: true, ResourceUpToDate: true, @@ -210,96 +114,24 @@ func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed } n.opTracker.LastOperation.Flush() - start := time.Now() - newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.opTracker.GetTfState(), n.ts.Meta) - metrics.ExternalAPITime.WithLabelValues("read").Observe(time.Since(start).Seconds()) - if diag != nil && diag.HasError() { - return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) - } - n.opTracker.SetTfState(newState) - // compute the instance diff - instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) - if err != nil { - return managed.ExternalObservation{}, errors.Wrap(err, "cannot compute the instance diff") - } - n.instanceDiff = instanceDiff - noDiff := instanceDiff.Empty() - - var connDetails managed.ConnectionDetails - resourceExists := newState != nil && newState.ID != "" - if !resourceExists && mg.GetDeletionTimestamp() != nil { - gvk := mg.GetObjectKind().GroupVersionKind() - metrics.DeletionTime.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(time.Since(mg.GetDeletionTimestamp().Time).Seconds()) - } - lateInitialized := false - if resourceExists { - if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown || - mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionFalse { - addTTR(mg) - } - // Set external name - en, err := n.config.ExternalName.GetExternalNameFn(map[string]any{ - "id": n.opTracker.GetTfID(), - }) - if err != nil { - return managed.ExternalObservation{}, errors.Wrapf(err, "failed to get the external-name from ID: %s", n.opTracker.GetTfID()) - } - // if external name is set for the first time or if it has changed, this is a spec update - // therefore managed reconciler needs to be informed to trigger a spec update - externalNameChanged := en != "" && mg.GetAnnotations()[meta.AnnotationKeyExternalName] != en - meta.SetExternalName(mg, en) - mg.SetConditions(xpv1.Available()) - stateValueMap, err := n.fromInstanceStateToJSONMap(newState) - if err != nil { - return managed.ExternalObservation{}, errors.Wrap(err, "cannot convert instance state to JSON map") - } - buff, err := json.TFParser.Marshal(stateValueMap) - if err != nil { - return managed.ExternalObservation{}, errors.Wrap(err, "cannot marshal the attributes of the new state for late-initialization") - } - lateInitialized, err = mg.(resource.Terraformed).LateInitialize(buff) - if err != nil { - return managed.ExternalObservation{}, errors.Wrap(err, "cannot late-initialize the managed resource") - } - // external name updates are considered as lateInitialized - lateInitialized = lateInitialized || externalNameChanged - err = mg.(resource.Terraformed).SetObservation(stateValueMap) - if err != nil { - return managed.ExternalObservation{}, errors.Errorf("could not set observation: %v", err) - } - connDetails, err = resource.GetConnectionDetails(stateValueMap, mg.(resource.Terraformed), n.config) - if err != nil { - return managed.ExternalObservation{}, errors.Wrap(err, "cannot get connection details") - } - - if noDiff { - n.metricRecorder.SetReconcileTime(mg.GetName()) - } - if !lateInitialized { - resource.SetUpToDateCondition(mg, noDiff) - } - } - - return managed.ExternalObservation{ - ResourceExists: resourceExists, - ResourceUpToDate: noDiff, - ConnectionDetails: connDetails, - ResourceLateInitialized: lateInitialized, - }, nil - + return n.noForkExternal.Observe(ctx, mg) } func (n *noForkAsyncExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { if !n.opTracker.LastOperation.MarkStart("create") { return managed.ExternalCreation{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return managed.ExternalCreation{}, err + } ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("create_async").Observe(time.Since(start).Seconds()) var tfErr error if diag != nil && diag.HasError() { @@ -322,13 +154,16 @@ func (n *noForkAsyncExternal) Update(ctx context.Context, mg xpresource.Managed) if !n.opTracker.LastOperation.MarkStart("update") { return managed.ExternalUpdate{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return managed.ExternalUpdate{}, err + } ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("update_async").Observe(time.Since(start).Seconds()) var tfErr error if diag != nil && diag.HasError() { @@ -352,20 +187,22 @@ func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) if !n.opTracker.LastOperation.MarkStart("destroy") { return errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - - if n.instanceDiff == nil { - n.instanceDiff = tf.NewInstanceDiff() + instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) + if err != nil { + return err + } + if instanceDiff == nil { + instanceDiff = tf.NewInstanceDiff() } - // TODO: sync - n.instanceDiff.Destroy = true + instanceDiff.Destroy = true ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() start := time.Now() tfID := n.opTracker.GetTfID() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("destroy_async").Observe(time.Since(start).Seconds()) var tfErr error if diag != nil && diag.HasError() { diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index f4668df2..c487b984 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -39,11 +39,12 @@ import ( ) type NoForkConnector struct { - getTerraformSetup terraform.SetupFn - kube client.Client - config *config.Resource - logger logging.Logger - metricRecorder *metrics.MetricRecorder + getTerraformSetup terraform.SetupFn + kube client.Client + config *config.Resource + logger logging.Logger + metricRecorder *metrics.MetricRecorder + operationTrackerStore *OperationTrackerStore } // NoForkOption allows you to configure NoForkConnector. @@ -64,11 +65,12 @@ func WithNoForkMetricRecorder(r *metrics.MetricRecorder) NoForkOption { } } -func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, opts ...NoForkOption) *NoForkConnector { +func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, ots *OperationTrackerStore, opts ...NoForkOption) *NoForkConnector { nfc := &NoForkConnector{ - kube: kube, - getTerraformSetup: sf, - config: cfg, + kube: kube, + getTerraformSetup: sf, + config: cfg, + operationTrackerStore: ots, } for _, f := range opts { f(nfc) @@ -104,16 +106,18 @@ type noForkExternal struct { resourceSchema *schema.Resource config *config.Resource kube client.Client - instanceState *tf.InstanceState instanceDiff *tf.InstanceDiff params map[string]any rawConfig cty.Value logger logging.Logger metricRecorder *metrics.MetricRecorder + opTracker *AsyncTracker } func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) + logger := c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()) + logger.Debug("Connecting to the service provider") start := time.Now() ts, err := c.getTerraformSetup(ctx, c.kube, mg) metrics.ExternalAPITime.WithLabelValues("connect").Observe(time.Since(start).Seconds()) @@ -123,6 +127,11 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m // To Compute the ResourceDiff: n.resourceSchema.Diff(...) tr := mg.(resource.Terraformed) + opTracker := c.operationTrackerStore.Tracker(tr) + externalName := meta.GetExternalName(tr) + if externalName == "" { + externalName = opTracker.GetTfID() + } params, err := tr.GetParameters() if err != nil { return nil, errors.Wrap(err, "cannot get parameters") @@ -130,7 +139,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { return nil, errors.Wrap(err, "cannot store sensitive parameters into params") } - c.config.ExternalName.SetIdentifierArgumentFn(params, meta.GetExternalName(tr)) + c.config.ExternalName.SetIdentifierArgumentFn(params, externalName) if c.config.TerraformConfigurationInjector != nil { m, err := getJSONMap(mg) if err != nil { @@ -139,7 +148,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m c.config.TerraformConfigurationInjector(m, params) } - tfID, err := c.config.ExternalName.GetIDFn(ctx, meta.GetExternalName(mg), params, ts.Map()) + tfID, err := c.config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) if err != nil { return nil, errors.Wrap(err, "cannot get ID") } @@ -153,45 +162,50 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m params["tags_all"] = params["tags"] } - tfState, err := tr.GetObservation() - if err != nil { - return nil, errors.Wrap(err, "failed to get the observation") - } - copyParams := len(tfState) == 0 - if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, tfState, tr.GetConnectionDetailsMapping()); err != nil { - return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") - } - c.config.ExternalName.SetIdentifierArgumentFn(tfState, meta.GetExternalName(tr)) - tfState["id"] = tfID - if copyParams { - tfState = copyParameters(tfState, params) - } + var rawConfig cty.Value + if !opTracker.HasState() { + logger.Debug("Instance state not found in cache, reconstructing...") + tfState, err := tr.GetObservation() + if err != nil { + return nil, errors.Wrap(err, "failed to get the observation") + } + copyParams := len(tfState) == 0 + if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, tfState, tr.GetConnectionDetailsMapping()); err != nil { + return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") + } + c.config.ExternalName.SetIdentifierArgumentFn(tfState, externalName) + tfState["id"] = tfID + if copyParams { + tfState = copyParameters(tfState, params) + } - tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, schemaBlock) - if err != nil { - return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") - } - s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) - if err != nil { - return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") - } - s.RawPlan = tfStateCtyValue - rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) - if err != nil { - return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") + tfStateCtyValue, err := schema.JSONMapToStateValue(tfState, schemaBlock) + if err != nil { + return nil, errors.Wrap(err, "cannot convert JSON map to state cty.Value") + } + s, err := c.config.TerraformResource.ShimInstanceStateFromValue(tfStateCtyValue) + if err != nil { + return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") + } + s.RawPlan = tfStateCtyValue + rawConfig, err = schema.JSONMapToStateValue(params, schemaBlock) + if err != nil { + return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") + } + s.RawConfig = rawConfig + opTracker.SetTfState(s) } - s.RawConfig = rawConfig return &noForkExternal{ ts: ts, resourceSchema: c.config.TerraformResource, config: c.config, kube: c.kube, - instanceState: s, params: params, rawConfig: rawConfig, - logger: c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()), + logger: logger, metricRecorder: c.metricRecorder, + opTracker: opTracker, }, nil } @@ -222,15 +236,16 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { + n.logger.Debug("Observing the external resource") start := time.Now() - newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.instanceState, n.ts.Meta) + newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.opTracker.GetTfState(), n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("read").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } - n.instanceState = newState + n.opTracker.SetTfState(newState) // TODO: missing RawConfig & RawPlan here... // compute the instance diff - instanceDiff, err := n.getResourceDataDiff(ctx, n.instanceState) + instanceDiff, err := n.getResourceDataDiff(ctx, newState) if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot compute the instance diff") } @@ -243,7 +258,7 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma gvk := mg.GetObjectKind().GroupVersionKind() metrics.DeletionTime.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(time.Since(mg.GetDeletionTimestamp().Time).Seconds()) } - lateInitialized := false + specUpdateRequired := false if resourceExists { if mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionUnknown || mg.GetCondition(xpv1.TypeReady).Status == corev1.ConditionFalse { @@ -259,7 +274,7 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot marshal the attributes of the new state for late-initialization") } - lateInitialized, err = mg.(resource.Terraformed).LateInitialize(buff) + specUpdateRequired, err = mg.(resource.Terraformed).LateInitialize(buff) if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot late-initialize the managed resource") } @@ -276,22 +291,47 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma if noDiff { n.metricRecorder.SetReconcileTime(mg.GetName()) } - if !lateInitialized { + if !specUpdateRequired { resource.SetUpToDateCondition(mg, noDiff) } + // check for an external-name change + if nameChanged, err := n.setExternalName(mg, newState); err != nil { + return managed.ExternalObservation{}, errors.Wrapf(err, "failed to set the external-name of the managed resource during observe") + } else { + specUpdateRequired = specUpdateRequired || nameChanged + } } return managed.ExternalObservation{ ResourceExists: resourceExists, ResourceUpToDate: noDiff, ConnectionDetails: connDetails, - ResourceLateInitialized: lateInitialized, + ResourceLateInitialized: specUpdateRequired, }, nil } +// sets the external-name on the MR. Returns `true` +// if the external-name of the MR has changed. +func (n *noForkExternal) setExternalName(mg xpresource.Managed, newState *tf.InstanceState) (bool, error) { + if newState.ID == "" { + return false, nil + } + newName, err := n.config.ExternalName.GetExternalNameFn(map[string]any{ + "id": newState.ID, + }) + if err != nil { + return false, errors.Wrapf(err, "failed to get the external-name from ID: %s", newState.ID) + } + oldName := meta.GetExternalName(mg) + // we have to make sure the newly set external-name is recorded + meta.SetExternalName(mg, newName) + return oldName != newName, nil +} + func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { + n.logger.Debug("Creating the external resource") start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("create").Observe(time.Since(start).Seconds()) // diag := n.resourceSchema.CreateWithoutTimeout(ctx, n.resourceData, n.ts.Meta) if diag != nil && diag.HasError() { @@ -301,15 +341,11 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man if newState == nil || newState.ID == "" { return managed.ExternalCreation{}, errors.New("failed to read the ID of the new resource") } + n.opTracker.SetTfState(newState) - en, err := n.config.ExternalName.GetExternalNameFn(map[string]any{ - "id": newState.ID, - }) - if err != nil { - return managed.ExternalCreation{}, errors.Wrapf(err, "failed to get the external-name from ID: %s", newState.ID) + if _, err := n.setExternalName(mg, newState); err != nil { + return managed.ExternalCreation{}, errors.Wrapf(err, "failed to set the external-name of the managed resource during create") } - // we have to make sure the newly set externa-name is recorded - meta.SetExternalName(mg, en) stateValueMap, err := n.fromInstanceStateToJSONMap(newState) if err != nil { return managed.ExternalCreation{}, err @@ -327,12 +363,14 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man } func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { + n.logger.Debug("Updating the external resource") start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("update").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return managed.ExternalUpdate{}, errors.Errorf("failed to update the resource: %v", diag) } + n.opTracker.SetTfState(newState) stateValueMap, err := n.fromInstanceStateToJSONMap(newState) if err != nil { @@ -347,17 +385,19 @@ func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (man } func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error { + n.logger.Debug("Deleting the external resource") if n.instanceDiff == nil { n.instanceDiff = tf.NewInstanceDiff() } n.instanceDiff.Destroy = true start := time.Now() - _, diag := n.resourceSchema.Apply(ctx, n.instanceState, n.instanceDiff, n.ts.Meta) + newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("delete").Observe(time.Since(start).Seconds()) if diag != nil && diag.HasError() { return errors.Errorf("failed to delete the resource: %v", diag) } + n.opTracker.SetTfState(newState) return nil } diff --git a/pkg/controller/nofork_common.go b/pkg/controller/nofork_common.go new file mode 100644 index 00000000..ca72768f --- /dev/null +++ b/pkg/controller/nofork_common.go @@ -0,0 +1,15 @@ +// Copyright 2023 Upbound Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index fba31db1..2c730241 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -1,13 +1,15 @@ package controller import ( + "sync" + "github.com/crossplane/crossplane-runtime/pkg/logging" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" tfsdk "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "k8s.io/apimachinery/pkg/types" + "github.com/upbound/upjet/pkg/resource" "github.com/upbound/upjet/pkg/terraform" - "k8s.io/apimachinery/pkg/types" - "sync" ) type AsyncTracker struct { @@ -44,6 +46,12 @@ func (a *AsyncTracker) GetTfState() *tfsdk.InstanceState { return a.tfState } +func (a *AsyncTracker) HasState() bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.tfState != nil && a.tfState.ID != "" +} + func (a *AsyncTracker) SetTfState(state *tfsdk.InstanceState) { a.mu.Lock() defer a.mu.Unlock() diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index a3adba80..b8ff620e 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -42,7 +42,7 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { } eventHandler := handler.NewEventHandler(handler.WithLogger(o.Logger.WithValues("gvk", {{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind))) {{- if .UseAsync }} - ac := tjcontroller.NewAPICallbacks(mgr, xpresource.ManagedKind({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind), tjcontroller.WithEventHandler(eventHandler)) + ac := tjcontroller.NewAPICallbacks(mgr, xpresource.ManagedKind({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind), tjcontroller.WithEventHandler(eventHandler){{ if .UseNoForkClient }}, tjcontroller.WithStatusUpdates(false){{ end }}) {{- end}} opts := []managed.ReconcilerOption{ managed.WithExternalConnecter( @@ -54,14 +54,14 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { tjcontroller.WithNoForkAsyncCallbackProvider(ac), tjcontroller.WithNoForkAsyncMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) {{- else -}} - tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithNoForkLogger(o.Logger), + tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], o.OperationTrackerStore, tjcontroller.WithNoForkLogger(o.Logger), tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) {{- end -}} {{- else -}} tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), {{- if .UseAsync }} tjcontroller.WithCallbackProvider(ac), - {{- end}} + {{- end }} ) {{- end -}} ), From 496f1fe1a272d31f27c4bf7a3349f456a83d075c Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 19 Oct 2023 14:29:48 +0300 Subject: [PATCH 28/60] Unconditionally compute InstanceState.RawConfig in the no-fork external clients Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index c487b984..59204590 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -162,7 +162,10 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m params["tags_all"] = params["tags"] } - var rawConfig cty.Value + rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) + if err != nil { + return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") + } if !opTracker.HasState() { logger.Debug("Instance state not found in cache, reconstructing...") tfState, err := tr.GetObservation() @@ -188,10 +191,6 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, errors.Wrap(err, "failed to convert cty.Value to terraform.InstanceState") } s.RawPlan = tfStateCtyValue - rawConfig, err = schema.JSONMapToStateValue(params, schemaBlock) - if err != nil { - return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") - } s.RawConfig = rawConfig opTracker.SetTfState(s) } From f31d1f7166acbbf42f58bb65bef91c81864f2ebe Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 19 Oct 2023 18:27:45 +0300 Subject: [PATCH 29/60] Refactor no-fork external client's Create/Update/Delete methods Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/api.go | 7 +- pkg/controller/external_async_nofork.go | 101 ++++++++---------------- 2 files changed, 39 insertions(+), 69 deletions(-) diff --git a/pkg/controller/api.go b/pkg/controller/api.go index e9946bee..f7d035ad 100644 --- a/pkg/controller/api.go +++ b/pkg/controller/api.go @@ -113,8 +113,13 @@ func (ac *APICallbacks) callbackFn(name, op string, requeue bool) terraform.Call if kErr := ac.kube.Get(ctx, nn, tr); kErr != nil { return errors.Wrapf(kErr, errGetFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) } + // For the no-fork architecture, we will need to be able to report + // reconciliation errors. The proper place is the `Synced` + // status condition but we need changes in the managed reconciler + // to do so. So we keep the `LastAsyncOperation` condition. + // TODO: move this to the `Synced` condition. + tr.SetConditions(resource.LastAsyncOperationCondition(err)) if ac.enableStatusUpdates { - tr.SetConditions(resource.LastAsyncOperationCondition(err)) tr.SetConditions(resource.AsyncOperationFinishedCondition()) } uErr := errors.Wrapf(ac.kube.Status().Update(ctx, tr), errUpdateStatusFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index 23695dd7..d996b23a 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -21,7 +21,6 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" - tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" @@ -117,105 +116,71 @@ func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed return n.noForkExternal.Observe(ctx, mg) } -func (n *noForkAsyncExternal) Create(ctx context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { +func (n *noForkAsyncExternal) Create(_ context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { if !n.opTracker.LastOperation.MarkStart("create") { return managed.ExternalCreation{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return managed.ExternalCreation{}, err - } - ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) + ctx, cancel := context.WithDeadline(context.Background(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() - start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) - metrics.ExternalAPITime.WithLabelValues("create_async").Observe(time.Since(start).Seconds()) - var tfErr error - if diag != nil && diag.HasError() { - tfErr = errors.Errorf("failed to create the resource: %v", diag) - n.opTracker.LastOperation.SetError(tfErr) + + n.opTracker.logger.Debug("Async create starting...", "tfID", n.opTracker.GetTfID()) + _, err := n.noForkExternal.Create(ctx, mg) + n.opTracker.LastOperation.SetError(errors.Wrap(err, "async create failed")) + n.opTracker.logger.Debug("Async create ended.", "error", err, "tfID", n.opTracker.GetTfID()) + + if cErr := n.callback.Create(mg.GetName())(err, ctx); cErr != nil { + n.opTracker.logger.Info("Async create callback failed", "error", cErr.Error()) } - n.opTracker.SetTfState(newState) - n.opTracker.logger.Debug("create async ended", "tfID", n.opTracker.GetTfID()) - - defer func() { - if cErr := n.callback.Create(mg.GetName())(tfErr, ctx); cErr != nil { - n.opTracker.logger.Info("create callback failed", "error", cErr.Error()) - } - }() }() + return managed.ExternalCreation{}, nil } -func (n *noForkAsyncExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { +func (n *noForkAsyncExternal) Update(_ context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { if !n.opTracker.LastOperation.MarkStart("update") { return managed.ExternalUpdate{}, errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return managed.ExternalUpdate{}, err - } - ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) + + ctx, cancel := context.WithDeadline(context.Background(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() - start := time.Now() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) - metrics.ExternalAPITime.WithLabelValues("update_async").Observe(time.Since(start).Seconds()) - var tfErr error - if diag != nil && diag.HasError() { - tfErr = errors.Errorf("failed to update the resource: %v", diag) - n.opTracker.LastOperation.SetError(tfErr) + + n.opTracker.logger.Debug("Async update starting...", "tfID", n.opTracker.GetTfID()) + _, err := n.noForkExternal.Update(ctx, mg) + n.opTracker.LastOperation.SetError(errors.Wrap(err, "async update failed")) + n.opTracker.logger.Debug("Async update ended.", "error", err, "tfID", n.opTracker.GetTfID()) + + if cErr := n.callback.Update(mg.GetName())(err, ctx); cErr != nil { + n.opTracker.logger.Info("Async update callback failed", "error", cErr.Error()) } - n.opTracker.SetTfState(newState) - n.opTracker.logger.Debug("update async ended", "tfID", n.opTracker.GetTfID()) - - defer func() { - if cErr := n.callback.Update(mg.GetName())(tfErr, ctx); cErr != nil { - n.opTracker.logger.Info("update callback failed", "error", cErr.Error()) - } - }() }() return managed.ExternalUpdate{}, nil } func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) error { - if !n.opTracker.LastOperation.MarkStart("destroy") { + if !n.opTracker.LastOperation.MarkStart("delete") { return errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } - instanceDiff, err := n.getResourceDataDiff(ctx, n.opTracker.GetTfState()) - if err != nil { - return err - } - if instanceDiff == nil { - instanceDiff = tf.NewInstanceDiff() - } - instanceDiff.Destroy = true - ctx, cancel := context.WithDeadline(context.TODO(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) + ctx, cancel := context.WithDeadline(context.Background(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() defer n.opTracker.LastOperation.MarkEnd() - start := time.Now() - tfID := n.opTracker.GetTfID() - newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), instanceDiff, n.ts.Meta) - metrics.ExternalAPITime.WithLabelValues("destroy_async").Observe(time.Since(start).Seconds()) - var tfErr error - if diag != nil && diag.HasError() { - tfErr = errors.Errorf("failed to destroy the resource: %v", diag) - n.opTracker.LastOperation.SetError(tfErr) + + n.opTracker.logger.Debug("Async delete starting...", "tfID", n.opTracker.GetTfID()) + err := n.noForkExternal.Delete(ctx, mg) + n.opTracker.LastOperation.SetError(errors.Wrap(err, "async delete failed")) + n.opTracker.logger.Debug("Async delete ended.", "error", err, "tfID", n.opTracker.GetTfID()) + + if cErr := n.callback.Destroy(mg.GetName())(err, ctx); cErr != nil { + n.opTracker.logger.Info("Async delete callback failed", "error", cErr.Error()) } - n.opTracker.SetTfState(newState) - n.opTracker.logger.Debug("destroy async ended", "tfID", tfID) - defer func() { - if cErr := n.callback.Destroy(mg.GetName())(tfErr, ctx); cErr != nil { - n.opTracker.logger.Info("destroy callback failed", "error", cErr.Error()) - } - }() }() + return nil } From d37e3a265c26eb4778e1337963c929faabe0a754 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 20 Oct 2023 17:29:22 +0300 Subject: [PATCH 30/60] Implement the missing prevent_destroy lifecycle meta-argument functionality Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 59204590..2f29a0ad 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -361,8 +361,33 @@ func (n *noForkExternal) Create(ctx context.Context, mg xpresource.Managed) (man return managed.ExternalCreation{ConnectionDetails: conn}, nil } +func (n *noForkExternal) assertNoForceNew() error { + if n.instanceDiff == nil { + return nil + } + for k, ad := range n.instanceDiff.Attributes { + if ad == nil { + continue + } + // TODO: use a multi-error implementation to report changes to + // all `ForceNew` arguments. + if ad.RequiresNew { + if ad.Sensitive { + return errors.Errorf("cannot change the value of the argument %q", k) + } + return errors.Errorf("cannot change the value of the argument %q from %q to %q", k, ad.Old, ad.New) + } + } + return nil +} + func (n *noForkExternal) Update(ctx context.Context, mg xpresource.Managed) (managed.ExternalUpdate, error) { n.logger.Debug("Updating the external resource") + + if err := n.assertNoForceNew(); err != nil { + return managed.ExternalUpdate{}, errors.Wrap(err, "refuse to update the external resource") + } + start := time.Now() newState, diag := n.resourceSchema.Apply(ctx, n.opTracker.GetTfState(), n.instanceDiff, n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("update").Observe(time.Since(start).Seconds()) From 43676d9e1ed689e7ed1dfbdc33fe1fd975a9c2dd Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Mon, 23 Oct 2023 17:38:57 +0300 Subject: [PATCH 31/60] Requeue an immediate reconcile request right after a successful async create, update or delete callback Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/api.go | 46 ++++++++++++------------- pkg/controller/external_async_nofork.go | 6 ++-- pkg/controller/handler/eventhandler.go | 24 ++++++++----- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/pkg/controller/api.go b/pkg/controller/api.go index f7d035ad..9a5808c5 100644 --- a/pkg/controller/api.go +++ b/pkg/controller/api.go @@ -106,7 +106,7 @@ type APICallbacks struct { enableStatusUpdates bool } -func (ac *APICallbacks) callbackFn(name, op string, requeue bool) terraform.CallbackFn { +func (ac *APICallbacks) callbackFn(name, op string) terraform.CallbackFn { return func(err error, ctx context.Context) error { nn := types.NamespacedName{Name: name} tr := ac.newTerraformed() @@ -123,17 +123,19 @@ func (ac *APICallbacks) callbackFn(name, op string, requeue bool) terraform.Call tr.SetConditions(resource.AsyncOperationFinishedCondition()) } uErr := errors.Wrapf(ac.kube.Status().Update(ctx, tr), errUpdateStatusFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) - if ac.eventHandler != nil && requeue { + if ac.eventHandler != nil { + rateLimiter := handler.NoRateLimiter switch { case err != nil: - // TODO: use the errors.Join from - // github.com/crossplane/crossplane-runtime. - if ok := ac.eventHandler.RequestReconcile(rateLimiterCallback, name, nil); !ok { - return errors.Errorf(errReconcileRequestFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) - } + rateLimiter = rateLimiterCallback default: ac.eventHandler.Forget(rateLimiterCallback, name) } + // TODO: use the errors.Join from + // github.com/crossplane/crossplane-runtime. + if ok := ac.eventHandler.RequestReconcile(rateLimiter, name, nil); !ok { + return errors.Errorf(errReconcileRequestFmt, tr.GetObjectKind().GroupVersionKind().String(), name, op) + } } return uErr } @@ -141,28 +143,26 @@ func (ac *APICallbacks) callbackFn(name, op string, requeue bool) terraform.Call // Create makes sure the error is saved in async operation condition. func (ac *APICallbacks) Create(name string) terraform.CallbackFn { - return func(err error, ctx context.Context) error { - // requeue is set to true although the managed reconciler already - // requeues with exponential back-off during the creation phase - // because the upjet external client returns ResourceExists & - // ResourceUpToDate both set to true, if an async operation is - // in-progress immediately following a Create call. This will - // delay a reobservation of the resource (while being created) - // for the poll period. - return ac.callbackFn(name, "create", true)(err, ctx) - } + // request will be requeued although the managed reconciler already + // requeues with exponential back-off during the creation phase + // because the upjet external client returns ResourceExists & + // ResourceUpToDate both set to true, if an async operation is + // in-progress immediately following a Create call. This will + // delay a reobservation of the resource (while being created) + // for the poll period. + return ac.callbackFn(name, "create") } // Update makes sure the error is saved in async operation condition. func (ac *APICallbacks) Update(name string) terraform.CallbackFn { - return func(err error, ctx context.Context) error { - return ac.callbackFn(name, "update", true)(err, ctx) - } + return ac.callbackFn(name, "update") } // Destroy makes sure the error is saved in async operation condition. func (ac *APICallbacks) Destroy(name string) terraform.CallbackFn { - // requeue is set to false because the managed reconciler already requeues - // with exponential back-off during the deletion phase. - return ac.callbackFn(name, "destroy", false) + // request will be requeued although the managed reconciler requeues + // with exponential back-off during the deletion phase because + // during the async deletion operation, external client's + // observe just returns success to the managed reconciler. + return ac.callbackFn(name, "destroy") } diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index d996b23a..fb800a83 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -124,13 +124,13 @@ func (n *noForkAsyncExternal) Create(_ context.Context, mg xpresource.Managed) ( ctx, cancel := context.WithDeadline(context.Background(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() - defer n.opTracker.LastOperation.MarkEnd() n.opTracker.logger.Debug("Async create starting...", "tfID", n.opTracker.GetTfID()) _, err := n.noForkExternal.Create(ctx, mg) n.opTracker.LastOperation.SetError(errors.Wrap(err, "async create failed")) n.opTracker.logger.Debug("Async create ended.", "error", err, "tfID", n.opTracker.GetTfID()) + n.opTracker.LastOperation.MarkEnd() if cErr := n.callback.Create(mg.GetName())(err, ctx); cErr != nil { n.opTracker.logger.Info("Async create callback failed", "error", cErr.Error()) } @@ -147,13 +147,13 @@ func (n *noForkAsyncExternal) Update(_ context.Context, mg xpresource.Managed) ( ctx, cancel := context.WithDeadline(context.Background(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() - defer n.opTracker.LastOperation.MarkEnd() n.opTracker.logger.Debug("Async update starting...", "tfID", n.opTracker.GetTfID()) _, err := n.noForkExternal.Update(ctx, mg) n.opTracker.LastOperation.SetError(errors.Wrap(err, "async update failed")) n.opTracker.logger.Debug("Async update ended.", "error", err, "tfID", n.opTracker.GetTfID()) + n.opTracker.LastOperation.MarkEnd() if cErr := n.callback.Update(mg.GetName())(err, ctx); cErr != nil { n.opTracker.logger.Info("Async update callback failed", "error", cErr.Error()) } @@ -170,13 +170,13 @@ func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) ctx, cancel := context.WithDeadline(context.Background(), n.opTracker.LastOperation.StartTime().Add(defaultAsyncTimeout)) go func() { defer cancel() - defer n.opTracker.LastOperation.MarkEnd() n.opTracker.logger.Debug("Async delete starting...", "tfID", n.opTracker.GetTfID()) err := n.noForkExternal.Delete(ctx, mg) n.opTracker.LastOperation.SetError(errors.Wrap(err, "async delete failed")) n.opTracker.logger.Debug("Async delete ended.", "error", err, "tfID", n.opTracker.GetTfID()) + n.opTracker.LastOperation.MarkEnd() if cErr := n.callback.Destroy(mg.GetName())(err, ctx); cErr != nil { n.opTracker.logger.Info("Async delete callback failed", "error", cErr.Error()) } diff --git a/pkg/controller/handler/eventhandler.go b/pkg/controller/handler/eventhandler.go index 734bc1b6..7d0c31f9 100644 --- a/pkg/controller/handler/eventhandler.go +++ b/pkg/controller/handler/eventhandler.go @@ -7,6 +7,7 @@ package handler import ( "context" "sync" + "time" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" @@ -17,6 +18,8 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/logging" ) +const NoRateLimiter = "" + // EventHandler handles Kubernetes events by queueing reconcile requests for // objects and allows upjet components to queue reconcile requests. type EventHandler struct { @@ -64,16 +67,19 @@ func (e *EventHandler) RequestReconcile(rateLimiterName, name string, failureLim Name: name, }, } - rateLimiter := e.rateLimiterMap[rateLimiterName] - if rateLimiter == nil { - rateLimiter = workqueue.DefaultControllerRateLimiter() - e.rateLimiterMap[rateLimiterName] = rateLimiter - } - if failureLimit != nil && rateLimiter.NumRequeues(item) > *failureLimit { - logger.Info("Failure limit has been exceeded.", "failureLimit", *failureLimit, "numRequeues", rateLimiter.NumRequeues(item)) - return false + var when time.Duration = 0 + if rateLimiterName != NoRateLimiter { + rateLimiter := e.rateLimiterMap[rateLimiterName] + if rateLimiter == nil { + rateLimiter = workqueue.DefaultControllerRateLimiter() + e.rateLimiterMap[rateLimiterName] = rateLimiter + } + if failureLimit != nil && rateLimiter.NumRequeues(item) > *failureLimit { + logger.Info("Failure limit has been exceeded.", "failureLimit", *failureLimit, "numRequeues", rateLimiter.NumRequeues(item)) + return false + } + when = rateLimiter.When(item) } - when := rateLimiter.When(item) e.queue.AddAfter(item, when) logger.Debug("Reconcile request has been requeued.", "rateLimiterName", rateLimiterName, "when", when) return true From 5cc8250c4c4f997de6ee7c591d9af20423621958 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 24 Oct 2023 20:23:53 +0300 Subject: [PATCH 32/60] Add support for logically deleting MRs Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 9 +++++++++ pkg/controller/nofork_store.go | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 2f29a0ad..5dded73f 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -236,6 +236,13 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { n.logger.Debug("Observing the external resource") + + if meta.WasDeleted(mg) && n.opTracker.IsDeleted() { + return managed.ExternalObservation{ + ResourceExists: false, + }, nil + } + start := time.Now() newState, diag := n.resourceSchema.RefreshWithoutUpgrade(ctx, n.opTracker.GetTfState(), n.ts.Meta) metrics.ExternalAPITime.WithLabelValues("read").Observe(time.Since(start).Seconds()) @@ -422,6 +429,8 @@ func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error return errors.Errorf("failed to delete the resource: %v", diag) } n.opTracker.SetTfState(newState) + // mark the resource as logically deleted if the TF call clears the state + n.opTracker.SetDeleted(newState == nil) return nil } diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index 2c730241..4b95b827 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -2,6 +2,7 @@ package controller import ( "sync" + "sync/atomic" "github.com/crossplane/crossplane-runtime/pkg/logging" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" @@ -18,6 +19,16 @@ type AsyncTracker struct { mu *sync.Mutex tfID string tfState *tfsdk.InstanceState + // lifecycle of certain external resources are bound to a parent resource's + // lifecycle, and they cannot be deleted without actually deleting + // the owning external resource (e.g., a database resource as the parent + // resource and a database configuration resource whose lifecycle is bound + // to it. For such resources, Terraform still removes the state for them + // after a successful delete call either by resetting to some defaults in + // the parent resource, or by a noop. We logically delete such resources as + // deleted after a successful delete call so that the next observe can + // tell the managed reconciler that the resource no longer "exists". + isDeleted atomic.Bool } type AsyncTrackerOption func(manager *AsyncTracker) @@ -73,6 +84,18 @@ func (a *AsyncTracker) SetTfID(tfId string) { a.tfID = tfId } +// IsDeleted returns whether the associated external resource +// has logically been deleted. +func (a *AsyncTracker) IsDeleted() bool { + return a.isDeleted.Load() +} + +// SetDeleted sets the logical deletion status of +// the associated external resource. +func (a *AsyncTracker) SetDeleted(deleted bool) { + a.isDeleted.Store(deleted) +} + type OperationTrackerStore struct { store map[types.UID]*AsyncTracker logger logging.Logger From 4835bbe3ffb4a906f6ecf194c6918faa551d8a93 Mon Sep 17 00:00:00 2001 From: Cem Mergenci Date: Thu, 19 Oct 2023 18:44:22 +0300 Subject: [PATCH 33/60] Partially support management policies in no-fork external client. Currently updates to `initProvider` after resource creation take effect, which is against specification. Next step is to fix it. Signed-off-by: Cem Mergenci Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 8 ++++++ pkg/controller/external_nofork.go | 24 ++++++++++++----- pkg/pipeline/templates/controller.go.tmpl | 15 ++++++++--- pkg/pipeline/templates/terraformed.go.tmpl | 31 ++++++++++++++++++++++ pkg/resource/interfaces.go | 1 + 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index fb800a83..f466fb51 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -95,6 +95,14 @@ func WithNoForkAsyncMetricRecorder(r *metrics.MetricRecorder) NoForkAsyncOption } } +// WithNoForkAsyncManagementPolicies configures whether the client should +// handle management policies. +func WithNoForkAsyncManagementPolicies(isManagementPoliciesEnabled bool) NoForkAsyncOption { + return func(c *NoForkAsyncConnector) { + c.isManagementPoliciesEnabled = isManagementPoliciesEnabled + } +} + type noForkAsyncExternal struct { *noForkExternal callback CallbackProvider diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 5dded73f..23caa5cd 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -39,12 +39,13 @@ import ( ) type NoForkConnector struct { - getTerraformSetup terraform.SetupFn - kube client.Client - config *config.Resource - logger logging.Logger - metricRecorder *metrics.MetricRecorder - operationTrackerStore *OperationTrackerStore + getTerraformSetup terraform.SetupFn + kube client.Client + config *config.Resource + logger logging.Logger + metricRecorder *metrics.MetricRecorder + operationTrackerStore *OperationTrackerStore + isManagementPoliciesEnabled bool } // NoForkOption allows you to configure NoForkConnector. @@ -65,6 +66,14 @@ func WithNoForkMetricRecorder(r *metrics.MetricRecorder) NoForkOption { } } +// WithNoForkManagementPolicies configures whether the client should +// handle management policies. +func WithNoForkManagementPolicies(isManagementPoliciesEnabled bool) NoForkOption { + return func(c *NoForkConnector) { + c.isManagementPoliciesEnabled = isManagementPoliciesEnabled + } +} + func NewNoForkConnector(kube client.Client, sf terraform.SetupFn, cfg *config.Resource, ots *OperationTrackerStore, opts ...NoForkOption) *NoForkConnector { nfc := &NoForkConnector{ kube: kube, @@ -132,7 +141,8 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m if externalName == "" { externalName = opTracker.GetTfID() } - params, err := tr.GetParameters() + + params, err := tr.GetMergedParameters(c.isManagementPoliciesEnabled) if err != nil { return nil, errors.Wrap(err, "cannot get parameters") } diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index b8ff620e..2df425e5 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -52,10 +52,19 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { tjcontroller.WithNoForkAsyncLogger(o.Logger), tjcontroller.WithNoForkAsyncConnectorEventHandler(eventHandler), tjcontroller.WithNoForkAsyncCallbackProvider(ac), - tjcontroller.WithNoForkAsyncMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) + tjcontroller.WithNoForkAsyncMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval)), + {{if .FeaturesPackageAlias -}} + tjcontroller.WithNoForkAsyncManagementPolicies(o.Features.Enabled({{ .FeaturesPackageAlias }}EnableAlphaManagementPolicies)) + {{- end -}} + ) {{- else -}} - tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], o.OperationTrackerStore, tjcontroller.WithNoForkLogger(o.Logger), - tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval))) + tjcontroller.NewNoForkConnector(mgr.GetClient(), o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], o.OperationTrackerStore, + tjcontroller.WithNoForkLogger(o.Logger), + tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval)), + {{if .FeaturesPackageAlias -}} + tjcontroller.WithNoForkManagementPolicies(o.Features.Enabled({{ .FeaturesPackageAlias }}EnableAlphaManagementPolicies)) + {{- end -}} + ) {{- end -}} {{- else -}} tjcontroller.NewConnector(mgr.GetClient(), o.WorkspaceStore, o.SetupFn, o.Provider.Resources["{{ .ResourceType }}"], tjcontroller.WithLogger(o.Logger), tjcontroller.WithConnectorEventHandler(eventHandler), diff --git a/pkg/pipeline/templates/terraformed.go.tmpl b/pkg/pipeline/templates/terraformed.go.tmpl index ee503fc4..7b1f561d 100644 --- a/pkg/pipeline/templates/terraformed.go.tmpl +++ b/pkg/pipeline/templates/terraformed.go.tmpl @@ -10,6 +10,7 @@ package {{ .APIVersion }} import ( "github.com/pkg/errors" + "dario.cat/mergo" "github.com/crossplane/upjet/pkg/resource" "github.com/crossplane/upjet/pkg/resource/json" @@ -86,6 +87,36 @@ import ( return base, json.TFParser.Unmarshal(p, &base) } + // GetInitParameters of this {{ .CRD.Kind }} + func (tr *{{ .CRD.Kind }}) GetMergedParameters(isBetaManagementPoliciesEnabled bool) (map[string]any, error) { + params, err := tr.GetParameters() + if err != nil { + return nil, errors.Wrapf(err, "cannot get parameters for resource '%q'", tr.GetName()) + } + if !isBetaManagementPoliciesEnabled { + return params, nil + } + + initParams, err := tr.GetInitParameters() + if err != nil { + return nil, errors.Wrapf(err, "cannot get init parameters for resource '%q'", tr.GetName()) + } + + // Note(lsviben): mergo.WithSliceDeepCopy is needed to merge the + // slices from the initProvider to forProvider. As it also sets + // overwrite to true, we need to set it back to false, we don't + // want to overwrite the forProvider fields with the initProvider + // fields. + err = mergo.Merge(¶ms, initParams, mergo.WithSliceDeepCopy, func(c *mergo.Config) { + c.Overwrite = false + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot merge spec.initProvider and spec.forProvider parameters for resource '%q'", tr.GetName()) + } + + return params, nil + } + // LateInitialize this {{ .CRD.Kind }} using its observed tfState. // returns True if there are any spec changes for the resource. func (tr *{{ .CRD.Kind }}) LateInitialize(attrs []byte) (bool, error) { diff --git a/pkg/resource/interfaces.go b/pkg/resource/interfaces.go index 2c68ea1d..c68d5263 100644 --- a/pkg/resource/interfaces.go +++ b/pkg/resource/interfaces.go @@ -21,6 +21,7 @@ type Parameterizable interface { GetParameters() (map[string]any, error) SetParameters(map[string]any) error GetInitParameters() (map[string]any, error) + GetMergedParameters(isBetaManagementPoliciesEnabled bool) (map[string]any, error) } // MetadataProvider provides Terraform metadata for the Terraform managed From e3a7c59a7c0ed589e47dd7f7ba67f78d074dd818 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 25 Oct 2023 11:23:19 +0300 Subject: [PATCH 34/60] Reviews for the management policies first phase Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 65 ++++++++++++---------- pkg/pipeline/templates/terraformed.go.tmpl | 6 +- pkg/resource/interfaces.go | 2 +- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 23caa5cd..2786cdbf 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -114,7 +114,6 @@ type noForkExternal struct { ts terraform.Setup resourceSchema *schema.Resource config *config.Resource - kube client.Client instanceDiff *tf.InstanceDiff params map[string]any rawConfig cty.Value @@ -123,6 +122,37 @@ type noForkExternal struct { opTracker *AsyncTracker } +func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externalName string, config *config.Resource, ts terraform.Setup, initParamsMerged bool, kube client.Client) (map[string]any, error) { + params, err := tr.GetMergedParameters(initParamsMerged) + if err != nil { + return nil, errors.Wrap(err, "cannot get merged parameters") + } + if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { + return nil, errors.Wrap(err, "cannot store sensitive parameters into params") + } + config.ExternalName.SetIdentifierArgumentFn(params, externalName) + if config.TerraformConfigurationInjector != nil { + m, err := getJSONMap(tr) + if err != nil { + return nil, errors.Wrap(err, "cannot get JSON map for the managed resource's spec.forProvider value") + } + config.TerraformConfigurationInjector(m, params) + } + + tfID, err := config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) + if err != nil { + return nil, errors.Wrap(err, "cannot get ID") + } + params["id"] = tfID + // we need to parameterize the following for a provider + // not all providers may have this attribute + // TODO: tags_all handling + if _, ok := config.TerraformResource.CoreConfigSchema().Attributes["tags_all"]; ok { + params["tags_all"] = params["tags"] + } + return params, nil +} + func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) logger := c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()) @@ -142,36 +172,12 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m externalName = opTracker.GetTfID() } - params, err := tr.GetMergedParameters(c.isManagementPoliciesEnabled) + params, err := getExtendedParameters(ctx, tr, externalName, c.config, ts, c.isManagementPoliciesEnabled, c.kube) if err != nil { - return nil, errors.Wrap(err, "cannot get parameters") - } - if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: c.kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { - return nil, errors.Wrap(err, "cannot store sensitive parameters into params") - } - c.config.ExternalName.SetIdentifierArgumentFn(params, externalName) - if c.config.TerraformConfigurationInjector != nil { - m, err := getJSONMap(mg) - if err != nil { - return nil, errors.Wrap(err, "cannot get JSON map for the managed resource's spec.forProvider value") - } - c.config.TerraformConfigurationInjector(m, params) + return nil, errors.Wrapf(err, "failed to get the extended parameters for resource %q", mg.GetName()) } - tfID, err := c.config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) - if err != nil { - return nil, errors.Wrap(err, "cannot get ID") - } - params["id"] = tfID - // we need to parameterize the following for a provider - // not all providers may have this attribute - // TODO: tags_all handling schemaBlock := c.config.TerraformResource.CoreConfigSchema() - attrs := schemaBlock.Attributes - if _, ok := attrs["tags_all"]; ok { - params["tags_all"] = params["tags"] - } - rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) if err != nil { return nil, errors.Wrap(err, "failed to convert params JSON map to cty.Value") @@ -187,7 +193,7 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m return nil, errors.Wrap(err, "cannot store sensitive parameters into tfState") } c.config.ExternalName.SetIdentifierArgumentFn(tfState, externalName) - tfState["id"] = tfID + tfState["id"] = params["id"] if copyParams { tfState = copyParameters(tfState, params) } @@ -209,7 +215,6 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m ts: ts, resourceSchema: c.config.TerraformResource, config: c.config, - kube: c.kube, params: params, rawConfig: rawConfig, logger: logger, @@ -237,7 +242,7 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } instanceDiff.RawPlan = v } - if instanceDiff != nil && len(instanceDiff.Attributes) > 0 { + if instanceDiff != nil && !instanceDiff.Empty() { n.logger.Debug("Diff detected", "instanceDiff", instanceDiff.GoString()) instanceDiff.RawConfig = n.rawConfig } diff --git a/pkg/pipeline/templates/terraformed.go.tmpl b/pkg/pipeline/templates/terraformed.go.tmpl index 7b1f561d..4681e523 100644 --- a/pkg/pipeline/templates/terraformed.go.tmpl +++ b/pkg/pipeline/templates/terraformed.go.tmpl @@ -9,8 +9,8 @@ package {{ .APIVersion }} import ( - "github.com/pkg/errors" "dario.cat/mergo" + "github.com/pkg/errors" "github.com/crossplane/upjet/pkg/resource" "github.com/crossplane/upjet/pkg/resource/json" @@ -88,12 +88,12 @@ import ( } // GetInitParameters of this {{ .CRD.Kind }} - func (tr *{{ .CRD.Kind }}) GetMergedParameters(isBetaManagementPoliciesEnabled bool) (map[string]any, error) { + func (tr *{{ .CRD.Kind }}) GetMergedParameters(shouldMergeInitProvider bool) (map[string]any, error) { params, err := tr.GetParameters() if err != nil { return nil, errors.Wrapf(err, "cannot get parameters for resource '%q'", tr.GetName()) } - if !isBetaManagementPoliciesEnabled { + if !shouldMergeInitProvider { return params, nil } diff --git a/pkg/resource/interfaces.go b/pkg/resource/interfaces.go index c68d5263..6243181e 100644 --- a/pkg/resource/interfaces.go +++ b/pkg/resource/interfaces.go @@ -21,7 +21,7 @@ type Parameterizable interface { GetParameters() (map[string]any, error) SetParameters(map[string]any) error GetInitParameters() (map[string]any, error) - GetMergedParameters(isBetaManagementPoliciesEnabled bool) (map[string]any, error) + GetMergedParameters(shouldMergeInitProvider bool) (map[string]any, error) } // MetadataProvider provides Terraform metadata for the Terraform managed From ff9c3540051a7638a9fb6f8785e8be925979e919 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 26 Oct 2023 17:07:57 +0300 Subject: [PATCH 35/60] Remove remaining upbound/upjet module references Signed-off-by: Alper Rifat Ulucinar --- go.mod | 1 - go.sum | 2 -- pkg/controller/external_async_nofork.go | 8 ++++---- pkg/controller/external_nofork.go | 10 +++++----- pkg/controller/nofork_store.go | 4 ++-- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index ed4d2938..57575eb6 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/spf13/afero v1.10.0 github.com/tmccombs/hcl2json v0.3.3 - github.com/upbound/upjet v0.10.0 github.com/yuin/goldmark v1.4.13 github.com/zclconf/go-cty v1.11.0 golang.org/x/net v0.15.0 diff --git a/go.sum b/go.sum index 3ce11661..c8cf1cdf 100644 --- a/go.sum +++ b/go.sum @@ -339,8 +339,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= -github.com/upbound/upjet v0.10.0 h1:6nxc0GUBcL4BDHxQUUZWjw4ROXu5KRK9jOpb7LeJ+NQ= -github.com/upbound/upjet v0.10.0/go.mod h1:2RXHgpIugCL/S/Use1QJAeVaev901RBeUByQh5gUtGk= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index f466fb51..f3f828b2 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -24,10 +24,10 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/upbound/upjet/pkg/config" - "github.com/upbound/upjet/pkg/controller/handler" - "github.com/upbound/upjet/pkg/metrics" - "github.com/upbound/upjet/pkg/terraform" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/controller/handler" + "github.com/crossplane/upjet/pkg/metrics" + "github.com/crossplane/upjet/pkg/terraform" ) var defaultAsyncTimeout = 1 * time.Hour diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 2786cdbf..056a2abe 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -31,11 +31,11 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/upbound/upjet/pkg/config" - "github.com/upbound/upjet/pkg/metrics" - "github.com/upbound/upjet/pkg/resource" - "github.com/upbound/upjet/pkg/resource/json" - "github.com/upbound/upjet/pkg/terraform" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/metrics" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/resource/json" + "github.com/crossplane/upjet/pkg/terraform" ) type NoForkConnector struct { diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index 4b95b827..e8a856bc 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -9,8 +9,8 @@ import ( tfsdk "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "k8s.io/apimachinery/pkg/types" - "github.com/upbound/upjet/pkg/resource" - "github.com/upbound/upjet/pkg/terraform" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/terraform" ) type AsyncTracker struct { From c89008b7ee9dea38d08604a3af25a7d5db00e209 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 26 Oct 2023 17:12:29 +0300 Subject: [PATCH 36/60] Fix tests Signed-off-by: Alper Rifat Ulucinar --- pkg/config/common_test.go | 88 +++++++++++++++++--------------- pkg/resource/fake/terraformed.go | 4 ++ 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/pkg/config/common_test.go b/pkg/config/common_test.go index db3fe5e5..2f9634bc 100644 --- a/pkg/config/common_test.go +++ b/pkg/config/common_test.go @@ -7,10 +7,11 @@ package config import ( "testing" - "github.com/crossplane/upjet/pkg/registry" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + + "github.com/crossplane/upjet/pkg/registry" ) func TestDefaultResource(t *testing.T) { @@ -32,14 +33,15 @@ func TestDefaultResource(t *testing.T) { name: "aws_ec2_instance", }, want: &Resource{ - Name: "aws_ec2_instance", - ShortGroup: "ec2", - Kind: "Instance", - Version: "v1alpha1", - ExternalName: NameAsIdentifier, - References: map[string]Reference{}, - Sensitive: NopSensitive, - UseAsync: true, + Name: "aws_ec2_instance", + ShortGroup: "ec2", + Kind: "Instance", + Version: "v1alpha1", + ExternalName: NameAsIdentifier, + References: map[string]Reference{}, + Sensitive: NopSensitive, + UseAsync: true, + SchemaElementOptions: SchemaElementOptions{}, }, }, "TwoSectionsName": { @@ -48,14 +50,15 @@ func TestDefaultResource(t *testing.T) { name: "aws_instance", }, want: &Resource{ - Name: "aws_instance", - ShortGroup: "aws", - Kind: "Instance", - Version: "v1alpha1", - ExternalName: NameAsIdentifier, - References: map[string]Reference{}, - Sensitive: NopSensitive, - UseAsync: true, + Name: "aws_instance", + ShortGroup: "aws", + Kind: "Instance", + Version: "v1alpha1", + ExternalName: NameAsIdentifier, + References: map[string]Reference{}, + Sensitive: NopSensitive, + UseAsync: true, + SchemaElementOptions: SchemaElementOptions{}, }, }, "NameWithPrefixAcronym": { @@ -64,14 +67,15 @@ func TestDefaultResource(t *testing.T) { name: "aws_db_sql_server", }, want: &Resource{ - Name: "aws_db_sql_server", - ShortGroup: "db", - Kind: "SQLServer", - Version: "v1alpha1", - ExternalName: NameAsIdentifier, - References: map[string]Reference{}, - Sensitive: NopSensitive, - UseAsync: true, + Name: "aws_db_sql_server", + ShortGroup: "db", + Kind: "SQLServer", + Version: "v1alpha1", + ExternalName: NameAsIdentifier, + References: map[string]Reference{}, + Sensitive: NopSensitive, + UseAsync: true, + SchemaElementOptions: SchemaElementOptions{}, }, }, "NameWithSuffixAcronym": { @@ -80,14 +84,15 @@ func TestDefaultResource(t *testing.T) { name: "aws_db_server_id", }, want: &Resource{ - Name: "aws_db_server_id", - ShortGroup: "db", - Kind: "ServerID", - Version: "v1alpha1", - ExternalName: NameAsIdentifier, - References: map[string]Reference{}, - Sensitive: NopSensitive, - UseAsync: true, + Name: "aws_db_server_id", + ShortGroup: "db", + Kind: "ServerID", + Version: "v1alpha1", + ExternalName: NameAsIdentifier, + References: map[string]Reference{}, + Sensitive: NopSensitive, + UseAsync: true, + SchemaElementOptions: SchemaElementOptions{}, }, }, "NameWithMultipleAcronyms": { @@ -96,14 +101,15 @@ func TestDefaultResource(t *testing.T) { name: "aws_db_sql_server_id", }, want: &Resource{ - Name: "aws_db_sql_server_id", - ShortGroup: "db", - Kind: "SQLServerID", - Version: "v1alpha1", - ExternalName: NameAsIdentifier, - References: map[string]Reference{}, - Sensitive: NopSensitive, - UseAsync: true, + Name: "aws_db_sql_server_id", + ShortGroup: "db", + Kind: "SQLServerID", + Version: "v1alpha1", + ExternalName: NameAsIdentifier, + References: map[string]Reference{}, + Sensitive: NopSensitive, + UseAsync: true, + SchemaElementOptions: SchemaElementOptions{}, }, }, } diff --git a/pkg/resource/fake/terraformed.go b/pkg/resource/fake/terraformed.go index 857788b8..6c97732e 100644 --- a/pkg/resource/fake/terraformed.go +++ b/pkg/resource/fake/terraformed.go @@ -46,6 +46,10 @@ type Parameterizable struct { InitParameters map[string]any } +func (t *Terraformed) GetMergedParameters(_ bool) (map[string]any, error) { + return t.Parameters, nil +} + // GetParameters is a mock. func (p *Parameterizable) GetParameters() (map[string]any, error) { return p.Parameters, nil From 6ca88de6dc9d6cc6204fd48abf380ee21b82604c Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 26 Oct 2023 18:46:20 +0300 Subject: [PATCH 37/60] Switch from alpha to beta management policisies in controller.go.tmpl Signed-off-by: Alper Rifat Ulucinar --- pkg/pipeline/templates/controller.go.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index 2df425e5..8c7cd5fe 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -54,7 +54,7 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { tjcontroller.WithNoForkAsyncCallbackProvider(ac), tjcontroller.WithNoForkAsyncMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval)), {{if .FeaturesPackageAlias -}} - tjcontroller.WithNoForkAsyncManagementPolicies(o.Features.Enabled({{ .FeaturesPackageAlias }}EnableAlphaManagementPolicies)) + tjcontroller.WithNoForkAsyncManagementPolicies(o.Features.Enabled({{ .FeaturesPackageAlias }}EnableBetaManagementPolicies)) {{- end -}} ) {{- else -}} @@ -62,7 +62,7 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { tjcontroller.WithNoForkLogger(o.Logger), tjcontroller.WithNoForkMetricRecorder(metrics.NewMetricRecorder({{ .TypePackageAlias }}{{ .CRD.Kind }}_GroupVersionKind, mgr, o.PollInterval)), {{if .FeaturesPackageAlias -}} - tjcontroller.WithNoForkManagementPolicies(o.Features.Enabled({{ .FeaturesPackageAlias }}EnableAlphaManagementPolicies)) + tjcontroller.WithNoForkManagementPolicies(o.Features.Enabled({{ .FeaturesPackageAlias }}EnableBetaManagementPolicies)) {{- end -}} ) {{- end -}} From d72fc1e60c5640a2bb8db1d7df858b6bceee6b7f Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Mon, 30 Oct 2023 13:46:24 +0300 Subject: [PATCH 38/60] Add support for hybrid CLI-based reconciling for configured resources - Add config.Provider.WithNoForkIncludeList to explicitly specify the set of resources to be reconciled under the no-fork architecture. Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 65 +++++++++++++++++++++++++++++++++++--- pkg/config/resource.go | 17 +++++++--- pkg/pipeline/controller.go | 7 ++-- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 6e406bb5..c72dc4b7 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -8,10 +8,12 @@ import ( "fmt" "regexp" + tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/pkg/errors" "github.com/crossplane/upjet/pkg/registry" + conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" ) // ResourceConfiguratorFn is a function that implements the ResourceConfigurator @@ -106,12 +108,22 @@ type Provider struct { skippedResourceNames []string // IncludeList is a list of regex for the Terraform resources to be - // included. For example, to include "aws_shield_protection_group" into + // included and reconciled via the Terraform CLI. + // For example, to include "aws_shield_protection_group" into // the generated resources, one can add "aws_shield_protection_group$". // To include whole aws waf group, one can add "aws_waf.*" to the list. // Defaults to []string{".+"} which would include all resources. IncludeList []string + // NoForkIncludeList is a list of regex for the Terraform resources to be + // included and reconciled in the no-fork architecture (without the + // Terraform CLI). + // For example, to include "aws_shield_protection_group" into + // the generated resources, one can add "aws_shield_protection_group$". + // To include whole aws waf group, one can add "aws_waf.*" to the list. + // Defaults to []string{".+"} which would include all resources. + NoForkIncludeList []string + // Resources is a map holding resource configurations where key is Terraform // resource name. Resources map[string]*Resource @@ -158,6 +170,20 @@ func WithIncludeList(l []string) ProviderOption { } } +// WithNoForkIncludeList configures IncludeList for this Provider. +func WithNoForkIncludeList(l []string) ProviderOption { + return func(p *Provider) { + p.NoForkIncludeList = l + } +} + +// WithTerraformProvider configures the TerraformProvider for this Provider. +func WithTerraformProvider(tp *schema.Provider) ProviderOption { + return func(p *Provider) { + p.TerraformProvider = tp + } +} + // WithSkipList configures SkipList for this Provider. func WithSkipList(l []string) ProviderOption { return func(p *Provider) { @@ -202,8 +228,23 @@ func WithMainTemplate(template string) ProviderOption { } } -// NewProvider builds and returns a new Provider from provider native schema. -func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo +// NewProvider builds and returns a new Provider from provider +// tfjson schema, that is generated using Terraform CLI with: +// `terraform providers schema --json` +func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo + ps := tfjson.ProviderSchemas{} + if err := ps.UnmarshalJSON(schema); err != nil { + panic(err) + } + if len(ps.Schemas) != 1 { + panic(fmt.Sprintf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas))) + } + var rs map[string]*tfjson.Schema + for _, v := range ps.Schemas { + rs = v.ResourceSchemas + break + } + resourceMap := conversiontfjson.GetV2ResourceMap(rs) providerMetadata, err := registry.NewProviderMetadataFromFile(metadata) if err != nil { panic(errors.Wrap(err, "cannot load provider metadata")) @@ -233,11 +274,27 @@ func NewProvider(resourceMap map[string]*schema.Resource, prefix string, moduleP // There are resources with no schema, that we will address later. fmt.Printf("Skipping resource %s because it has no schema\n", name) } - if len(terraformResource.Schema) == 0 || matches(name, p.SkipList) || !matches(name, p.IncludeList) { + // if in both of the include lists, the new behavior prevails + isNoFork := matches(name, p.NoForkIncludeList) + if len(terraformResource.Schema) == 0 || matches(name, p.SkipList) || (!matches(name, p.IncludeList) && !isNoFork) { p.skippedResourceNames = append(p.skippedResourceNames, name) continue } + if isNoFork { + if p.TerraformProvider == nil || p.TerraformProvider.ResourcesMap[name] == nil { + panic(errors.Errorf("resource %q is configured to be reconciled without the Terraform CLI"+ + "but either config.Provider.TerraformProvider is not configured or the Go schema does not exist for the resource", name)) + } + terraformResource = p.TerraformProvider.ResourcesMap[name] + // TODO: we will need to bump the terraform-plugin-sdk dependency to handle + // schema.Resource.SchemaFunc + if terraformResource.Schema == nil { + p.skippedResourceNames = append(p.skippedResourceNames, name) + continue + } + } p.Resources[name] = DefaultResource(name, terraformResource, providerMetadata.Resources[name], p.DefaultResourceOptions...) + p.Resources[name].useNoForkClient = isNoFork } for i, refInjector := range p.refInjectors { if err := refInjector.InjectReferences(p.Resources); err != nil { diff --git a/pkg/config/resource.go b/pkg/config/resource.go index dd5e5aeb..14b31ac2 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -9,7 +9,6 @@ import ( "fmt" "time" - "github.com/crossplane/upjet/pkg/registry" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" @@ -22,6 +21,8 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + + "github.com/crossplane/upjet/pkg/registry" ) // SetIdentifierArgumentsFn sets the name of the resource in Terraform attributes map, @@ -294,10 +295,8 @@ type Resource struct { // databases. UseAsync bool - // UseNoForkClient indicates that a no-fork external client should - // be generated instead of the Terraform CLI-forking client. - UseNoForkClient bool - + // InitializerFns specifies the initializer functions to be used + // for this Resource. InitializerFns []NewInitializerFn // OperationTimeouts allows configuring resource operation timeouts. @@ -339,6 +338,14 @@ type Resource struct { // TerraformCustomDiff allows a resource.Terraformed to customize how its // Terraform InstanceDiff is computed during reconciliation. TerraformCustomDiff CustomDiff + + // useNoForkClient indicates that a no-fork external client should + // be generated instead of the Terraform CLI-forking client. + useNoForkClient bool +} + +func (r *Resource) ShouldUseNoForkClient() bool { + return r.useNoForkClient } // CustomDiff customizes the computed Terraform InstanceDiff. This can be used diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 8305e951..40a7c2d0 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -9,10 +9,11 @@ import ( "path/filepath" "strings" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/pipeline/templates" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/pipeline/templates" ) // NewControllerGenerator returns a new ControllerGenerator. @@ -49,7 +50,7 @@ func (cg *ControllerGenerator) Generate(cfg *config.Resource, typesPkgPath strin "DisableNameInitializer": cfg.ExternalName.DisableNameInitializer, "TypePackageAlias": ctrlFile.Imports.UsePackage(typesPkgPath), "UseAsync": cfg.UseAsync, - "UseNoForkClient": cfg.UseNoForkClient, + "UseNoForkClient": cfg.ShouldUseNoForkClient(), "ResourceType": cfg.Name, "Initializers": cfg.InitializerFns, } From 8016470229679c88bf1f502f3cd1e656eac4a641 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Mon, 30 Oct 2023 16:03:18 +0300 Subject: [PATCH 39/60] apply state functions to MR spec parameters Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 55 +++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 056a2abe..71018015 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -153,6 +153,57 @@ func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externa return params, nil } +func (c *NoForkConnector) processParamsWithStateFunc(schemaMap map[string]*schema.Schema, params map[string]any) map[string]any { + for key, param := range params { + if sc, ok := schemaMap[key]; ok { + params[key] = c.applyStateFuncToParam(sc, param) + } else { + params[key] = param + } + } + return params +} + +func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) any { + switch sc.Type { + case schema.TypeMap: + if sc.Elem == nil { + return param + } + // TypeMap only supports schema in Elem + if mapSchema, ok := sc.Elem.(*schema.Schema); ok { + pmap := param.(map[string]any) + for pk, pv := range pmap { + pmap[pk] = c.applyStateFuncToParam(mapSchema, pv) + } + return pmap + } + case schema.TypeSet, schema.TypeList: + pArray := param.([]any) + if setSchema, ok := sc.Elem.(*schema.Schema); ok { + for i, p := range pArray { + pArray[i] = c.applyStateFuncToParam(setSchema, p) + } + return pArray + } else if setResource, ok := sc.Elem.(*schema.Resource); ok { + for i, p := range pArray { + resParam := p.(map[string]any) + pArray[i] = c.processParamsWithStateFunc(setResource.Schema, resParam) + } + } + case schema.TypeBool, schema.TypeInt, schema.TypeFloat, schema.TypeString: + if sc.StateFunc != nil { + return sc.StateFunc(param) + } + return param + case schema.TypeInvalid: + return param + default: + return param + } + return param +} + func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (managed.ExternalClient, error) { c.metricRecorder.ObserveReconcileDelay(mg.GetObjectKind().GroupVersionKind(), mg.GetName()) logger := c.logger.WithValues("uid", mg.GetUID(), "name", mg.GetName(), "gvk", mg.GetObjectKind().GroupVersionKind().String()) @@ -168,14 +219,12 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m tr := mg.(resource.Terraformed) opTracker := c.operationTrackerStore.Tracker(tr) externalName := meta.GetExternalName(tr) - if externalName == "" { - externalName = opTracker.GetTfID() - } params, err := getExtendedParameters(ctx, tr, externalName, c.config, ts, c.isManagementPoliciesEnabled, c.kube) if err != nil { return nil, errors.Wrapf(err, "failed to get the extended parameters for resource %q", mg.GetName()) } + params = c.processParamsWithStateFunc(c.config.TerraformResource.Schema, params) schemaBlock := c.config.TerraformResource.CoreConfigSchema() rawConfig, err := schema.JSONMapToStateValue(params, schemaBlock) From b6929906f5b9a2433058fcb03c15fbf5f7c1cc09 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Mon, 30 Oct 2023 16:22:29 +0300 Subject: [PATCH 40/60] nil check Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 71018015..d63a363c 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -179,6 +179,9 @@ func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) an return pmap } case schema.TypeSet, schema.TypeList: + if sc.Elem == nil { + return param + } pArray := param.([]any) if setSchema, ok := sc.Elem.(*schema.Schema); ok { for i, p := range pArray { From eb72995c5e3aee4382f574cdd4963fe490b004b4 Mon Sep 17 00:00:00 2001 From: Cem Mergenci Date: Tue, 31 Oct 2023 01:33:34 +0300 Subject: [PATCH 41/60] Improve management policies support. Changes to initProvider parameters are ignored, with edge cases to be handled. Signed-off-by: Cem Mergenci Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 172 ++++++++++++++++++++++++++---- 1 file changed, 151 insertions(+), 21 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index d63a363c..a79fe256 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -16,6 +16,9 @@ package controller import ( "context" + "fmt" + "strconv" + "strings" "time" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" @@ -111,15 +114,16 @@ func getJSONMap(mg xpresource.Managed) (map[string]any, error) { } type noForkExternal struct { - ts terraform.Setup - resourceSchema *schema.Resource - config *config.Resource - instanceDiff *tf.InstanceDiff - params map[string]any - rawConfig cty.Value - logger logging.Logger - metricRecorder *metrics.MetricRecorder - opTracker *AsyncTracker + ts terraform.Setup + resourceSchema *schema.Resource + config *config.Resource + instanceDiff *tf.InstanceDiff + params map[string]any + initProviderExclusiveParamKeys []string + rawConfig cty.Value + logger logging.Logger + metricRecorder *metrics.MetricRecorder + opTracker *AsyncTracker } func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externalName string, config *config.Resource, ts terraform.Setup, initParamsMerged bool, kube client.Client) (map[string]any, error) { @@ -223,6 +227,16 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m opTracker := c.operationTrackerStore.Tracker(tr) externalName := meta.GetExternalName(tr) + paramsForProvider, err := tr.GetParameters() + if err != nil { + errors.Wrap(err, "cannot get forProvider parameters.") + } + paramsInitProvider, err := tr.GetInitParameters() + if err != nil { + errors.Wrap(err, "cannot get initProvider parameters.") + } + initProviderExclusiveParams := GetTerraformIgnoreChanges(paramsForProvider, paramsInitProvider) + params, err := getExtendedParameters(ctx, tr, externalName, c.config, ts, c.isManagementPoliciesEnabled, c.kube) if err != nil { return nil, errors.Wrapf(err, "failed to get the extended parameters for resource %q", mg.GetName()) @@ -264,18 +278,53 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m } return &noForkExternal{ - ts: ts, - resourceSchema: c.config.TerraformResource, - config: c.config, - params: params, - rawConfig: rawConfig, - logger: logger, - metricRecorder: c.metricRecorder, - opTracker: opTracker, + ts: ts, + resourceSchema: c.config.TerraformResource, + config: c.config, + params: params, + initProviderExclusiveParamKeys: initProviderExclusiveParams, + rawConfig: rawConfig, + logger: logger, + metricRecorder: c.metricRecorder, + opTracker: opTracker, }, nil } -func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState) (*tf.InstanceDiff, error) { +func deleteInstanceDiffAttribute(instanceDiff *tf.InstanceDiff, paramKey string) error { + delete(instanceDiff.Attributes, paramKey) + + keyComponents := strings.Split(paramKey, ".") + if len(keyComponents) < 2 { + return nil + } + + keyComponents[len(keyComponents)-1] = "%" + lengthKey := strings.Join(keyComponents, ".") + if lengthValue, ok := instanceDiff.Attributes[lengthKey]; ok { + newValue, err := strconv.Atoi(lengthValue.New) + if err != nil { + return errors.Wrap(err, "cannot convert instance diff attribute to integer") + } + + // TODO: consider what happens if oldValue = "" + oldValue, err := strconv.Atoi(lengthValue.Old) + if err != nil { + return errors.Wrap(err, "cannot convert instance diff attribute to integer") + } + + newValue -= 1 + if oldValue == newValue { + delete(instanceDiff.Attributes, lengthKey) + } else { + // TODO: consider what happens if oldValue = "" + lengthValue.New = strconv.Itoa(newValue) + } + } + + return nil +} + +func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, tf.NewResourceConfigRaw(n.params), nil, n.ts.Meta, false) if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") @@ -286,6 +335,29 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance return nil, errors.Wrap(err, "failed to compute the customized terraform.InstanceDiff") } } + if !instanceDiff.Empty() && resourceExists { + for _, keyToIgnore := range n.initProviderExclusiveParamKeys { + for attributeKey := range instanceDiff.Attributes { + if keyToIgnore != attributeKey { + continue + } + + if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { + return nil, errors.Wrapf(err, "cannot delete key (%v) from instance diff.", keyToIgnore) + } + + keyComponents := strings.Split(keyToIgnore, ".") + if keyComponents[0] != "tags" { + continue + } + keyComponents[0] = "tags_all" + keyToIgnore = strings.Join(keyComponents, ".") + if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { + return nil, errors.Wrapf(err, "cannot delete key (%v) from instance diff.", keyToIgnore) + } + } + } + } if instanceDiff != nil { v := cty.EmptyObjectVal v, err = instanceDiff.ApplyToValue(v, n.resourceSchema.CoreConfigSchema()) @@ -296,6 +368,8 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance } if instanceDiff != nil && !instanceDiff.Empty() { n.logger.Debug("Diff detected", "instanceDiff", instanceDiff.GoString()) + // Assumption: Source of truth when applying diffs, for instance on updates, is instanceDiff.Attributes. + // Setting instanceDiff.RawConfig has no effect on diff application. instanceDiff.RawConfig = n.rawConfig } return instanceDiff, nil @@ -317,8 +391,9 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma return managed.ExternalObservation{}, errors.Errorf("failed to observe the resource: %v", diag) } n.opTracker.SetTfState(newState) // TODO: missing RawConfig & RawPlan here... - // compute the instance diff - instanceDiff, err := n.getResourceDataDiff(ctx, newState) + + resourceExists := newState != nil && newState.ID != "" + instanceDiff, err := n.getResourceDataDiff(ctx, newState, resourceExists) if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot compute the instance diff") } @@ -326,7 +401,6 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma noDiff := instanceDiff.Empty() var connDetails managed.ConnectionDetails - resourceExists := newState != nil && newState.ID != "" if !resourceExists && mg.GetDeletionTimestamp() != nil { gvk := mg.GetObjectKind().GroupVersionKind() metrics.DeletionTime.WithLabelValues(gvk.Group, gvk.Version, gvk.Kind).Observe(time.Since(mg.GetDeletionTimestamp().Time).Seconds()) @@ -513,3 +587,59 @@ func (n *noForkExternal) fromInstanceStateToJSONMap(newState *tf.InstanceState) } return stateValueMap, nil } + +// GetTerraformIgnoreChanges returns a sorted Terraform `ignore_changes` +// lifecycle meta-argument expression by looking for differences between +// the `initProvider` and `forProvider` maps. The ignored fields are the ones +// that are present in initProvider, but not in forProvider. +// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. +// Consider merging this implementation with the original one. +func GetTerraformIgnoreChanges(forProvider, initProvider map[string]any) []string { + ignored := getIgnoredFieldsMap("%s", forProvider, initProvider) + return ignored +} + +// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. +// Consider merging this implementation with the original one. +func getIgnoredFieldsMap(format string, forProvider, initProvider map[string]any) []string { + ignored := []string{} + + for k := range initProvider { + if _, ok := forProvider[k]; !ok { + ignored = append(ignored, fmt.Sprintf(format, k)) + } else { + // both are the same type so we dont need to check for forProvider type + if _, ok = initProvider[k].(map[string]any); ok { + ignored = append(ignored, getIgnoredFieldsMap(fmt.Sprintf(format, k)+".%v", forProvider[k].(map[string]any), initProvider[k].(map[string]any))...) + } + // if its an array, we need to check if its an array of maps or not + if _, ok = initProvider[k].([]any); ok { + ignored = append(ignored, getIgnoredFieldsArray(fmt.Sprintf(format, k), forProvider[k].([]any), initProvider[k].([]any))...) + } + + } + } + return ignored +} + +// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. +// Consider merging this implementation with the original one. +func getIgnoredFieldsArray(format string, forProvider, initProvider []any) []string { + ignored := []string{} + for i := range initProvider { + // Construct the full field path with array index and prefix. + fieldPath := fmt.Sprintf("%s[%d]", format, i) + if i < len(forProvider) { + if _, ok := initProvider[i].(map[string]any); ok { + ignored = append(ignored, getIgnoredFieldsMap(fieldPath+".%s", forProvider[i].(map[string]any), initProvider[i].(map[string]any))...) + } + if _, ok := initProvider[i].([]any); ok { + ignored = append(ignored, getIgnoredFieldsArray(fieldPath+"%s", forProvider[i].([]any), initProvider[i].([]any))...) + } + } else { + ignored = append(ignored, fieldPath) + } + + } + return ignored +} From 3cfd053162372b654de8238489d76db1e306218d Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 31 Oct 2023 09:28:31 +0300 Subject: [PATCH 42/60] Reviews for the granular management policies second phase Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 122 ++++++++++++++++-------------- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index a79fe256..9727d4c5 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -114,16 +114,15 @@ func getJSONMap(mg xpresource.Managed) (map[string]any, error) { } type noForkExternal struct { - ts terraform.Setup - resourceSchema *schema.Resource - config *config.Resource - instanceDiff *tf.InstanceDiff - params map[string]any - initProviderExclusiveParamKeys []string - rawConfig cty.Value - logger logging.Logger - metricRecorder *metrics.MetricRecorder - opTracker *AsyncTracker + ts terraform.Setup + resourceSchema *schema.Resource + config *config.Resource + instanceDiff *tf.InstanceDiff + params map[string]any + rawConfig cty.Value + logger logging.Logger + metricRecorder *metrics.MetricRecorder + opTracker *AsyncTracker } func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externalName string, config *config.Resource, ts terraform.Setup, initParamsMerged bool, kube client.Client) (map[string]any, error) { @@ -226,17 +225,6 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m tr := mg.(resource.Terraformed) opTracker := c.operationTrackerStore.Tracker(tr) externalName := meta.GetExternalName(tr) - - paramsForProvider, err := tr.GetParameters() - if err != nil { - errors.Wrap(err, "cannot get forProvider parameters.") - } - paramsInitProvider, err := tr.GetInitParameters() - if err != nil { - errors.Wrap(err, "cannot get initProvider parameters.") - } - initProviderExclusiveParams := GetTerraformIgnoreChanges(paramsForProvider, paramsInitProvider) - params, err := getExtendedParameters(ctx, tr, externalName, c.config, ts, c.isManagementPoliciesEnabled, c.kube) if err != nil { return nil, errors.Wrapf(err, "failed to get the extended parameters for resource %q", mg.GetName()) @@ -278,15 +266,14 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m } return &noForkExternal{ - ts: ts, - resourceSchema: c.config.TerraformResource, - config: c.config, - params: params, - initProviderExclusiveParamKeys: initProviderExclusiveParams, - rawConfig: rawConfig, - logger: logger, - metricRecorder: c.metricRecorder, - opTracker: opTracker, + ts: ts, + resourceSchema: c.config.TerraformResource, + config: c.config, + params: params, + rawConfig: rawConfig, + logger: logger, + metricRecorder: c.metricRecorder, + opTracker: opTracker, }, nil } @@ -303,13 +290,13 @@ func deleteInstanceDiffAttribute(instanceDiff *tf.InstanceDiff, paramKey string) if lengthValue, ok := instanceDiff.Attributes[lengthKey]; ok { newValue, err := strconv.Atoi(lengthValue.New) if err != nil { - return errors.Wrap(err, "cannot convert instance diff attribute to integer") + return errors.Wrapf(err, "cannot convert instance diff attribute %q to integer", lengthValue.New) } // TODO: consider what happens if oldValue = "" oldValue, err := strconv.Atoi(lengthValue.Old) if err != nil { - return errors.Wrap(err, "cannot convert instance diff attribute to integer") + return errors.Wrapf(err, "cannot convert instance diff attribute %q to integer", lengthValue.Old) } newValue -= 1 @@ -324,7 +311,45 @@ func deleteInstanceDiffAttribute(instanceDiff *tf.InstanceDiff, paramKey string) return nil } -func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { +func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.InstanceDiff) error { + if instanceDiff == nil || instanceDiff.Empty() { + return nil + } + paramsForProvider, err := tr.GetParameters() + if err != nil { + return errors.Wrap(err, "cannot get spec.forProvider parameters") + } + paramsInitProvider, err := tr.GetInitParameters() + if err != nil { + return errors.Wrap(err, "cannot get spec.initProvider parameters") + } + initProviderExclusiveParamKeys := getTerraformIgnoreChanges(paramsForProvider, paramsInitProvider) + + for _, keyToIgnore := range initProviderExclusiveParamKeys { + for attributeKey := range instanceDiff.Attributes { + if keyToIgnore != attributeKey { + continue + } + + if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { + return errors.Wrapf(err, "cannot delete key %q from instance diff", keyToIgnore) + } + + keyComponents := strings.Split(keyToIgnore, ".") + if keyComponents[0] != "tags" { + continue + } + keyComponents[0] = "tags_all" + keyToIgnore = strings.Join(keyComponents, ".") + if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { + return errors.Wrapf(err, "cannot delete key %q from instance diff", keyToIgnore) + } + } + } + return nil +} + +func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, tf.NewResourceConfigRaw(n.params), nil, n.ts.Meta, false) if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") @@ -335,27 +360,10 @@ func (n *noForkExternal) getResourceDataDiff(ctx context.Context, s *tf.Instance return nil, errors.Wrap(err, "failed to compute the customized terraform.InstanceDiff") } } - if !instanceDiff.Empty() && resourceExists { - for _, keyToIgnore := range n.initProviderExclusiveParamKeys { - for attributeKey := range instanceDiff.Attributes { - if keyToIgnore != attributeKey { - continue - } - - if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { - return nil, errors.Wrapf(err, "cannot delete key (%v) from instance diff.", keyToIgnore) - } - - keyComponents := strings.Split(keyToIgnore, ".") - if keyComponents[0] != "tags" { - continue - } - keyComponents[0] = "tags_all" - keyToIgnore = strings.Join(keyComponents, ".") - if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { - return nil, errors.Wrapf(err, "cannot delete key (%v) from instance diff.", keyToIgnore) - } - } + + if resourceExists { + if err := filterInitExclusiveDiffs(tr, instanceDiff); err != nil { + return nil, errors.Wrap(err, "failed to filter the diffs exclusive to spec.initProvider in the terraform.InstanceDiff") } } if instanceDiff != nil { @@ -393,7 +401,7 @@ func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (ma n.opTracker.SetTfState(newState) // TODO: missing RawConfig & RawPlan here... resourceExists := newState != nil && newState.ID != "" - instanceDiff, err := n.getResourceDataDiff(ctx, newState, resourceExists) + instanceDiff, err := n.getResourceDataDiff(mg.(resource.Terraformed), ctx, newState, resourceExists) if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot compute the instance diff") } @@ -588,13 +596,13 @@ func (n *noForkExternal) fromInstanceStateToJSONMap(newState *tf.InstanceState) return stateValueMap, nil } -// GetTerraformIgnoreChanges returns a sorted Terraform `ignore_changes` +// getTerraformIgnoreChanges returns a sorted Terraform `ignore_changes` // lifecycle meta-argument expression by looking for differences between // the `initProvider` and `forProvider` maps. The ignored fields are the ones // that are present in initProvider, but not in forProvider. // TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. // Consider merging this implementation with the original one. -func GetTerraformIgnoreChanges(forProvider, initProvider map[string]any) []string { +func getTerraformIgnoreChanges(forProvider, initProvider map[string]any) []string { ignored := getIgnoredFieldsMap("%s", forProvider, initProvider) return ignored } From 1bad32b35f491867095654ca39bc726a0497c225 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 1 Nov 2023 15:30:24 +0300 Subject: [PATCH 43/60] Fix REUSE issues Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 14 ++------------ pkg/controller/external_nofork.go | 14 ++------------ pkg/controller/nofork_common.go | 15 --------------- pkg/controller/nofork_finalizer.go | 4 ++++ pkg/controller/nofork_store.go | 4 ++++ 5 files changed, 12 insertions(+), 39 deletions(-) delete mode 100644 pkg/controller/nofork_common.go diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index f3f828b2..7d8eaa4a 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -1,16 +1,6 @@ -// Copyright 2023 Upbound Inc. +// SPDX-FileCopyrightText: 2023 The Crossplane Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package controller diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 9727d4c5..de7e281a 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -1,16 +1,6 @@ -// Copyright 2023 Upbound Inc. +// SPDX-FileCopyrightText: 2023 The Crossplane Authors // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// SPDX-License-Identifier: Apache-2.0 package controller diff --git a/pkg/controller/nofork_common.go b/pkg/controller/nofork_common.go deleted file mode 100644 index ca72768f..00000000 --- a/pkg/controller/nofork_common.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2023 Upbound Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller diff --git a/pkg/controller/nofork_finalizer.go b/pkg/controller/nofork_finalizer.go index 2da9a655..7476f454 100644 --- a/pkg/controller/nofork_finalizer.go +++ b/pkg/controller/nofork_finalizer.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2023 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + package controller import ( diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index e8a856bc..a02b5261 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2023 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + package controller import ( From 4aeb57e0d7c17bfae4f1a648041365efa4ae832c Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 1 Nov 2023 18:29:42 +0300 Subject: [PATCH 44/60] Fix tests Signed-off-by: Alper Rifat Ulucinar --- pkg/config/common_test.go | 1 + pkg/config/provider.go | 2 +- pkg/controller/external_nofork.go | 6 +++--- pkg/controller/nofork_store.go | 6 ------ pkg/metrics/metrics.go | 2 +- pkg/types/field.go | 9 +++++---- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pkg/config/common_test.go b/pkg/config/common_test.go index 2f9634bc..3ad62cab 100644 --- a/pkg/config/common_test.go +++ b/pkg/config/common_test.go @@ -119,6 +119,7 @@ func TestDefaultResource(t *testing.T) { cmpopts.IgnoreFields(Sensitive{}, "fieldPaths", "AdditionalConnectionDetailsFn"), cmpopts.IgnoreFields(LateInitializer{}, "ignoredCanonicalFieldPaths"), cmpopts.IgnoreFields(ExternalName{}, "SetIdentifierArgumentFn", "GetExternalNameFn", "GetIDFn"), + cmpopts.IgnoreFields(Resource{}, "useNoForkClient"), } for name, tc := range cases { diff --git a/pkg/config/provider.go b/pkg/config/provider.go index c72dc4b7..29ae740a 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -231,7 +231,7 @@ func WithMainTemplate(template string) ProviderOption { // NewProvider builds and returns a new Provider from provider // tfjson schema, that is generated using Terraform CLI with: // `terraform providers schema --json` -func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo +func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { //nolint:gocyclo ps := tfjson.ProviderSchemas{} if err := ps.UnmarshalJSON(schema); err != nil { panic(err) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index de7e281a..2484d8e9 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -157,7 +157,7 @@ func (c *NoForkConnector) processParamsWithStateFunc(schemaMap map[string]*schem return params } -func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) any { +func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) any { //nolint:gocyclo switch sc.Type { case schema.TypeMap: if sc.Elem == nil { @@ -301,7 +301,7 @@ func deleteInstanceDiffAttribute(instanceDiff *tf.InstanceDiff, paramKey string) return nil } -func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.InstanceDiff) error { +func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.InstanceDiff) error { //nolint:gocyclo if instanceDiff == nil || instanceDiff.Empty() { return nil } @@ -373,7 +373,7 @@ func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx contex return instanceDiff, nil } -func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { +func (n *noForkExternal) Observe(ctx context.Context, mg xpresource.Managed) (managed.ExternalObservation, error) { //nolint:gocyclo n.logger.Debug("Observing the external resource") if meta.WasDeleted(mg) && n.opTracker.IsDeleted() { diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index a02b5261..dfcdd5ca 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -82,12 +82,6 @@ func (a *AsyncTracker) GetTfID() string { return a.tfState.ID } -func (a *AsyncTracker) SetTfID(tfId string) { - //a.mu.Lock() - //defer a.mu.Unlock() - a.tfID = tfId -} - // IsDeleted returns whether the associated external resource // has logically been deleted. func (a *AsyncTracker) IsDeleted() bool { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a06d41aa..762c80b6 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -140,7 +140,7 @@ func (r *MetricRecorder) ObserveReconcileDelay(gvk schema.GroupVersionKind, name if o == nil || !o.(*Observations).observeReconcileDelay || o.(*Observations).expectedReconcileTime == nil { return } - d := time.Now().Sub(*o.(*Observations).expectedReconcileTime) + d := time.Since(*o.(*Observations).expectedReconcileTime) if d < 0 { d = 0 } diff --git a/pkg/types/field.go b/pkg/types/field.go index c127affa..4a130eb7 100644 --- a/pkg/types/field.go +++ b/pkg/types/field.go @@ -12,13 +12,14 @@ import ( "sort" "strings" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/pkg/errors" + "k8s.io/utils/ptr" + "github.com/crossplane/upjet/pkg" "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/types/comments" "github.com/crossplane/upjet/pkg/types/name" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/pkg/errors" - "k8s.io/utils/ptr" ) var parentheses = regexp.MustCompile(`\(([^)]+)\)`) @@ -213,7 +214,7 @@ func NewReferenceField(g *Builder, cfg *config.Resource, r *resource, sch *schem } // AddToResource adds built field to the resource. -func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames, addToObservation bool) { +func (f *Field) AddToResource(g *Builder, r *resource, typeNames *TypeNames, addToObservation bool) { //nolint:gocyclo if f.Comment.UpjetOptions.FieldJSONTag != nil { f.JSONTag = *f.Comment.UpjetOptions.FieldJSONTag } From b93f12782ebc61c167a1b527ddcc16413855b0f2 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 2 Nov 2023 00:56:16 +0300 Subject: [PATCH 45/60] Replace custom gci section prefixes github.com/crossplane/crossplane-* with github.com/crossplane/upjet - Run `gci write` to organize imports with the above change. Signed-off-by: Alper Rifat Ulucinar --- .golangci.yml | 3 +-- cmd/scraper/main.go | 3 ++- pkg/config/common.go | 3 ++- pkg/config/externalname_test.go | 3 +-- pkg/config/resource.go | 9 ++++---- pkg/config/resource_test.go | 5 ++--- pkg/controller/api.go | 10 ++++----- pkg/controller/api_test.go | 12 +++++------ pkg/controller/external.go | 16 +++++++------- pkg/controller/external_test.go | 22 ++++++++++---------- pkg/controller/handler/eventhandler.go | 3 +-- pkg/controller/options.go | 6 +++--- pkg/migration/api_steps.go | 7 +++---- pkg/migration/configurationmetadata_steps.go | 3 +-- pkg/migration/configurationpackage_steps.go | 5 ++--- pkg/migration/converter.go | 18 +++++++--------- pkg/migration/fake/objects.go | 4 ++-- pkg/migration/fork_executor.go | 3 +-- pkg/migration/fork_executor_test.go | 3 +-- pkg/migration/interfaces.go | 1 - pkg/migration/patches.go | 3 +-- pkg/migration/plan_generator.go | 12 +++++------ pkg/migration/plan_generator_test.go | 17 +++++++-------- pkg/migration/provider_package_steps.go | 3 +-- pkg/migration/registry.go | 8 +++---- pkg/pipeline/register.go | 3 ++- pkg/pipeline/run.go | 4 ++-- pkg/pipeline/setup.go | 5 +++-- pkg/pipeline/terraformed.go | 3 ++- pkg/pipeline/version.go | 3 ++- pkg/registry/meta_test.go | 5 ++--- pkg/registry/reference/references.go | 3 ++- pkg/registry/reference/resolver.go | 6 +++--- pkg/registry/resource.go | 4 ++-- pkg/resource/conditions.go | 6 +++--- pkg/resource/fake/terraformed.go | 3 +-- pkg/resource/lateinit.go | 6 +++--- pkg/resource/sensitive.go | 10 ++++----- pkg/resource/sensitive_test.go | 14 ++++++------- pkg/terraform/files.go | 5 ++--- pkg/terraform/files_test.go | 16 +++++++------- pkg/terraform/finalizer.go | 3 +-- pkg/terraform/finalizer_test.go | 8 +++---- pkg/terraform/provider_runner.go | 3 +-- pkg/terraform/provider_runner_test.go | 5 ++--- pkg/terraform/provider_scheduler.go | 4 ++-- pkg/terraform/store.go | 14 ++++++------- pkg/terraform/timeouts.go | 4 ++-- pkg/terraform/timeouts_test.go | 3 +-- pkg/terraform/workspace.go | 10 ++++----- pkg/terraform/workspace_test.go | 6 +++--- pkg/types/builder.go | 4 ++-- pkg/types/builder_test.go | 4 ++-- pkg/types/comments/comment_test.go | 6 +++--- pkg/types/markers/crossplane_test.go | 3 ++- pkg/types/markers/terrajet_test.go | 3 +-- pkg/types/reference.go | 3 ++- pkg/types/reference_test.go | 5 +++-- 58 files changed, 174 insertions(+), 192 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a6cc3f56..fb24e160 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,8 +44,7 @@ linters-settings: sections: - standard - default - - prefix(github.com/crossplane/crossplane-runtime) - - prefix(github.com/crossplane/crossplane) + - prefix(github.com/crossplane/upjet) - blank - dot diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index 077c3170..692697e1 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -8,8 +8,9 @@ import ( "os" "path/filepath" - "github.com/crossplane/upjet/pkg/registry" "gopkg.in/alecthomas/kingpin.v2" + + "github.com/crossplane/upjet/pkg/registry" ) func main() { diff --git a/pkg/config/common.go b/pkg/config/common.go index 7bf2226a..ee291e39 100644 --- a/pkg/config/common.go +++ b/pkg/config/common.go @@ -7,9 +7,10 @@ package config import ( "strings" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/crossplane/upjet/pkg/registry" tjname "github.com/crossplane/upjet/pkg/types/name" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) const ( diff --git a/pkg/config/externalname_test.go b/pkg/config/externalname_test.go index c9f39b2a..f090a0f6 100644 --- a/pkg/config/externalname_test.go +++ b/pkg/config/externalname_test.go @@ -8,10 +8,9 @@ import ( "context" "testing" - "github.com/google/go-cmp/cmp" - "github.com/crossplane/crossplane-runtime/pkg/errors" "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" ) func TestGetExternalNameFromTemplated(t *testing.T) { diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 14b31ac2..7e106af8 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -9,6 +9,10 @@ import ( "fmt" "time" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" @@ -17,11 +21,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" - "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" - "github.com/crossplane/upjet/pkg/registry" ) diff --git a/pkg/config/resource_test.go b/pkg/config/resource_test.go index 4663781b..a94366c7 100644 --- a/pkg/config/resource_test.go +++ b/pkg/config/resource_test.go @@ -9,14 +9,13 @@ import ( "fmt" "testing" - "github.com/google/go-cmp/cmp" - "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/crossplane/crossplane-runtime/pkg/errors" "github.com/crossplane/crossplane-runtime/pkg/fieldpath" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/crossplane/crossplane-runtime/pkg/resource/fake" "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( diff --git a/pkg/controller/api.go b/pkg/controller/api.go index 9a5808c5..f8efda9b 100644 --- a/pkg/controller/api.go +++ b/pkg/controller/api.go @@ -7,9 +7,8 @@ package controller import ( "context" - "github.com/crossplane/upjet/pkg/controller/handler" - "github.com/crossplane/upjet/pkg/resource" - "github.com/crossplane/upjet/pkg/terraform" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -17,8 +16,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ctrl "sigs.k8s.io/controller-runtime/pkg/manager" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/upjet/pkg/controller/handler" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/terraform" ) const ( diff --git a/pkg/controller/api_test.go b/pkg/controller/api_test.go index b42e662c..7bd60b3c 100644 --- a/pkg/controller/api_test.go +++ b/pkg/controller/api_test.go @@ -8,17 +8,17 @@ import ( "context" "testing" - "github.com/crossplane/upjet/pkg/resource" - "github.com/crossplane/upjet/pkg/resource/fake" - tjerrors "github.com/crossplane/upjet/pkg/terraform/errors" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + xpfake "github.com/crossplane/crossplane-runtime/pkg/resource/fake" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" ctrl "sigs.k8s.io/controller-runtime/pkg/manager" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" - xpfake "github.com/crossplane/crossplane-runtime/pkg/resource/fake" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/resource/fake" + tjerrors "github.com/crossplane/upjet/pkg/terraform/errors" ) func TestAPICallbacksCreate(t *testing.T) { diff --git a/pkg/controller/external.go b/pkg/controller/external.go index 62580935..4eff3812 100644 --- a/pkg/controller/external.go +++ b/pkg/controller/external.go @@ -8,6 +8,14 @@ import ( "context" "time" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/controller/handler" "github.com/crossplane/upjet/pkg/metrics" @@ -15,14 +23,6 @@ import ( "github.com/crossplane/upjet/pkg/resource/json" "github.com/crossplane/upjet/pkg/terraform" tferrors "github.com/crossplane/upjet/pkg/terraform/errors" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - "github.com/crossplane/crossplane-runtime/pkg/logging" - "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" ) const ( diff --git a/pkg/controller/external_test.go b/pkg/controller/external_test.go index 7710e705..68e06c16 100644 --- a/pkg/controller/external_test.go +++ b/pkg/controller/external_test.go @@ -8,17 +8,6 @@ import ( "context" "testing" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/resource" - "github.com/crossplane/upjet/pkg/resource/fake" - "github.com/crossplane/upjet/pkg/resource/json" - "github.com/crossplane/upjet/pkg/terraform" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/logging" xpmeta "github.com/crossplane/crossplane-runtime/pkg/meta" @@ -26,6 +15,17 @@ import ( xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" xpfake "github.com/crossplane/crossplane-runtime/pkg/resource/fake" "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/resource/fake" + "github.com/crossplane/upjet/pkg/resource/json" + "github.com/crossplane/upjet/pkg/terraform" ) const ( diff --git a/pkg/controller/handler/eventhandler.go b/pkg/controller/handler/eventhandler.go index 7d0c31f9..7032e3ca 100644 --- a/pkg/controller/handler/eventhandler.go +++ b/pkg/controller/handler/eventhandler.go @@ -9,13 +9,12 @@ import ( "sync" "time" + "github.com/crossplane/crossplane-runtime/pkg/logging" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/crossplane/crossplane-runtime/pkg/logging" ) const NoRateLimiter = "" diff --git a/pkg/controller/options.go b/pkg/controller/options.go index 491e12e2..353ef866 100644 --- a/pkg/controller/options.go +++ b/pkg/controller/options.go @@ -8,11 +8,11 @@ import ( "crypto/tls" "time" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/terraform" + "github.com/crossplane/crossplane-runtime/pkg/controller" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/crossplane/crossplane-runtime/pkg/controller" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/terraform" ) // Options contains incriminating options for a given Upjet controller instance. diff --git a/pkg/migration/api_steps.go b/pkg/migration/api_steps.go index f077fc89..de1082cd 100644 --- a/pkg/migration/api_steps.go +++ b/pkg/migration/api_steps.go @@ -8,14 +8,13 @@ import ( "fmt" "strconv" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - v1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/crossplane/crossplane-runtime/pkg/resource/unstructured/claim" "github.com/crossplane/crossplane-runtime/pkg/resource/unstructured/composite" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) const ( diff --git a/pkg/migration/configurationmetadata_steps.go b/pkg/migration/configurationmetadata_steps.go index ea18ff33..a28ad611 100644 --- a/pkg/migration/configurationmetadata_steps.go +++ b/pkg/migration/configurationmetadata_steps.go @@ -8,10 +8,9 @@ import ( "fmt" "strconv" - "github.com/pkg/errors" - xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" + "github.com/pkg/errors" ) const ( diff --git a/pkg/migration/configurationpackage_steps.go b/pkg/migration/configurationpackage_steps.go index d8103eaa..dbabd470 100644 --- a/pkg/migration/configurationpackage_steps.go +++ b/pkg/migration/configurationpackage_steps.go @@ -7,11 +7,10 @@ package migration import ( "fmt" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - v1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) const ( diff --git a/pkg/migration/converter.go b/pkg/migration/converter.go index b6cbe41f..d137e21f 100644 --- a/pkg/migration/converter.go +++ b/pkg/migration/converter.go @@ -7,24 +7,22 @@ package migration import ( "fmt" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/json" - k8sjson "sigs.k8s.io/json" - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" xpmeta "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/crossplane/crossplane-runtime/pkg/resource" - xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" xppkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" xppkgv1beta1 "github.com/crossplane/crossplane/apis/pkg/v1beta1" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/json" + k8sjson "sigs.k8s.io/json" ) const ( diff --git a/pkg/migration/fake/objects.go b/pkg/migration/fake/objects.go index c9b1a59e..81c7e614 100644 --- a/pkg/migration/fake/objects.go +++ b/pkg/migration/fake/objects.go @@ -7,10 +7,10 @@ package fake import ( - "github.com/crossplane/upjet/pkg/migration/fake/mocks" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "k8s.io/apimachinery/pkg/runtime/schema" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/upjet/pkg/migration/fake/mocks" ) const ( diff --git a/pkg/migration/fork_executor.go b/pkg/migration/fork_executor.go index 61e46281..f756f622 100644 --- a/pkg/migration/fork_executor.go +++ b/pkg/migration/fork_executor.go @@ -7,10 +7,9 @@ package migration import ( "os" + "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/pkg/errors" "k8s.io/utils/exec" - - "github.com/crossplane/crossplane-runtime/pkg/logging" ) const ( diff --git a/pkg/migration/fork_executor_test.go b/pkg/migration/fork_executor_test.go index cc146156..14538ff4 100644 --- a/pkg/migration/fork_executor_test.go +++ b/pkg/migration/fork_executor_test.go @@ -7,12 +7,11 @@ package migration import ( "testing" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" k8sExec "k8s.io/utils/exec" testingexec "k8s.io/utils/exec/testing" - - "github.com/crossplane/crossplane-runtime/pkg/test" ) var ( diff --git a/pkg/migration/interfaces.go b/pkg/migration/interfaces.go index a8771588..466b4b05 100644 --- a/pkg/migration/interfaces.go +++ b/pkg/migration/interfaces.go @@ -6,7 +6,6 @@ package migration import ( "github.com/crossplane/crossplane-runtime/pkg/resource" - xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" diff --git a/pkg/migration/patches.go b/pkg/migration/patches.go index d5d84188..e09ea770 100644 --- a/pkg/migration/patches.go +++ b/pkg/migration/patches.go @@ -11,11 +11,10 @@ import ( "regexp" "strings" + xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - - xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" ) var ( diff --git a/pkg/migration/plan_generator.go b/pkg/migration/plan_generator.go index fcd8d4a6..70c73bfa 100644 --- a/pkg/migration/plan_generator.go +++ b/pkg/migration/plan_generator.go @@ -9,20 +9,18 @@ import ( "reflect" "time" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/rand" - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/crossplane/crossplane-runtime/pkg/resource" - xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" xppkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" xppkgv1beta1 "github.com/crossplane/crossplane/apis/pkg/v1beta1" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/rand" ) const ( diff --git a/pkg/migration/plan_generator_test.go b/pkg/migration/plan_generator_test.go index eb16bdff..791d7cbf 100644 --- a/pkg/migration/plan_generator_test.go +++ b/pkg/migration/plan_generator_test.go @@ -11,7 +11,13 @@ import ( "regexp" "testing" - "github.com/crossplane/upjet/pkg/migration/fake" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/crossplane-runtime/pkg/test" + v1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" + xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" + xppkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" + xppkgv1beta1 "github.com/crossplane/crossplane/apis/pkg/v1beta1" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,14 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" k8syaml "sigs.k8s.io/yaml" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" - "github.com/crossplane/crossplane-runtime/pkg/test" - - v1 "github.com/crossplane/crossplane/apis/apiextensions/v1" - xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" - xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" - xppkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" - xppkgv1beta1 "github.com/crossplane/crossplane/apis/pkg/v1beta1" + "github.com/crossplane/upjet/pkg/migration/fake" ) func TestGeneratePlan(t *testing.T) { diff --git a/pkg/migration/provider_package_steps.go b/pkg/migration/provider_package_steps.go index 5064b086..5e1f160e 100644 --- a/pkg/migration/provider_package_steps.go +++ b/pkg/migration/provider_package_steps.go @@ -8,10 +8,9 @@ import ( "fmt" "strings" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" ) const ( diff --git a/pkg/migration/registry.go b/pkg/migration/registry.go index 617dfd7b..c4970f0f 100644 --- a/pkg/migration/registry.go +++ b/pkg/migration/registry.go @@ -7,17 +7,15 @@ package migration import ( "regexp" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/crossplane/crossplane-runtime/pkg/resource" - xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" xpmetav1 "github.com/crossplane/crossplane/apis/pkg/meta/v1" xpmetav1alpha1 "github.com/crossplane/crossplane/apis/pkg/meta/v1alpha1" xppkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" xppkgv1beta1 "github.com/crossplane/crossplane/apis/pkg/v1beta1" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ) var ( diff --git a/pkg/pipeline/register.go b/pkg/pipeline/register.go index 5cd80fbb..f814547b 100644 --- a/pkg/pipeline/register.go +++ b/pkg/pipeline/register.go @@ -9,9 +9,10 @@ import ( "path/filepath" "sort" - "github.com/crossplane/upjet/pkg/pipeline/templates" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/pipeline/templates" ) // NewRegisterGenerator returns a new RegisterGenerator. diff --git a/pkg/pipeline/run.go b/pkg/pipeline/run.go index 4b1e3353..4d707984 100644 --- a/pkg/pipeline/run.go +++ b/pkg/pipeline/run.go @@ -11,10 +11,10 @@ import ( "sort" "strings" + "github.com/crossplane/crossplane-runtime/pkg/errors" + "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/examples" - - "github.com/crossplane/crossplane-runtime/pkg/errors" ) type terraformedInput struct { diff --git a/pkg/pipeline/setup.go b/pkg/pipeline/setup.go index 121f6cba..2b85b276 100644 --- a/pkg/pipeline/setup.go +++ b/pkg/pipeline/setup.go @@ -12,10 +12,11 @@ import ( "sort" "text/template" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/pipeline/templates" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/pipeline/templates" ) // NewProviderGenerator returns a new ProviderGenerator. diff --git a/pkg/pipeline/terraformed.go b/pkg/pipeline/terraformed.go index 10796b22..fea56e6a 100644 --- a/pkg/pipeline/terraformed.go +++ b/pkg/pipeline/terraformed.go @@ -10,9 +10,10 @@ import ( "path/filepath" "strings" - "github.com/crossplane/upjet/pkg/pipeline/templates" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/pipeline/templates" ) // NewTerraformedGenerator returns a new TerraformedGenerator. diff --git a/pkg/pipeline/version.go b/pkg/pipeline/version.go index c155a061..d0093b82 100644 --- a/pkg/pipeline/version.go +++ b/pkg/pipeline/version.go @@ -10,9 +10,10 @@ import ( "path/filepath" "strings" - "github.com/crossplane/upjet/pkg/pipeline/templates" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/pipeline/templates" ) // NewVersionGenerator returns a new VersionGenerator. diff --git a/pkg/registry/meta_test.go b/pkg/registry/meta_test.go index 4fb46975..84a6717d 100644 --- a/pkg/registry/meta_test.go +++ b/pkg/registry/meta_test.go @@ -8,12 +8,11 @@ import ( "os" "testing" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + xptest "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "gopkg.in/yaml.v3" - - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" - xptest "github.com/crossplane/crossplane-runtime/pkg/test" ) func TestScrapeRepo(t *testing.T) { diff --git a/pkg/registry/reference/references.go b/pkg/registry/reference/references.go index f15b6dec..fc9fcc99 100644 --- a/pkg/registry/reference/references.go +++ b/pkg/registry/reference/references.go @@ -8,10 +8,11 @@ import ( "fmt" "strings" + "github.com/pkg/errors" + "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/registry" "github.com/crossplane/upjet/pkg/types" - "github.com/pkg/errors" ) const ( diff --git a/pkg/registry/reference/resolver.go b/pkg/registry/reference/resolver.go index 63941112..6fc6dccf 100644 --- a/pkg/registry/reference/resolver.go +++ b/pkg/registry/reference/resolver.go @@ -10,12 +10,12 @@ import ( "strconv" "strings" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/pkg/errors" + "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/registry" "github.com/crossplane/upjet/pkg/resource/json" - "github.com/pkg/errors" - - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" ) const ( diff --git a/pkg/registry/resource.go b/pkg/registry/resource.go index 53e9f781..e8f949b5 100644 --- a/pkg/registry/resource.go +++ b/pkg/registry/resource.go @@ -5,11 +5,11 @@ package registry import ( - "github.com/crossplane/upjet/pkg/resource/json" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/pkg/errors" "gopkg.in/yaml.v2" - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/crossplane/upjet/pkg/resource/json" ) const ( diff --git a/pkg/resource/conditions.go b/pkg/resource/conditions.go index 62e8462b..05a5877c 100644 --- a/pkg/resource/conditions.go +++ b/pkg/resource/conditions.go @@ -5,12 +5,12 @@ package resource import ( - tferrors "github.com/crossplane/upjet/pkg/terraform/errors" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + tferrors "github.com/crossplane/upjet/pkg/terraform/errors" ) // Condition constants. diff --git a/pkg/resource/fake/terraformed.go b/pkg/resource/fake/terraformed.go index 6c97732e..ed71b356 100644 --- a/pkg/resource/fake/terraformed.go +++ b/pkg/resource/fake/terraformed.go @@ -5,11 +5,10 @@ package fake import ( + "github.com/crossplane/crossplane-runtime/pkg/resource/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/json" - - "github.com/crossplane/crossplane-runtime/pkg/resource/fake" ) // Observable is mock Observable. diff --git a/pkg/resource/lateinit.go b/pkg/resource/lateinit.go index 9d9ffa9a..6d592306 100644 --- a/pkg/resource/lateinit.go +++ b/pkg/resource/lateinit.go @@ -10,12 +10,12 @@ import ( "runtime/debug" "strings" - "github.com/crossplane/upjet/pkg/config" + xpmeta "github.com/crossplane/crossplane-runtime/pkg/meta" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xpmeta "github.com/crossplane/crossplane-runtime/pkg/meta" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/upjet/pkg/config" ) const ( diff --git a/pkg/resource/sensitive.go b/pkg/resource/sensitive.go index 70b21454..229ddd97 100644 --- a/pkg/resource/sensitive.go +++ b/pkg/resource/sensitive.go @@ -10,15 +10,15 @@ import ( "regexp" "strings" - "github.com/crossplane/upjet/pkg/config" - "github.com/pkg/errors" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - v1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/pkg/errors" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/crossplane/upjet/pkg/config" ) const ( diff --git a/pkg/resource/sensitive_test.go b/pkg/resource/sensitive_test.go index 4aa71919..de2aabc8 100644 --- a/pkg/resource/sensitive_test.go +++ b/pkg/resource/sensitive_test.go @@ -8,10 +8,9 @@ import ( "context" "testing" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/resource/fake" - "github.com/crossplane/upjet/pkg/resource/fake/mocks" - "github.com/crossplane/upjet/pkg/resource/json" + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/golang/mock/gomock" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" @@ -20,9 +19,10 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/resource/fake" + "github.com/crossplane/upjet/pkg/resource/fake/mocks" + "github.com/crossplane/upjet/pkg/resource/json" ) var ( diff --git a/pkg/terraform/files.go b/pkg/terraform/files.go index 404488aa..78adac16 100644 --- a/pkg/terraform/files.go +++ b/pkg/terraform/files.go @@ -12,15 +12,14 @@ import ( "strings" "dario.cat/mergo" + "github.com/crossplane/crossplane-runtime/pkg/feature" + "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/pkg/errors" "github.com/spf13/afero" "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/resource" "github.com/crossplane/upjet/pkg/resource/json" - - "github.com/crossplane/crossplane-runtime/pkg/feature" - "github.com/crossplane/crossplane-runtime/pkg/meta" ) const ( diff --git a/pkg/terraform/files_test.go b/pkg/terraform/files_test.go index 975b687d..926ac1f2 100644 --- a/pkg/terraform/files_test.go +++ b/pkg/terraform/files_test.go @@ -11,19 +11,19 @@ import ( "testing" "time" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/resource" - "github.com/crossplane/upjet/pkg/resource/fake" - "github.com/crossplane/upjet/pkg/resource/json" + "github.com/crossplane/crossplane-runtime/pkg/feature" + "github.com/crossplane/crossplane-runtime/pkg/meta" + xpfake "github.com/crossplane/crossplane-runtime/pkg/resource/fake" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/crossplane/crossplane-runtime/pkg/feature" - "github.com/crossplane/crossplane-runtime/pkg/meta" - xpfake "github.com/crossplane/crossplane-runtime/pkg/resource/fake" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/resource/fake" + "github.com/crossplane/upjet/pkg/resource/json" ) const ( diff --git a/pkg/terraform/finalizer.go b/pkg/terraform/finalizer.go index bff2607e..cf0043f1 100644 --- a/pkg/terraform/finalizer.go +++ b/pkg/terraform/finalizer.go @@ -7,9 +7,8 @@ package terraform import ( "context" - "github.com/pkg/errors" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/pkg/errors" ) const ( diff --git a/pkg/terraform/finalizer_test.go b/pkg/terraform/finalizer_test.go index f2be6d11..1803950a 100644 --- a/pkg/terraform/finalizer_test.go +++ b/pkg/terraform/finalizer_test.go @@ -8,13 +8,13 @@ import ( "context" "testing" - "github.com/crossplane/upjet/pkg/resource" - "github.com/google/go-cmp/cmp" - "github.com/pkg/errors" - "github.com/crossplane/crossplane-runtime/pkg/logging" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" + "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/resource" ) var ( diff --git a/pkg/terraform/provider_runner.go b/pkg/terraform/provider_runner.go index 6653995e..e9753fca 100644 --- a/pkg/terraform/provider_runner.go +++ b/pkg/terraform/provider_runner.go @@ -12,11 +12,10 @@ import ( "sync" "time" + "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/pkg/errors" "k8s.io/utils/clock" "k8s.io/utils/exec" - - "github.com/crossplane/crossplane-runtime/pkg/logging" ) const ( diff --git a/pkg/terraform/provider_runner_test.go b/pkg/terraform/provider_runner_test.go index bd558110..fd584c4d 100644 --- a/pkg/terraform/provider_runner_test.go +++ b/pkg/terraform/provider_runner_test.go @@ -14,14 +14,13 @@ import ( "testing" "time" + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" clock "k8s.io/utils/clock/testing" "k8s.io/utils/exec" testingexec "k8s.io/utils/exec/testing" - - "github.com/crossplane/crossplane-runtime/pkg/logging" - "github.com/crossplane/crossplane-runtime/pkg/test" ) func TestStartSharedServer(t *testing.T) { diff --git a/pkg/terraform/provider_scheduler.go b/pkg/terraform/provider_scheduler.go index 60abc288..38240f9e 100644 --- a/pkg/terraform/provider_scheduler.go +++ b/pkg/terraform/provider_scheduler.go @@ -7,10 +7,10 @@ package terraform import ( "sync" - tferrors "github.com/crossplane/upjet/pkg/terraform/errors" + "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/pkg/errors" - "github.com/crossplane/crossplane-runtime/pkg/logging" + tferrors "github.com/crossplane/upjet/pkg/terraform/errors" ) // ProviderHandle represents native plugin (Terraform provider) process diff --git a/pkg/terraform/store.go b/pkg/terraform/store.go index 2e65ad06..862dab7c 100644 --- a/pkg/terraform/store.go +++ b/pkg/terraform/store.go @@ -15,9 +15,10 @@ import ( "sync" "time" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/metrics" - "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/crossplane-runtime/pkg/feature" + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/meta" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/mitchellh/go-ps" "github.com/pkg/errors" "github.com/spf13/afero" @@ -25,10 +26,9 @@ import ( "k8s.io/utils/exec" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/crossplane/crossplane-runtime/pkg/feature" - "github.com/crossplane/crossplane-runtime/pkg/logging" - "github.com/crossplane/crossplane-runtime/pkg/meta" - xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/metrics" + "github.com/crossplane/upjet/pkg/resource" ) const ( diff --git a/pkg/terraform/timeouts.go b/pkg/terraform/timeouts.go index 93c3d240..c3defebd 100644 --- a/pkg/terraform/timeouts.go +++ b/pkg/terraform/timeouts.go @@ -5,10 +5,10 @@ package terraform import ( + "github.com/crossplane/crossplane-runtime/pkg/errors" + "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/resource/json" - - "github.com/crossplane/crossplane-runtime/pkg/errors" ) // "e2bfb730-ecaa-11e6-8f88-34363bc7c4c0" is a hardcoded string for Terraform diff --git a/pkg/terraform/timeouts_test.go b/pkg/terraform/timeouts_test.go index cc70cd72..3da97f9f 100644 --- a/pkg/terraform/timeouts_test.go +++ b/pkg/terraform/timeouts_test.go @@ -8,10 +8,9 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" - "github.com/crossplane/crossplane-runtime/pkg/errors" "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" ) func TestTimeoutsAsParameter(t *testing.T) { diff --git a/pkg/terraform/workspace.go b/pkg/terraform/workspace.go index beb6d2da..63b39d51 100644 --- a/pkg/terraform/workspace.go +++ b/pkg/terraform/workspace.go @@ -13,15 +13,15 @@ import ( "sync" "time" - "github.com/crossplane/upjet/pkg/metrics" - "github.com/crossplane/upjet/pkg/resource" - "github.com/crossplane/upjet/pkg/resource/json" - tferrors "github.com/crossplane/upjet/pkg/terraform/errors" + "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/pkg/errors" "github.com/spf13/afero" k8sExec "k8s.io/utils/exec" - "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/upjet/pkg/metrics" + "github.com/crossplane/upjet/pkg/resource" + "github.com/crossplane/upjet/pkg/resource/json" + tferrors "github.com/crossplane/upjet/pkg/terraform/errors" ) const ( diff --git a/pkg/terraform/workspace_test.go b/pkg/terraform/workspace_test.go index b33c88ee..b0efaeed 100644 --- a/pkg/terraform/workspace_test.go +++ b/pkg/terraform/workspace_test.go @@ -9,15 +9,15 @@ import ( "testing" "time" - "github.com/crossplane/upjet/pkg/resource/json" - tferrors "github.com/crossplane/upjet/pkg/terraform/errors" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/spf13/afero" k8sExec "k8s.io/utils/exec" testingexec "k8s.io/utils/exec/testing" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/upjet/pkg/resource/json" + tferrors "github.com/crossplane/upjet/pkg/terraform/errors" ) var ( diff --git a/pkg/types/builder.go b/pkg/types/builder.go index cae32821..b4dbc3ef 100644 --- a/pkg/types/builder.go +++ b/pkg/types/builder.go @@ -11,13 +11,13 @@ import ( "sort" "strings" - "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" twtypes "github.com/muvaf/typewriter/pkg/types" "github.com/pkg/errors" "k8s.io/utils/ptr" - "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/crossplane/upjet/pkg/config" ) const ( diff --git a/pkg/types/builder_test.go b/pkg/types/builder_test.go index 86fba91e..0322f2e9 100644 --- a/pkg/types/builder_test.go +++ b/pkg/types/builder_test.go @@ -10,12 +10,12 @@ import ( "go/types" "testing" - "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/pkg/errors" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/upjet/pkg/config" ) func TestBuilder_generateTypeName(t *testing.T) { diff --git a/pkg/types/comments/comment_test.go b/pkg/types/comments/comment_test.go index 8050d203..8f561d1b 100644 --- a/pkg/types/comments/comment_test.go +++ b/pkg/types/comments/comment_test.go @@ -8,12 +8,12 @@ import ( "reflect" "testing" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/types/markers" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/types/markers" ) func TestComment_Build(t *testing.T) { diff --git a/pkg/types/markers/crossplane_test.go b/pkg/types/markers/crossplane_test.go index 58269280..dbf4e1c6 100644 --- a/pkg/types/markers/crossplane_test.go +++ b/pkg/types/markers/crossplane_test.go @@ -7,8 +7,9 @@ package markers import ( "testing" - "github.com/crossplane/upjet/pkg/config" "github.com/google/go-cmp/cmp" + + "github.com/crossplane/upjet/pkg/config" ) func TestCrossplaneOptions_String(t *testing.T) { diff --git a/pkg/types/markers/terrajet_test.go b/pkg/types/markers/terrajet_test.go index 88d526e5..e196a4af 100644 --- a/pkg/types/markers/terrajet_test.go +++ b/pkg/types/markers/terrajet_test.go @@ -8,10 +8,9 @@ import ( "fmt" "testing" + "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" - - "github.com/crossplane/crossplane-runtime/pkg/test" ) func Test_parseAsUpjetOption(t *testing.T) { diff --git a/pkg/types/reference.go b/pkg/types/reference.go index 9fcf78f2..72c14c5c 100644 --- a/pkg/types/reference.go +++ b/pkg/types/reference.go @@ -11,10 +11,11 @@ import ( "reflect" "strings" + "k8s.io/utils/ptr" + "github.com/crossplane/upjet/pkg/types/comments" "github.com/crossplane/upjet/pkg/types/markers" "github.com/crossplane/upjet/pkg/types/name" - "k8s.io/utils/ptr" ) const ( diff --git a/pkg/types/reference_test.go b/pkg/types/reference_test.go index cf012348..ed239af8 100644 --- a/pkg/types/reference_test.go +++ b/pkg/types/reference_test.go @@ -9,10 +9,11 @@ import ( "go/types" "testing" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/types/name" "github.com/google/go-cmp/cmp" twtypes "github.com/muvaf/typewriter/pkg/types" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/types/name" ) func TestBuilder_generateReferenceFields(t *testing.T) { From e4d7b4eb9b645e04232aabcd8d713dc3c4958722 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 2 Nov 2023 01:18:15 +0300 Subject: [PATCH 46/60] Fix linter issues Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 2 +- pkg/controller/nofork_store.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index 7d8eaa4a..6b4a1494 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -160,7 +160,7 @@ func (n *noForkAsyncExternal) Update(_ context.Context, mg xpresource.Managed) ( return managed.ExternalUpdate{}, nil } -func (n *noForkAsyncExternal) Delete(ctx context.Context, mg xpresource.Managed) error { +func (n *noForkAsyncExternal) Delete(_ context.Context, mg xpresource.Managed) error { if !n.opTracker.LastOperation.MarkStart("delete") { return errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } diff --git a/pkg/controller/nofork_store.go b/pkg/controller/nofork_store.go index dfcdd5ca..4ec2e082 100644 --- a/pkg/controller/nofork_store.go +++ b/pkg/controller/nofork_store.go @@ -21,7 +21,6 @@ type AsyncTracker struct { LastOperation *terraform.Operation logger logging.Logger mu *sync.Mutex - tfID string tfState *tfsdk.InstanceState // lifecycle of certain external resources are bound to a parent resource's // lifecycle, and they cannot be deleted without actually deleting From 229c2734b4a559c1d7d6dccba407aa25261f6cea Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Thu, 2 Nov 2023 17:01:46 +0300 Subject: [PATCH 47/60] change TF CustomDiff func signature to accept state and config Signed-off-by: Alper Rifat Ulucinar --- pkg/config/resource.go | 2 +- pkg/controller/external_nofork.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 7e106af8..876ec6f5 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -350,7 +350,7 @@ func (r *Resource) ShouldUseNoForkClient() bool { // CustomDiff customizes the computed Terraform InstanceDiff. This can be used // in cases where, for example, changes in a certain argument should just be // dismissed. The new InstanceDiff is returned along with any errors. -type CustomDiff func(diff *terraform.InstanceDiff) (*terraform.InstanceDiff, error) +type CustomDiff func(diff *terraform.InstanceDiff, state *terraform.InstanceState, config *terraform.ResourceConfig) (*terraform.InstanceDiff, error) // ConfigurationInjector is a function that injects Terraform configuration // values from the specified managed resource into the specified configuration diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 2484d8e9..bb559d40 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -340,12 +340,13 @@ func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.Instance } func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { - instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, tf.NewResourceConfigRaw(n.params), nil, n.ts.Meta, false) + resourceConfig := tf.NewResourceConfigRaw(n.params) + instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, resourceConfig, nil, n.ts.Meta, false) if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") } if n.config.TerraformCustomDiff != nil { - instanceDiff, err = n.config.TerraformCustomDiff(instanceDiff) + instanceDiff, err = n.config.TerraformCustomDiff(instanceDiff, s, resourceConfig) if err != nil { return nil, errors.Wrap(err, "failed to compute the customized terraform.InstanceDiff") } From 82a4ad94280e8dc0446736ed687242c1d5bc40f2 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Thu, 2 Nov 2023 18:08:40 +0300 Subject: [PATCH 48/60] Add support for configuring Terraform resource timeouts both from the schema and config.Resource.OperationTimeouts - The timeouts specified in upjet resource configuration (config.Resource.OperationTimeouts) prevails any defaults configured in the Terraform schema. Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 45 ++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index bb559d40..056cd9d3 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -339,8 +339,44 @@ func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.Instance return nil } +// resource timeouts configuration +func getTimeoutParameters(config *config.Resource) map[string]any { + timeouts := make(map[string]any) + // first use the timeout overrides specified in + // the Terraform resource schema + if config.TerraformResource.Timeouts != nil { + if config.TerraformResource.Timeouts.Create != nil && *config.TerraformResource.Timeouts.Create != 0 { + timeouts[schema.TimeoutCreate] = config.TerraformResource.Timeouts.Create.Nanoseconds() + } + if config.TerraformResource.Timeouts.Update != nil && *config.TerraformResource.Timeouts.Update != 0 { + timeouts[schema.TimeoutUpdate] = config.TerraformResource.Timeouts.Update.Nanoseconds() + } + if config.TerraformResource.Timeouts.Delete != nil && *config.TerraformResource.Timeouts.Delete != 0 { + timeouts[schema.TimeoutDelete] = config.TerraformResource.Timeouts.Delete.Nanoseconds() + } + if config.TerraformResource.Timeouts.Read != nil && *config.TerraformResource.Timeouts.Read != 0 { + timeouts[schema.TimeoutRead] = config.TerraformResource.Timeouts.Read.Nanoseconds() + } + } + // then, override any Terraform defaults using any upjet + // resource configuration overrides + if config.OperationTimeouts.Create != 0 { + timeouts[schema.TimeoutCreate] = config.OperationTimeouts.Create.Nanoseconds() + } + if config.OperationTimeouts.Update != 0 { + timeouts[schema.TimeoutUpdate] = config.OperationTimeouts.Update.Nanoseconds() + } + if config.OperationTimeouts.Delete != 0 { + timeouts[schema.TimeoutDelete] = config.OperationTimeouts.Delete.Nanoseconds() + } + if config.OperationTimeouts.Read != 0 { + timeouts[schema.TimeoutRead] = config.OperationTimeouts.Read.Nanoseconds() + } + return timeouts +} + func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { - resourceConfig := tf.NewResourceConfigRaw(n.params) + resourceConfig := tf.NewResourceConfigRaw(n.params) instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, resourceConfig, nil, n.ts.Meta, false) if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") @@ -371,6 +407,13 @@ func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx contex // Setting instanceDiff.RawConfig has no effect on diff application. instanceDiff.RawConfig = n.rawConfig } + timeouts := getTimeoutParameters(n.config) + if instanceDiff.Meta == nil && len(timeouts) > 0 { + instanceDiff.Meta = make(map[string]interface{}) + } + if len(timeouts) > 0 { + instanceDiff.Meta[schema.TimeoutKey] = timeouts + } return instanceDiff, nil } From b4ecd2240192c3a03759870586f9cb078bb3a24c Mon Sep 17 00:00:00 2001 From: Cem Mergenci Date: Thu, 2 Nov 2023 02:31:43 +0300 Subject: [PATCH 49/60] Handle top-level sets while ignoring initProvider diffs. * Remove all length keys, which are suffixed with % or #, from initProvider-exclusive diffs. Map and set diffs are successfully applied without these length keys. List diffs require length keys. To be addressed later. Signed-off-by: Cem Mergenci --- pkg/controller/external_nofork.go | 99 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 056cd9d3..fd099bfb 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -7,7 +7,6 @@ package controller import ( "context" "fmt" - "strconv" "strings" "time" @@ -139,7 +138,8 @@ func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externa params["id"] = tfID // we need to parameterize the following for a provider // not all providers may have this attribute - // TODO: tags_all handling + // TODO: tags-tags_all implementation is AWS specific. + // Consider making this logic independent of provider. if _, ok := config.TerraformResource.CoreConfigSchema().Attributes["tags_all"]; ok { params["tags_all"] = params["tags"] } @@ -267,44 +267,12 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m }, nil } -func deleteInstanceDiffAttribute(instanceDiff *tf.InstanceDiff, paramKey string) error { - delete(instanceDiff.Attributes, paramKey) - - keyComponents := strings.Split(paramKey, ".") - if len(keyComponents) < 2 { - return nil - } - - keyComponents[len(keyComponents)-1] = "%" - lengthKey := strings.Join(keyComponents, ".") - if lengthValue, ok := instanceDiff.Attributes[lengthKey]; ok { - newValue, err := strconv.Atoi(lengthValue.New) - if err != nil { - return errors.Wrapf(err, "cannot convert instance diff attribute %q to integer", lengthValue.New) - } - - // TODO: consider what happens if oldValue = "" - oldValue, err := strconv.Atoi(lengthValue.Old) - if err != nil { - return errors.Wrapf(err, "cannot convert instance diff attribute %q to integer", lengthValue.Old) - } - - newValue -= 1 - if oldValue == newValue { - delete(instanceDiff.Attributes, lengthKey) - } else { - // TODO: consider what happens if oldValue = "" - lengthValue.New = strconv.Itoa(newValue) - } - } - - return nil -} - +// TODO: Remeasure cyclomatic complexity of this function and consider addressing. func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.InstanceDiff) error { //nolint:gocyclo if instanceDiff == nil || instanceDiff.Empty() { return nil } + paramsForProvider, err := tr.GetParameters() if err != nil { return errors.Wrap(err, "cannot get spec.forProvider parameters") @@ -313,27 +281,61 @@ func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.Instance if err != nil { return errors.Wrap(err, "cannot get spec.initProvider parameters") } - initProviderExclusiveParamKeys := getTerraformIgnoreChanges(paramsForProvider, paramsInitProvider) + initProviderExclusiveParamKeys := getTerraformIgnoreChanges(paramsForProvider, paramsInitProvider) for _, keyToIgnore := range initProviderExclusiveParamKeys { for attributeKey := range instanceDiff.Attributes { - if keyToIgnore != attributeKey { + keyToIgnoreAsPrefix := fmt.Sprintf("%s.", keyToIgnore) + if keyToIgnore != attributeKey && !strings.HasPrefix(attributeKey, keyToIgnoreAsPrefix) { continue } - if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { - return errors.Wrapf(err, "cannot delete key %q from instance diff", keyToIgnore) - } + delete(instanceDiff.Attributes, attributeKey) - keyComponents := strings.Split(keyToIgnore, ".") + // TODO: tags-tags_all implementation is AWS specific. + // Consider making this logic independent of provider. + keyComponents := strings.Split(attributeKey, ".") if keyComponents[0] != "tags" { continue } keyComponents[0] = "tags_all" - keyToIgnore = strings.Join(keyComponents, ".") - if err := deleteInstanceDiffAttribute(instanceDiff, keyToIgnore); err != nil { - return errors.Wrapf(err, "cannot delete key %q from instance diff", keyToIgnore) - } + tagsAllAttributeKey := strings.Join(keyComponents, ".") + delete(instanceDiff.Attributes, tagsAllAttributeKey) + } + } + + // Delete length keys, such as "tags.%" (schema.TypeMap) and + // "cidrBlocks.#" (schema.TypeSet), because of two reasons: + // + // 1. Diffs are applied successfully without them, except for + // schema.TypeList. + // + // 2. If only length keys remain in the diff, after ignored + // attributes are removed above, they cause diff to be considered + // non-empty, even though it is effectively empty, therefore causing + // an infinite update loop. + for _, keyToIgnore := range initProviderExclusiveParamKeys { + keyComponents := strings.Split(keyToIgnore, ".") + if len(keyComponents) < 2 { + continue + } + + // TODO: Consider locating the schema corresponding to keyToIgnore + // and checking whether it's a collection, before attempting to + // delete its length key. + for _, lengthSymbol := range []string{"%", "#"} { + keyComponents[len(keyComponents)-1] = lengthSymbol + lengthKey := strings.Join(keyComponents, ".") + delete(instanceDiff.Attributes, lengthKey) + } + + // TODO: tags-tags_all implementation is AWS specific. + // Consider making this logic independent of provider. + if keyComponents[0] == "tags" { + keyComponents[0] = "tags_all" + keyComponents[len(keyComponents)-1] = "%" + lengthKey := strings.Join(keyComponents, ".") + delete(instanceDiff.Attributes, lengthKey) } } return nil @@ -670,18 +672,17 @@ func getIgnoredFieldsArray(format string, forProvider, initProvider []any) []str ignored := []string{} for i := range initProvider { // Construct the full field path with array index and prefix. - fieldPath := fmt.Sprintf("%s[%d]", format, i) + fieldPath := fmt.Sprintf("%s.%d", format, i) if i < len(forProvider) { if _, ok := initProvider[i].(map[string]any); ok { ignored = append(ignored, getIgnoredFieldsMap(fieldPath+".%s", forProvider[i].(map[string]any), initProvider[i].(map[string]any))...) } if _, ok := initProvider[i].([]any); ok { - ignored = append(ignored, getIgnoredFieldsArray(fieldPath+"%s", forProvider[i].([]any), initProvider[i].([]any))...) + ignored = append(ignored, getIgnoredFieldsArray(fieldPath, forProvider[i].([]any), initProvider[i].([]any))...) } } else { ignored = append(ignored, fieldPath) } - } return ignored } From 5c527e47a82b66a4ac07cb0fac96ebb4df58f138 Mon Sep 17 00:00:00 2001 From: Cem Mergenci Date: Fri, 3 Nov 2023 15:50:33 +0300 Subject: [PATCH 50/60] Move `ignore_changes` helper functions to a separate file. Signed-off-by: Cem Mergenci --- pkg/controller/external_nofork.go | 55 --------------------------- pkg/controller/ignored_nofork.go | 62 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 55 deletions(-) create mode 100644 pkg/controller/ignored_nofork.go diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index fd099bfb..327c4611 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -631,58 +631,3 @@ func (n *noForkExternal) fromInstanceStateToJSONMap(newState *tf.InstanceState) } return stateValueMap, nil } - -// getTerraformIgnoreChanges returns a sorted Terraform `ignore_changes` -// lifecycle meta-argument expression by looking for differences between -// the `initProvider` and `forProvider` maps. The ignored fields are the ones -// that are present in initProvider, but not in forProvider. -// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. -// Consider merging this implementation with the original one. -func getTerraformIgnoreChanges(forProvider, initProvider map[string]any) []string { - ignored := getIgnoredFieldsMap("%s", forProvider, initProvider) - return ignored -} - -// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. -// Consider merging this implementation with the original one. -func getIgnoredFieldsMap(format string, forProvider, initProvider map[string]any) []string { - ignored := []string{} - - for k := range initProvider { - if _, ok := forProvider[k]; !ok { - ignored = append(ignored, fmt.Sprintf(format, k)) - } else { - // both are the same type so we dont need to check for forProvider type - if _, ok = initProvider[k].(map[string]any); ok { - ignored = append(ignored, getIgnoredFieldsMap(fmt.Sprintf(format, k)+".%v", forProvider[k].(map[string]any), initProvider[k].(map[string]any))...) - } - // if its an array, we need to check if its an array of maps or not - if _, ok = initProvider[k].([]any); ok { - ignored = append(ignored, getIgnoredFieldsArray(fmt.Sprintf(format, k), forProvider[k].([]any), initProvider[k].([]any))...) - } - - } - } - return ignored -} - -// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. -// Consider merging this implementation with the original one. -func getIgnoredFieldsArray(format string, forProvider, initProvider []any) []string { - ignored := []string{} - for i := range initProvider { - // Construct the full field path with array index and prefix. - fieldPath := fmt.Sprintf("%s.%d", format, i) - if i < len(forProvider) { - if _, ok := initProvider[i].(map[string]any); ok { - ignored = append(ignored, getIgnoredFieldsMap(fieldPath+".%s", forProvider[i].(map[string]any), initProvider[i].(map[string]any))...) - } - if _, ok := initProvider[i].([]any); ok { - ignored = append(ignored, getIgnoredFieldsArray(fieldPath, forProvider[i].([]any), initProvider[i].([]any))...) - } - } else { - ignored = append(ignored, fieldPath) - } - } - return ignored -} diff --git a/pkg/controller/ignored_nofork.go b/pkg/controller/ignored_nofork.go new file mode 100644 index 00000000..74230698 --- /dev/null +++ b/pkg/controller/ignored_nofork.go @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2023 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import "fmt" + +// getTerraformIgnoreChanges returns a sorted Terraform `ignore_changes` +// lifecycle meta-argument expression by looking for differences between +// the `initProvider` and `forProvider` maps. The ignored fields are the ones +// that are present in initProvider, but not in forProvider. +// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. +// Consider merging this implementation with the original one. +func getTerraformIgnoreChanges(forProvider, initProvider map[string]any) []string { + ignored := getIgnoredFieldsMap("%s", forProvider, initProvider) + return ignored +} + +// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. +// Consider merging this implementation with the original one. +func getIgnoredFieldsMap(format string, forProvider, initProvider map[string]any) []string { + ignored := []string{} + + for k := range initProvider { + if _, ok := forProvider[k]; !ok { + ignored = append(ignored, fmt.Sprintf(format, k)) + } else { + // both are the same type so we dont need to check for forProvider type + if _, ok = initProvider[k].(map[string]any); ok { + ignored = append(ignored, getIgnoredFieldsMap(fmt.Sprintf(format, k)+".%v", forProvider[k].(map[string]any), initProvider[k].(map[string]any))...) + } + // if its an array, we need to check if its an array of maps or not + if _, ok = initProvider[k].([]any); ok { + ignored = append(ignored, getIgnoredFieldsArray(fmt.Sprintf(format, k), forProvider[k].([]any), initProvider[k].([]any))...) + } + + } + } + return ignored +} + +// TODO: This method is copy-pasted from `pkg/resource/ignored.go` and adapted. +// Consider merging this implementation with the original one. +func getIgnoredFieldsArray(format string, forProvider, initProvider []any) []string { + ignored := []string{} + for i := range initProvider { + // Construct the full field path with array index and prefix. + fieldPath := fmt.Sprintf("%s.%d", format, i) + if i < len(forProvider) { + if _, ok := initProvider[i].(map[string]any); ok { + ignored = append(ignored, getIgnoredFieldsMap(fieldPath+".%s", forProvider[i].(map[string]any), initProvider[i].(map[string]any))...) + } + if _, ok := initProvider[i].([]any); ok { + ignored = append(ignored, getIgnoredFieldsArray(fieldPath, forProvider[i].([]any), initProvider[i].([]any))...) + } + } else { + ignored = append(ignored, fieldPath) + } + } + return ignored +} From 146ba718baebe7c4d6dee6652314ce704a6ae63e Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 3 Nov 2023 16:30:55 +0300 Subject: [PATCH 51/60] Suppress linter issues Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_nofork.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 327c4611..78a91954 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -267,7 +267,6 @@ func (c *NoForkConnector) Connect(ctx context.Context, mg xpresource.Managed) (m }, nil } -// TODO: Remeasure cyclomatic complexity of this function and consider addressing. func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.InstanceDiff) error { //nolint:gocyclo if instanceDiff == nil || instanceDiff.Empty() { return nil @@ -342,7 +341,7 @@ func filterInitExclusiveDiffs(tr resource.Terraformed, instanceDiff *tf.Instance } // resource timeouts configuration -func getTimeoutParameters(config *config.Resource) map[string]any { +func getTimeoutParameters(config *config.Resource) map[string]any { //nolint:gocyclo timeouts := make(map[string]any) // first use the timeout overrides specified in // the Terraform resource schema @@ -377,7 +376,7 @@ func getTimeoutParameters(config *config.Resource) map[string]any { return timeouts } -func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { +func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { //nolint:gocyclo resourceConfig := tf.NewResourceConfigRaw(n.params) instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, resourceConfig, nil, n.ts.Meta, false) if err != nil { From 280b4b0fdd1f0d78181fa0f88b03c6a4aa1492ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Fri, 3 Nov 2023 16:37:55 +0300 Subject: [PATCH 52/60] Add nil check for instanceDiff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- pkg/controller/external_nofork.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 78a91954..e580e29c 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -409,10 +409,13 @@ func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx contex instanceDiff.RawConfig = n.rawConfig } timeouts := getTimeoutParameters(n.config) - if instanceDiff.Meta == nil && len(timeouts) > 0 { - instanceDiff.Meta = make(map[string]interface{}) - } if len(timeouts) > 0 { + if instanceDiff == nil { + instanceDiff = tf.NewInstanceDiff() + } + if instanceDiff.Meta == nil { + instanceDiff.Meta = make(map[string]interface{}) + } instanceDiff.Meta[schema.TimeoutKey] = timeouts } return instanceDiff, nil From 520fe966ae6c9a6b6894f01b9eedb6075dd8763c Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 7 Nov 2023 13:51:23 +0200 Subject: [PATCH 53/60] Add new error types for no-fork async mode Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 11 +++-- pkg/resource/conditions.go | 39 ++++++++++++--- pkg/terraform/errors/errors.go | 63 +++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 10 deletions(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index 6b4a1494..a6d1f7f8 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -18,6 +18,7 @@ import ( "github.com/crossplane/upjet/pkg/controller/handler" "github.com/crossplane/upjet/pkg/metrics" "github.com/crossplane/upjet/pkg/terraform" + tferrors "github.com/crossplane/upjet/pkg/terraform/errors" ) var defaultAsyncTimeout = 1 * time.Hour @@ -125,7 +126,8 @@ func (n *noForkAsyncExternal) Create(_ context.Context, mg xpresource.Managed) ( n.opTracker.logger.Debug("Async create starting...", "tfID", n.opTracker.GetTfID()) _, err := n.noForkExternal.Create(ctx, mg) - n.opTracker.LastOperation.SetError(errors.Wrap(err, "async create failed")) + err = tferrors.NewAsyncCreateFailed(err) + n.opTracker.LastOperation.SetError(err) n.opTracker.logger.Debug("Async create ended.", "error", err, "tfID", n.opTracker.GetTfID()) n.opTracker.LastOperation.MarkEnd() @@ -148,7 +150,8 @@ func (n *noForkAsyncExternal) Update(_ context.Context, mg xpresource.Managed) ( n.opTracker.logger.Debug("Async update starting...", "tfID", n.opTracker.GetTfID()) _, err := n.noForkExternal.Update(ctx, mg) - n.opTracker.LastOperation.SetError(errors.Wrap(err, "async update failed")) + err = tferrors.NewAsyncUpdateFailed(err) + n.opTracker.LastOperation.SetError(err) n.opTracker.logger.Debug("Async update ended.", "error", err, "tfID", n.opTracker.GetTfID()) n.opTracker.LastOperation.MarkEnd() @@ -170,8 +173,8 @@ func (n *noForkAsyncExternal) Delete(_ context.Context, mg xpresource.Managed) e defer cancel() n.opTracker.logger.Debug("Async delete starting...", "tfID", n.opTracker.GetTfID()) - err := n.noForkExternal.Delete(ctx, mg) - n.opTracker.LastOperation.SetError(errors.Wrap(err, "async delete failed")) + err := tferrors.NewAsyncDeleteFailed(n.noForkExternal.Delete(ctx, mg)) + n.opTracker.LastOperation.SetError(err) n.opTracker.logger.Debug("Async delete ended.", "error", err, "tfID", n.opTracker.GetTfID()) n.opTracker.LastOperation.MarkEnd() diff --git a/pkg/resource/conditions.go b/pkg/resource/conditions.go index 05a5877c..24ae35ab 100644 --- a/pkg/resource/conditions.go +++ b/pkg/resource/conditions.go @@ -18,12 +18,15 @@ const ( TypeLastAsyncOperation = "LastAsyncOperation" TypeAsyncOperation = "AsyncOperation" - ReasonApplyFailure xpv1.ConditionReason = "ApplyFailure" - ReasonDestroyFailure xpv1.ConditionReason = "DestroyFailure" - ReasonSuccess xpv1.ConditionReason = "Success" - ReasonOngoing xpv1.ConditionReason = "Ongoing" - ReasonFinished xpv1.ConditionReason = "Finished" - ReasonResourceUpToDate xpv1.ConditionReason = "UpToDate" + ReasonApplyFailure xpv1.ConditionReason = "ApplyFailure" + ReasonDestroyFailure xpv1.ConditionReason = "DestroyFailure" + ReasonAsyncCreateFailure xpv1.ConditionReason = "AsyncCreateFailure" + ReasonAsyncUpdateFailure xpv1.ConditionReason = "AsyncUpdateFailure" + ReasonAsyncDeleteFailure xpv1.ConditionReason = "AsyncDeleteFailure" + ReasonSuccess xpv1.ConditionReason = "Success" + ReasonOngoing xpv1.ConditionReason = "Ongoing" + ReasonFinished xpv1.ConditionReason = "Finished" + ReasonResourceUpToDate xpv1.ConditionReason = "UpToDate" ) // LastAsyncOperationCondition returns the condition depending on the content @@ -53,6 +56,30 @@ func LastAsyncOperationCondition(err error) xpv1.Condition { Reason: ReasonDestroyFailure, Message: err.Error(), } + case tferrors.IsAsyncCreateFailed(err): + return xpv1.Condition{ + Type: TypeLastAsyncOperation, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: ReasonAsyncCreateFailure, + Message: err.Error(), + } + case tferrors.IsAsyncUpdateFailed(err): + return xpv1.Condition{ + Type: TypeLastAsyncOperation, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: ReasonAsyncUpdateFailure, + Message: err.Error(), + } + case tferrors.IsAsyncDeleteFailed(err): + return xpv1.Condition{ + Type: TypeLastAsyncOperation, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), + Reason: ReasonAsyncDeleteFailure, + Message: err.Error(), + } default: return xpv1.Condition{ Type: "Unknown", diff --git a/pkg/terraform/errors/errors.go b/pkg/terraform/errors/errors.go index 58bb4d6c..66fa95e5 100644 --- a/pkg/terraform/errors/errors.go +++ b/pkg/terraform/errors/errors.go @@ -184,3 +184,66 @@ func IsRetryScheduleError(err error) bool { r := &retrySchedule{} return errors.As(err, &r) } + +type asyncCreateFailed struct { + error +} + +// NewAsyncCreateFailed returns a new async crate failure. +func NewAsyncCreateFailed(err error) error { + if err == nil { + return nil + } + return &asyncCreateFailed{ + error: errors.Wrap(err, "async create failed"), + } +} + +// IsAsyncCreateFailed returns whether error is due to failure of +// an async create operation. +func IsAsyncCreateFailed(err error) bool { + r := &asyncCreateFailed{} + return errors.As(err, &r) +} + +type asyncUpdateFailed struct { + error +} + +// NewAsyncUpdateFailed returns a new async update failure. +func NewAsyncUpdateFailed(err error) error { + if err == nil { + return nil + } + return &asyncUpdateFailed{ + error: errors.Wrap(err, "async update failed"), + } +} + +// IsAsyncUpdateFailed returns whether error is due to failure of +// an async update operation. +func IsAsyncUpdateFailed(err error) bool { + r := &asyncUpdateFailed{} + return errors.As(err, &r) +} + +type asyncDeleteFailed struct { + error +} + +// NewAsyncDeleteFailed returns a new async delete failure. +func NewAsyncDeleteFailed(err error) error { + if err == nil { + return nil + } + return &asyncDeleteFailed{ + error: errors.Wrap(err, "async delete failed"), + } +} + +// IsAsyncDeleteFailed returns whether error is due to failure of +// an async delete operation. +func IsAsyncDeleteFailed(err error) bool { + r := &asyncDeleteFailed{} + return errors.As(err, &r) +} From 00618d78c11a9947e3781b80f35f3ed1b2b76f68 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Wed, 8 Nov 2023 16:38:36 +0200 Subject: [PATCH 54/60] Clear errors from async operations upon successful observation Signed-off-by: Alper Rifat Ulucinar --- pkg/controller/external_async_nofork.go | 11 ++++++++++- pkg/pipeline/templates/controller.go.tmpl | 1 - 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index a6d1f7f8..c5d8e364 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -9,6 +9,7 @@ import ( "time" "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/meta" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/pkg/errors" @@ -17,6 +18,7 @@ import ( "github.com/crossplane/upjet/pkg/config" "github.com/crossplane/upjet/pkg/controller/handler" "github.com/crossplane/upjet/pkg/metrics" + "github.com/crossplane/upjet/pkg/resource" "github.com/crossplane/upjet/pkg/terraform" tferrors "github.com/crossplane/upjet/pkg/terraform/errors" ) @@ -112,7 +114,14 @@ func (n *noForkAsyncExternal) Observe(ctx context.Context, mg xpresource.Managed } n.opTracker.LastOperation.Flush() - return n.noForkExternal.Observe(ctx, mg) + o, err := n.noForkExternal.Observe(ctx, mg) + // clear any previously reported LastAsyncOperation error condition here, + // because there are no pending updates on the existing resource and it's + // not scheduled to be deleted. + if err == nil && o.ResourceExists && o.ResourceUpToDate && !meta.WasDeleted(mg) { + mg.(resource.Terraformed).SetConditions(resource.LastAsyncOperationCondition(nil)) + } + return o, err } func (n *noForkAsyncExternal) Create(_ context.Context, mg xpresource.Managed) (managed.ExternalCreation, error) { diff --git a/pkg/pipeline/templates/controller.go.tmpl b/pkg/pipeline/templates/controller.go.tmpl index 8c7cd5fe..e75d3714 100644 --- a/pkg/pipeline/templates/controller.go.tmpl +++ b/pkg/pipeline/templates/controller.go.tmpl @@ -83,7 +83,6 @@ func Setup(mgr ctrl.Manager, o tjcontroller.Options) error { {{- else }} managed.WithFinalizer(terraform.NewWorkspaceFinalizer(o.WorkspaceStore, xpresource.NewAPIFinalizer(mgr.GetClient(), managed.FinalizerName))), {{- end }} - managed.WithTimeout(3*time.Minute), managed.WithInitializers(initializers), managed.WithConnectionPublishers(cps...), From d9420d3eee1ff7a7940cffd78c94ec53c9b5e433 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici Date: Fri, 10 Nov 2023 10:48:46 +0300 Subject: [PATCH 55/60] add nil & type assertion checks in param statefunc processor Signed-off-by: Erhan Cagirici --- pkg/controller/external_nofork.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index e580e29c..59903fa8 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -147,6 +147,9 @@ func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externa } func (c *NoForkConnector) processParamsWithStateFunc(schemaMap map[string]*schema.Schema, params map[string]any) map[string]any { + if params == nil { + return params + } for key, param := range params { if sc, ok := schemaMap[key]; ok { params[key] = c.applyStateFuncToParam(sc, param) @@ -158,14 +161,17 @@ func (c *NoForkConnector) processParamsWithStateFunc(schemaMap map[string]*schem } func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) any { //nolint:gocyclo + if param == nil { + return param + } switch sc.Type { case schema.TypeMap: if sc.Elem == nil { return param } + pmap, okParam := param.(map[string]any) // TypeMap only supports schema in Elem - if mapSchema, ok := sc.Elem.(*schema.Schema); ok { - pmap := param.(map[string]any) + if mapSchema, ok := sc.Elem.(*schema.Schema); ok && okParam { for pk, pv := range pmap { pmap[pk] = c.applyStateFuncToParam(mapSchema, pv) } @@ -175,16 +181,17 @@ func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) an if sc.Elem == nil { return param } - pArray := param.([]any) - if setSchema, ok := sc.Elem.(*schema.Schema); ok { + pArray, okParam := param.([]any) + if setSchema, ok := sc.Elem.(*schema.Schema); ok && okParam { for i, p := range pArray { pArray[i] = c.applyStateFuncToParam(setSchema, p) } return pArray } else if setResource, ok := sc.Elem.(*schema.Resource); ok { for i, p := range pArray { - resParam := p.(map[string]any) - pArray[i] = c.processParamsWithStateFunc(setResource.Schema, resParam) + if resParam, okRParam := p.(map[string]any); okRParam { + pArray[i] = c.processParamsWithStateFunc(setResource.Schema, resParam) + } } } case schema.TypeBool, schema.TypeInt, schema.TypeFloat, schema.TypeString: From 8e74a8ec894b84bb35c27abbf8f917ea14a64cd7 Mon Sep 17 00:00:00 2001 From: Erhan Cagirici <105722117+erhancagirici@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:57:18 +0200 Subject: [PATCH 56/60] add support for hcl functions in string params (#9) Signed-off-by: Alper Rifat Ulucinar --- go.mod | 1 + go.sum | 2 + pkg/controller/external_nofork.go | 16 ++- pkg/controller/hcl.go | 163 ++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 pkg/controller/hcl.go diff --git a/go.mod b/go.mod index 57575eb6..865d0191 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/tmccombs/hcl2json v0.3.3 github.com/yuin/goldmark v1.4.13 github.com/zclconf/go-cty v1.11.0 + github.com/zclconf/go-cty-yaml v1.0.3 golang.org/x/net v0.15.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index c8cf1cdf..c4735a54 100644 --- a/go.sum +++ b/go.sum @@ -362,6 +362,8 @@ github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uU github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0= github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA= github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/zclconf/go-cty-yaml v1.0.3 h1:og/eOQ7lvA/WWhHGFETVWNduJM7Rjsv2RRpx1sdFMLc= +github.com/zclconf/go-cty-yaml v1.0.3/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index 59903fa8..c6f9ed01 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -194,7 +194,21 @@ func (c *NoForkConnector) applyStateFuncToParam(sc *schema.Schema, param any) an } } } - case schema.TypeBool, schema.TypeInt, schema.TypeFloat, schema.TypeString: + case schema.TypeString: + // For String types check if it is an HCL string and process + if isHCLSnippetPattern.MatchString(param.(string)) { + hclProccessedParam, err := processHCLParam(param.(string)) + if err != nil { + c.logger.Debug("could not process param, returning original", "param", sc.GoString()) + } else { + param = hclProccessedParam + } + } + if sc.StateFunc != nil { + return sc.StateFunc(param) + } + return param + case schema.TypeBool, schema.TypeInt, schema.TypeFloat: if sc.StateFunc != nil { return sc.StateFunc(param) } diff --git a/pkg/controller/hcl.go b/pkg/controller/hcl.go new file mode 100644 index 00000000..63853ecc --- /dev/null +++ b/pkg/controller/hcl.go @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: 2023 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "encoding/base64" + "fmt" + "log" + "regexp" + "unicode/utf8" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclparse" + ctyyaml "github.com/zclconf/go-cty-yaml" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" + ctyfuncstdlib "github.com/zclconf/go-cty/cty/function/stdlib" +) + +var Base64DecodeFunc = function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "str", + Type: cty.String, + AllowMarked: true, + }, + }, + Type: function.StaticReturnType(cty.String), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + str, strMarks := args[0].Unmark() + s := str.AsString() + sDec, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return cty.UnknownVal(cty.String), fmt.Errorf("failed to decode base64 data %s", s) + } + if !utf8.Valid(sDec) { + log.Printf("[DEBUG] the result of decoding the provided string is not valid UTF-8: %s", s) + return cty.UnknownVal(cty.String), fmt.Errorf("the result of decoding the provided string is not valid UTF-8") + } + return cty.StringVal(string(sDec)).WithMarks(strMarks), nil + }, +}) + +var Base64EncodeFunc = function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "str", + Type: cty.String, + }, + }, + Type: function.StaticReturnType(cty.String), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + return cty.StringVal(base64.StdEncoding.EncodeToString([]byte(args[0].AsString()))), nil + }, +}) + +// evalCtx registers the known functions for HCL processing +// variable interpolation is not supported, as in our case they are irrelevant +var evalCtx = &hcl.EvalContext{ + Variables: map[string]cty.Value{}, + Functions: map[string]function.Function{ + "abs": ctyfuncstdlib.AbsoluteFunc, + "ceil": ctyfuncstdlib.CeilFunc, + "chomp": ctyfuncstdlib.ChompFunc, + "coalescelist": ctyfuncstdlib.CoalesceListFunc, + "compact": ctyfuncstdlib.CompactFunc, + "concat": ctyfuncstdlib.ConcatFunc, + "contains": ctyfuncstdlib.ContainsFunc, + "csvdecode": ctyfuncstdlib.CSVDecodeFunc, + "distinct": ctyfuncstdlib.DistinctFunc, + "element": ctyfuncstdlib.ElementFunc, + "chunklist": ctyfuncstdlib.ChunklistFunc, + "flatten": ctyfuncstdlib.FlattenFunc, + "floor": ctyfuncstdlib.FloorFunc, + "format": ctyfuncstdlib.FormatFunc, + "formatdate": ctyfuncstdlib.FormatDateFunc, + "formatlist": ctyfuncstdlib.FormatListFunc, + "indent": ctyfuncstdlib.IndentFunc, + "join": ctyfuncstdlib.JoinFunc, + "jsondecode": ctyfuncstdlib.JSONDecodeFunc, + "jsonencode": ctyfuncstdlib.JSONEncodeFunc, + "keys": ctyfuncstdlib.KeysFunc, + "log": ctyfuncstdlib.LogFunc, + "lower": ctyfuncstdlib.LowerFunc, + "max": ctyfuncstdlib.MaxFunc, + "merge": ctyfuncstdlib.MergeFunc, + "min": ctyfuncstdlib.MinFunc, + "parseint": ctyfuncstdlib.ParseIntFunc, + "pow": ctyfuncstdlib.PowFunc, + "range": ctyfuncstdlib.RangeFunc, + "regex": ctyfuncstdlib.RegexFunc, + "regexall": ctyfuncstdlib.RegexAllFunc, + "reverse": ctyfuncstdlib.ReverseListFunc, + "setintersection": ctyfuncstdlib.SetIntersectionFunc, + "setproduct": ctyfuncstdlib.SetProductFunc, + "setsubtract": ctyfuncstdlib.SetSubtractFunc, + "setunion": ctyfuncstdlib.SetUnionFunc, + "signum": ctyfuncstdlib.SignumFunc, + "slice": ctyfuncstdlib.SliceFunc, + "sort": ctyfuncstdlib.SortFunc, + "split": ctyfuncstdlib.SplitFunc, + "strrev": ctyfuncstdlib.ReverseFunc, + "substr": ctyfuncstdlib.SubstrFunc, + "timeadd": ctyfuncstdlib.TimeAddFunc, + "title": ctyfuncstdlib.TitleFunc, + "trim": ctyfuncstdlib.TrimFunc, + "trimprefix": ctyfuncstdlib.TrimPrefixFunc, + "trimspace": ctyfuncstdlib.TrimSpaceFunc, + "trimsuffix": ctyfuncstdlib.TrimSuffixFunc, + "upper": ctyfuncstdlib.UpperFunc, + "values": ctyfuncstdlib.ValuesFunc, + "zipmap": ctyfuncstdlib.ZipmapFunc, + "yamldecode": ctyyaml.YAMLDecodeFunc, + "yamlencode": ctyyaml.YAMLEncodeFunc, + "base64encode": Base64EncodeFunc, + "base64decode": Base64DecodeFunc, + }, +} + +// hclBlock is the target type for decoding the specially-crafted HCL document. +// interested in processing HCL snippets for a single parameter +type hclBlock struct { + Parameter string `hcl:"parameter"` +} + +// isHCLSnippetPattern is the regex pattern for determining whether +// the param is an HCL template +var isHCLSnippetPattern = regexp.MustCompile(`\$\{\w+\s*\([\S\s]*\}`) + +// processHCLParam processes the given string parameter +// with HCL format and including HCL functions, +// coming from the Managed Resource spec parameters. +// It prepares a tailored HCL snippet which consist of only a single attribute +// parameter = theGivenParameterValueInHCLSyntax +// It only operates on string parameters, and returns a string. +// caller should ensure that the given parameter is an HCL snippet +func processHCLParam(param string) (string, error) { + param = fmt.Sprintf("parameter = \"%s\"\n", param) + return processHCLParamBytes([]byte(param)) +} + +// processHCLParamBytes parses and decodes the HCL snippet +func processHCLParamBytes(paramValueBytes []byte) (string, error) { + hclParser := hclparse.NewParser() + // here the filename argument is not important, + // used by the hcl parser lib for tracking caching purposes + // it is just a name reference + hclFile, diag := hclParser.ParseHCL(paramValueBytes, "dummy.hcl") + if diag.HasErrors() { + return "", diag + } + + var paramWrapper hclBlock + diags := gohcl.DecodeBody(hclFile.Body, evalCtx, ¶mWrapper) + if diags.HasErrors() { + return "", diags + } + + return paramWrapper.Parameter, nil +} From 67ddc679d5735c8ec7753b211f5fd3340ab89727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= <44261342+sergenyalcin@users.noreply.github.com> Date: Fri, 10 Nov 2023 14:02:51 +0300 Subject: [PATCH 57/60] Fix delete condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın <44261342+sergenyalcin@users.noreply.github.com> --- pkg/controller/external_async_nofork.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controller/external_async_nofork.go b/pkg/controller/external_async_nofork.go index c5d8e364..6829f5d1 100644 --- a/pkg/controller/external_async_nofork.go +++ b/pkg/controller/external_async_nofork.go @@ -173,7 +173,11 @@ func (n *noForkAsyncExternal) Update(_ context.Context, mg xpresource.Managed) ( } func (n *noForkAsyncExternal) Delete(_ context.Context, mg xpresource.Managed) error { - if !n.opTracker.LastOperation.MarkStart("delete") { + switch { + case n.opTracker.LastOperation.Type == "delete": + n.opTracker.logger.Debug("The previous delete operation is still ongoing", "tfID", n.opTracker.GetTfID()) + return nil + case !n.opTracker.LastOperation.MarkStart("delete"): return errors.Errorf("%s operation that started at %s is still running", n.opTracker.LastOperation.Type, n.opTracker.LastOperation.StartTime().String()) } From 7294c9255e3a089cd4e3c614347c0fa1c583a7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Wed, 8 Nov 2023 13:10:49 +0300 Subject: [PATCH 58/60] Add unit tests for no-fork arch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- go.mod | 3 + go.sum | 9 + pkg/controller/external_async_nofork_test.go | 332 ++++++++++++++++++ pkg/controller/external_nofork.go | 14 +- pkg/controller/external_nofork_test.go | 342 +++++++++++++++++++ 5 files changed, 696 insertions(+), 4 deletions(-) create mode 100644 pkg/controller/external_async_nofork_test.go create mode 100644 pkg/controller/external_nofork_test.go diff --git a/go.mod b/go.mod index 865d0191..89b10cbf 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -104,6 +105,8 @@ require ( github.com/vmihailenco/tagparser v0.1.1 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect diff --git a/go.sum b/go.sum index c4735a54..0e81176e 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/ github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -108,6 +109,7 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -372,9 +374,15 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -599,6 +607,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/controller/external_async_nofork_test.go b/pkg/controller/external_async_nofork_test.go new file mode 100644 index 00000000..47f3737a --- /dev/null +++ b/pkg/controller/external_async_nofork_test.go @@ -0,0 +1,332 @@ +package controller + +import ( + "context" + "testing" + + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/resource/fake" + "github.com/crossplane/upjet/pkg/terraform" +) + +var ( + cfgAsync = &config.Resource{ + TerraformResource: &schema.Resource{ + Timeouts: &schema.ResourceTimeout{ + Create: &timeout, + Read: &timeout, + Update: &timeout, + Delete: &timeout, + }, + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + }, + "id": { + Type: schema.TypeString, + Computed: true, + Required: false, + }, + "map": { + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "list": { + Type: schema.TypeList, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + ExternalName: config.IdentifierFromProvider, + Sensitive: config.Sensitive{AdditionalConnectionDetailsFn: func(attr map[string]any) (map[string][]byte, error) { + return nil, nil + }}, + } + objAsync = &fake.Terraformed{ + Parameterizable: fake.Parameterizable{ + Parameters: map[string]any{ + "name": "example", + "map": map[string]any{ + "key": "value", + }, + "list": []any{"elem1", "elem2"}, + }, + }, + Observable: fake.Observable{ + Observation: map[string]any{}, + }, + } +) + +func prepareNoForkAsyncExternal(r Resource, cfg *config.Resource, fns CallbackFns) *noForkAsyncExternal { + schemaBlock := cfg.TerraformResource.CoreConfigSchema() + rawConfig, err := schema.JSONMapToStateValue(map[string]any{"name": "example"}, schemaBlock) + if err != nil { + panic(err) + } + return &noForkAsyncExternal{ + noForkExternal: &noForkExternal{ + ts: terraform.Setup{}, + resourceSchema: r, + config: cfg, + params: map[string]any{ + "name": "example", + }, + rawConfig: rawConfig, + logger: log, + opTracker: NewAsyncTracker(), + }, + callback: fns, + } +} + +func TestAsyncNoForkConnect(t *testing.T) { + type args struct { + setupFn terraform.SetupFn + cfg *config.Resource + ots *OperationTrackerStore + obj xpresource.Managed + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + setupFn: func(_ context.Context, _ client.Client, _ xpresource.Managed) (terraform.Setup, error) { + return terraform.Setup{}, nil + }, + cfg: cfgAsync, + obj: objAsync, + ots: ots, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := NewNoForkAsyncConnector(nil, tc.args.ots, tc.args.setupFn, tc.args.cfg, WithNoForkAsyncLogger(log)) + _, err := c.Connect(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestAsyncNoForkObserve(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + } + type want struct { + obs managed.ExternalObservation + err error + } + cases := map[string]struct { + args + want + }{ + "NotExists": { + args: args{ + r: mockResource{ + RefreshWithoutUpgradeFn: func(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return nil, nil + }, + }, + cfg: cfgAsync, + obj: objAsync, + }, + want: want{ + obs: managed.ExternalObservation{ + ResourceExists: false, + ResourceUpToDate: false, + ResourceLateInitialized: false, + ConnectionDetails: nil, + Diff: "", + }, + }, + }, + "UpToDate": { + args: args{ + r: mockResource{ + RefreshWithoutUpgradeFn: func(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id", Attributes: map[string]string{"name": "example"}}, nil + }, + }, + cfg: cfgAsync, + obj: objAsync, + }, + want: want{ + obs: managed.ExternalObservation{ + ResourceExists: true, + ResourceUpToDate: true, + ResourceLateInitialized: true, + ConnectionDetails: nil, + Diff: "", + }, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkAsyncExternal := prepareNoForkAsyncExternal(tc.args.r, tc.args.cfg, CallbackFns{}) + observation, err := noForkAsyncExternal.Observe(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.obs, observation); diff != "" { + t.Errorf("\n%s\nObserve(...): -want observation, +got observation:\n", diff) + } + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestAsyncNoForkCreate(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + fns CallbackFns + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id"}, nil + }, + }, + cfg: cfgAsync, + obj: objAsync, + fns: CallbackFns{ + CreateFn: func(s string) terraform.CallbackFn { + return func(err error, ctx context.Context) error { + return nil + } + }, + }, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkAsyncExternal := prepareNoForkAsyncExternal(tc.args.r, tc.args.cfg, tc.args.fns) + _, err := noForkAsyncExternal.Create(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestAsyncNoForkUpdate(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + fns CallbackFns + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id"}, nil + }, + }, + cfg: cfgAsync, + obj: objAsync, + fns: CallbackFns{ + UpdateFn: func(s string) terraform.CallbackFn { + return func(err error, ctx context.Context) error { + return nil + } + }, + }, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkAsyncExternal := prepareNoForkAsyncExternal(tc.args.r, tc.args.cfg, tc.args.fns) + _, err := noForkAsyncExternal.Update(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestAsyncNoForkDelete(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + fns CallbackFns + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id"}, nil + }, + }, + cfg: cfgAsync, + obj: objAsync, + fns: CallbackFns{ + DestroyFn: func(s string) terraform.CallbackFn { + return func(err error, ctx context.Context) error { + return nil + } + }, + }, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkAsyncExternal := prepareNoForkAsyncExternal(tc.args.r, tc.args.cfg, tc.args.fns) + err := noForkAsyncExternal.Delete(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} diff --git a/pkg/controller/external_nofork.go b/pkg/controller/external_nofork.go index c6f9ed01..3bcd674b 100644 --- a/pkg/controller/external_nofork.go +++ b/pkg/controller/external_nofork.go @@ -17,6 +17,7 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/hashicorp/go-cty/cty" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" @@ -102,9 +103,14 @@ func getJSONMap(mg xpresource.Managed) (map[string]any, error) { return v.(map[string]any), nil } +type Resource interface { + Apply(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) + RefreshWithoutUpgrade(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) +} + type noForkExternal struct { ts terraform.Setup - resourceSchema *schema.Resource + resourceSchema Resource config *config.Resource instanceDiff *tf.InstanceDiff params map[string]any @@ -399,7 +405,7 @@ func getTimeoutParameters(config *config.Resource) map[string]any { //nolint:goc func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx context.Context, s *tf.InstanceState, resourceExists bool) (*tf.InstanceDiff, error) { //nolint:gocyclo resourceConfig := tf.NewResourceConfigRaw(n.params) - instanceDiff, err := schema.InternalMap(n.resourceSchema.Schema).Diff(ctx, s, resourceConfig, nil, n.ts.Meta, false) + instanceDiff, err := schema.InternalMap(n.config.TerraformResource.Schema).Diff(ctx, s, resourceConfig, nil, n.ts.Meta, false) if err != nil { return nil, errors.Wrap(err, "failed to get *terraform.InstanceDiff") } @@ -417,7 +423,7 @@ func (n *noForkExternal) getResourceDataDiff(tr resource.Terraformed, ctx contex } if instanceDiff != nil { v := cty.EmptyObjectVal - v, err = instanceDiff.ApplyToValue(v, n.resourceSchema.CoreConfigSchema()) + v, err = instanceDiff.ApplyToValue(v, n.config.TerraformResource.CoreConfigSchema()) if err != nil { return nil, errors.Wrap(err, "cannot apply Terraform instance diff to an empty value") } @@ -643,7 +649,7 @@ func (n *noForkExternal) Delete(ctx context.Context, _ xpresource.Managed) error } func (n *noForkExternal) fromInstanceStateToJSONMap(newState *tf.InstanceState) (map[string]interface{}, error) { - impliedType := n.resourceSchema.CoreConfigSchema().ImpliedType() + impliedType := n.config.TerraformResource.CoreConfigSchema().ImpliedType() attrsAsCtyValue, err := newState.AttrsAsObjectValue(impliedType) if err != nil { return nil, errors.Wrap(err, "could not convert attrs to cty value") diff --git a/pkg/controller/external_nofork_test.go b/pkg/controller/external_nofork_test.go new file mode 100644 index 00000000..cd854ea5 --- /dev/null +++ b/pkg/controller/external_nofork_test.go @@ -0,0 +1,342 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + tf "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + "github.com/pkg/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/resource/fake" + "github.com/crossplane/upjet/pkg/terraform" +) + +var ( + zl = zap.New(zap.UseDevMode(true)) + log = logging.NewLogrLogger(zl.WithName("provider-aws")) + ots = NewOperationStore(log) + timeout = time.Duration(1200000000000) + cfg = &config.Resource{ + TerraformResource: &schema.Resource{ + Timeouts: &schema.ResourceTimeout{ + Create: &timeout, + Read: &timeout, + Update: &timeout, + Delete: &timeout, + }, + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + }, + "id": { + Type: schema.TypeString, + Computed: true, + Required: false, + }, + "map": { + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "list": { + Type: schema.TypeList, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + ExternalName: config.IdentifierFromProvider, + Sensitive: config.Sensitive{AdditionalConnectionDetailsFn: func(attr map[string]any) (map[string][]byte, error) { + return nil, nil + }}, + } + obj = &fake.Terraformed{ + Parameterizable: fake.Parameterizable{ + Parameters: map[string]any{ + "name": "example", + "map": map[string]any{ + "key": "value", + }, + "list": []any{"elem1", "elem2"}, + }, + }, + Observable: fake.Observable{ + Observation: map[string]any{}, + }, + } +) + +func prepareNoForkExternal(r Resource, cfg *config.Resource) *noForkExternal { + schemaBlock := cfg.TerraformResource.CoreConfigSchema() + rawConfig, err := schema.JSONMapToStateValue(map[string]any{"name": "example"}, schemaBlock) + if err != nil { + panic(err) + } + return &noForkExternal{ + ts: terraform.Setup{}, + resourceSchema: r, + config: cfg, + params: map[string]any{ + "name": "example", + }, + rawConfig: rawConfig, + logger: log, + opTracker: NewAsyncTracker(), + } +} + +type mockResource struct { + ApplyFn func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) + RefreshWithoutUpgradeFn func(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) +} + +func (m mockResource) Apply(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return m.ApplyFn(ctx, s, d, meta) +} + +func (m mockResource) RefreshWithoutUpgrade(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return m.RefreshWithoutUpgradeFn(ctx, s, meta) +} + +func TestNoForkConnect(t *testing.T) { + type args struct { + setupFn terraform.SetupFn + cfg *config.Resource + ots *OperationTrackerStore + obj xpresource.Managed + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + setupFn: func(_ context.Context, _ client.Client, _ xpresource.Managed) (terraform.Setup, error) { + return terraform.Setup{}, nil + }, + cfg: cfg, + obj: obj, + ots: ots, + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := NewNoForkConnector(nil, tc.args.setupFn, tc.args.cfg, tc.args.ots, WithNoForkLogger(log)) + _, err := c.Connect(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestNoForkObserve(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + } + type want struct { + obs managed.ExternalObservation + err error + } + cases := map[string]struct { + args + want + }{ + "NotExists": { + args: args{ + r: mockResource{ + RefreshWithoutUpgradeFn: func(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return nil, nil + }, + }, + cfg: cfg, + obj: obj, + }, + want: want{ + obs: managed.ExternalObservation{ + ResourceExists: false, + ResourceUpToDate: false, + ResourceLateInitialized: false, + ConnectionDetails: nil, + Diff: "", + }, + }, + }, + "UpToDate": { + args: args{ + r: mockResource{ + RefreshWithoutUpgradeFn: func(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id", Attributes: map[string]string{"name": "example"}}, nil + }, + }, + cfg: cfg, + obj: obj, + }, + want: want{ + obs: managed.ExternalObservation{ + ResourceExists: true, + ResourceUpToDate: true, + ResourceLateInitialized: true, + ConnectionDetails: nil, + Diff: "", + }, + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkExternal := prepareNoForkExternal(tc.args.r, tc.args.cfg) + observation, err := noForkExternal.Observe(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.obs, observation); diff != "" { + t.Errorf("\n%s\nObserve(...): -want observation, +got observation:\n", diff) + } + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestNoForkCreate(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Unsuccessful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return nil, nil + }, + }, + cfg: cfg, + obj: obj, + }, + want: want{ + err: errors.New("failed to read the ID of the new resource"), + }, + }, + "Successful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id"}, nil + }, + }, + cfg: cfg, + obj: obj, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkExternal := prepareNoForkExternal(tc.args.r, tc.args.cfg) + _, err := noForkExternal.Create(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestNoForkUpdate(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id"}, nil + }, + }, + cfg: cfg, + obj: obj, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkExternal := prepareNoForkExternal(tc.args.r, tc.args.cfg) + _, err := noForkExternal.Update(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} + +func TestNoForkDelete(t *testing.T) { + type args struct { + r Resource + cfg *config.Resource + obj xpresource.Managed + } + type want struct { + err error + } + cases := map[string]struct { + args + want + }{ + "Successful": { + args: args{ + r: mockResource{ + ApplyFn: func(ctx context.Context, s *tf.InstanceState, d *tf.InstanceDiff, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id"}, nil + }, + }, + cfg: cfg, + obj: obj, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + noForkExternal := prepareNoForkExternal(tc.args.r, tc.args.cfg) + err := noForkExternal.Delete(context.TODO(), tc.args.obj) + if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) + } + }) + } +} From 6d324c10d4e1cb33d42be68bb1308ca92bf0bdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Mon, 13 Nov 2023 12:15:32 +0300 Subject: [PATCH 59/60] Add licence statements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- pkg/controller/external_async_nofork_test.go | 8 ++++++-- pkg/controller/external_nofork_test.go | 12 ++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/controller/external_async_nofork_test.go b/pkg/controller/external_async_nofork_test.go index 47f3737a..7acd0ae4 100644 --- a/pkg/controller/external_async_nofork_test.go +++ b/pkg/controller/external_async_nofork_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2023 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + package controller import ( @@ -87,7 +91,7 @@ func prepareNoForkAsyncExternal(r Resource, cfg *config.Resource, fns CallbackFn "name": "example", }, rawConfig: rawConfig, - logger: log, + logger: logTest, opTracker: NewAsyncTracker(), }, callback: fns, @@ -121,7 +125,7 @@ func TestAsyncNoForkConnect(t *testing.T) { } for name, tc := range cases { t.Run(name, func(t *testing.T) { - c := NewNoForkAsyncConnector(nil, tc.args.ots, tc.args.setupFn, tc.args.cfg, WithNoForkAsyncLogger(log)) + c := NewNoForkAsyncConnector(nil, tc.args.ots, tc.args.setupFn, tc.args.cfg, WithNoForkAsyncLogger(logTest)) _, err := c.Connect(context.TODO(), tc.args.obj) if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) diff --git a/pkg/controller/external_nofork_test.go b/pkg/controller/external_nofork_test.go index cd854ea5..e6f96dd8 100644 --- a/pkg/controller/external_nofork_test.go +++ b/pkg/controller/external_nofork_test.go @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2023 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + package controller import ( @@ -24,8 +28,8 @@ import ( var ( zl = zap.New(zap.UseDevMode(true)) - log = logging.NewLogrLogger(zl.WithName("provider-aws")) - ots = NewOperationStore(log) + logTest = logging.NewLogrLogger(zl.WithName("provider-aws")) + ots = NewOperationStore(logTest) timeout = time.Duration(1200000000000) cfg = &config.Resource{ TerraformResource: &schema.Resource{ @@ -94,7 +98,7 @@ func prepareNoForkExternal(r Resource, cfg *config.Resource) *noForkExternal { "name": "example", }, rawConfig: rawConfig, - logger: log, + logger: logTest, opTracker: NewAsyncTracker(), } } @@ -140,7 +144,7 @@ func TestNoForkConnect(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { - c := NewNoForkConnector(nil, tc.args.setupFn, tc.args.cfg, tc.args.ots, WithNoForkLogger(log)) + c := NewNoForkConnector(nil, tc.args.setupFn, tc.args.cfg, tc.args.ots, WithNoForkLogger(logTest)) _, err := c.Connect(context.TODO(), tc.args.obj) if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { t.Errorf("\n%s\nConnect(...): -want error, +got error:\n", diff) From 911e290126da5747c8b8f13af348e135ce6395db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Mon, 13 Nov 2023 15:41:10 +0300 Subject: [PATCH 60/60] Add unit tests for errors package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add unit test for InitProvider Signed-off-by: Sergen Yalçın --- pkg/controller/external_nofork_test.go | 59 +++++ pkg/terraform/errors/errors_test.go | 288 +++++++++++++++++++++++++ 2 files changed, 347 insertions(+) diff --git a/pkg/controller/external_nofork_test.go b/pkg/controller/external_nofork_test.go index e6f96dd8..3109f669 100644 --- a/pkg/controller/external_nofork_test.go +++ b/pkg/controller/external_nofork_test.go @@ -140,6 +140,29 @@ func TestNoForkConnect(t *testing.T) { ots: ots, }, }, + "HCL": { + args: args{ + setupFn: func(_ context.Context, _ client.Client, _ xpresource.Managed) (terraform.Setup, error) { + return terraform.Setup{}, nil + }, + cfg: cfg, + obj: &fake.Terraformed{ + Parameterizable: fake.Parameterizable{ + Parameters: map[string]any{ + "name": " ${jsonencode({\n type = \"object\"\n })}", + "map": map[string]any{ + "key": "value", + }, + "list": []any{"elem1", "elem2"}, + }, + }, + Observable: fake.Observable{ + Observation: map[string]any{}, + }, + }, + ots: ots, + }, + }, } for name, tc := range cases { @@ -207,6 +230,42 @@ func TestNoForkObserve(t *testing.T) { }, }, }, + "InitProvider": { + args: args{ + r: mockResource{ + RefreshWithoutUpgradeFn: func(ctx context.Context, s *tf.InstanceState, meta interface{}) (*tf.InstanceState, diag.Diagnostics) { + return &tf.InstanceState{ID: "example-id", Attributes: map[string]string{"name": "example2"}}, nil + }, + }, + cfg: cfg, + obj: &fake.Terraformed{ + Parameterizable: fake.Parameterizable{ + Parameters: map[string]any{ + "name": "example", + "map": map[string]any{ + "key": "value", + }, + "list": []any{"elem1", "elem2"}, + }, + InitParameters: map[string]any{ + "list": []any{"elem1", "elem2", "elem3"}, + }, + }, + Observable: fake.Observable{ + Observation: map[string]any{}, + }, + }, + }, + want: want{ + obs: managed.ExternalObservation{ + ResourceExists: true, + ResourceUpToDate: false, + ResourceLateInitialized: true, + ConnectionDetails: nil, + Diff: "", + }, + }, + }, } for name, tc := range cases { diff --git a/pkg/terraform/errors/errors_test.go b/pkg/terraform/errors/errors_test.go index e1602254..a554c24b 100644 --- a/pkg/terraform/errors/errors_test.go +++ b/pkg/terraform/errors/errors_test.go @@ -321,3 +321,291 @@ func TestNewPlanFailed(t *testing.T) { }) } } + +func TestNewRetryScheduleError(t *testing.T) { + type args struct { + invocationCount, ttl int + } + tests := map[string]struct { + args + wantErrMessage string + }{ + "Successful": { + args: args{ + invocationCount: 101, + ttl: 100, + }, + wantErrMessage: "native provider reuse budget has been exceeded: invocationCount: 101, ttl: 100", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := NewRetryScheduleError(tc.args.invocationCount, tc.args.ttl) + got := err.Error() + if diff := cmp.Diff(tc.wantErrMessage, got); diff != "" { + t.Errorf("\nNewRetryScheduleError(...): -want message, +got message:\n%s", diff) + } + }) + } +} + +func TestIsRetryScheduleError(t *testing.T) { + var nilErr *retrySchedule + type args struct { + err error + } + tests := map[string]struct { + args + want bool + }{ + "NilError": { + args: args{}, + want: false, + }, + "NilRetryScheduleError": { + args: args{ + err: nilErr, + }, + want: true, + }, + "NonRetryScheduleError": { + args: args{ + err: errorBoom, + }, + want: false, + }, + "Successful": { + args: args{err: NewRetryScheduleError(101, 100)}, + want: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + if got := IsRetryScheduleError(tc.args.err); got != tc.want { + t.Errorf("IsRetryScheduleError() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestNewAsyncCreateFailed(t *testing.T) { + type args struct { + err error + } + tests := map[string]struct { + args + wantErrMessage string + }{ + "Successful": { + args: args{ + err: errors.New("already exists"), + }, + wantErrMessage: "async create failed: already exists", + }, + "Nil": { + args: args{ + err: nil, + }, + wantErrMessage: "", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := NewAsyncCreateFailed(tc.args.err) + got := "" + if err != nil { + got = err.Error() + } + if diff := cmp.Diff(tc.wantErrMessage, got); diff != "" { + t.Errorf("\nNewAsyncCreateFailed(...): -want message, +got message:\n%s", diff) + } + }) + } +} + +func TestIsAsyncCreateFailed(t *testing.T) { + var nilErr *asyncCreateFailed + type args struct { + err error + } + tests := map[string]struct { + args + want bool + }{ + "NilError": { + args: args{}, + want: false, + }, + "NilAsyncCreateError": { + args: args{ + err: nilErr, + }, + want: true, + }, + "NonAsyncCreateError": { + args: args{ + err: errorBoom, + }, + want: false, + }, + "Successful": { + args: args{err: NewAsyncCreateFailed(errors.New("test"))}, + want: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + if got := IsAsyncCreateFailed(tc.args.err); got != tc.want { + t.Errorf("IsAsyncCreateFailed() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestAsyncUpdateFailed(t *testing.T) { + type args struct { + err error + } + tests := map[string]struct { + args + wantErrMessage string + }{ + "Successful": { + args: args{ + err: errors.New("immutable field"), + }, + wantErrMessage: "async update failed: immutable field", + }, + "Nil": { + args: args{ + err: nil, + }, + wantErrMessage: "", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := NewAsyncUpdateFailed(tc.args.err) + got := "" + if err != nil { + got = err.Error() + } + if diff := cmp.Diff(tc.wantErrMessage, got); diff != "" { + t.Errorf("\nAsyncUpdateFailed(...): -want message, +got message:\n%s", diff) + } + }) + } +} + +func TestIsAsyncUpdateFailed(t *testing.T) { + var nilErr *asyncUpdateFailed + type args struct { + err error + } + tests := map[string]struct { + args + want bool + }{ + "NilError": { + args: args{}, + want: false, + }, + "NilAsyncUpdateError": { + args: args{ + err: nilErr, + }, + want: true, + }, + "NonAsyncUpdateError": { + args: args{ + err: errorBoom, + }, + want: false, + }, + "Successful": { + args: args{err: NewAsyncUpdateFailed(errors.New("test"))}, + want: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + if got := IsAsyncUpdateFailed(tc.args.err); got != tc.want { + t.Errorf("IsAsyncUpdateFailed() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestAsyncDeleteFailed(t *testing.T) { + type args struct { + err error + } + tests := map[string]struct { + args + wantErrMessage string + }{ + "Successful": { + args: args{ + err: errors.New("dependency violation"), + }, + wantErrMessage: "async delete failed: dependency violation", + }, + "Nil": { + args: args{ + err: nil, + }, + wantErrMessage: "", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := NewAsyncDeleteFailed(tc.args.err) + got := "" + if err != nil { + got = err.Error() + } + if diff := cmp.Diff(tc.wantErrMessage, got); diff != "" { + t.Errorf("\nAsyncDeleteFailed(...): -want message, +got message:\n%s", diff) + } + }) + } +} + +func TestIsAsyncDeleteFailed(t *testing.T) { + var nilErr *asyncDeleteFailed + type args struct { + err error + } + tests := map[string]struct { + args + want bool + }{ + "NilError": { + args: args{}, + want: false, + }, + "NilAsyncUpdateError": { + args: args{ + err: nilErr, + }, + want: true, + }, + "NonAsyncUpdateError": { + args: args{ + err: errorBoom, + }, + want: false, + }, + "Successful": { + args: args{err: NewAsyncDeleteFailed(errors.New("test"))}, + want: true, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + if got := IsAsyncDeleteFailed(tc.args.err); got != tc.want { + t.Errorf("IsAsyncDeleteFailed() = %v, want %v", got, tc.want) + } + }) + } +}