Consume upjet ProviderScheduler

cluster/images/provider-aws/Dockerfile

@@ -4,8 +4,6 @@ RUN apk --no-cache add ca-certificates bash
 ARG TARGETOS
 ARG TARGETARCH
 
-ADD "bin/${TARGETOS}_${TARGETARCH}/provider" /usr/local/bin/provider
-
 ENV USER_ID=65532
 
 # Setup Terraform environment
@@ -44,8 +42,11 @@ ENV TERRAFORM_PROVIDER_SOURCE ${TERRAFORM_PROVIDER_SOURCE}
 ENV TERRAFORM_PROVIDER_VERSION ${TERRAFORM_PROVIDER_VERSION}
 # ENV TERRAFORM_NATIVE_PROVIDER_PATH ${PLUGIN_DIR}/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_v${TERRAFORM_PROVIDER_VERSION}_x5
 ENV TF_APPEND_USER_AGENT "crossplane-provider-aws/${CROSSPLANE_PROVIDER_VERSION} upbound-provider-aws/${CROSSPLANE_PROVIDER_VERSION}"
+ENV TERRAFORM_NATIVE_PROVIDER_PATH ${PLUGIN_DIR}/${TERRAFORM_PROVIDER_DOWNLOAD_NAME}_v${TERRAFORM_PROVIDER_VERSION}_x5
 
 USER ${USER_ID}
 EXPOSE 8080
 
+ADD "bin/${TARGETOS}_${TARGETARCH}/provider" /usr/local/bin/provider
+
 ENTRYPOINT ["provider"]

cmd/provider/main.go

@@ -35,20 +35,22 @@ import (
 
 func main() {
 	var (
-		app                = kingpin.New(filepath.Base(os.Args[0]), "AWS support for Crossplane.").DefaultEnvars()
-		debug              = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
-		syncInterval       = app.Flag("sync", "Sync interval controls how often all resources will be double checked for drift.").Short('s').Default("1h").Duration()
-		pollInterval       = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration()
-		leaderElection     = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
-		maxReconcileRate   = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may be checked for drift from the desired state.").Default("10").Int()
-		terraformVersion   = app.Flag("terraform-version", "Terraform version.").Required().Envar("TERRAFORM_VERSION").String()
-		providerSource     = app.Flag("terraform-provider-source", "Terraform provider source.").Required().Envar("TERRAFORM_PROVIDER_SOURCE").String()
-		providerVersion    = app.Flag("terraform-provider-version", "Terraform provider version.").Required().Envar("TERRAFORM_PROVIDER_VERSION").String()
-		nativeProviderPath = app.Flag("terraform-native-provider-path", "Terraform native provider path for shared execution.").Default("").Envar("TERRAFORM_NATIVE_PROVIDER_PATH").String()
+		app              = kingpin.New(filepath.Base(os.Args[0]), "AWS support for Crossplane.").DefaultEnvars()
+		debug            = app.Flag("debug", "Run with debug logging.").Short('d').Bool()
+		syncInterval     = app.Flag("sync", "Sync interval controls how often all resources will be double checked for drift.").Short('s').Default("1h").Duration()
+		pollInterval     = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration()
+		leaderElection   = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").OverrideDefaultFromEnvar("LEADER_ELECTION").Bool()
+		maxReconcileRate = app.Flag("max-reconcile-rate", "The global maximum rate per second at which resources may be checked for drift from the desired state.").Default("10").Int()
+		pluginProcessTTL = app.Flag("provider-ttl", "TTL for the native plugin processes before they are replaced. Changing the default may increase memory consumption.").Default("100").Int()
 
 		namespace                  = app.Flag("namespace", "Namespace used to set as default scope in default secret store config.").Default("crossplane-system").Envar("POD_NAMESPACE").String()
 		enableExternalSecretStores = app.Flag("enable-external-secret-stores", "Enable support for ExternalSecretStores.").Default("false").Envar("ENABLE_EXTERNAL_SECRET_STORES").Bool()
 	)
+	setupConfig := &clients.SetupConfig{}
+	setupConfig.TerraformVersion = app.Flag("terraform-version", "Terraform version.").Required().Envar("TERRAFORM_VERSION").String()
+	setupConfig.NativeProviderSource = app.Flag("terraform-provider-source", "Terraform provider source.").Required().Envar("TERRAFORM_PROVIDER_SOURCE").String()
+	setupConfig.NativeProviderVersion = app.Flag("terraform-provider-version", "Terraform provider version.").Required().Envar("TERRAFORM_PROVIDER_VERSION").String()
+	setupConfig.NativeProviderPath = app.Flag("terraform-native-provider-path", "Terraform native provider path for shared execution.").Default("").Envar("TERRAFORM_NATIVE_PROVIDER_PATH").String()
 
 	kingpin.MustParse(app.Parse(os.Args[1:]))
 
@@ -84,9 +86,9 @@ func main() {
 	// we do not use the shared gRPC server and default to the regular
 	// Terraform CLI behaviour (of forking a plugin process per invocation).
 	// This removes some complexity for setting up development environments.
-	var runner terraform.ProviderRunner = terraform.NewNoOpProviderRunner()
-	if len(*nativeProviderPath) != 0 {
-		runner = terraform.NewSharedProvider(log, *nativeProviderPath, "registry.terraform.io/"+*providerSource)
+	setupConfig.DefaultScheduler = terraform.NewNoOpProviderScheduler()
+	if len(*setupConfig.NativeProviderPath) != 0 {
+		setupConfig.DefaultScheduler = terraform.NewSharedProviderScheduler(log, *pluginProcessTTL, terraform.WithNativeProviderPath(*setupConfig.NativeProviderPath), terraform.WithNativeProviderName("registry.terraform.io/"+*setupConfig.NativeProviderSource))
 	}
 
 	o := tjcontroller.Options{
@@ -98,8 +100,8 @@ func main() {
 			Features:                &feature.Flags{},
 		},
 		Provider:       config.GetProvider(),
-		WorkspaceStore: terraform.NewWorkspaceStore(log, terraform.WithProviderRunner(runner), terraform.WithProcessReportInterval(*pollInterval)),
-		SetupFn:        clients.SelectTerraformSetup(*terraformVersion, *providerSource, *providerVersion),
+		WorkspaceStore: terraform.NewWorkspaceStore(log, terraform.WithDisableInit(len(*setupConfig.NativeProviderPath) != 0), terraform.WithProcessReportInterval(*pollInterval)),
+		SetupFn:        clients.SelectTerraformSetup(log, setupConfig),
 	}
 
 	if *enableExternalSecretStores {

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.24.0
 	github.com/pkg/errors v0.9.1
-	github.com/upbound/upjet v0.9.0-rc.0.0.20230308103251-5377e5db7994
+	github.com/upbound/upjet v0.9.0-rc.0.0.20230327151245-05c3d628e791
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	k8s.io/apimachinery v0.25.0
 	k8s.io/client-go v0.25.0

go.sum

@@ -694,8 +694,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1
 github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ=
 github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
-github.com/upbound/upjet v0.9.0-rc.0.0.20230308103251-5377e5db7994 h1:sehUwDpJIZgM9JjDrY2UvAjmxqeli8r9/cjQbEI9laY=
-github.com/upbound/upjet v0.9.0-rc.0.0.20230308103251-5377e5db7994/go.mod h1:wwCuupQRfs+SL6LGAlAHu5Z/oPffvWaBQ9Luf+7CIZ8=
+github.com/upbound/upjet v0.9.0-rc.0.0.20230327151245-05c3d628e791 h1:Y2+LQwPlfgRFbe3Yev6P+7aumifXRvGl+I3nryrU8qM=
+github.com/upbound/upjet v0.9.0-rc.0.0.20230327151245-05c3d628e791/go.mod h1:wwCuupQRfs+SL6LGAlAHu5Z/oPffvWaBQ9Luf+7CIZ8=
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=

internal/clients/aws.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"k8s.io/apimachinery/pkg/types"
 
+	"github.com/crossplane/crossplane-runtime/pkg/logging"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 	"github.com/pkg/errors"
 	"github.com/upbound/upjet/pkg/terraform"
@@ -36,19 +37,29 @@ const (
 	keyExternalID                = "external_id"
 )
 
-func SelectTerraformSetup(version, providerSource, providerVersion string) terraform.SetupFn {
+type SetupConfig struct {
+	NativeProviderPath    *string
+	NativeProviderSource  *string
+	NativeProviderVersion *string
+	TerraformVersion      *string
+	DefaultScheduler      terraform.ProviderScheduler
+}
+
+func SelectTerraformSetup(log logging.Logger, config *SetupConfig) terraform.SetupFn {
 	return func(ctx context.Context, c client.Client, mg resource.Managed) (terraform.Setup, error) {
 		pc := &v1beta1.ProviderConfig{}
 		if err := c.Get(ctx, types.NamespacedName{Name: mg.GetProviderConfigReference().Name}, pc); err != nil {
 			return terraform.Setup{}, errors.Wrapf(err, "cannot get referenced Provider: %s", mg.GetProviderConfigReference().Name)
 		}
 		ps := terraform.Setup{
-			Version: version,
+			Version: *config.TerraformVersion,
 			Requirement: terraform.ProviderRequirement{
-				Source:  providerSource,
-				Version: providerVersion,
+				Source:  *config.NativeProviderSource,
+				Version: *config.NativeProviderVersion,
 			},
+			Scheduler: config.DefaultScheduler,
 		}
+
 		account, err := getAccountId(ctx, c, mg)
 		if err != nil {
 			return terraform.Setup{}, errors.Wrap(err, "cannot get account id")
@@ -62,6 +73,11 @@ func SelectTerraformSetup(version, providerSource, providerVersion string) terra
 			if err != nil {
 				return terraform.Setup{}, errors.Wrap(err, "cannot build terraform configuration")
 			}
+			// we cannot use the shared scheduler here.
+			// We will force a workspace scheduler if we can configure one.
+			if len(*config.NativeProviderPath) != 0 {
+				ps.Scheduler = terraform.NewWorkspaceProviderScheduler(log, terraform.WithNativeProviderPath(*config.NativeProviderPath), terraform.WithNativeProviderName("registry.terraform.io/"+*config.NativeProviderSource))
+			}
 		} else {
 			err = pushDownTerraformSetupBuilder(ctx, c, mg, pc, &ps)
 			if err != nil {