Implement basic provider upgrade testing

examples/examples_nodejs_test.go

@@ -6,9 +6,12 @@ package examples
 
 import (
 	"path"
+	"path/filepath"
 	"testing"
 
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+
+	"github.com/pulumi/pulumi-tls/examples/v5/internal/testutil"
 )
 
 func TestAccPrivateKeyTs(t *testing.T) {
@@ -29,6 +32,25 @@ func TestAccSelfSignedCert(t *testing.T) {
 	integration.ProgramTest(t, &test)
 }
 
+// This test is a bit special as it is not an example, but a way to detect changes in the current version of provider or
+// provider SDK that would generate unexpected plans for users that have stacks provisioned on the baseline version
+// which is typically the latest released version.
+//
+// Currently the baseline version is specified by editing ./provider-update/ts/package.json
+// .dependencies["@pulumi/tls"].
+//
+// Note that this is currently pointing at v5.0.0, as pointing it at "^4.0.0" breaks the test. We do not expect the
+// update test to pass across major versions in general, although in the specific case of this provider there is some
+// compensating PreStateUpgradeHook code that makes the plans more tolerable (Update instead of Replace).
+func TestProviderUpdate(t *testing.T) {
+	test := getJSBaseOptions(t).
+		With(integration.ProgramTestOptions{
+			Dir: filepath.Join(getCwd(t), "provider-update", "ts"),
+		})
+
+	testutil.ProviderUpdateTest(t, "pulumi-resource-tls", test)
+}
+
 func getJSBaseOptions(t *testing.T) integration.ProgramTestOptions {
 	base := getBaseOptions()
 	baseJS := base.With(integration.ProgramTestOptions{

examples/internal/testutil/testutil.go

@@ -0,0 +1,157 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pulumi/pulumi/pkg/v3/engine"
+	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	pulumi_testing "github.com/pulumi/pulumi/sdk/v3/go/common/testing"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/retry"
+)
+
+// Test that upgrading the binary and language SDK is safe and results in no-change preview and update.
+//
+// providerBinary: name of the binary that may be in PATH, such as "pulumi-resource-random".
+//
+// Ideally this should be expressed in ProgramTest facilities, but it is currently not possible.
+//
+// Also, currently this only works for Node Pulumi programs.
+func ProviderUpdateTest(t *testing.T, providerBinary string, opts integration.ProgramTestOptions) {
+	opts.SkipEmptyPreviewUpdate = true
+	opts.SkipExportImport = true
+
+	// The environment may have providerBinary in PATH, but the first update should use a different version.
+	// Restrict the PATH so pulumi CLI does not see it.
+	opts = opts.With(restrictPath(t, "node", "pulumi", "yarn"))
+
+	// Capture workdir from PrepareProject, as integration.ProgramTester does not expose it.
+	var workdir string
+
+	// This will be called only once. Call yarn install. Assume package.json refs a prod dependency like this:
+	//
+	//    "@pulumi/random": "^4.0.0"
+	//
+	// Then this should pull the latest prod version of the provider Node SDK, and then Pulumi will auto-install the
+	// matching provider binary version.
+	opts.PrepareProject = func(info *engine.Projinfo) (err error) {
+		workdir = info.Root
+		yarn(t, opts, workdir, "install")
+		return nil
+	}
+
+	pt := integration.ProgramTestManualLifeCycle(t, &opts)
+	err := pt.TestLifeCyclePrepare()
+	require.NoError(t, err)
+
+	pt.TestFinished = false
+	defer pt.TestCleanUp()
+
+	err = pt.TestLifeCycleInitialize()
+	require.NoError(t, err)
+
+	defer func() {
+		destroyErr := pt.TestLifeCycleDestroy()
+		require.NoError(t, destroyErr)
+	}()
+
+	// This should preview and update the stack using the stable version of the provider and its SDK.
+	err = pt.TestPreviewUpdateAndEdits()
+	require.NoError(t, err)
+
+	// Now bring the local binary being tested back in PATH.
+	opts = opts.With(restrictPath(t, "node", "pulumi", "yarn", providerBinary))
+
+	// Also bring local Node SDK versions in for testing via yarn link.
+	for _, d := range opts.Dependencies {
+		yarn(t, opts, workdir, "link", d)
+	}
+
+	// Test that local provider and SDK versions generate nop preview/udpate on the stack.
+	err = pt.PreviewAndUpdate(workdir, "upgraded",
+		false, /* shouldFail */
+		true,  /* expectNopPreview */
+		true /* expectNopUpdate */)
+	require.NoError(t, err)
+
+	pt.TestFinished = true
+}
+
+func yarn(t *testing.T, opts integration.ProgramTestOptions, wd string, args ...string) {
+	name := "yarn " + strings.Join(args, " ")
+
+	// Yarn will time out if multiple processes are trying to install packages at the same time.
+	pulumi_testing.YarnInstallMutex.Lock()
+	defer t.Log("released yarn install lock")
+	defer pulumi_testing.YarnInstallMutex.Unlock()
+	t.Log("acquired yarn install lock")
+
+	var yarnBin string = opts.YarnBin
+	if yarnBin == "" {
+		var err error
+		yarnBin, err = exec.LookPath("yarn")
+		require.NoError(t, err)
+		require.NotEmpty(t, yarnBin)
+	}
+
+	_, _, err := retry.Until(context.Background(), retry.Acceptor{
+		Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) {
+			runerr := integration.RunCommand(t, name, append([]string{yarnBin}, args...), wd, &opts)
+			if runerr == nil {
+				return true, nil, nil
+			} else if _, ok := runerr.(*exec.ExitError); ok {
+				// yarn failed, let's try again, assuming we haven't failed a few times.
+				if try+1 >= 3 {
+					return false, nil, fmt.Errorf("%v did not complete after %v tries", name, try+1)
+				}
+
+				return false, nil, nil
+			}
+
+			// someother error, fail
+			return false, nil, runerr
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func restrictPath(t *testing.T, commands ...string) integration.ProgramTestOptions {
+	paths := lookPaths(t, commands...)
+	pathVar := strings.Join(paths, string(filepath.ListSeparator))
+	return integration.ProgramTestOptions{
+		Env: []string{"PATH=" + pathVar},
+	}
+}
+
+func lookPaths(t *testing.T, commands ...string) (ret []string) {
+	for _, c := range commands {
+		cPath, err := exec.LookPath(c)
+		require.NoError(t, err)
+		require.NotEmpty(t, cPath)
+		ret = append(ret, filepath.Dir(cPath))
+	}
+	return
+}

examples/provider-update/ts/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: provider-update
+runtime: nodejs
+description: This stack helps testing provider upgrades and is not a proper example.

examples/provider-update/ts/index.ts

@@ -0,0 +1,29 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as tls from "@pulumi/tls";
+
+export const key = new tls.PrivateKey("my-private-key", {
+    algorithm: "ECDSA",
+    ecdsaCurve: "P384",
+});
+
+export const ssCert = new tls.SelfSignedCert("ssoCert", {
+    allowedUses: ["cert_signing"],
+    privateKeyPem: key.privateKeyPem,
+    subject: {
+        commonName: `api.example.com`,
+    },
+    validityPeriodHours: (365*24),
+})

examples/provider-update/ts/package.json

@@ -0,0 +1,18 @@
+{
+    "name": "providerupdate",
+    "version": "0.0.1",
+    "license": "Apache-2.0",
+    "main": "bin/index.js",
+    "typings": "bin/index.d.ts",
+    "scripts": {
+        "build": "tsc"
+    },
+    "dependencies": {
+        "@pulumi/pulumi": "^3.0.0",
+        "@pulumi/tls": "v5.0.0-alpha.0"
+    },
+    "devDependencies": {
+        "@types/node": "^8.0.27",
+        "typescript": "^3.0.0"
+    }
+}

examples/provider-update/ts/tsconfig.json

@@ -0,0 +1,22 @@
+{
+    "compilerOptions": {
+        "outDir": "bin",
+        "target": "es6",
+        "lib": [
+            "es6"
+        ],        
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "sourceMap": true,
+        "experimentalDecorators": true,
+        "pretty": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitAny": true,
+        "noImplicitReturns": true,
+        "forceConsistentCasingInFileNames": true,
+        "strictNullChecks": true
+    },
+    "files": [
+        "index.ts"
+    ]
+}

examples/self-signed-cert/ts/index.ts

@@ -24,4 +24,3 @@ new tls.SelfSignedCert("ssoCert", {
     },
     validityPeriodHours: (365*24),
 })
-