From b520a98024725e95a11fb58500349e2eaa3cab4e Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Tue, 14 Mar 2023 11:52:03 -0500 Subject: [PATCH 1/2] Implement basic provider upgrade testing --- examples/examples_nodejs_test.go | 22 +++ examples/internal/testutil/testutil.go | 157 ++++++++++++++++++++++ examples/provider-update/ts/Pulumi.yaml | 3 + examples/provider-update/ts/index.ts | 29 ++++ examples/provider-update/ts/package.json | 18 +++ examples/provider-update/ts/tsconfig.json | 22 +++ examples/self-signed-cert/ts/index.ts | 1 - 7 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 examples/internal/testutil/testutil.go create mode 100644 examples/provider-update/ts/Pulumi.yaml create mode 100644 examples/provider-update/ts/index.ts create mode 100644 examples/provider-update/ts/package.json create mode 100644 examples/provider-update/ts/tsconfig.json diff --git a/examples/examples_nodejs_test.go b/examples/examples_nodejs_test.go index 00aec5a4..7943beb3 100644 --- a/examples/examples_nodejs_test.go +++ b/examples/examples_nodejs_test.go @@ -6,9 +6,12 @@ package examples import ( "path" + "path/filepath" "testing" "github.com/pulumi/pulumi/pkg/v3/testing/integration" + + "github.com/pulumi/pulumi-tls/examples/v5/internal/testutil" ) func TestAccPrivateKeyTs(t *testing.T) { @@ -29,6 +32,25 @@ func TestAccSelfSignedCert(t *testing.T) { integration.ProgramTest(t, &test) } +// This test is a bit special as it is not an example, but a way to detect changes in the current version of provider or +// provider SDK that would generate unexpected plans for users that have stacks provisioned on the baseline version +// which is typically the latest released version. +// +// Currently the baseline version is specified by editing ./provider-update/ts/package.json +// .dependencies["@pulumi/tls"]. +// +// Note that this is currently pointing at v5.0.0, as pointing it at "^4.0.0" breaks the test. We do not expect the +// update test to pass across major versions in general, although in the specific case of this provider there is some +// compensating PreStateUpgradeHook code that makes the plans more tolerable (Update instead of Replace). +func TestProviderUpdate(t *testing.T) { + test := getJSBaseOptions(t). + With(integration.ProgramTestOptions{ + Dir: filepath.Join(getCwd(t), "provider-update", "ts"), + }) + + testutil.ProviderUpdateTest(t, "pulumi-resource-tls", test) +} + func getJSBaseOptions(t *testing.T) integration.ProgramTestOptions { base := getBaseOptions() baseJS := base.With(integration.ProgramTestOptions{ diff --git a/examples/internal/testutil/testutil.go b/examples/internal/testutil/testutil.go new file mode 100644 index 00000000..366c70a7 --- /dev/null +++ b/examples/internal/testutil/testutil.go @@ -0,0 +1,157 @@ +// Copyright 2016-2023, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/pulumi/pulumi/pkg/v3/engine" + "github.com/pulumi/pulumi/pkg/v3/testing/integration" + pulumi_testing "github.com/pulumi/pulumi/sdk/v3/go/common/testing" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/retry" +) + +// Test that upgrading the binary and language SDK is safe and results in no-change preview and update. +// +// providerBinary: name of the binary that may be in PATH, such as "pulumi-resource-random". +// +// Ideally this should be expressed in ProgramTest facilities, but it is currently not possible. +// +// Also, currently this only works for Node Pulumi programs. +func ProviderUpdateTest(t *testing.T, providerBinary string, opts integration.ProgramTestOptions) { + opts.SkipEmptyPreviewUpdate = true + opts.SkipExportImport = true + + // The environment may have providerBinary in PATH, but the first update should use a different version. + // Restrict the PATH so pulumi CLI does not see it. + opts = opts.With(restrictPath(t, "node", "pulumi", "yarn")) + + // Capture workdir from PrepareProject, as integration.ProgramTester does not expose it. + var workdir string + + // This will be called only once. Call yarn install. Assume package.json refs a prod dependency like this: + // + // "@pulumi/random": "^4.0.0" + // + // Then this should pull the latest prod version of the provider Node SDK, and then Pulumi will auto-install the + // matching provider binary version. + opts.PrepareProject = func(info *engine.Projinfo) (err error) { + workdir = info.Root + yarn(t, opts, workdir, "install") + return nil + } + + pt := integration.ProgramTestManualLifeCycle(t, &opts) + err := pt.TestLifeCyclePrepare() + require.NoError(t, err) + + pt.TestFinished = false + defer pt.TestCleanUp() + + err = pt.TestLifeCycleInitialize() + require.NoError(t, err) + + defer func() { + destroyErr := pt.TestLifeCycleDestroy() + require.NoError(t, destroyErr) + }() + + // This should preview and update the stack using the stable version of the provider and its SDK. + err = pt.TestPreviewUpdateAndEdits() + require.NoError(t, err) + + // Now bring the local binary being tested back in PATH. + opts = opts.With(restrictPath(t, "node", "pulumi", "yarn", providerBinary)) + + // Also bring local Node SDK versions in for testing via yarn link. + for _, d := range opts.Dependencies { + yarn(t, opts, workdir, "link", d) + } + + // Test that local provider and SDK versions generate nop preview/udpate on the stack. + err = pt.PreviewAndUpdate(workdir, "upgraded", + false, /* shouldFail */ + true, /* expectNopPreview */ + true /* expectNopUpdate */) + require.NoError(t, err) + + pt.TestFinished = true +} + +func yarn(t *testing.T, opts integration.ProgramTestOptions, wd string, args ...string) { + name := "yarn " + strings.Join(args, " ") + + // Yarn will time out if multiple processes are trying to install packages at the same time. + pulumi_testing.YarnInstallMutex.Lock() + defer pulumi_testing.YarnInstallMutex.Unlock() + t.Log("acquired yarn install lock") + t.Log("released yarn install lock") + + var yarnBin string = opts.YarnBin + if yarnBin == "" { + var err error + yarnBin, err = exec.LookPath("yarn") + require.NoError(t, err) + require.NotEmpty(t, yarnBin) + } + + _, _, err := retry.Until(context.Background(), retry.Acceptor{ + Accept: func(try int, nextRetryTime time.Duration) (bool, interface{}, error) { + runerr := integration.RunCommand(t, name, append([]string{yarnBin}, args...), wd, &opts) + if runerr == nil { + return true, nil, nil + } else if _, ok := runerr.(*exec.ExitError); ok { + // yarn failed, let's try again, assuming we haven't failed a few times. + if try+1 >= 3 { + return false, nil, fmt.Errorf("%v did not complete after %v tries", name, try+1) + } + + return false, nil, nil + } + + // someother error, fail + return false, nil, runerr + }, + }) + if err != nil { + t.Fatal(err) + } +} + +func restrictPath(t *testing.T, commands ...string) integration.ProgramTestOptions { + paths := lookPaths(t, commands...) + pathVar := strings.Join(paths, string(filepath.ListSeparator)) + return integration.ProgramTestOptions{ + Env: []string{"PATH=" + pathVar}, + } +} + +func lookPaths(t *testing.T, commands ...string) (ret []string) { + for _, c := range commands { + cPath, err := exec.LookPath(c) + require.NoError(t, err) + require.NotEmpty(t, cPath) + ret = append(ret, filepath.Dir(cPath)) + } + return +} diff --git a/examples/provider-update/ts/Pulumi.yaml b/examples/provider-update/ts/Pulumi.yaml new file mode 100644 index 00000000..e22b729e --- /dev/null +++ b/examples/provider-update/ts/Pulumi.yaml @@ -0,0 +1,3 @@ +name: provider-update +runtime: nodejs +description: This stack helps testing provider upgrades and is not a proper example. diff --git a/examples/provider-update/ts/index.ts b/examples/provider-update/ts/index.ts new file mode 100644 index 00000000..dea12257 --- /dev/null +++ b/examples/provider-update/ts/index.ts @@ -0,0 +1,29 @@ +// Copyright 2016-2023, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as tls from "@pulumi/tls"; + +export const key = new tls.PrivateKey("my-private-key", { + algorithm: "ECDSA", + ecdsaCurve: "P384", +}); + +export const ssCert = new tls.SelfSignedCert("ssoCert", { + allowedUses: ["cert_signing"], + privateKeyPem: key.privateKeyPem, + subject: { + commonName: `api.example.com`, + }, + validityPeriodHours: (365*24), +}) diff --git a/examples/provider-update/ts/package.json b/examples/provider-update/ts/package.json new file mode 100644 index 00000000..03843261 --- /dev/null +++ b/examples/provider-update/ts/package.json @@ -0,0 +1,18 @@ +{ + "name": "providerupdate", + "version": "0.0.1", + "license": "Apache-2.0", + "main": "bin/index.js", + "typings": "bin/index.d.ts", + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@pulumi/pulumi": "^3.0.0", + "@pulumi/tls": "v5.0.0-alpha.0" + }, + "devDependencies": { + "@types/node": "^8.0.27", + "typescript": "^3.0.0" + } +} diff --git a/examples/provider-update/ts/tsconfig.json b/examples/provider-update/ts/tsconfig.json new file mode 100644 index 00000000..275ad886 --- /dev/null +++ b/examples/provider-update/ts/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "outDir": "bin", + "target": "es6", + "lib": [ + "es6" + ], + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "experimentalDecorators": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "forceConsistentCasingInFileNames": true, + "strictNullChecks": true + }, + "files": [ + "index.ts" + ] +} diff --git a/examples/self-signed-cert/ts/index.ts b/examples/self-signed-cert/ts/index.ts index 53825709..b0b8c452 100644 --- a/examples/self-signed-cert/ts/index.ts +++ b/examples/self-signed-cert/ts/index.ts @@ -24,4 +24,3 @@ new tls.SelfSignedCert("ssoCert", { }, validityPeriodHours: (365*24), }) - From e533a2b2a11e52ff83201cdd33665de326606392 Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Wed, 15 Mar 2023 09:19:10 -0500 Subject: [PATCH 2/2] Update examples/internal/testutil/testutil.go Co-authored-by: Ian Wahbe --- examples/internal/testutil/testutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/internal/testutil/testutil.go b/examples/internal/testutil/testutil.go index 366c70a7..8becba94 100644 --- a/examples/internal/testutil/testutil.go +++ b/examples/internal/testutil/testutil.go @@ -103,9 +103,9 @@ func yarn(t *testing.T, opts integration.ProgramTestOptions, wd string, args ... // Yarn will time out if multiple processes are trying to install packages at the same time. pulumi_testing.YarnInstallMutex.Lock() + defer t.Log("released yarn install lock") defer pulumi_testing.YarnInstallMutex.Unlock() t.Log("acquired yarn install lock") - t.Log("released yarn install lock") var yarnBin string = opts.YarnBin if yarnBin == "" {