Skip to content

Commit

Permalink
fix(pulumi): include build deps in plugin commands (#6260)
Browse files Browse the repository at this point in the history
In 0.12, we included build dependencies for Pulumi modules (e.g. when
previewing). A build dependency (e.g. on an `exec` Module or Build)
is a common way to ensure that the relevant language SDK is installed
for the module/action before running Pulumi.

We didn't bring this logic along when we adapted the Pulumi plugin
commands for 0.13—this is fixed here.

Also added a lightweight test project to test this logic (which should
also serve us well when making further fixes to this plugin).
  • Loading branch information
thsig committed Jul 8, 2024
1 parent 0a12df4 commit 1e5c1df
Show file tree
Hide file tree
Showing 33 changed files with 11,119 additions and 3,379 deletions.
3,886 changes: 2,249 additions & 1,637 deletions examples/pulumi/k8s-deployment/package-lock.json

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions examples/pulumi/k8s-namespace/Pulumi.k8s-namespace.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
config:
kubernetes:context: docker-desktop
pulumi-k8s:namespace: pulumi-k8s
backend:
url: https://api.pulumi.com
2 changes: 2 additions & 0 deletions examples/pulumi/k8s-namespace/garden.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ kind: Deploy
type: pulumi
name: k8s-namespace
description: Creates a k8s namespace.
dependencies:
- build.ensure-pulumi-sdk-for-k8s-namespace
spec:
createStack: true
cacheStatus: true
Expand Down
3,898 changes: 2,254 additions & 1,644 deletions examples/pulumi/k8s-namespace/package-lock.json

Large diffs are not rendered by default.

6 changes: 5 additions & 1 deletion examples/pulumi/project.garden.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,14 @@ apiVersion: garden.io/v1
kind: Project
name: pulumi
variables:
pulumiAppOrg: garden # <--- replace with your own org name
pulumiAppOrg: thsig
environments:
- name: local
providers:
- name: exec
# Ensure that the node SDK is installed for the k8s-namespace and k8s-deployment projects before we run pulumi.
initScript: "for dir in k8s-namespace k8s-deployment; do [ ! -d $dir/node_modules ] && cd $dir && npm install && cd ..; done"
- name: pulumi
dependencies: [exec]
environments: [local]
orgName: ${var.pulumiAppOrg}
169 changes: 94 additions & 75 deletions plugins/pulumi/src/commands.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ import type {
PluginCommandParams,
PluginContext,
} from "@garden-io/sdk/build/src/types.js"
import { PluginActionTask } from "@garden-io/sdk/build/src/types.js"
import { BuildTask, PluginActionTask } from "@garden-io/sdk/build/src/types.js"

import type { PulumiDeploy } from "./action.js"
import type { PulumiProvider } from "./provider.js"
import { Profile } from "@garden-io/core/build/src/util/profiling.js"
import type { PulumiParams } from "./helpers.js"
import type { OperationCounts, PreviewResult, PulumiParams } from "./helpers.js"
import {
cancelUpdate,
getModifiedPlansDirPath,
Expand All @@ -33,18 +33,19 @@ import {
import { dedent, deline } from "@garden-io/sdk/build/src/util/string.js"
import { BooleanParameter, parsePluginCommandArgs } from "@garden-io/sdk/build/src/util/cli.js"
import fsExtra from "fs-extra"

const { copy, emptyDir } = fsExtra
const { copy, emptyDir, writeJSON } = fsExtra
import { join } from "path"
import { isBuildAction } from "@garden-io/core/build/src/actions/build.js"
import { isDeployAction } from "@garden-io/core/build/src/actions/deploy.js"
import { TemplatableConfigContext } from "@garden-io/core/build/src/config/template-contexts/project.js"
import type { ActionTaskProcessParams, ValidResultType } from "@garden-io/core/build/src/tasks/base.js"
import type { ActionTaskProcessParams, BaseTask, ValidResultType } from "@garden-io/core/build/src/tasks/base.js"
import { deletePulumiDeploy } from "./handlers.js"
import type { ActionLog, Log } from "@garden-io/core/build/src/logger/log-entry.js"
import { createActionLog } from "@garden-io/core/build/src/logger/log-entry.js"
import { ActionSpecContext } from "@garden-io/core/build/src/config/template-contexts/actions.js"
import type { ProviderMap } from "@garden-io/core/build/src/config/provider.js"
import { styles } from "@garden-io/core/build/src/logger/styles.js"
import { isTruthy } from "@garden-io/core/build/src/util/util.js"

type PulumiBaseParams = Omit<PulumiParams, "action">

Expand All @@ -68,23 +69,22 @@ interface PulumiCommandSpec {
}) => Promise<any>
}

// TODO-G2-thor: Re-enable and test when 0.13 is stable enough to run commands.
// interface TotalSummary {
// /**
// * The ISO timestamp of when the plan was completed.
// */
// completedAt: string
// /**
// * The total number of operations by step type (excluding `same` steps).
// */
// totalStepCounts: OperationCounts
// /**
// * A more detailed summary for each pulumi service affected by the plan.
// */
// results: {
// [serviceName: string]: PreviewResult
// }
// }
interface TotalSummary {
/**
* The ISO timestamp of when the plan was completed.
*/
completedAt: string
/**
* The total number of operations by step type (excluding `same` steps).
*/
totalStepCounts: OperationCounts
/**
* A more detailed summary for each pulumi service affected by the plan.
*/
results: {
[serviceName: string]: PreviewResult
}
}

const pulumiCommandSpecs: PulumiCommandSpec[] = [
{
Expand Down Expand Up @@ -128,31 +128,35 @@ const pulumiCommandSpecs: PulumiCommandSpec[] = [
}
}
},
// TODO-G2-thor: Re-enable and test when 0.13 is stable enough to run commands.
// afterFn: async ({ ctx, log, results, pulumiTasks }) => {
// // No-op plans (i.e. where no resources were changed) are omitted here.
// const pulumiTaskResults = Object.fromEntries(
// pulumiTasks.map((t) => [t.getName(), results.getResult(t)?.outputs || null])
// )
// const totalStepCounts: OperationCounts = {}
// for (const result of Object.values(pulumiTaskResults)) {
// const opCounts = (<PreviewResult>result).operationCounts
// for (const [stepType, count] of Object.entries(opCounts)) {
// totalStepCounts[stepType] = (totalStepCounts[stepType] || 0) + count
// }
// }
// const totalSummary: TotalSummary = {
// completedAt: new Date().toISOString(),
// totalStepCounts,
// results: pulumiTaskResults,
// }
// const previewDirPath = getPreviewDirPath(ctx)
// const summaryPath = join(previewDirPath, "plan-summary.json")
// await writeJSON(summaryPath, totalSummary, { spaces: 2 })
// log.info("")
// log.info(styles.success(`Wrote plan summary to ${styles.accent(summaryPath)}`))
// return totalSummary
// },
afterFn: async ({ ctx, log, results, pulumiTasks }) => {
// No-op plans (i.e. where no resources were changed) are omitted here.
const pulumiTaskResults: { [name: string]: PreviewResult } = Object.fromEntries(
pulumiTasks
.map((t) => {
const outputs = results.getResult(t)?.outputs
return outputs && Object.keys(outputs).length > 0 ? [t.getName(), outputs] : null
})
.filter(isTruthy)
)
const totalStepCounts: OperationCounts = {}
for (const result of Object.values(pulumiTaskResults)) {
const opCounts = result.operationCounts
for (const [stepType, count] of Object.entries(opCounts)) {
totalStepCounts[stepType] = (totalStepCounts[stepType] || 0) + count
}
}
const totalSummary: TotalSummary = {
completedAt: new Date().toISOString(),
totalStepCounts,
results: pulumiTaskResults,
}
const previewDirPath = getPreviewDirPath(ctx)
const summaryPath = join(previewDirPath, "plan-summary.json")
await writeJSON(summaryPath, totalSummary, { spaces: 2 })
log.info("")
log.info(styles.success(`Wrote plan summary to ${styles.accent(summaryPath)}`))
return totalSummary
},
},
{
name: "cancel",
Expand Down Expand Up @@ -272,41 +276,56 @@ class PulumiPluginCommandTask extends PluginActionTask<PulumiDeploy, PulumiComma
* Override the base method to be sure that `garden plugins pulumi preview` happens in dependency order.
*/
override resolveProcessDependencies() {
const currentTask = this.getResolveTask(this.action)
if (this.skipRuntimeDependencies) {
return [currentTask]
}

const pulumiDeployNames = this.graph
.getDeploys()
.filter((d) => d.type === "pulumi")
.map((d) => d.name)

const deps = this.graph
const buildTasks = this.graph
.getDependencies({
kind: "Deploy",
name: this.getName(),
recursive: false,
filter: (depNode) => pulumiDeployNames.includes(depNode.name),
})
.filter(isDeployAction)

const depTasks = deps.map((action) => {
return new PulumiPluginCommandTask({
garden: this.garden,
graph: this.graph,
log: this.log,
action,
commandName: this.commandName,
commandDescription: this.commandDescription,
skipRuntimeDependencies: this.skipRuntimeDependencies,
runFn: this.runFn,
pulumiParams: this.pulumiParams,
resolvedProviders: this.resolvedProviders,
.filter(isBuildAction)
.map((action) => {
return new BuildTask({
garden: this.garden,
log: this.log,
action,
graph: this.graph,
force: false,
})
})
})
const tasks: BaseTask[] = [this.getResolveTask(this.action), ...buildTasks]

const pulumiDeployNames = this.graph
.getDeploys()
.filter((d) => d.type === "pulumi")
.map((d) => d.name)

if (!this.skipRuntimeDependencies) {
const deployTasks = this.graph
.getDependencies({
kind: "Deploy",
name: this.getName(),
recursive: false,
filter: (depNode) => pulumiDeployNames.includes(depNode.name),
})
.filter(isDeployAction)
.map((action) => {
return new PulumiPluginCommandTask({
garden: this.garden,
graph: this.graph,
log: this.log,
action,
commandName: this.commandName,
commandDescription: this.commandDescription,
skipRuntimeDependencies: this.skipRuntimeDependencies,
runFn: this.runFn,
pulumiParams: this.pulumiParams,
resolvedProviders: this.resolvedProviders,
})
})
tasks.push(...deployTasks)
}

return [currentTask, ...depTasks]
return tasks
}

async getStatus() {
Expand Down
3 changes: 3 additions & 0 deletions plugins/pulumi/test/.eslintrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"ignorePatterns": ["test-project*/"]
}
87 changes: 87 additions & 0 deletions plugins/pulumi/test/commands.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright (C) 2018-2024 Garden Technologies, Inc. <info@garden.io>
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

import { dirname, join, resolve } from "path"
import { fileURLToPath } from "node:url"
import fsExtra from "fs-extra"
import type { ResolvedConfigGraph } from "@garden-io/core/build/src/graph/config-graph.js"
import type { PluginContext } from "@garden-io/core/build/src/plugin-context.js"
import { makeTestGarden, type TestGarden } from "@garden-io/sdk/build/src/testing.js"
import type { Log } from "@garden-io/sdk/build/src/types.js"
import type { PulumiProvider } from "../src/provider.js"
import { gardenPlugin as pulumiPlugin } from "../src/index.js"
import { ensureNodeModules } from "./test-helpers.js"
import { getPulumiCommands } from "../src/commands.js"
import { expect } from "chai"

const moduleDirName = dirname(fileURLToPath(import.meta.url))

// Careful here!
// We have some packages in the test directory but when this here runs we're a subfolder of '/build'
// so to actually find the files we need to traverse back to the source folder.
// TODO: Find a better way to do this.
const projectRoot = resolve(moduleDirName, "../../test/", "test-project-local-script")

const deployARoot = join(projectRoot, "deploy-a")
const deployBRoot = join(projectRoot, "deploy-b")

// Looking for log entries indicating that these exec actions had run proved to be flaky, so we're using the
// more robust method of touching a file in the source dir to indicate that the action was run.
const buildAFile = join(deployARoot, "build-a.txt")
const runAFile = join(deployARoot, "run-a.txt")

const buildBFile = join(deployBRoot, "build-b.txt")
const runBFile = join(deployBRoot, "run-b.txt")

async function clearGeneratedFiles() {
await Promise.all(
[buildAFile, runAFile, buildBFile, runBFile].map(async (path) => {
try {
await fsExtra.remove(path)
} catch (err) {
// This file may not exist, we're just cleaning up in case of repeated test runs.
}
})
)
}

describe("pulumi plugin commands", () => {
let garden: TestGarden
let graph: ResolvedConfigGraph
let ctx: PluginContext
let log: Log
let provider: PulumiProvider

before(async () => {
await ensureNodeModules([deployARoot, deployBRoot])
const plugin = pulumiPlugin()
garden = await makeTestGarden(projectRoot, { plugins: [plugin] })
log = garden.log
provider = (await garden.resolveProvider({ log, name: "pulumi" })) as PulumiProvider
ctx = await garden.getPluginContext({ provider, templateContext: undefined, events: undefined })
graph = await garden.getResolvedConfigGraph({ log, emit: false })
await clearGeneratedFiles()
})

after(async () => {
await clearGeneratedFiles()
})

// Note: Since the stacks in this test project don't have any side-effects, we don't need an after-cleanup step here.

describe("preview command", () => {
it("executes Build dependencies, but not Run dependencies", async () => {
const previewCmd = getPulumiCommands().find((cmd) => cmd.name === "preview")!
await previewCmd.handler({ garden, ctx, args: [], graph, log })
expect(await fsExtra.pathExists(buildAFile), "build-a.txt should exist").to.eql(true)
expect(await fsExtra.pathExists(buildBFile), "build-b.txt should exist").to.eql(true)
expect(await fsExtra.pathExists(runAFile), "run-a.txt should not exist").to.eql(false)
expect(await fsExtra.pathExists(runBFile), "run-b.txt should not exist").to.eql(false)
})
})
})
20 changes: 2 additions & 18 deletions plugins/pulumi/test/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@
import type { Log, PluginContext } from "@garden-io/sdk/build/src/types.js"
import type { TestGarden } from "@garden-io/sdk/build/src/testing.js"
import { makeTestGarden } from "@garden-io/sdk/build/src/testing.js"
import { execa } from "execa"
import fsExtra from "fs-extra"
const { pathExists } = fsExtra
import { dirname, join, resolve } from "node:path"
import { deployPulumi, getPulumiDeployStatus } from "../src/handlers.js"
import type { PulumiProvider } from "../src/provider.js"
Expand All @@ -21,6 +18,7 @@ import { getStackVersionTag } from "../src/helpers.js"
import { getPulumiCommands } from "../src/commands.js"
import type { ResolvedConfigGraph } from "@garden-io/core/build/src/graph/config-graph.js"
import { fileURLToPath } from "node:url"
import { ensureNodeModules } from "./test-helpers.js"

const moduleDirName = dirname(fileURLToPath(import.meta.url))

Expand All @@ -33,20 +31,6 @@ const projectRoot = resolve(moduleDirName, "../../test/", "test-project-k8s")
const nsModuleRoot = join(projectRoot, "k8s-namespace")
const deploymentModuleRoot = join(projectRoot, "k8s-deployment")

// Here, pulumi needs node modules to be installed (to use the TS SDK in the pulumi program).
const ensureNodeModules = async () => {
await Promise.all(
[nsModuleRoot, deploymentModuleRoot].map(async (moduleRoot) => {
if (await pathExists(join(moduleRoot, "node_modules"))) {
return
}
await execa("npm", ["install"], { cwd: moduleRoot })
})
)
}

// TODO: Write + finish unit and integ tests

// Note: By default, this test suite assumes that PULUMI_ACCESS_TOKEN is present in the environment (which is the case
// in CI). To run this test suite with your own pulumi org, replace the `orgName` variable in
// `test-project-k8s/project.garden.yml` with your own org's name and make sure you've logged in via `pulumi login`.
Expand All @@ -58,7 +42,7 @@ describe("pulumi plugin handlers", () => {
let provider: PulumiProvider

before(async () => {
await ensureNodeModules()
await ensureNodeModules([nsModuleRoot, deploymentModuleRoot])
const plugin = pulumiPlugin()
garden = await makeTestGarden(projectRoot, { plugins: [plugin] })
log = garden.log
Expand Down
Loading

0 comments on commit 1e5c1df

Please sign in to comment.