feat(telemetry): add wasm-tools wasm binary annotations (#1016)

* refactor(main): simplify/clean-up flow

* refactor: pass wasmtools versioner throughout codebase

* refactor(version): clarify versioner used

* refactor(compute/init): sort flags

* refactor: move InstallDir to github package

* refactor: move setBinPerms to github package

* remove(compute/init): wasm-tools

NOTE: It's only needed for after a Wasm binary is produced.

* feat: add wasm-tools versioner

* feat: add skeleton wasm-tools annotation

* feat: add more data to wasm-tools annotation

* feat: support data collection for all languages

* remove(config): don't increment version

* fix(telemetry): disable unless hidden flag is set

* fix(compute/test): ignore enable-telemetry flag

* feat(telemetry): disable telemetry via env var

* debugging

* fix(compute/build): fix broken ci test

* debugging

* fix(compute/build): fix broken ci test

* Revert "debugging"

This reverts commit fb3f840.

* Revert "debugging"

This reverts commit 456126f.

* feat(telemetry): track starter kit

* feat(telemetry): capture imports for Go/Rust from source files

* feat(telemetry): add --show-metadata to compute build

* fix: remove temporary path

* refactor(env): replace SplitN with Cut

* feat(telemetry): parse JavaScript imports

* refactor(compute/build): namespace metadata

* remove: Imports()

* refactor: rename env var to disable wasm metadata collection

* fix(compute/build): switch metadata from go.sum to go.mod

* feat: set only CLI version in metadata when metadata disabled

* fix: move placement of wasmtools install

* fix: compute tests

* fix: Windows test

* debugging windows

* refactor: annotate public methods

* refactor(compute/build): use gomod parser

* refactor(compute/build): use processed-by flag for CLI version

* refactor(compute/build): replace individual flags with single json data

* fix(compute/build): record only current memory allocation

* fix(compute/build): invert conditional and move show-metadata

* fix(compute/build): don't hide show-metadata flag

This is because we now always annotate the Wasm binary. If the user
enables metadata then we annotate lots of data, otherwise we'll default
to annotating the binary with the CLI version and the project language.

* feat(compute/build): display IMPORTANT message related to wasm-tools

* feat(compute/build): filter specified env vars

* fix(compute/build): add new flag to test suite

* fix(compute/build): fix formatting line breaks + use correct CPU value

* temp

* refactor(compute/build): clean-up debug log

* fix(compute/build): trufflehog

* fix(compute/build): only display secrets result once

* doc(compute/build): annotate Result type

* fix(compute/init): prevent runtime panic by removing duplicate flag

* fix(compute): add ignore rules for new build flag to existing tests

* refactor(compute/build): clean-up annotation logic

* feat(compute/build): better secret scrubbing

* fix: add missing metadata flags to composite commands

* fix(compute/hashsum): pass correct variable

* fix: correct main package after bad rebase

* fix(compute/build): avoid empty config value

.fastly/config.toml

@@ -14,5 +14,8 @@ toolchain_constraint_tinygo = ">= 1.18"    # Go toolchain constraint for use wit
 toolchain_constraint = ">= 1.56.1"
 wasm_wasi_target = "wasm32-wasi"
 
+[wasm-tools]
+ttl = "24h"
+
 [viceroy]
 ttl = "24h"

.github/workflows/pr_test.yml

@@ -152,7 +152,7 @@ jobs:
         shell: bash
         env:
           # NOTE: The following lets us focus the test run while debugging.
-          # TEST_ARGS: "-run TestDeploy ./pkg/commands/compute/..."
+          # TEST_ARGS: "-run TestBuild ./pkg/commands/compute/..."
           TEST_COMPUTE_INIT: true
           TEST_COMPUTE_BUILD: true
           TEST_COMPUTE_DEPLOY: true

cmd/fastly/main.go

@@ -6,13 +6,16 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"strings"
 	"time"
 
+	"github.com/fastly/go-fastly/v8/fastly"
 	"github.com/fatih/color"
 
+	"github.com/fastly/cli/pkg/api"
 	"github.com/fastly/cli/pkg/app"
+	"github.com/fastly/cli/pkg/commands/compute"
 	"github.com/fastly/cli/pkg/config"
+	"github.com/fastly/cli/pkg/env"
 	fsterr "github.com/fastly/cli/pkg/errors"
 	"github.com/fastly/cli/pkg/github"
 	"github.com/fastly/cli/pkg/manifest"
@@ -21,38 +24,34 @@ import (
 )
 
 func main() {
-	// Some configuration options can come from env vars.
-	var env config.Environment
-	env.Read(parseEnv(os.Environ()))
+	// Parse the arguments provided by the user via the command-line interface.
+	args := os.Args[1:]
 
-	// All of the work of building the set of commands and subcommands, wiring
-	// them together, picking which one to call, and executing it, occurs in a
-	// helper function, Run. We parameterize all of the dependencies so we can
-	// test it more easily. Here, we declare all of the dependencies, using
-	// the "real" versions that pull e.g. actual commandline arguments, the
-	// user's real environment, etc.
+	// Define a HTTP client that will be used for making arbitrary HTTP requests.
+	httpClient := &http.Client{Timeout: time.Minute * 2}
+
+	// Define the standard input/output streams.
 	var (
-		args                    = os.Args[1:]
-		clientFactory           = app.FastlyAPIClient
-		httpClient              = &http.Client{Timeout: time.Minute * 2}
-		in            io.Reader = os.Stdin
-		out           io.Writer = sync.NewWriter(color.Output)
+		in  io.Reader = os.Stdin
+		out io.Writer = sync.NewWriter(color.Output)
 	)
 
-	// We have to manually handle the inclusion of the verbose flag here because
-	// Kingpin doesn't evaluate the provided arguments until app.Run which
-	// happens later in the file and yet we need to know if we should be printing
-	// output related to the application configuration file in this file.
+	// Read relevant configuration options from the user's environment.
+	var e config.Environment
+	e.Read(env.Parse(os.Environ()))
+
+	// Identify verbose flag early (before Kingpin parser has executed) so we can
+	// print additional output related to the CLI configuration.
 	var verboseOutput bool
 	for _, seg := range args {
 		if seg == "-v" || seg == "--verbose" {
 			verboseOutput = true
 		}
 	}
 
-	// Similarly for the --auto-yes/--non-interactive flags, we need access to
-	// these for handling interactive error prompts to the user, in case the CLI
-	// is being run in a CI environment.
+	// Identify auto-yes/non-interactive flag early (before Kingpin parser has
+	// executed) so we can handle the interactive prompts appropriately with
+	// regards to processing the CLI configuration.
 	var autoYes, nonInteractive bool
 	for _, seg := range args {
 		if seg == "-y" || seg == "--auto-yes" {
@@ -63,20 +62,19 @@ func main() {
 		}
 	}
 
-	// Extract a subset of configuration options from the local application directory.
+	// Extract a subset of configuration options from the local app directory.
 	var cfg config.File
 	cfg.SetAutoYes(autoYes)
 	cfg.SetNonInteractive(nonInteractive)
-
 	// The CLI relies on a valid configuration, otherwise we can't continue.
 	err := cfg.Read(config.FilePath, in, out, fsterr.Log, verboseOutput)
 	if err != nil {
 		fsterr.Deduce(err).Print(color.Error)
-
 		// WARNING: os.Exit will exit, and any `defer` calls will not be run.
 		os.Exit(1)
 	}
 
+	// Extract user's project configuration from the fastly.toml manifest.
 	var md manifest.Data
 	md.File.Args = args
 	md.File.SetErrLog(fsterr.Log)
@@ -85,18 +83,25 @@ func main() {
 	// NOTE: We skip handling the error because not all commands relate to Compute.
 	_ = md.File.Read(manifest.Filename)
 
-	// Main is basically just a shim to call Run, so we do that here.
-	opts := app.RunOpts{
-		APIClient:  clientFactory,
-		Args:       args,
-		ConfigFile: cfg,
-		ConfigPath: config.FilePath,
-		Env:        env,
-		ErrLog:     fsterr.Log,
-		HTTPClient: httpClient,
-		Manifest:   &md,
-		Stdin:      in,
-		Stdout:     out,
+	// The `main` function is a shim for calling `app.Run()`.
+	err = app.Run(app.RunOpts{
+		APIClient: func(token, endpoint string, debugMode bool) (api.Interface, error) {
+			client, err := fastly.NewClientForEndpoint(token, endpoint)
+			if debugMode {
+				client.DebugMode = true
+			}
+			return client, err
+		},
+		Args:             args,
+		ConfigFile:       cfg,
+		ConfigPath:       config.FilePath,
+		Env:              e,
+		ErrLog:           fsterr.Log,
+		ExecuteWasmTools: compute.ExecuteWasmTools,
+		HTTPClient:       httpClient,
+		Manifest:         &md,
+		Stdin:            in,
+		Stdout:           out,
 		Versioners: app.Versioners{
 			CLI: github.New(github.Opts{
 				HTTPClient: httpClient,
@@ -111,9 +116,16 @@ func main() {
 				Binary:     "viceroy",
 				Version:    md.File.LocalServer.ViceroyVersion,
 			}),
+			WasmTools: github.New(github.Opts{
+				HTTPClient: httpClient,
+				Org:        "bytecodealliance",
+				Repo:       "wasm-tools",
+				Binary:     "wasm-tools",
+				External:   true,
+				Nested:     true,
+			}),
 		},
-	}
-	err = app.Run(opts)
+	})
 
 	// NOTE: We persist any error log entries to disk before attempting to handle
 	// a possible error response from app.Run as there could be errors recorded
@@ -124,7 +136,6 @@ func main() {
 	if logErr != nil {
 		fsterr.Deduce(logErr).Print(color.Error)
 	}
-
 	if err != nil {
 		text.Break(out)
 		fsterr.Deduce(err).Print(color.Error)
@@ -137,16 +148,3 @@ func main() {
 		os.Exit(1)
 	}
 }
-
-func parseEnv(environ []string) map[string]string {
-	env := map[string]string{}
-	for _, kv := range environ {
-		toks := strings.SplitN(kv, "=", 2)
-		if len(toks) != 2 {
-			continue
-		}
-		k, v := toks[0], toks[1]
-		env[k] = v
-	}
-	return env
-}