housekeeping: Fixing frontend scaffolding (#3137)

### Description
Frontend scaffolding has been broken for a bit but it also wasn't smart
enough to handle cases where it was extending functionality for the
internal application versus an external.

- Broke apart the logic to make it easier to parse and update
- Added flags to overwrite templates as well as an internal flag

### Testing Performed
manual

Makefile

@@ -76,24 +76,24 @@ backend-verify:
 backend-config-validation:
 	cd backend && go run main.go -validate -c clutch-config.yaml
 
-.PHONY: yarn-install # Install frontend dependencies.
-yarn-install: yarn-ensure
-	$(YARN) --cwd frontend install --immutable
-
 .PHONY: backend-integration-test
 backend-integration-test:
 	cd backend/internal/test/integration/chaos && ./do_integration_test.sh
 
+.PHONY: frontend-install # Install frontend dependencies.
+frontend-install: yarn-ensure
+	$(YARN) --cwd frontend install --immutable
+
 .PHONY: frontend # Build production frontend assets.
-frontend: yarn-ensure preflight-checks-frontend yarn-install
+frontend: yarn-ensure preflight-checks-frontend frontend-install
 	$(YARN) --cwd frontend build
 
-.PHONY: frontend-dev-build # Build development frontend assets.
-frontend-dev-build: yarn-install
+.PHONY: frontend-compile # Build development frontend assets.
+frontend-compile: frontend-install
 	$(YARN) --cwd frontend build:dev
 
 .PHONY: frontend-dev # Start the frontend in development mode.
-frontend-dev: setup-git-hooks yarn-install
+frontend-dev: setup-git-hooks frontend-install
 	$(YARN) --cwd frontend start
 
 .PHONY: frontend-lint # Lint the frontend code.
@@ -151,11 +151,11 @@ scaffold-workflow:
 	cd tools/scaffolding && go run scaffolder.go -m frontend-plugin
 
 .PHONY: storybook # Start storybook locally.
-storybook: yarn-install
+storybook: frontend-install
 	$(YARN) --cwd frontend storybook
 
 .PHONY: storybook-build # Build storybook assets for deploy.
-storybook-build: yarn-install
+storybook-build: frontend-install
 	$(YARN) --cwd frontend storybook:build
 
 .PHONY: test # Unit test all of the code.

tools/scaffolding/scaffold/api.go

@@ -0,0 +1,44 @@
+package scaffold
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+func GenerateAPI(args *Args, tmpFolder string, dest string) {
+	log.Println("Adding clutch dependencies to go.mod...")
+	if err := os.Chdir(filepath.Join(tmpFolder, "backend")); err != nil {
+		log.Fatal(err)
+	}
+	cmd := exec.Command("go", "get", fmt.Sprintf("github.com/%s/clutch/backend@%s", args.Org, args.GoPin))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`go get` backend in the destination dir returned the above error")
+	}
+
+	cmd = exec.Command("go", "get", fmt.Sprintf("github.com/%s/clutch/tools@%s", args.Org, args.GoPin))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`go get` tools in the destination dir returned the above error")
+	}
+
+	if err := os.Chdir(tmpFolder); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("Generating API code from protos...")
+	log.Println("cd", tmpFolder, "&& make api")
+	cmd = exec.Command("make", "api")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`make api` in the destination dir returned the above error")
+	}
+	log.Println("API generation complete")
+
+	fmt.Println("*** All done!")
+	fmt.Println("\n*** Try the following command to get started developing the custom gateway:")
+	fmt.Printf("cd %s && make\n", dest)
+}

tools/scaffolding/scaffold/args.go

@@ -0,0 +1,28 @@
+package scaffold
+
+import (
+	"flag"
+)
+
+const yarnInstallVersion = "4.3.1"
+
+type Args struct {
+	Internal          bool
+	Mode              string
+	GoPin             string
+	Org               string
+	TemplateOverwrite string
+	YarnPin           string
+}
+
+func ParseArgs() *Args {
+	f := &Args{}
+	flag.BoolVar(&f.Internal, "i", false, "when creating workflows in clutch")
+	flag.StringVar(&f.Mode, "m", "gateway", "oneof gateway, frontend-plugin")
+	flag.StringVar(&f.GoPin, "p", "main", "sha or other github ref to version of tools used in scaffolding")
+	flag.StringVar(&f.Org, "o", "lyft", "overrides the github organization (for use in fork testing)")
+	flag.StringVar(&f.TemplateOverwrite, "templates", "", "directory to use for template overwrites")
+	flag.StringVar(&f.YarnPin, "y", "4.3.1", "version of yarn to use")
+	flag.Parse()
+	return f
+}

tools/scaffolding/scaffold/determine.go

@@ -0,0 +1,58 @@
+package scaffold
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strings"
+)
+
+func determineGoPath() string {
+	goPathOut, err := exec.Command("go", "env", "GOPATH").Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	return strings.TrimSpace(string(goPathOut))
+}
+
+func DetermineYarnPath() string {
+	path, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	yarnScript := filepath.Join(path, "../..", "build", "bin", "yarn.sh")
+	scriptPath, err := exec.LookPath(yarnScript)
+	if err != nil {
+		cmd := exec.Command("make", "yarn-ensure")
+		cmd.Dir = filepath.Join(path, "../..")
+		if out, err := cmd.CombinedOutput(); err != nil {
+			log.Println(string(out))
+			log.Fatal("`make yarn-ensure` returned the above error")
+		}
+
+		return yarnScript
+	}
+	return scriptPath
+}
+
+func determineUsername() string {
+	u, err := user.Current()
+	if err != nil {
+		log.Fatal(err)
+	}
+	return u.Username
+}
+
+func determineUserEmail() string {
+	gitEmail, err := exec.Command("git", "config", "user.email").Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	email := strings.TrimSpace(string(gitEmail))
+	if email == "" {
+		email = "unknown@example.com"
+	}
+	return email
+}

tools/scaffolding/scaffold/frontend.go

@@ -0,0 +1,143 @@
+package scaffold
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+type workflowTemplateValues struct {
+	Name             string
+	PackageName      string
+	Description      string
+	DeveloperName    string
+	DeveloperEmail   string
+	URLRoot          string
+	URLPath          string
+	IsWizardTemplate bool
+}
+
+func GetFrontendPluginTemplateValues() (*workflowTemplateValues, string) {
+	log.Println("Welcome!")
+	fmt.Println("*** Analyzing environment...")
+
+	dest := filepath.Join(os.Getenv("OLDPWD"), "frontend", "workflows")
+
+	fmt.Println("\n*** Based on your environment, we've picked the following destination for your new workflow:")
+	fmt.Println(">", dest)
+	okay := promptOrDefault("Is this okay?", "Y/n")
+	if !strings.HasPrefix(strings.ToLower(okay), "y") {
+		dest = promptOrDefault("Enter the destination folder", dest)
+	}
+
+	data := &workflowTemplateValues{}
+
+	data.IsWizardTemplate = true
+
+	wizard := promptOrDefault("Is this a wizard workflow?", "Y/n")
+	if strings.HasPrefix(strings.ToLower(wizard), "n") {
+		data.IsWizardTemplate = false
+	}
+
+	data.Name = strings.Title(promptOrDefault("Enter the name of this workflow", "Hello World"))
+	description := promptOrDefault("Enter a description of the workflow", "Greet the world")
+	data.Description = strings.ToUpper(description[:1]) + description[1:]
+	data.DeveloperName = promptOrDefault("Enter the developer's name", determineUsername())
+	data.DeveloperEmail = promptOrDefault("Enter the developer's email", determineUserEmail())
+
+	// n.b. transform workflow name into package name, e.g. foo bar baz -> fooBarBaz
+	packageName := strings.ToLower(data.Name[:1]) + strings.Title(data.Name)[1:]
+	data.PackageName = strings.Replace(packageName, " ", "", -1)
+
+	data.URLRoot = strings.Replace(strings.ToLower(data.Name), " ", "", -1)
+	data.URLPath = "/"
+
+	return data, filepath.Join(dest, data.PackageName)
+}
+
+func GenerateFrontend(args *Args, tmpFolder string, dest string) {
+	fmt.Println("Compiling workflow, this may take a few minutes...")
+	yarn := DetermineYarnPath()
+	fmt.Println("cd", tmpFolder, "&&", yarn, "install &&", yarn, "tsc && ", yarn, "compile")
+	if err := os.Chdir(tmpFolder); err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println("Running yarn install in", tmpFolder, "with yarn path", yarn)
+	installCmd := exec.Command(yarn, "install")
+	if out, err := installCmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Println("`yarn install` returned the above error")
+
+		YarnInstall(yarn, args.YarnPin)
+	}
+
+	fmt.Println("Running yarn tsc in", tmpFolder, "with yarn path", yarn)
+	compileTypesCmd := exec.Command(yarn, "tsc")
+	if out, err := compileTypesCmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`yarn tsc` returned the above error")
+	}
+
+	fmt.Println("Running yarn compile in", tmpFolder, "with yarn path", yarn)
+	compileDevCmd := exec.Command(yarn, "compile")
+	if out, err := compileDevCmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`yarn compile` returned the above error")
+	}
+
+	fmt.Println("*** All done!")
+	fmt.Printf("\n*** Your new workflow can be found here: %s\n", dest)
+	fmt.Println("For information on how to register this new workflow see our configuration guide: https://clutch.sh/docs/configuration")
+}
+
+func GenerateInternalFrontend(args *Args, tmpFolder string, dest string) {
+	root := os.Getenv("OLDPWD")
+	feRoot := filepath.Join(root, "frontend")
+
+	fmt.Println("Compiling workflow, this may take a few minutes...")
+
+	fmt.Println("Running `make frontend-install` in", root)
+	installCmd := exec.Command("make", "frontend-install")
+	installCmd.Dir = root
+	if out, err := installCmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`make frontend-install` returned the above error")
+	}
+
+	// Move tmpdir contents to destination.
+	MoveTempFilesToDest(tmpFolder, dest)
+
+	fmt.Println("Running `yarn install` in", feRoot)
+	yarnInstallCmd := exec.Command("yarn", "install")
+	yarnInstallCmd.Dir = feRoot
+	if out, err := yarnInstallCmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`yarn install` returned the above error")
+	}
+
+	fmt.Println("Running `make frontend-compile` in", root)
+	compileDevCmd := exec.Command("make", "frontend-compile")
+	compileDevCmd.Dir = root
+	if out, err := compileDevCmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		log.Fatal("`make frontend-compile` returned the above error")
+	}
+
+	fmt.Println("*** All done!")
+	fmt.Printf("\n*** Your new workflow can be found here: %s\n", dest)
+	fmt.Println("For information on how to register this new workflow see our configuration guide: https://clutch.sh/docs/configuration")
+}
+
+func PostProcessFrontend(flags *Args, tmpFolder string, dest string) {
+	GenerateFrontend(flags, tmpFolder, dest)
+
+	MoveTempFilesToDest(tmpFolder, dest)
+}
+
+func PostProcessFrontendInternal(flags *Args, tmpFolder string, dest string) {
+	GenerateInternalFrontend(flags, tmpFolder, dest)
+}

tools/scaffolding/scaffold/gateway.go

@@ -0,0 +1,59 @@
+package scaffold
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	defaultSourceControlProvider = "github.com"
+	defaultRepoName              = "clutch-custom-gateway"
+)
+
+type gatewayTemplateValues struct {
+	RepoOwner    string
+	RepoName     string
+	RepoProvider string
+}
+
+func GetGatewayTemplateValues() (*gatewayTemplateValues, string) {
+	// Ask the user if assumptions are correct or a new destination is needed.
+	log.Println("Welcome!")
+	fmt.Println("*** Analyzing environment...")
+
+	gopath := determineGoPath()
+	username := determineUsername()
+
+	fmt.Println("> GOPATH:", gopath)
+	fmt.Println("> User:", username)
+
+	dest := filepath.Join(gopath, "src", defaultSourceControlProvider, username, defaultRepoName)
+	fmt.Println("\n*** Based on your environment, we've picked the following destination for your new repo:")
+	fmt.Println(">", dest)
+	fmt.Println("\nNote: please pay special attention to see if the username matches your provider's username.")
+	okay := promptOrDefault("Is this okay?", "Y/n")
+	data := &gatewayTemplateValues{
+		RepoOwner:    username,
+		RepoName:     defaultRepoName,
+		RepoProvider: defaultSourceControlProvider,
+	}
+	if !strings.HasPrefix(strings.ToLower(okay), "y") {
+		data.RepoProvider = promptOrDefault("Enter the name of the source control provider", data.RepoProvider)
+		data.RepoOwner = promptOrDefault("Enter the name of the repository owner or org", data.RepoOwner)
+		data.RepoName = promptOrDefault("Enter the desired repository name", data.RepoName)
+		dest = promptOrDefault(
+			"Enter the destination folder",
+			filepath.Join(gopath, "src", data.RepoProvider, data.RepoOwner, data.RepoName),
+		)
+	}
+
+	return data, dest
+}
+
+func PostProcessGateway(flags *Args, tmpFolder string, dest string) {
+	GenerateAPI(flags, tmpFolder, dest)
+
+	MoveTempFilesToDest(tmpFolder, dest)
+}