From a8bb5bdeb8438f08dd1012de109c63989be722c0 Mon Sep 17 00:00:00 2001 From: Alyx Holms Date: Tue, 19 Sep 2023 11:43:29 -0600 Subject: [PATCH] chore: add St Bernard to help with module syncing in the workspace (#58) * feat: add proper variant of St Bernard * chore: use modsync instead of build for init * chore: add config files to air reloads * chore: change foo to a more useful example command * chore: cleanup * chore: start up integration testing services as part of init * chore: add volume clearing and clean build conditionally * chore: add wiping mechanic to integration testing services * fix: possible infinite loop if running modsync in a directory with no go.work anywhere above it * fix: remove bh-testing service start from init --- .air.debug.toml | 2 +- .air.toml | 2 +- .vscode/extensions.json | 74 ++++++------ go.work | 1 + justfile | 30 ++++- packages/go/stbernard/command/command.go | 106 +++++++++++++++++ .../go/stbernard/command/envdump/envdump.go | 58 +++++++++ .../go/stbernard/command/modsync/modsync.go | 68 +++++++++++ packages/go/stbernard/command/registration.go | 42 +++++++ packages/go/stbernard/go.mod | 5 + packages/go/stbernard/go.sum | 1 + packages/go/stbernard/main.go | 22 ++++ packages/go/stbernard/workspace/sync.go | 110 ++++++++++++++++++ 13 files changed, 476 insertions(+), 45 deletions(-) create mode 100644 packages/go/stbernard/command/command.go create mode 100644 packages/go/stbernard/command/envdump/envdump.go create mode 100644 packages/go/stbernard/command/modsync/modsync.go create mode 100644 packages/go/stbernard/command/registration.go create mode 100644 packages/go/stbernard/go.mod create mode 100644 packages/go/stbernard/go.sum create mode 100644 packages/go/stbernard/main.go create mode 100644 packages/go/stbernard/workspace/sync.go diff --git a/.air.debug.toml b/.air.debug.toml index 09cdc3a2c..dd2b54ad8 100644 --- a/.air.debug.toml +++ b/.air.debug.toml @@ -29,7 +29,7 @@ exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false full_bin = "dlv exec --accept-multiclient --log --headless --listen :2345 --api-version 2 ../tmp/main --" -include_dir = ["cmd/api/src", "packages/go"] +include_dir = ["cmd/api/src", "packages/go", "local-harnesses"] include_ext = ["go", "json"] include_file = [] kill_delay = "0s" diff --git a/.air.toml b/.air.toml index 0eccaa748..d836e914b 100644 --- a/.air.toml +++ b/.air.toml @@ -29,7 +29,7 @@ exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false full_bin = "" -include_dir = ["cmd/api/src", "packages/go"] +include_dir = ["cmd/api/src", "packages/go", "local-harnesses"] include_ext = ["go", "json"] include_file = [] kill_delay = "0s" diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 5b54e8d3b..c7e78ee5c 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,38 +1,38 @@ { - "recommendations": [ - "alefragnani.bookmarks", - "nickgo.cuelang", - "hediet.vscode-drawio", - "ms-toolsai.jupyter-keymap", - "pkief.material-icon-theme", - "ms-vscode-remote.vscode-remote-extensionpack", - "ms-vscode.remote-explorer", - "ms-python.black-formatter", - "jakeboone02.cypher-query-language", - "redhat.fabric8-analytics", - "ms-azuretools.vscode-docker", - "tombonnike.vscode-status-bar-format-toggle", - "felipecaputo.git-project-manager", - "eamodio.gitlens", - "golang.go", - "bierner.markdown-checkbox", - "ryu1kn.partial-diff", - "jebbs.plantuml", - "ms-ossdata.vscode-postgresql", - "ms-python.vscode-pylance", - "ms-python.python", - "buenon.scratchpads", - "richie5um2.vscode-sort-json", - "luisfontes19.vscode-swissknife", - "tabnine.tabnine-vscode", - "gruntfuggly.todo-tree", - "tomsaunders.vscode-workspace-explorer", - "tonybaloney.vscode-pets", - "gitlab.gitlab-workflow", - "yoavbls.pretty-ts-errors", - "esbenp.prettier-vscode", - "prisma.prisma", - "skellock.just", - "github.vscode-github-actions" - ] -} \ No newline at end of file + "recommendations": [ + "alefragnani.bookmarks", + "nickgo.cuelang", + "hediet.vscode-drawio", + "ms-toolsai.jupyter-keymap", + "pkief.material-icon-theme", + "ms-vscode-remote.vscode-remote-extensionpack", + "ms-vscode.remote-explorer", + "ms-python.black-formatter", + "jakeboone02.cypher-query-language", + "redhat.fabric8-analytics", + "ms-azuretools.vscode-docker", + "tombonnike.vscode-status-bar-format-toggle", + "felipecaputo.git-project-manager", + "eamodio.gitlens", + "golang.go", + "bierner.markdown-checkbox", + "ryu1kn.partial-diff", + "jebbs.plantuml", + "ms-ossdata.vscode-postgresql", + "ms-python.vscode-pylance", + "ms-python.python", + "buenon.scratchpads", + "richie5um2.vscode-sort-json", + "luisfontes19.vscode-swissknife", + "tabnine.tabnine-vscode", + "gruntfuggly.todo-tree", + "tomsaunders.vscode-workspace-explorer", + "tonybaloney.vscode-pets", + "gitlab.gitlab-workflow", + "yoavbls.pretty-ts-errors", + "esbenp.prettier-vscode", + "prisma.prisma", + "kokakiwi.vscode-just", + "github.vscode-github-actions" + ] +} diff --git a/go.work b/go.work index 529eae18d..34308b2f4 100644 --- a/go.work +++ b/go.work @@ -34,4 +34,5 @@ use ( ./packages/go/params ./packages/go/schemagen ./packages/go/slices + ./packages/go/stbernard ) diff --git a/justfile b/justfile index bde72757a..7fda84c93 100644 --- a/justfile +++ b/justfile @@ -15,6 +15,7 @@ set positional-arguments # Initialize your dev environment (use "just init clean" to reset your config files) init wipe="": #!/usr/bin/env bash + echo "Init BloodHound CE" echo "Make local copies of configuration files" if [[ -d "./local-harnesses/build.config.json" ]]; then rm -rf "./local-harnesses/build.config.json" @@ -37,13 +38,26 @@ init wipe="": echo "Install additional Go tools" go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 - echo "Run a build to ensure go.work.sum is valid" - just build -vf + echo "Run modsync to ensure workspace is up to date" + just modsync echo "Ensure containers have been rebuilt" - just bh-dev build + if [[ "{{wipe}}" != "clean" ]]; then + just bh-dev build + else + echo "Clear volumes and rebuild without cache" + just bh-clear-volumes + just bh-clean-docker-build + fi - echo "Init Complete" + echo "Start integration testing services" + if [[ "{{wipe}}" == "clean" ]]; then + echo "Clear volumes and restart testing services without cache" + just bh-testing-clear-volumes + just bh-testing build --no-cache + fi + + echo "BloodHound CE Init Complete" # Show available targets for this context. show *FLAGS: @@ -63,6 +77,10 @@ test *FLAGS: set -euo pipefail python3 packages/python/beagle/main.py test {{FLAGS}} +# sync modules in workspace +modsync: + @go run github.com/specterops/bloodhound/packages/go/stbernard modsync + # updates favicon.ico, logo192.png and logo512.png from logo.svg update-favicon: @just imagemagick convert -background none ./cmd/ui/public/logo-light.svg -define icon:auto-resize ./cmd/ui/public/favicon-light.ico @@ -142,8 +160,8 @@ bh-testing-clear-volumes *ARGS='': @docker compose --project-name bh-testing -f docker-compose.testing.yml down -v {{ARGS}} # clear BH docker compose volumes (pass --remove-orphans if troubleshooting) -bh-clear-volumes *ARGS='': - @docker compose -f docker-compose.dev.yml down -v {{ARGS}} +bh-clear-volumes target='dev' *ARGS='': + @docker compose --profile {{target}} -f docker-compose.dev.yml down -v {{ARGS}} # build BH target cleanly (default profile dev with --no-cache flag) bh-clean-docker-build target='dev' *ARGS='': diff --git a/packages/go/stbernard/command/command.go b/packages/go/stbernard/command/command.go new file mode 100644 index 000000000..cb1a4eae1 --- /dev/null +++ b/packages/go/stbernard/command/command.go @@ -0,0 +1,106 @@ +package command + +import ( + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/specterops/bloodhound/packages/go/stbernard/command/envdump" + "github.com/specterops/bloodhound/packages/go/stbernard/command/modsync" +) + +// Commander is an interface for commands, allowing commands to implement the minimum +// set of requirements to observe and run the command from above. It is used as a return +// type to allow passing a usable command to the caller after parsing and creating +// the command implementation +type Commander interface { + Name() string + Usage() string + Run() error +} + +var NoCmdErr = errors.New("no command specified") +var InvalidCmdErr = errors.New("invalid command specified") +var FailedCreateCmdErr = errors.New("failed to create command") + +// ParseCLI parses for a subcommand as the first argument to the calling binary, +// and initializes the command (if it exists). It also provides the default usage +// statement. +// +// It does not support flags of its own, each subcommand is responsible for parsing +// their flags. +func ParseCLI() (Commander, error) { + // Generate a nice usage message + flag.Usage = usage + + // Default usage if no arguments provided + if len(os.Args) < 2 { + flag.Usage() + return nil, NoCmdErr + } + + switch os.Args[1] { + case ModSync.String(): + config := modsync.Config{Environment: environment()} + if cmd, err := modsync.Create(config); err != nil { + return nil, fmt.Errorf("%w: %w", FailedCreateCmdErr, err) + } else { + return cmd, nil + } + + case EnvDump.String(): + config := envdump.Config{Environment: environment()} + if cmd, err := envdump.Create(config); err != nil { + return nil, fmt.Errorf("%w: %w", FailedCreateCmdErr, err) + } else { + return cmd, nil + } + + default: + flag.Parse() + flag.Usage() + return nil, InvalidCmdErr + } +} + +// usage creates a pretty usage message for our main command +func usage() { + var longestCmdLen int + + w := flag.CommandLine.Output() + fmt.Fprint(w, "A BloodHound Swiss Army Knife\n\nUsage: stbernard COMMAND\n\nCommands:\n") + + for _, cmd := range Commands() { + if len(cmd.String()) > longestCmdLen { + longestCmdLen = len(cmd.String()) + } + } + + for cmd, usage := range CommandsUsage() { + cmdStr := Command(cmd).String() + padding := strings.Repeat(" ", longestCmdLen-len(cmdStr)) + fmt.Fprintf(w, " %s%s %s\n", cmdStr, padding, usage) + } +} + +// environment is used to add default env vars as needed to the existing environment variables +func environment() []string { + var envMap = make(map[string]string) + + for _, env := range os.Environ() { + envTuple := strings.SplitN(env, "=", 2) + envMap[envTuple[0]] = envTuple[1] + } + + // Make any changes here + envMap["FOO"] = "foo" // For illustrative purposes only + + var envSlice = make([]string, 0, len(envMap)) + for key, val := range envMap { + envSlice = append(envSlice, strings.Join([]string{key, val}, "=")) + } + + return envSlice +} diff --git a/packages/go/stbernard/command/envdump/envdump.go b/packages/go/stbernard/command/envdump/envdump.go new file mode 100644 index 000000000..56b2974a0 --- /dev/null +++ b/packages/go/stbernard/command/envdump/envdump.go @@ -0,0 +1,58 @@ +package envdump + +import ( + "flag" + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +const ( + Name = "envdump" + Usage = "Dump your environment variables" +) + +type Config struct { + Environment []string +} + +type command struct { + config Config +} + +func (s command) Name() string { + return Name +} + +func (s command) Usage() string { + return Usage +} + +func (s command) Run() error { + log.Printf("Environment:\n\n") + for _, env := range s.config.Environment { + envTuple := strings.SplitN(env, "=", 2) + log.Printf("%s: %s\n", envTuple[0], envTuple[1]) + } + log.Printf("\n") + + return nil +} + +func Create(config Config) (command, error) { + envdumpCmd := flag.NewFlagSet(Name, flag.ExitOnError) + + envdumpCmd.Usage = func() { + w := flag.CommandLine.Output() + fmt.Fprintf(w, "%s\n\nUsage: %s %s [OPTIONS]\n", Usage, filepath.Base(os.Args[0]), Name) + } + + if err := envdumpCmd.Parse(os.Args[2:]); err != nil { + envdumpCmd.Usage() + return command{}, fmt.Errorf("failed to parse %s command: %w", Name, err) + } else { + return command{config: config}, nil + } +} diff --git a/packages/go/stbernard/command/modsync/modsync.go b/packages/go/stbernard/command/modsync/modsync.go new file mode 100644 index 000000000..6fe3c2d67 --- /dev/null +++ b/packages/go/stbernard/command/modsync/modsync.go @@ -0,0 +1,68 @@ +package modsync + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/specterops/bloodhound/packages/go/stbernard/workspace" +) + +const ( + Name = "modsync" + Usage = "Sync all modules in current workspace" +) + +type flags struct { + verbose bool +} + +type Config struct { + flags flags + Environment []string +} + +type command struct { + config Config +} + +func (s command) Usage() string { + return Usage +} + +func (s command) Name() string { + return Name +} + +func (s command) Run() error { + if cwd, err := workspace.FindRoot(); err != nil { + return fmt.Errorf("could not find workspace root: %w", err) + } else if modPaths, err := workspace.ParseModulesAbsPaths(cwd); err != nil { + return fmt.Errorf("could not parse module absolute paths: %w", err) + } else if err := workspace.DownloadModules(modPaths, s.config.Environment); err != nil { + return fmt.Errorf("could not download modules: %w", err) + } else if err := workspace.SyncWorkspace(cwd, s.config.Environment); err != nil { + return fmt.Errorf("could not sync workspace: %w", err) + } else { + return nil + } +} + +func Create(config Config) (command, error) { + modsyncCmd := flag.NewFlagSet(Name, flag.ExitOnError) + modsyncCmd.BoolVar(&config.flags.verbose, "v", false, "Print verbose logs") + + modsyncCmd.Usage = func() { + w := flag.CommandLine.Output() + fmt.Fprintf(w, "%s\n\nUsage: %s %s [OPTIONS]\n\nOptions:\n", Usage, filepath.Base(os.Args[0]), Name) + modsyncCmd.PrintDefaults() + } + + if err := modsyncCmd.Parse(os.Args[2:]); err != nil { + modsyncCmd.Usage() + return command{}, fmt.Errorf("failed to parse modsync command: %w", err) + } else { + return command{config: config}, nil + } +} diff --git a/packages/go/stbernard/command/registration.go b/packages/go/stbernard/command/registration.go new file mode 100644 index 000000000..13c83c000 --- /dev/null +++ b/packages/go/stbernard/command/registration.go @@ -0,0 +1,42 @@ +package command + +import ( + "github.com/specterops/bloodhound/packages/go/stbernard/command/envdump" + "github.com/specterops/bloodhound/packages/go/stbernard/command/modsync" +) + +// Command enum represents our subcommands +type Command int + +const ( + InvalidCommand Command = iota - 1 + ModSync + EnvDump +) + +// String implements Stringer for the Command enum +func (s Command) String() string { + switch s { + case ModSync: + return modsync.Name + case EnvDump: + return envdump.Name + default: + return "invalid command" + } +} + +// Commands returns our valid set of Command options +func Commands() []Command { + return []Command{ModSync, EnvDump} +} + +// Commands usage returns a slice of Command usage statements indexed by their enum +func CommandsUsage() []string { + var usage = make([]string, len(Commands())) + + usage[ModSync] = modsync.Usage + usage[EnvDump] = envdump.Usage + + return usage +} diff --git a/packages/go/stbernard/go.mod b/packages/go/stbernard/go.mod new file mode 100644 index 000000000..98cd8f21b --- /dev/null +++ b/packages/go/stbernard/go.mod @@ -0,0 +1,5 @@ +module github.com/specterops/bloodhound/packages/go/stbernard + +go 1.20 + +require golang.org/x/mod v0.11.0 diff --git a/packages/go/stbernard/go.sum b/packages/go/stbernard/go.sum new file mode 100644 index 000000000..76e5a3716 --- /dev/null +++ b/packages/go/stbernard/go.sum @@ -0,0 +1 @@ +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= diff --git a/packages/go/stbernard/main.go b/packages/go/stbernard/main.go new file mode 100644 index 000000000..ce07f0ca8 --- /dev/null +++ b/packages/go/stbernard/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "errors" + "log" + + "github.com/specterops/bloodhound/packages/go/stbernard/command" +) + +func main() { + if cmd, err := command.ParseCLI(); err != nil { + if errors.Is(err, command.NoCmdErr) { + log.Fatal("No command specified") + } else { + log.Fatalf("Error while parsing command: %v", err) + } + } else if err := cmd.Run(); err != nil { + log.Fatalf("Failed to run command %s: %v", cmd.Name(), err) + } else { + log.Printf("%s completed successfully", cmd.Name()) + } +} diff --git a/packages/go/stbernard/workspace/sync.go b/packages/go/stbernard/workspace/sync.go new file mode 100644 index 000000000..16b1f0289 --- /dev/null +++ b/packages/go/stbernard/workspace/sync.go @@ -0,0 +1,110 @@ +package workspace + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + + "golang.org/x/mod/modfile" +) + +// FindRoot will attempt to crawl up the path until it finds a go.work file +func FindRoot() (string, error) { + if cwd, err := os.Getwd(); err != nil { + return "", fmt.Errorf("could not get current working directory: %w", err) + } else { + var found bool + + for !found { + found, err = WorkFileExists(cwd) + if err != nil { + return cwd, fmt.Errorf("error while trying to find go.work file: %w", err) + } + + if found { + break + } + + prevCwd := cwd + + // Go up a directory before retrying + cwd = filepath.Dir(cwd) + + if cwd == prevCwd { + return cwd, errors.New("found root path without finding go.work file") + } + } + + return cwd, nil + } +} + +// WorkFileExists checks if a go.work file exists in the given directory +func WorkFileExists(cwd string) (bool, error) { + if _, err := os.Stat(filepath.Join(cwd, "go.work")); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("could not stat go.work file: %w", err) + } else { + return true, nil + } +} + +// ParseModulesAbsPaths parses the modules listed in the go.work file from the given +// directory and returns a list of absolute paths to those modules +func ParseModulesAbsPaths(cwd string) ([]string, error) { + var workfilePath = filepath.Join(cwd, "go.work") + // go.work files aren't particularly heavy, so we'll just read into memory + if data, err := os.ReadFile(workfilePath); err != nil { + return nil, fmt.Errorf("could not read go.work file: %w", err) + } else if workfile, err := modfile.ParseWork(workfilePath, data, nil); err != nil { + return nil, fmt.Errorf("could not parse go.work file: %w", err) + } else { + var ( + modulePaths = make([]string, 0, len(workfile.Use)) + workDir = filepath.Dir(workfilePath) + ) + + for _, use := range workfile.Use { + modulePaths = append(modulePaths, filepath.Join(workDir, use.Path)) + } + + return modulePaths, nil + } +} + +// DownloadModules runs go mod download for all module paths passed with a given +// set of environment variables +func DownloadModules(modPaths []string, env []string) error { + var errs = make([]error, 0) + + for _, modPath := range modPaths { + cmd := exec.Command("go", "mod", "download") + cmd.Env = env + cmd.Dir = modPath + if err := cmd.Run(); err != nil { + errs = append(errs, fmt.Errorf("failure when running command: %w", err)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("failed to download all modules: %w", errors.Join(errs...)) + } else { + return nil + } +} + +// SyncWorkspace runs go work sync in the given directory with a given set of environment +// variables +func SyncWorkspace(cwd string, env []string) error { + cmd := exec.Command("go", "work", "sync") + cmd.Env = env + cmd.Dir = cwd + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed running go work sync: %w", err) + } else { + return nil + } +}