From e7c0a6bbb69029d8acb0688405675469463b1dc3 Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Sun, 3 Nov 2024 20:29:19 +0100 Subject: [PATCH 1/3] runnerv2client: add client and "beta run --server" (#672) With `beta run --server` it's possible to execute a Markdown code block on the server from the CLI. This is a prerequisite for supporting the execution of code blocks in the remote locations using the CLI, even if the remote location is a well-defined sandbox Docker container running locally. ## Testing First, start the server: ``` go run . beta server start ``` Next, execute a code block on the server using the CLI: ``` go run . beta run -s=local print-name ``` --------- Co-authored-by: Sebastian (Tiedtke) Huckleberry --- .vscode/settings.json | 3 +- cmd/gqltool/main.go | 1 - internal/cmd/beta/beta_cmd.go | 1 + internal/cmd/beta/run_cmd.go | 127 ++++++++++++--- internal/command/command_terminal_test.go | 20 ++- internal/command/command_virtual.go | 16 -- internal/command/command_windows.go | 2 +- internal/command/config_code_block.go | 31 +++- internal/command/factory.go | 16 ++ internal/config/autoconfig/autoconfig.go | 6 +- .../project/projectservice/project_service.go | 65 +++++--- .../projectservice/project_service_test.go | 55 ++----- internal/runner/service_test.go | 28 +--- internal/runnerv2client/client.go | 150 +++++++++++++++++- internal/runnerv2client/client_test.go | 137 ++++++++++++++++ internal/runnerv2client/client_unix_test.go | 54 +++++++ internal/runnerv2service/execution.go | 2 +- .../runnerv2service/service_execute_test.go | 50 ++---- .../service_resolve_program_test.go | 7 +- .../runnerv2service/service_sessions_test.go | 7 +- internal/testutils/grpc.go | 51 ++++++ internal/tls/tls.go | 3 +- pkg/document/block.go | 3 +- .../editor/editorservice/service_test.go | 18 +-- 24 files changed, 651 insertions(+), 202 deletions(-) create mode 100644 internal/runnerv2client/client_test.go create mode 100644 internal/runnerv2client/client_unix_test.go create mode 100644 internal/testutils/grpc.go diff --git a/.vscode/settings.json b/.vscode/settings.json index d37c537e..d186bb77 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,7 +13,8 @@ "--proto_path=/usr/local/include/protoc" ] }, - "go.buildTags": "test_with_docker,test_with_txtar" + "go.buildTags": "test_with_docker,test_with_txtar", + "makefile.configureOnOpen": false // Uncomment if you want to work on files in ./web. // "go.buildTags": "js,wasm", // Uncomment if you want to check compilation errors on Windows. diff --git a/cmd/gqltool/main.go b/cmd/gqltool/main.go index bcd1115c..772fd793 100644 --- a/cmd/gqltool/main.go +++ b/cmd/gqltool/main.go @@ -11,7 +11,6 @@ import ( "github.com/stateful/runme/v3/internal/client/graphql" ) -// var tokenDir = flag.String("token-dir", cmd.GetUserConfigHome(), "The directory with tokens") var apiURL = flag.String("api-url", "http://localhost:4000", "The API base address") func init() { diff --git a/internal/cmd/beta/beta_cmd.go b/internal/cmd/beta/beta_cmd.go index 5412819a..65620a89 100644 --- a/internal/cmd/beta/beta_cmd.go +++ b/internal/cmd/beta/beta_cmd.go @@ -59,6 +59,7 @@ All commands use the runme.yaml configuration file.`, return nil }) + // Print the error to stderr but don't return it because error modes // are neither fully baked yet nor ready for users to consume. if err != nil { diff --git a/internal/cmd/beta/run_cmd.go b/internal/cmd/beta/run_cmd.go index c5fd80f5..4789faaf 100644 --- a/internal/cmd/beta/run_cmd.go +++ b/internal/cmd/beta/run_cmd.go @@ -2,7 +2,10 @@ package beta import ( "context" + "io" + "os" + "github.com/creack/pty" "github.com/pkg/errors" "github.com/spf13/cobra" "go.uber.org/zap" @@ -10,6 +13,7 @@ import ( "github.com/stateful/runme/v3/internal/command" "github.com/stateful/runme/v3/internal/config/autoconfig" rcontext "github.com/stateful/runme/v3/internal/runner/context" + "github.com/stateful/runme/v3/internal/runnerv2client" "github.com/stateful/runme/v3/internal/session" runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" "github.com/stateful/runme/v3/pkg/document" @@ -17,6 +21,8 @@ import ( ) func runCmd(*commonFlags) *cobra.Command { + var remote bool + cmd := cobra.Command{ Use: "run [command1 command2 ...]", Aliases: []string{"exec"}, @@ -36,6 +42,7 @@ Run all blocks from the "setup" and "teardown" tags: RunE: func(cmd *cobra.Command, args []string) error { return autoconfig.InvokeForCommand( func( + clientFactory autoconfig.ClientFactory, cmdFactory command.Factory, filters []project.Filter, logger *zap.Logger, @@ -67,21 +74,57 @@ Run all blocks from the "setup" and "teardown" tags: return errors.WithStack(err) } - session, err := session.New( - session.WithOwl(false), - session.WithProject(proj), - session.WithSeedEnv(nil), - ) - if err != nil { - return err - } - options := getCommandOptions(cmd, session) + ctx := cmd.Context() + + if remote { + client, err := clientFactory() + if err != nil { + return err + } + + sessionResp, err := client.CreateSession( + ctx, + &runnerv2.CreateSessionRequest{ + Project: &runnerv2.Project{ + Root: proj.Root(), + EnvLoadOrder: proj.EnvFilesReadOrder(), + }, + }, + ) + if err != nil { + return errors.WithMessage(err, "failed to create session") + } - for _, t := range tasks { - err := runCodeBlock(cmd.Context(), t.CodeBlock, cmdFactory, options) + for _, t := range tasks { + err := runCodeBlockWithClient( + ctx, + cmd, + client, + t.CodeBlock, + sessionResp.GetSession().GetId(), + ) + if err != nil { + return err + } + } + } else { + session, err := session.New( + session.WithOwl(false), + session.WithProject(proj), + session.WithSeedEnv(nil), + ) if err != nil { return err } + + options := createCommandOptions(cmd, session) + + for _, t := range tasks { + err := runCodeBlock(ctx, t.CodeBlock, cmdFactory, options) + if err != nil { + return err + } + } } return nil @@ -90,10 +133,12 @@ Run all blocks from the "setup" and "teardown" tags: }, } + cmd.Flags().BoolVarP(&remote, "remote", "r", false, "Run commands on a remote server.") + return &cmd } -func getCommandOptions( +func createCommandOptions( cmd *cobra.Command, sess *session.Session, ) command.CommandOptions { @@ -105,12 +150,7 @@ func getCommandOptions( } } -func runCodeBlock( - ctx context.Context, - block *document.CodeBlock, - factory command.Factory, - options command.CommandOptions, -) error { +func createProgramConfigFromCodeBlock(block *document.CodeBlock, opts ...command.ConfigBuilderOption) (*command.ProgramConfig, error) { // TODO(adamb): [command.Config] is generated exclusively from the [document.CodeBlock]. // As we introduce some document- and block-related configs in runme.yaml (root but also nested), // this [Command.Config] should be further extended. @@ -121,7 +161,16 @@ func runCodeBlock( // the last element of the returned config chain. Finally, [command.Config] should be updated. // This algorithm should be likely encapsulated in the [internal/config] and [internal/command] // packages. - cfg, err := command.NewProgramConfigFromCodeBlock(block) + return command.NewProgramConfigFromCodeBlock(block, opts...) +} + +func runCodeBlock( + ctx context.Context, + block *document.CodeBlock, + factory command.Factory, + options command.CommandOptions, +) error { + cfg, err := createProgramConfigFromCodeBlock(block) if err != nil { return err } @@ -138,9 +187,45 @@ func runCodeBlock( if err != nil { return err } - err = cmd.Start(ctx) - if err != nil { + if err := cmd.Start(ctx); err != nil { return err } return cmd.Wait(ctx) } + +func runCodeBlockWithClient( + ctx context.Context, + cobraCommand *cobra.Command, + client *runnerv2client.Client, + block *document.CodeBlock, + sessionID string, +) error { + cfg, err := createProgramConfigFromCodeBlock(block, command.WithInteractiveLegacy()) + if err != nil { + return err + } + + opts := runnerv2client.ExecuteProgramOptions{ + SessionID: sessionID, + Stdin: io.NopCloser(cobraCommand.InOrStdin()), + Stdout: cobraCommand.OutOrStdout(), + Stderr: cobraCommand.ErrOrStderr(), + StoreStdoutInEnv: true, + } + + if stdin, ok := cobraCommand.InOrStdin().(*os.File); ok { + size, err := pty.GetsizeFull(stdin) + if err != nil { + return errors.WithMessage(err, "failed to get terminal size") + } + + opts.Winsize = &runnerv2.Winsize{ + Rows: uint32(size.Rows), + Cols: uint32(size.Cols), + X: uint32(size.X), + Y: uint32(size.Y), + } + } + + return client.ExecuteProgram(ctx, cfg, opts) +} diff --git a/internal/command/command_terminal_test.go b/internal/command/command_terminal_test.go index 178f4ccb..80278581 100644 --- a/internal/command/command_terminal_test.go +++ b/internal/command/command_terminal_test.go @@ -52,12 +52,12 @@ func TestTerminalCommand_EnvPropagation(t *testing.T) { // Terminal command sets up a trap on EXIT. // Wait for it before starting to send commands. - expectContainLines(t, stdout, []string{"trap -- \"__cleanup\" EXIT"}) + expectContainLines(ctx, t, stdout, []string{"trap -- \"__cleanup\" EXIT"}) _, err = stdinW.Write([]byte("export TEST_ENV=1\n")) require.NoError(t, err) // Wait for the prompt before sending the next command. - expectContainLines(t, stdout, []string{"$"}) + expectContainLines(ctx, t, stdout, []string{"$"}) _, err = stdinW.Write([]byte("exit\n")) require.NoError(t, err) @@ -96,15 +96,19 @@ func TestTerminalCommand_Intro(t *testing.T) { require.NoError(t, cmd.Start(ctx)) - expectContainLines(t, stdout, []string{envSourceCmd, introSecondLine}) + expectContainLines(ctx, t, stdout, []string{envSourceCmd, introSecondLine}) } -func expectContainLines(t *testing.T, r io.Reader, expected []string) { +func expectContainLines(ctx context.Context, t *testing.T, r io.Reader, expected []string) { t.Helper() - var output strings.Builder + hits := make(map[string]bool, len(expected)) + output := new(strings.Builder) for { + buf := new(bytes.Buffer) + r := io.TeeReader(r, buf) + scanner := bufio.NewScanner(r) for scanner.Scan() { _, _ = output.WriteString(scanner.Text()) @@ -121,7 +125,11 @@ func expectContainLines(t *testing.T, r io.Reader, expected []string) { return } - time.Sleep(time.Millisecond * 400) + select { + case <-time.After(100 * time.Millisecond): + case <-ctx.Done(): + t.Fatalf("error waiting for line %q, instead read %q: %s", expected, buf.Bytes(), ctx.Err()) + } } } diff --git a/internal/command/command_virtual.go b/internal/command/command_virtual.go index bc62333f..5b726cfd 100644 --- a/internal/command/command_virtual.go +++ b/internal/command/command_virtual.go @@ -5,7 +5,6 @@ import ( "io" "os" "os/exec" - "reflect" "sync" "syscall" @@ -224,21 +223,6 @@ func SetWinsize(cmd Command, winsize *Winsize) (err error) { return errors.WithStack(err) } -func isNil(val any) bool { - if val == nil { - return true - } - - v := reflect.ValueOf(val) - - switch v.Type().Kind() { - case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer: - return v.IsNil() - default: - return false - } -} - // readCloser wraps [io.Reader] into [io.ReadCloser]. // // When Close is called, the underlying read operation is ignored. diff --git a/internal/command/command_windows.go b/internal/command/command_windows.go index 59b74899..7a8ca640 100644 --- a/internal/command/command_windows.go +++ b/internal/command/command_windows.go @@ -28,4 +28,4 @@ func disableEcho(uintptr) error { "and join our Discord server at https://discord.gg/runme if you have further questions!") } -func signalPgid(int, os.Signal) error { return errors.New("unsupported") } +func signalPgid(int, os.Signal) error { return errors.New("signalPgid: unsupported") } diff --git a/internal/command/config_code_block.go b/internal/command/config_code_block.go index 44344ffb..01040b22 100644 --- a/internal/command/config_code_block.go +++ b/internal/command/config_code_block.go @@ -9,12 +9,30 @@ import ( "github.com/stateful/runme/v3/pkg/document" ) -func NewProgramConfigFromCodeBlock(block *document.CodeBlock) (*ProgramConfig, error) { - return (&configBuilder{block: block}).Build() +type ConfigBuilderOption func(*configBuilder) error + +func WithInteractiveLegacy() ConfigBuilderOption { + return func(b *configBuilder) error { + b.useInteractiveLegacy = true + return nil + } +} + +func NewProgramConfigFromCodeBlock(block *document.CodeBlock, opts ...ConfigBuilderOption) (*ProgramConfig, error) { + b := &configBuilder{block: block} + + for _, opt := range opts { + if err := opt(b); err != nil { + return nil, err + } + } + + return b.Build() } type configBuilder struct { - block *document.CodeBlock + block *document.CodeBlock + useInteractiveLegacy bool } func (b *configBuilder) Build() (*ProgramConfig, error) { @@ -22,7 +40,12 @@ func (b *configBuilder) Build() (*ProgramConfig, error) { ProgramName: b.programPath(), LanguageId: b.block.Language(), Directory: b.dir(), - Interactive: b.block.Interactive(), + } + + if b.useInteractiveLegacy { + cfg.Interactive = b.block.InteractiveLegacy() + } else { + cfg.Interactive = b.block.Interactive() } if isShell(cfg) { diff --git a/internal/command/factory.go b/internal/command/factory.go index 1338c976..8d996a28 100644 --- a/internal/command/factory.go +++ b/internal/command/factory.go @@ -2,6 +2,7 @@ package command import ( "io" + "reflect" "go.uber.org/zap" @@ -303,3 +304,18 @@ func (f *commandFactory) getLogger(name string) *zap.Logger { id := ulid.GenerateID() return f.logger.Named(name).With(zap.String("instanceID", id)) } + +func isNil(val any) bool { + if val == nil { + return true + } + + v := reflect.ValueOf(val) + + switch v.Type().Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer: + return v.IsNil() + default: + return false + } +} diff --git a/internal/config/autoconfig/autoconfig.go b/internal/config/autoconfig/autoconfig.go index 9ab399e8..7d9353e1 100644 --- a/internal/config/autoconfig/autoconfig.go +++ b/internal/config/autoconfig/autoconfig.go @@ -98,6 +98,7 @@ func getClient(cfg *config.Config, logger *zap.Logger) (*runnerv2client.Client, return runnerv2client.New( cfg.Server.Address, + logger, opts..., ) } @@ -185,6 +186,10 @@ func getProject(c *config.Config, logger *zap.Logger) (*project.Project, error) project.WithLogger(logger), } + if env := c.Project.Env; env != nil { + opts = append(opts, project.WithEnvFilesReadOrder(env.Sources)) + } + if c.Project.Filename != "" { return project.NewFileProject(c.Project.Filename, opts...) } @@ -199,7 +204,6 @@ func getProject(c *config.Config, logger *zap.Logger) (*project.Project, error) opts, project.WithIgnoreFilePatterns(c.Project.Ignore...), project.WithRespectGitignore(!c.Project.DisableGitignore), - project.WithEnvFilesReadOrder(c.Project.Env.Sources), ) if c.Project.FindRepoUpward { diff --git a/internal/project/projectservice/project_service.go b/internal/project/projectservice/project_service.go index ae45c1d4..af4e9b24 100644 --- a/internal/project/projectservice/project_service.go +++ b/internal/project/projectservice/project_service.go @@ -6,6 +6,8 @@ import ( "sync" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" projectv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/project/v1" "github.com/stateful/runme/v3/pkg/project" @@ -36,34 +38,45 @@ func (s *projectServiceServer) Load(req *projectv1.LoadRequest, srv projectv1.Pr go func() { defer wg.Done() defer close(errc) - for event := range eventc { - msg := &projectv1.LoadResponse{ - Type: projectv1.LoadEventType(event.Type), - } - - if err := setDataForLoadResponseFromLoadEvent(msg, event); err != nil { - errc <- err - goto errhandler - } - - if err := srv.Send(msg); err != nil { - errc <- err - goto errhandler - } - continue - - errhandler: - cancel() - // Project.Load() should be notified that it should exit early - // via cancel(). eventc will be closed, but it should be drained too - // in order to clean up any in-flight events. - // In theory, this is not necessary provided that all sends to eventc - // are wrapped in selects which observe ctx.Done(). - //revive:disable:empty-block - for range eventc { + for { + select { + case <-ctx.Done(): + errc <- status.Error(codes.Canceled, ctx.Err().Error()) + return + case event, ok := <-eventc: + if !ok { + return + } + + msg := &projectv1.LoadResponse{ + Type: projectv1.LoadEventType(event.Type), + } + + if err := setDataForLoadResponseFromLoadEvent(msg, event); err != nil { + errc <- err + goto errhandler + } + + if err := srv.Send(msg); err != nil { + errc <- err + goto errhandler + } + + continue + + errhandler: + cancel() + // Project.Load() should be notified that it should exit early + // via cancel(). eventc will be closed, but it should be drained too + // in order to clean up any in-flight events. + // In theory, this is not necessary provided that all sends to eventc + // are wrapped in selects which observe ctx.Done(). + //revive:disable:empty-block + for range eventc { + } + //revive:enable:empty-block } - //revive:enable:empty-block } }() diff --git a/internal/project/projectservice/project_service_test.go b/internal/project/projectservice/project_service_test.go index 4675bac0..61df0bb3 100644 --- a/internal/project/projectservice/project_service_test.go +++ b/internal/project/projectservice/project_service_test.go @@ -9,30 +9,26 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" + "go.uber.org/zap/zaptest" "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" "google.golang.org/grpc/test/bufconn" "github.com/stateful/runme/v3/internal/project/projectservice" + "github.com/stateful/runme/v3/internal/testutils" projectv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/project/v1" "github.com/stateful/runme/v3/pkg/project/teststub" - "github.com/stateful/runme/v3/pkg/project/testutils" + projtestutils "github.com/stateful/runme/v3/pkg/project/testutils" ) -func init() { - resolver.SetDefaultScheme("passthrough") -} - func TestProjectServiceServer_Load(t *testing.T) { t.Parallel() lis, stop := testStartProjectServiceServer(t) t.Cleanup(stop) - _, client := testCreateProjectServiceClient(t, lis) + + _, client := testutils.NewTestGRPCClient(t, lis, projectv1.NewProjectServiceClient) t.Run("GitProject", func(t *testing.T) { t.Parallel() @@ -45,7 +41,7 @@ func TestProjectServiceServer_Load(t *testing.T) { Directory: &projectv1.DirectoryProjectOptions{ Path: testData.GitProjectPath(), SkipGitignore: false, - IgnoreFilePatterns: testutils.IgnoreFilePatternsWithDefaults("ignored.md"), + IgnoreFilePatterns: projtestutils.IgnoreFilePatternsWithDefaults("ignored.md"), SkipRepoLookupUpward: false, }, }, @@ -56,7 +52,7 @@ func TestProjectServiceServer_Load(t *testing.T) { eventTypes, err := collectLoadEventTypes(loadClient) require.NoError(t, err) - assert.Len(t, eventTypes, len(testutils.GitProjectLoadOnlyNotIgnoredFilesEvents)) + assert.Len(t, eventTypes, len(projtestutils.GitProjectLoadOnlyNotIgnoredFilesEvents)) }) t.Run("FileProject", func(t *testing.T) { @@ -78,7 +74,7 @@ func TestProjectServiceServer_Load(t *testing.T) { eventTypes, err := collectLoadEventTypes(loadClient) require.NoError(t, err) - assert.Len(t, eventTypes, len(testutils.FileProjectEvents)) + assert.Len(t, eventTypes, len(projtestutils.FileProjectEvents)) }) } @@ -90,7 +86,8 @@ func TestProjectServiceServer_Load_ClientConnClosed(t *testing.T) { lis, stop := testStartProjectServiceServer(t) t.Cleanup(stop) - clientConn, client := testCreateProjectServiceClient(t, lis) + + clientConn, client := testutils.NewTestGRPCClient(t, lis, projectv1.NewProjectServiceClient) req := &projectv1.LoadRequest{ Kind: &projectv1.LoadRequest_File{ @@ -99,14 +96,11 @@ func TestProjectServiceServer_Load_ClientConnClosed(t *testing.T) { }, }, } - loadClient, err := client.Load(context.Background(), req) require.NoError(t, err) - errc := make(chan error, 1) - go func() { - errc <- clientConn.Close() - }() + err = clientConn.Close() + require.NoError(t, err) for { _, err := loadClient.Recv() @@ -115,8 +109,6 @@ func TestProjectServiceServer_Load_ClientConnClosed(t *testing.T) { break } } - - require.NoError(t, <-errc) } func collectLoadEventTypes(client projectv1.ProjectService_LoadClient) ([]projectv1.LoadEventType, error) { @@ -140,31 +132,14 @@ func testStartProjectServiceServer(t *testing.T) ( interface{ Dial() (net.Conn, error) }, func(), ) { - logger, err := zap.NewDevelopment() - require.NoError(t, err) + t.Helper() server := grpc.NewServer() - service := projectservice.NewProjectServiceServer(logger) + + service := projectservice.NewProjectServiceServer(zaptest.NewLogger(t)) projectv1.RegisterProjectServiceServer(server, service) lis := bufconn.Listen(1024 << 10) - go server.Serve(lis) - return lis, server.Stop } - -func testCreateProjectServiceClient( - t *testing.T, - lis interface{ Dial() (net.Conn, error) }, -) (*grpc.ClientConn, projectv1.ProjectServiceClient) { - conn, err := grpc.NewClient( - "passthrough", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithContextDialer(func(ctx context.Context, s string) (net.Conn, error) { - return lis.Dial() - }), - ) - require.NoError(t, err) - return conn, projectv1.NewProjectServiceClient(conn) -} diff --git a/internal/runner/service_test.go b/internal/runner/service_test.go index 988fd801..d6351877 100644 --- a/internal/runner/service_test.go +++ b/internal/runner/service_test.go @@ -20,26 +20,24 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" "google.golang.org/grpc/test/bufconn" "google.golang.org/protobuf/proto" + "github.com/stateful/runme/v3/internal/testutils" "github.com/stateful/runme/v3/internal/ulid" runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1" ) +// TODO(adamb): remove global state. It prevents from running tests in parallel. var ( logger *zap.Logger logFile string ) -func init() { - resolver.SetDefaultScheme("passthrough") -} - +// TODO(adamb): remove and use [zaptest.NewLogger] instead. func testCreateLogger(t *testing.T) *zap.Logger { + t.Helper() logger, err := zap.NewDevelopment() require.NoError(t, err) t.Cleanup(func() { _ = logger.Sync() }) @@ -75,21 +73,6 @@ func testStartRunnerServiceServer(t *testing.T) ( return lis, server.Stop } -func testCreateRunnerServiceClient( - t *testing.T, - lis interface{ Dial() (net.Conn, error) }, -) (*grpc.ClientConn, runnerv1.RunnerServiceClient) { - conn, err := grpc.NewClient( - "passthrough", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithContextDialer(func(ctx context.Context, s string) (net.Conn, error) { - return lis.Dial() - }), - ) - require.NoError(t, err) - return conn, runnerv1.NewRunnerServiceClient(conn) -} - type executeResult struct { Stdout []byte Stderr []byte @@ -129,7 +112,8 @@ func getExecuteResult( func Test_runnerService(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + + _, client := testutils.NewTestGRPCClient(t, lis, runnerv1.NewRunnerServiceClient) t.Run("Sessions", func(t *testing.T) { t.Parallel() diff --git a/internal/runnerv2client/client.go b/internal/runnerv2client/client.go index d253c859..d0e6305e 100644 --- a/internal/runnerv2client/client.go +++ b/internal/runnerv2client/client.go @@ -1,22 +1,164 @@ package runnerv2client import ( + "context" + "io" + "reflect" + "github.com/pkg/errors" - runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" + "go.uber.org/zap" "google.golang.org/grpc" + + runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" ) type Client struct { runnerv2.RunnerServiceClient + conn *grpc.ClientConn + logger *zap.Logger } -func New(target string, opts ...grpc.DialOption) (*Client, error) { +func New(target string, logger *zap.Logger, opts ...grpc.DialOption) (*Client, error) { client, err := grpc.NewClient(target, opts...) if err != nil { return nil, errors.WithStack(err) } + serviceClient := &Client{ + RunnerServiceClient: runnerv2.NewRunnerServiceClient(client), + conn: client, + logger: logger, + } + return serviceClient, nil +} - serviceClient := &Client{RunnerServiceClient: runnerv2.NewRunnerServiceClient(client)} +func (c *Client) Close() error { + return c.conn.Close() +} - return serviceClient, nil +type ExecuteProgramOptions struct { + InputData []byte + SessionID string + Stdin io.ReadCloser + Stdout io.Writer + Stderr io.Writer + StoreStdoutInEnv bool + Winsize *runnerv2.Winsize +} + +func (c *Client) ExecuteProgram( + ctx context.Context, + cfg *runnerv2.ProgramConfig, + opts ExecuteProgramOptions, +) error { + return c.executeProgram(ctx, cfg, opts) +} + +func (c *Client) executeProgram( + ctx context.Context, + cfg *runnerv2.ProgramConfig, + opts ExecuteProgramOptions, +) error { + stream, err := c.Execute(ctx) + if err != nil { + return errors.WithMessage(err, "failed to call Execute()") + } + + // Send the initial request. + req := &runnerv2.ExecuteRequest{ + Config: cfg, + InputData: opts.InputData, + SessionId: opts.SessionID, + StoreStdoutInEnv: opts.StoreStdoutInEnv, + Winsize: opts.Winsize, + } + if err := stream.Send(req); err != nil { + return errors.WithMessage(err, "failed to send initial request") + } + + if stdin := opts.Stdin; !isNil(stdin) { + // TODO(adamb): reimplement it. There should be a singleton + // handling and forwarding the stdin. The current implementation + // does not temrinate multiple stdin readers in the case of + // running multiple commands using "beta run command1 command2 ... commandN". + go func() { + defer func() { + c.logger.Info("finishing reading stdin") + err := stream.CloseSend() + if err != nil { + c.logger.Info("failed to close send", zap.Error(err)) + } + }() + + c.logger.Info("reading stdin") + + buf := make([]byte, 2*1024*1024) + + for { + n, err := stdin.Read(buf) + if err != nil { + c.logger.Info("failed to read stdin", zap.Error(err)) + break + } + + c.logger.Info("sending stdin", zap.Int("size", n)) + + err = stream.Send(&runnerv2.ExecuteRequest{ + InputData: buf[:n], + }) + if err != nil { + c.logger.Info("failed to send stdin", zap.Error(err)) + break + } + } + }() + } + + for { + resp, err := stream.Recv() + if err != nil { + if !errors.Is(err, io.EOF) { + c.logger.Info("failed to receive response", zap.Error(err)) + } + break + } + + if pid := resp.Pid; pid != nil { + c.logger.Info("server started a process with PID", zap.Uint32("pid", pid.GetValue()), zap.String("mime", resp.MimeType)) + } + + if stdout := opts.Stdout; !isNil(stdout) { + _, err = stdout.Write(resp.StdoutData) + if err != nil { + return errors.WithMessage(err, "failed to write stdout") + } + } + + if stderr := opts.Stderr; !isNil(stderr) { + _, err = stderr.Write(resp.StderrData) + if err != nil { + return errors.WithMessage(err, "failed to write stderr") + } + } + + if code := resp.GetExitCode(); code != nil && code.GetValue() != 0 { + return errors.WithMessagef(err, "exit with code %d", code.GetValue()) + } + } + + return nil +} + +func isNil(val any) bool { + if val == nil { + return true + } + + v := reflect.ValueOf(val) + + switch v.Type().Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer: + return v.IsNil() + default: + return false + } } diff --git a/internal/runnerv2client/client_test.go b/internal/runnerv2client/client_test.go new file mode 100644 index 00000000..ad99a050 --- /dev/null +++ b/internal/runnerv2client/client_test.go @@ -0,0 +1,137 @@ +package runnerv2client + +import ( + "bytes" + "context" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + + "github.com/stateful/runme/v3/internal/command" + "github.com/stateful/runme/v3/internal/runnerv2service" + runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" +) + +func init() { + command.SetEnvDumpCommand("env -0") +} + +func TestClient_ExecuteProgram(t *testing.T) { + t.Parallel() + + lis, stop := testStartRunnerServiceServer(t) + t.Cleanup(stop) + + t.Run("OutputWithSession", func(t *testing.T) { + t.Parallel() + + client := testCreateClient(t, lis) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + sessionResp, err := client.CreateSession( + ctx, + &runnerv2.CreateSessionRequest{ + Env: []string{"TEST=test-output-with-session-env"}, + }, + ) + require.NoError(t, err) + + cfg := &command.ProgramConfig{ + ProgramName: "bash", + Source: &runnerv2.ProgramConfig_Commands{ + Commands: &runnerv2.ProgramConfig_CommandList{ + Items: []string{ + "echo -n $TEST", + ">&2 echo -n test-output-with-session-stderr", + }, + }, + }, + Mode: runnerv2.CommandMode_COMMAND_MODE_INLINE, + } + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + err = client.ExecuteProgram( + ctx, + cfg, + ExecuteProgramOptions{ + SessionID: sessionResp.GetSession().GetId(), + Stdout: stdout, + Stderr: stderr, + }, + ) + require.NoError(t, err) + require.Equal(t, "test-output-with-session-env", stdout.String()) + require.Equal(t, "test-output-with-session-stderr", stderr.String()) + }) + + t.Run("InputNonInteractive", func(t *testing.T) { + t.Parallel() + + client := testCreateClient(t, lis) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cfg := &command.ProgramConfig{ + ProgramName: "bash", + Source: &runnerv2.ProgramConfig_Commands{ + Commands: &runnerv2.ProgramConfig_CommandList{ + Items: []string{ + "read -r name", + "echo $name", + }, + }, + }, + Mode: runnerv2.CommandMode_COMMAND_MODE_INLINE, + } + stdout := new(bytes.Buffer) + err := client.ExecuteProgram( + ctx, + cfg, + ExecuteProgramOptions{ + InputData: []byte("test-input-non-interactive\n"), + Stdout: stdout, + }, + ) + require.NoError(t, err) + require.Equal(t, "test-input-non-interactive\n", stdout.String()) + }) +} + +// TODO(adamb): it's copied from internal/runnerv2service. +func testStartRunnerServiceServer(t *testing.T) (*bufconn.Listener, func()) { + t.Helper() + + logger := zaptest.NewLogger(t) + factory := command.NewFactory(command.WithLogger(logger)) + + server := grpc.NewServer() + + runnerService, err := runnerv2service.NewRunnerService(factory, logger) + require.NoError(t, err) + runnerv2.RegisterRunnerServiceServer(server, runnerService) + + lis := bufconn.Listen(1024 << 10) + go server.Serve(lis) + + return lis, server.Stop +} + +func testCreateClient(t *testing.T, lis *bufconn.Listener) *Client { + client, err := New( + "passthrough://bufconn", + zaptest.NewLogger(t), + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return lis.Dial() + }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + return client +} diff --git a/internal/runnerv2client/client_unix_test.go b/internal/runnerv2client/client_unix_test.go new file mode 100644 index 00000000..05cf77c4 --- /dev/null +++ b/internal/runnerv2client/client_unix_test.go @@ -0,0 +1,54 @@ +//go:build !windows +// +build !windows + +package runnerv2client + +import ( + "bytes" + "context" + "io" + "testing" + "time" + + "github.com/stateful/runme/v3/internal/command" + runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" + "github.com/stretchr/testify/require" +) + +func TestClient_ExecuteProgram_InputInteractive(t *testing.T) { + t.Parallel() + + lis, stop := testStartRunnerServiceServer(t) + t.Cleanup(stop) + + client := testCreateClient(t, lis) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cfg := &command.ProgramConfig{ + ProgramName: "bash", + Source: &runnerv2.ProgramConfig_Commands{ + Commands: &runnerv2.ProgramConfig_CommandList{ + Items: []string{ + "read -r name", + "echo $name", + }, + }, + }, + Interactive: true, + Mode: runnerv2.CommandMode_COMMAND_MODE_INLINE, + } + stdout := new(bytes.Buffer) + err := client.ExecuteProgram( + ctx, + cfg, + ExecuteProgramOptions{ + Stdin: io.NopCloser(bytes.NewBufferString("test-input-interactive\n")), + Stdout: stdout, + }, + ) + require.NoError(t, err) + // Using [require.Contains] because on Linux the input is repeated. + // Unclear why it passes fine on macOS. + require.Contains(t, stdout.String(), "test-input-interactive\r\n") +} diff --git a/internal/runnerv2service/execution.go b/internal/runnerv2service/execution.go index 7328ce02..d3100622 100644 --- a/internal/runnerv2service/execution.go +++ b/internal/runnerv2service/execution.go @@ -29,7 +29,7 @@ const ( // small. // In the future, it might be worth to implement // variable-sized buffers. - msgBufferSize = 2048 << 10 // 2 MiB + msgBufferSize = 2 * 1024 * 1024 // 2 MiB ) var opininatedEnvVarNamingRegexp = regexp.MustCompile(`^[A-Z_][A-Z0-9_]{1}[A-Z0-9_]*[A-Z][A-Z0-9_]*$`) diff --git a/internal/runnerv2service/service_execute_test.go b/internal/runnerv2service/service_execute_test.go index 3957b1f3..20ad89a3 100644 --- a/internal/runnerv2service/service_execute_test.go +++ b/internal/runnerv2service/service_execute_test.go @@ -18,22 +18,19 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/resolver" "google.golang.org/grpc/test/bufconn" "github.com/stateful/runme/v3/internal/command" "github.com/stateful/runme/v3/internal/command/testdata" "github.com/stateful/runme/v3/internal/config" "github.com/stateful/runme/v3/internal/config/autoconfig" + "github.com/stateful/runme/v3/internal/testutils" runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" ) func init() { command.SetEnvDumpCommand("env -0") - resolver.SetDefaultScheme("passthrough") - // Server uses autoconfig to get necessary dependencies. // One of them, implicit, is [config.Config]. With the default // [config.Loader] it won't be found during testing, so @@ -89,7 +86,7 @@ func Test_conformsOpinionatedEnvVarNaming(t *testing.T) { func TestRunnerServiceServerExecute_Response(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) stream, err := client.Execute(context.Background()) require.NoError(t, err) @@ -160,7 +157,7 @@ func TestRunnerServiceServerExecute_Response(t *testing.T) { func TestRunnerServiceServerExecute_MimeType(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) stream, err := client.Execute(context.Background()) require.NoError(t, err) @@ -199,7 +196,7 @@ func TestRunnerServiceServerExecute_MimeType(t *testing.T) { func TestRunnerServiceServerExecute_StoreLastStdout(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) sessionResp, err := client.CreateSession(context.Background(), &runnerv2.CreateSessionRequest{}) require.NoError(t, err) @@ -276,7 +273,7 @@ func TestRunnerServiceServerExecute_LastStdoutExceedsEnvLimit(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) sessionResp, err := client.CreateSession(context.Background(), &runnerv2.CreateSessionRequest{}) require.NoError(t, err) @@ -353,7 +350,7 @@ func TestRunnerServiceServerExecute_LargeOutput(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) stream, err := client.Execute(context.Background()) require.NoError(t, err) @@ -387,7 +384,7 @@ func TestRunnerServiceServerExecute_LargeOutput(t *testing.T) { func TestRunnerServiceServerExecute_StoreKnownName(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) sessionResp, err := client.CreateSession(context.Background(), &runnerv2.CreateSessionRequest{}) require.NoError(t, err) @@ -460,7 +457,7 @@ func TestRunnerServiceServerExecute_StoreKnownName(t *testing.T) { func TestRunnerServiceServerExecute_Configs(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) testCases := []struct { name string @@ -617,7 +614,7 @@ func TestRunnerServiceServerExecute_Configs(t *testing.T) { func TestRunnerServiceServerExecute_CommandMode_Terminal(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) sessResp, err := client.CreateSession(context.Background(), &runnerv2.CreateSessionRequest{}) require.NoError(t, err) @@ -696,7 +693,7 @@ func TestRunnerServiceServerExecute_CommandMode_Terminal(t *testing.T) { func TestRunnerServiceServerExecute_PathEnvInSession(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) sessionResp, err := client.CreateSession(context.Background(), &runnerv2.CreateSessionRequest{}) require.NoError(t, err) @@ -759,7 +756,7 @@ func TestRunnerServiceServerExecute_PathEnvInSession(t *testing.T) { func TestRunnerServiceServerExecute_WithInput(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) t.Run("ContinuousInput", func(t *testing.T) { stream, err := client.Execute(context.Background()) @@ -876,7 +873,7 @@ func TestRunnerServiceServerExecute_WithInput(t *testing.T) { func TestRunnerServiceServerExecute_WithSession(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) t.Run("WithEnvAndMostRecentSessionStrategy", func(t *testing.T) { { @@ -942,7 +939,7 @@ func TestRunnerServiceServerExecute_WithSession(t *testing.T) { func TestRunnerServiceServerExecute_WithStop(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) stream, err := client.Execute(context.Background()) require.NoError(t, err) @@ -1009,7 +1006,8 @@ func TestRunnerServiceServerExecute_WithStop(t *testing.T) { func TestRunnerServiceServerExecute_Winsize(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) t.Run("DefaultWinsize", func(t *testing.T) { t.Parallel() @@ -1104,24 +1102,6 @@ func testStartRunnerServiceServer(t *testing.T) ( return lis, server.Stop } -func testCreateRunnerServiceClient( - t *testing.T, - lis interface{ Dial() (net.Conn, error) }, -) (*grpc.ClientConn, runnerv2.RunnerServiceClient) { - t.Helper() - - conn, err := grpc.NewClient( - "passthrough", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithContextDialer(func(ctx context.Context, s string) (net.Conn, error) { - return lis.Dial() - }), - ) - require.NoError(t, err) - - return conn, runnerv2.NewRunnerServiceClient(conn) -} - type executeResult struct { Stdout []byte Stderr []byte diff --git a/internal/runnerv2service/service_resolve_program_test.go b/internal/runnerv2service/service_resolve_program_test.go index 8ab8dcb5..1ce0a5c3 100644 --- a/internal/runnerv2service/service_resolve_program_test.go +++ b/internal/runnerv2service/service_resolve_program_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + "github.com/stateful/runme/v3/internal/testutils" runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" ) @@ -16,7 +17,7 @@ import ( func TestRunnerServiceResolveProgram(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) testCases := []struct { name string @@ -93,7 +94,7 @@ func TestRunnerResolveProgram_CommandsWithNewLines(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) request := &runnerv2.ResolveProgramRequest{ Env: []string{"FILE_NAME=my-file.txt"}, @@ -145,7 +146,7 @@ func TestRunnerResolveProgram_CommandsWithNewLines(t *testing.T) { func TestRunnerResolveProgram_OnlyShellLanguages(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) t.Run("Javascript passed as script", func(t *testing.T) { script := "console.log('test');" diff --git a/internal/runnerv2service/service_sessions_test.go b/internal/runnerv2service/service_sessions_test.go index ff51e4e8..43d6727a 100644 --- a/internal/runnerv2service/service_sessions_test.go +++ b/internal/runnerv2service/service_sessions_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc/resolver" + "github.com/stateful/runme/v3/internal/testutils" runnerv2 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v2" "github.com/stateful/runme/v3/pkg/project/teststub" ) @@ -25,7 +26,8 @@ func TestRunnerServiceSessions(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) EnvStoreSeedingNone := runnerv2.CreateSessionRequest_Config_SESSION_ENV_STORE_SEEDING_UNSPECIFIED.Enum() @@ -117,7 +119,8 @@ func TestRunnerServiceSessions(t *testing.T) { func TestRunnerServiceSessions_StrategyMostRecent(t *testing.T) { lis, stop := testStartRunnerServiceServer(t) t.Cleanup(stop) - _, client := testCreateRunnerServiceClient(t, lis) + + _, client := testutils.NewTestGRPCClient(t, lis, runnerv2.NewRunnerServiceClient) // Create a session with env. sessResp, err := client.CreateSession( diff --git a/internal/testutils/grpc.go b/internal/testutils/grpc.go new file mode 100644 index 00000000..266cdb35 --- /dev/null +++ b/internal/testutils/grpc.go @@ -0,0 +1,51 @@ +package testutils + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func NewTestGRPCClient[T any]( + t *testing.T, + lis interface{ Dial() (net.Conn, error) }, + fn func(grpc.ClientConnInterface) T, +) (*grpc.ClientConn, T) { + t.Helper() + conn, client, err := newGRPCClient(lis, fn) + require.NoError(t, err) + return conn, client +} + +func NewGRPCClient[T any]( + lis interface{ Dial() (net.Conn, error) }, + fn func(grpc.ClientConnInterface) T, +) (*grpc.ClientConn, T) { + conn, client, err := newGRPCClient(lis, fn) + if err != nil { + panic(err) + } + return conn, client +} + +func newGRPCClient[T any]( + lis interface{ Dial() (net.Conn, error) }, + fn func(grpc.ClientConnInterface) T, +) (*grpc.ClientConn, T, error) { + conn, err := grpc.NewClient( + "passthrough://bufconn", + grpc.WithContextDialer(func(ctx context.Context, s string) (net.Conn, error) { + return lis.Dial() + }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + var result T + return nil, result, err + } + return conn, fn(conn), nil +} diff --git a/internal/tls/tls.go b/internal/tls/tls.go index 2e6cfbc2..aecded44 100644 --- a/internal/tls/tls.go +++ b/internal/tls/tls.go @@ -91,7 +91,8 @@ func LoadOrGenerateConfig(certFile, keyFile string, logger *zap.Logger) (*tls.Co } if config != nil { - if ttl, err := validateTLSConfig(config); err == nil { + ttl, err := validateTLSConfig(config) + if err == nil { logger.Info("certificate is valid", zap.Duration("ttl", ttl), zap.String("certFile", certFile), zap.String("keyFile", keyFile)) return config, nil } diff --git a/pkg/document/block.go b/pkg/document/block.go index 8e104f0f..9cf98230 100644 --- a/pkg/document/block.go +++ b/pkg/document/block.go @@ -176,7 +176,8 @@ func (b *CodeBlock) Interactive() bool { } // InteractiveLegacy returns true as a default value. -// Deprecated: use Interactive instead. +// Deprecated: use Interactive instead, however, keep using +// if you want to align with the VS Code extension. func (b *CodeBlock) InteractiveLegacy() bool { val, err := strconv.ParseBool(b.Attributes()["interactive"]) if err != nil { diff --git a/pkg/document/editor/editorservice/service_test.go b/pkg/document/editor/editorservice/service_test.go index fadf883f..cc72e166 100644 --- a/pkg/document/editor/editorservice/service_test.go +++ b/pkg/document/editor/editorservice/service_test.go @@ -3,19 +3,17 @@ package editorservice import ( "context" "fmt" - "net" "os" "strings" "testing" + "github.com/stateful/runme/v3/internal/testutils" "github.com/stateful/runme/v3/internal/ulid" "github.com/stateful/runme/v3/internal/version" parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1" "github.com/stretchr/testify/assert" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/resolver" "google.golang.org/grpc/test/bufconn" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/wrapperspb" @@ -56,27 +54,15 @@ var ( func TestMain(m *testing.M) { ulid.MockGenerator(testMockID) - resolver.SetDefaultScheme("passthrough") lis := bufconn.Listen(2048) server := grpc.NewServer() parserv1.RegisterParserServiceServer(server, NewParserServiceServer(zap.NewNop())) go server.Serve(lis) - conn, err := grpc.NewClient( - "passthrough", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithContextDialer(func(ctx context.Context, s string) (net.Conn, error) { - return lis.Dial() - }), - ) - if err != nil { - panic(err) - } + _, client = testutils.NewGRPCClient(lis, parserv1.NewParserServiceClient) - client = parserv1.NewParserServiceClient(conn) code := m.Run() - ulid.ResetGenerator() os.Exit(code) } From 4cc5606224540a260a5c3b604fa2365e824739db Mon Sep 17 00:00:00 2001 From: "Sebastian (Tiedtke) Huckleberry" Date: Mon, 4 Nov 2024 10:34:15 -0800 Subject: [PATCH 2/3] =?UTF-8?q?Owl=20=F0=9F=A6=89=20refactor=20spec=20(#69?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor env spec definitions loading from instead of hard-coding them. --- go.mod | 2 +- internal/cmd/environment.go | 155 ++++-- internal/owl/envSpecDefs.defaults.yaml | 111 ++++ internal/owl/graph.go | 493 ++++++++++++------ internal/owl/graph_test.go | 6 +- internal/owl/parse.go | 10 +- internal/owl/parse_test.go | 22 +- internal/owl/query.go | 268 ++++++++-- internal/owl/store.go | 177 ++++++- internal/owl/store_test.go | 103 +++- internal/owl/testdata/graph/dotenv.graphql | 2 +- .../testdata/graph/env_without_specs.graphql | 2 +- .../owl/testdata/graph/insecureget.graphql | 2 +- .../testdata/graph/query_complex_env.graphql | 2 +- .../testdata/graph/query_simple_env.graphql | 2 +- .../graph/reconcile_operationless.graphql | 2 +- .../owl/testdata/graph/sensitive_keys.graphql | 2 +- .../owl/testdata/graph/store_update.graphql | 2 +- .../graph/validate_simple_env.graphql | 2 +- internal/owl/testdata/resolve/.env.example | 7 +- internal/owl/validate.go | 386 ++++---------- internal/owl/validate_test.go | 10 +- 22 files changed, 1176 insertions(+), 592 deletions(-) create mode 100644 internal/owl/envSpecDefs.defaults.yaml diff --git a/go.mod b/go.mod index ebded417..67e80fc3 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sys v0.26.0 golang.org/x/term v0.25.0 + google.golang.org/api v0.196.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 google.golang.org/protobuf v1.35.1 mvdan.cc/sh/v3 v3.10.0 @@ -93,7 +94,6 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/time v0.7.0 // indirect - google.golang.org/api v0.196.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect gotest.tools/v3 v3.5.1 // indirect diff --git a/internal/cmd/environment.go b/internal/cmd/environment.go index cfe09867..7f6be005 100644 --- a/internal/cmd/environment.go +++ b/internal/cmd/environment.go @@ -42,7 +42,16 @@ func environmentCmd() *cobra.Command { return &cmd } +type envStoreFlags struct { + serverAddr string + sessionID string + sessionStrategy string + tlsDir string +} + func storeCmd() *cobra.Command { + var storeFlags envStoreFlags + cmd := cobra.Command{ Hidden: true, Use: "store", @@ -50,20 +59,109 @@ func storeCmd() *cobra.Command { Long: "Owl Store", } - cmd.AddCommand(storeSnapshotCmd()) + cmd.Flags().StringVar(&storeFlags.serverAddr, "server-address", os.Getenv("RUNME_SERVER_ADDR"), "The Server ServerAddress to connect to, i.e. 127.0.0.1:7865") + cmd.Flags().StringVar(&storeFlags.tlsDir, "tls-dir", os.Getenv("RUNME_TLS_DIR"), "Path to tls files") + cmd.Flags().StringVar(&storeFlags.sessionID, "session", os.Getenv("RUNME_SESSION"), "Session Id") + cmd.Flags().StringVar(&storeFlags.sessionStrategy, "session-strategy", func() string { + if val, ok := os.LookupEnv("RUNME_SESSION_STRATEGY"); ok { + return val + } + return "manual" + }(), "Strategy for session selection. Options are manual, recent. Defaults to manual") + + cmd.AddCommand(storeSnapshotCmd(storeFlags)) + cmd.AddCommand(storeSourceCmd(storeFlags)) cmd.AddCommand(storeCheckCmd()) return &cmd } -func storeSnapshotCmd() *cobra.Command { +func storeSourceCmd(storeFlags envStoreFlags) *cobra.Command { var ( - serverAddr string - sessionID string - sessionStrategy string - tlsDir string - limit int - all bool + insecure bool + export bool + sessionID = "" + ) + + cmd := cobra.Command{ + Use: "source", + Short: "Source environment variables from session", + Long: "Source environment variables from session", + RunE: func(cmd *cobra.Command, args []string) error { + // discard any stderr in silent mode + if !insecure { + return errors.New("must be run in insecure mode to prevent misuse; enable by adding --insecure flag") + } + + tlsConfig, err := runmetls.LoadClientConfigFromDir(storeFlags.tlsDir) + if err != nil { + return err + } + + credentials := credentials.NewTLS(tlsConfig) + conn, err := grpc.NewClient( + storeFlags.serverAddr, + grpc.WithTransportCredentials(credentials), + ) + if err != nil { + return errors.Wrap(err, "failed to connect") + } + defer conn.Close() + + client := runnerv1.NewRunnerServiceClient(conn) + + // todo(sebastian): would it be better to require a specific session? + if strings.ToLower(storeFlags.sessionStrategy) == "recent" { + req := &runnerv1.ListSessionsRequest{} + resp, err := client.ListSessions(cmd.Context(), req) + if err != nil { + return err + } + l := len(resp.Sessions) + if l == 0 { + return errors.New("no sessions found") + } + // potentially unreliable + sessionID = resp.Sessions[l-1].Id + } + + req := &runnerv1.GetSessionRequest{Id: sessionID} + resp, err := client.GetSession(cmd.Context(), req) + if err != nil { + return err + } + + for _, kv := range resp.Session.Envs { + parts := strings.Split(kv, "=") + if len(parts) < 2 { + return errors.Errorf("invalid key-value pair: %s", kv) + } + + envVar := fmt.Sprintf("%s=%q", parts[0], strings.Join(parts[1:], "=")) + if export { + envVar = fmt.Sprintf("export %s", envVar) + } + + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s\n", envVar); err != nil { + return err + } + } + + return nil + }, + } + + cmd.Flags().BoolVarP(&export, "export", "", false, "export variables") + cmd.Flags().BoolVar(&insecure, "insecure", false, "Explicitly allow delicate operations to prevent misuse") + + return &cmd +} + +func storeSnapshotCmd(storeFlags envStoreFlags) *cobra.Command { + var ( + limit int + reveal bool + all bool ) cmd := cobra.Command{ @@ -72,14 +170,18 @@ func storeSnapshotCmd() *cobra.Command { Short: "Takes a snapshot of the smart env store", Long: "Connects with a running server to inspect the environment variables of a session and returns a snapshot of the smart env store.", RunE: func(cmd *cobra.Command, args []string) error { - tlsConfig, err := runmetls.LoadClientConfigFromDir(tlsDir) + tlsConfig, err := runmetls.LoadClientConfigFromDir(storeFlags.tlsDir) if err != nil { return err } + if reveal && !fInsecure { + return errors.New("must be run in insecure mode to prevent misuse; enable by adding --insecure flag") + } + credentials := credentials.NewTLS(tlsConfig) conn, err := grpc.NewClient( - serverAddr, + storeFlags.serverAddr, grpc.WithTransportCredentials(credentials), ) if err != nil { @@ -90,7 +192,7 @@ func storeSnapshotCmd() *cobra.Command { client := runnerv1.NewRunnerServiceClient(conn) // todo(sebastian): this should move into API as part of v2 - if strings.ToLower(sessionStrategy) == "recent" { + if strings.ToLower(storeFlags.sessionStrategy) == "recent" { req := &runnerv1.ListSessionsRequest{} resp, err := client.ListSessions(cmd.Context(), req) if err != nil { @@ -101,11 +203,11 @@ func storeSnapshotCmd() *cobra.Command { return errors.New("no sessions found") } // potentially unreliable - sessionID = resp.Sessions[l-1].Id + storeFlags.sessionID = resp.Sessions[l-1].Id } req := &runnerv1.MonitorEnvStoreRequest{ - Session: &runnerv1.Session{Id: sessionID}, + Session: &runnerv1.Session{Id: storeFlags.sessionID}, } meClient, err := client.MonitorEnvStore(cmd.Context(), req) if err != nil { @@ -119,25 +221,17 @@ func storeSnapshotCmd() *cobra.Command { } if msgData, ok := msg.Data.(*runnerv1.MonitorEnvStoreResponse_Snapshot); ok { - return errors.Wrap(printStore(cmd, msgData, limit, all), "failed to render") + insecureReveal := reveal && fInsecure + return errors.Wrap(printStore(cmd, msgData, limit, insecureReveal, all), "failed to render") } return nil }, } - cmd.Flags().StringVar(&serverAddr, "server-address", os.Getenv("RUNME_SERVER_ADDR"), "The Server ServerAddress to connect to, i.e. 127.0.0.1:7865") - cmd.Flags().StringVar(&tlsDir, "tls-dir", os.Getenv("RUNME_TLS_DIR"), "Path to tls files") - cmd.Flags().StringVar(&sessionID, "session", os.Getenv("RUNME_SESSION"), "Session Id") - cmd.Flags().StringVar(&sessionStrategy, "session-strategy", func() string { - if val, ok := os.LookupEnv("RUNME_SESSION_STRATEGY"); ok { - return val - } - - return "manual" - }(), "Strategy for session selection. Options are manual, recent. Defaults to manual") cmd.Flags().IntVar(&limit, "limit", 50, "Limit the number of lines") cmd.Flags().BoolVarP(&all, "all", "A", false, "Show all lines") + cmd.Flags().BoolVarP(&reveal, "reveal", "r", false, "Reveal hidden values") return &cmd } @@ -225,7 +319,7 @@ func environmentDumpCmd() *cobra.Command { return &cmd } -func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Snapshot, lines int, all bool) error { +func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Snapshot, lines int, reveal, all bool) error { term := term.FromIO(cmd.InOrStdin(), cmd.OutOrStdout(), cmd.ErrOrStderr()) width, _, err := term.Size() @@ -238,7 +332,7 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn table.AddField(strings.ToUpper("Value")) table.AddField(strings.ToUpper("Description")) table.AddField(strings.ToUpper("Spec")) - table.AddField(strings.ToUpper("Origin")) + table.AddField(strings.ToUpper("Source")) table.AddField(strings.ToUpper("Updated")) table.EndRow() @@ -247,8 +341,8 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn break } - specless := msgData.Snapshot.Envs[0].Spec != owl.SpecNameOpaque - if !all && specless && env.Spec == owl.SpecNameOpaque { + specless := msgData.Snapshot.Envs[0].Spec != owl.AtomicNameOpaque + if !all && specless && env.Spec == owl.AtomicNameOpaque { break } @@ -260,7 +354,10 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn case runnerv1.MonitorEnvStoreResponseSnapshot_STATUS_MASKED: value = "[masked]" case runnerv1.MonitorEnvStoreResponseSnapshot_STATUS_HIDDEN: - value = "******" + value = "[hidden]" + if reveal { + value = env.GetOriginalValue() + } } strippedVal := strings.ReplaceAll(strings.ReplaceAll(value, "\n", " "), "\r", "") diff --git a/internal/owl/envSpecDefs.defaults.yaml b/internal/owl/envSpecDefs.defaults.yaml new file mode 100644 index 00000000..a76641e7 --- /dev/null +++ b/internal/owl/envSpecDefs.defaults.yaml @@ -0,0 +1,111 @@ +apiVersion: runme.stateful.com/v1beta1 +kind: EnvSpecDefinitions +metadata: + name: runme + namespace: stateful + annotations: + github.com/repo-url: https://github.com/stateful/runme +spec: + type: owl + envSpecs: + - name: Auth0 + breaker: AUTH0 + atomics: + - key: AUDIENCE + atomic: Plain + rules: url + required: true + - key: CLIENT_ID + atomic: Secret + rules: alphanum,min=32,max=32 + required: true + - key: DOMAIN + atomic: Secret + rules: fqdn + required: true + - key: COOKIE_DOMAIN + atomic: Secret + rules: min=8 + required: true + - name: Auth0Mgmt + breaker: AUTH0_MANAGEMENT + atomics: + - key: CLIENT_ID + atomic: Plain + rules: alphanum,min=32,max=32 + required: true + - key: CLIENT_SECRET + atomic: Secret + rules: ascii,min=64,max=64 + required: true + - key: AUDIENCE + atomic: Plain + rules: url + required: true + - name: DatabaseUrl + breaker: DATABASE + atomics: + - key: URL + atomic: Secret + rules: database_url + required: true + - name: OpenAI + breaker: OPENAI + atomics: + - key: ORG_ID + atomic: Opaque + rules: ascii,min=28,max=28,startswith=org- + required: true + - key: API_KEY + atomic: Secret + rules: ascii,min=34,startswith=sk- + required: true + - name: Redis + breaker: REDIS + atomics: + - key: HOST + atomic: Plain + rules: ip|hostname + required: true + - key: PORT + atomic: Plain + rules: number + required: true + - key: PASSWORD + atomic: Password + rules: min=18,max=32 + required: false + - name: Slack + breaker: SLACK + atomics: + - key: CLIENT_ID + atomic: Plain + rules: min=24,max=24 + required: true + - key: CLIENT_SECRET + atomic: Secret + rules: min=32,max=32 + required: true + - key: REDIRECT_URL + atomic: Secret + rules: url + required: true + - name: UserHub + breaker: USERHUB + atomics: + - key: WEBHOOK_SECRET + atomic: Secret + rules: min=8,max=64 + required: true + - key: CONN_PROVIDER_ID + atomic: Plain + rules: + required: true + - key: SKIP_CONNECTION + atomic: Plain + rules: + required: false + - key: API_KEY + atomic: Secret + rules: min=16,max=64 + required: true diff --git a/internal/owl/graph.go b/internal/owl/graph.go index a9c4db20..32ebcbf3 100644 --- a/internal/owl/graph.go +++ b/internal/owl/graph.go @@ -2,7 +2,6 @@ package owl import ( "bytes" - "context" "encoding/json" "fmt" "log" @@ -13,19 +12,21 @@ import ( exprlang "github.com/expr-lang/expr" "github.com/graphql-go/graphql" "github.com/pkg/errors" + "golang.org/x/oauth2/google" + "google.golang.org/api/idtoken" ) // Constants representing different spec names. -// These constants are of type SpecName and are assigned string values. +// These constants are of type AtomicName and are assigned string values. const ( - SpecNameOpaque string = "Opaque" // SpecNameOpaque specifies an opaque specification. - SpecNamePlain string = "Plain" // SpecNamePlain specifies a plain specification. - SpecNameSecret string = "Secret" // SpecNameSecret specifies a secret specification. - SpecNamePassword string = "Password" // SpecNamePassword specifies a password specification. - SpecNameDefault = SpecNameOpaque + AtomicNameOpaque string = "Opaque" // SpecNameOpaque specifies an opaque specification. + AtomicNamePlain string = "Plain" // SpecNamePlain specifies a plain specification. + AtomicNameSecret string = "Secret" // SpecNameSecret specifies a secret specification. + AtomicNamePassword string = "Password" // SpecNamePassword specifies a password specification. + AtomicNameDefault = AtomicNameOpaque ) -type specType struct { +type atomicType struct { typeName string typeObject *graphql.Object resolveFn graphql.FieldResolveFn @@ -33,8 +34,8 @@ type specType struct { var ( Schema graphql.Schema - SpecTypes map[string]*specType - ComplexType *specType + AtomicTypes map[string]*atomicType + SpecType *atomicType ) var EnvironmentType, @@ -44,8 +45,8 @@ var EnvironmentType, SpecTypeErrorsType *graphql.Object // todo(sebastian): use gql interface? -func registerSpecFields(fields graphql.Fields) { - for _, t := range SpecTypes { +func registerAtomicFields(fields graphql.Fields) { + for _, t := range AtomicTypes { fields[t.typeName] = &graphql.Field{ Type: t.typeObject, Resolve: t.resolveFn, @@ -61,9 +62,9 @@ func registerSpecFields(fields graphql.Fields) { } } - fields[ComplexType.typeName] = &graphql.Field{ - Type: ComplexType.typeObject, - Resolve: ComplexType.resolveFn, + fields[SpecType.typeName] = &graphql.Field{ + Type: SpecType.typeObject, + Resolve: SpecType.resolveFn, Args: graphql.FieldConfigArgument{ "name": &graphql.ArgumentConfig{ Type: graphql.NewNonNull(graphql.String), @@ -78,9 +79,9 @@ func registerSpecFields(fields graphql.Fields) { } } -func registerSpec(name string, sensitive, mask bool, resolver graphql.FieldResolveFn) *specType { +func registerAtomicType(name string, sensitive, mask bool, resolver graphql.FieldResolveFn) *atomicType { typ := graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("SpecType%s", name), + Name: fmt.Sprintf("AtomicType%s", name), Fields: (graphql.FieldsThunk)(func() graphql.Fields { fields := graphql.Fields{ "name": &graphql.Field{ @@ -109,8 +110,8 @@ func registerSpec(name string, sensitive, mask bool, resolver graphql.FieldResol switch p.Source.(type) { case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet default: return nil, errors.New("source does not contain an OperationSet") } @@ -141,21 +142,21 @@ func registerSpec(name string, sensitive, mask bool, resolver graphql.FieldResol }, } - registerSpecFields(fields) + registerAtomicFields(fields) return fields }), }) - return &specType{ + return &atomicType{ typeName: name, typeObject: typ, resolveFn: resolver, } } -func registerComplexType(resolver graphql.FieldResolveFn) *specType { - name := ComplexSpecType +func registerSpecType(resolver graphql.FieldResolveFn) *atomicType { + name := SpecTypeKey typ := graphql.NewObject(graphql.ObjectConfig{ Name: fmt.Sprintf("SpecType%s", name), Fields: (graphql.FieldsThunk)(func() graphql.Fields { @@ -174,8 +175,8 @@ func registerComplexType(resolver graphql.FieldResolveFn) *specType { switch p.Source.(type) { case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet default: return nil, errors.New("source does not contain an OperationSet") } @@ -206,39 +207,39 @@ func registerComplexType(resolver graphql.FieldResolveFn) *specType { }, } - registerSpecFields(fields) + registerAtomicFields(fields) return fields }), }) - return &specType{ + return &atomicType{ typeName: name, typeObject: typ, resolveFn: resolver, } } -type SpecResolverMutator func(val *SetVarValue, spec *SetVarSpec, insecure bool) +type AtomicResolverMutator func(val *SetVarValue, spec *SetVarSpec, insecure bool) -func specResolver(mutator SpecResolverMutator) graphql.FieldResolveFn { +func atomicResolver(mutator AtomicResolverMutator) graphql.FieldResolveFn { return func(p graphql.ResolveParams) (interface{}, error) { insecure := p.Args["insecure"].(bool) keysArg := p.Args["keys"].([]interface{}) var opSet *OperationSet - complexName := "" - complexNs := "" + specName := "" + specNs := "" switch p.Source.(type) { case *OperationSet: opSet = p.Source.(*OperationSet) - complexName = "" - complexNs = "" - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet - complexName = p.Source.(*ComplexOperationSet).Name - complexNs = p.Source.(*ComplexOperationSet).Namespace + specName = "" + specNs = "" + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet + specName = p.Source.(*SpecOperationSet).Name + specNs = p.Source.(*SpecOperationSet).Namespace default: return nil, errors.New("source does not contain an OperationSet") } @@ -252,8 +253,8 @@ func specResolver(mutator SpecResolverMutator) graphql.FieldResolveFn { continue } - spec.Spec.Complex = complexName - spec.Spec.Namespace = complexNs + spec.Spec.Spec = specName + spec.Spec.Namespace = specNs spec.Spec.Checked = true // skip if last known status was DELETED @@ -268,7 +269,7 @@ func specResolver(mutator SpecResolverMutator) graphql.FieldResolveFn { continue } - if spec.Spec.Required { + if spec.Spec.Required && spec.Spec.Error == nil { spec.Spec.Error = NewRequiredError(&SetVarItem{Var: val.Var, Value: val.Value, Spec: spec.Spec}) continue } @@ -296,8 +297,8 @@ func resolveSensitiveKeys() graphql.FieldResolveFn { return sensitive, nil case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet default: return nil, errors.New("source does not contain an OperationSet") } @@ -334,8 +335,8 @@ func resolveDotEnv() graphql.FieldResolveFn { return dotenv, nil case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet default: return nil, errors.New("source does not contain an OperationSet") } @@ -376,8 +377,8 @@ func resolveGetter() graphql.FieldResolveFn { return kv, nil case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet default: return nil, errors.New("source is does not contain an OperationSet") } @@ -412,8 +413,8 @@ func resolveSnapshot() graphql.FieldResolveFn { return snapshot, nil case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - opSet = p.Source.(*ComplexOperationSet).OperationSet + case *SpecOperationSet: + opSet = p.Source.(*SpecOperationSet).OperationSet default: return nil, errors.New("source does not contain an OperationSet") } @@ -516,10 +517,13 @@ func mutateLoadOrUpdate(revived SetVarItems, resolverOpSet *OperationSet, hasSpe if old, ok := resolverOpSet.values[r.Var.Key]; ok { oldCreated := old.Var.Created r.Var.Created = oldCreated - if old.Var.Origin != "" { - source = old.Var.Origin - } else if old.Value.Operation != nil && old.Value.Operation.Kind != ReconcileSetOperation { - source = old.Value.Operation.Source + // todo(sebastian): are source and origin the same? + if source == "" { + if old.Var.Origin != "" { + source = old.Var.Origin + } else if old.Value.Operation != nil && old.Value.Operation.Kind != ReconcileSetOperation { + source = old.Value.Operation.Source + } } } r.Var.Origin = source @@ -575,10 +579,10 @@ func mutateDelete(vars SetVarItems, resolverOpSet *OperationSet, _ bool) error { } func init() { - SpecTypes = make(map[string]*specType) + AtomicTypes = make(map[string]*atomicType) - SpecTypes[SpecNameSecret] = registerSpec(SpecNameSecret, true, true, - specResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { + AtomicTypes[AtomicNameSecret] = registerAtomicType(AtomicNameSecret, true, true, + atomicResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { if insecure { original := val.Value.Original val.Value.Resolved = original @@ -596,8 +600,8 @@ func init() { }), ) - SpecTypes[SpecNamePassword] = registerSpec(SpecNamePassword, true, true, - specResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { + AtomicTypes[AtomicNamePassword] = registerAtomicType(AtomicNamePassword, true, true, + atomicResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { if insecure { original := val.Value.Original val.Value.Resolved = original @@ -611,8 +615,8 @@ func init() { val.Value.Resolved = strings.Repeat("*", max(8, len(original))) }), ) - SpecTypes[SpecNameOpaque] = registerSpec(SpecNameOpaque, true, false, - specResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { + AtomicTypes[AtomicNameOpaque] = registerAtomicType(AtomicNameOpaque, true, false, + atomicResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { if insecure { original := val.Value.Original val.Value.Resolved = original @@ -624,8 +628,8 @@ func init() { val.Value.Resolved = "" }), ) - SpecTypes[SpecNamePlain] = registerSpec(SpecNamePlain, false, false, - specResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { + AtomicTypes[AtomicNamePlain] = registerAtomicType(AtomicNamePlain, false, false, + atomicResolver(func(val *SetVarValue, spec *SetVarSpec, insecure bool) { if insecure { original := val.Value.Original val.Value.Resolved = original @@ -638,23 +642,23 @@ func init() { }), ) - ComplexType = registerComplexType( + SpecType = registerSpecType( func(p graphql.ResolveParams) (interface{}, error) { name := p.Args["name"].(string) ns := p.Args["namespace"].(string) keys := p.Args["keys"].([]interface{}) - var complexOpSet *ComplexOperationSet + var specOpSet *SpecOperationSet switch p.Source.(type) { case *OperationSet: - complexOpSet = &ComplexOperationSet{ + specOpSet = &SpecOperationSet{ OperationSet: p.Source.(*OperationSet), Name: name, Namespace: ns, } - case *ComplexOperationSet: - complexOpSet = p.Source.(*ComplexOperationSet) + case *SpecOperationSet: + specOpSet = p.Source.(*SpecOperationSet) default: return nil, errors.New("source does not contain an OperationSet") } @@ -668,21 +672,26 @@ func init() { valuekeys = append(valuekeys, v) } - complexOpSet.Name = name - complexOpSet.Namespace = ns - complexOpSet.Keys = valuekeys + specOpSet.Name = name + specOpSet.Namespace = ns + specOpSet.Keys = valuekeys + + specDefs, ok := p.Context.Value(OwlEnvSpecDefsKey).(SpecDefs) + if !ok { + return nil, errors.New("missing specDefs in context") + } - validationErrs, err := complexOpSet.validate() + validationErrs, err := specOpSet.validate(specDefs) if err != nil { return nil, err } for _, verr := range validationErrs { key := verr.VarItem().Var.Key - complexOpSet.specs[key].Spec.Error = verr + specOpSet.specs[key].Spec.Error = verr } - return complexOpSet, nil + return specOpSet, nil }) SpecTypeErrorsType = graphql.NewObject(graphql.ObjectConfig{ @@ -705,7 +714,7 @@ func init() { Type: EnvironmentType, }, } - registerSpecFields(fields) + registerAtomicFields(fields) return fields }), }) @@ -714,93 +723,206 @@ func init() { Name: "ResolveType", Fields: (graphql.FieldsThunk)(func() graphql.Fields { fields := graphql.Fields{ - "transform": &graphql.Field{ - Type: ResolveType, + "GcpProvider": &graphql.Field{ + Type: graphql.NewObject(graphql.ObjectConfig{ + Name: "GCPResolveType", + Fields: graphql.Fields{ + "transform": &graphql.Field{ + Type: ResolveType, + Args: graphql.FieldConfigArgument{ + "expr": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + }, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + var opSet *OperationSet + var specOpSet *SpecOperationSet + var resolveOpSet *ResolveOperationSet + + switch p.Source.(type) { + case *OperationSet: + opSet = p.Source.(*OperationSet) + case *SpecOperationSet: + specOpSet = p.Source.(*SpecOperationSet) + opSet = specOpSet.OperationSet + case *ResolveOperationSet: + resolveOpSet = p.Source.(*ResolveOperationSet) + specOpSet = resolveOpSet.SpecOperationSet + opSet = specOpSet.OperationSet + default: + return nil, errors.New("source does not contain an OperationSet") + } + + expr, ok := p.Args["expr"].(string) + if !ok { + return nil, errors.New("transform without expr") + } + + if resolveOpSet.Mapping == nil { + resolveOpSet.Mapping = make(map[string]string) + } + + for _, v := range opSet.values { + if v.Value.Status != "UNRESOLVED" { + v.Value.Status = "DELETED" + continue + } + + env := map[string]string{"key": v.Var.Key} + + program, err := exprlang.Compile(expr, exprlang.Env(env)) + if err != nil { + return nil, errors.Wrap(err, "failed to compile transform program") + } + + output, err := exprlang.Run(program, env) + if err != nil { + return nil, errors.Wrap(err, "failed to run transform program") + } + + transformed, ok := output.(string) + if !ok { + return nil, errors.New("transform output is not a string") + } + + spec, ok := opSet.specs[v.Var.Key] + if !ok { + return nil, fmt.Errorf("missing spec for %s", v.Var.Key) + } + + specDefs, ok := p.Context.Value(OwlEnvSpecDefsKey).(SpecDefs) + if !ok { + return nil, errors.New("missing specDefs in context") + } + + _, aitem, err := specOpSet.GetAtomic(spec, specDefs) + if err != nil { + return nil, err + } + + if aitem.Spec.Name != AtomicNameSecret && aitem.Spec.Name != AtomicNamePassword && !aitem.Spec.Required { + v.Value.Status = "DELETED" + continue + } + + resolveOpSet.Mapping[v.Var.Key] = transformed + } + + return resolveOpSet, nil + }, + }, + }, + }), Args: graphql.FieldConfigArgument{ - "expr": &graphql.ArgumentConfig{ - Type: graphql.NewNonNull(graphql.String), + "api": &graphql.ArgumentConfig{ + Type: graphql.String, + DefaultValue: "secretmanager", + }, + "project": &graphql.ArgumentConfig{ + Type: graphql.String, + DefaultValue: "", }, }, Resolve: func(p graphql.ResolveParams) (interface{}, error) { var opSet *OperationSet - var complexOpSet *ComplexOperationSet + var specOpSet *SpecOperationSet switch p.Source.(type) { case *OperationSet: opSet = p.Source.(*OperationSet) - case *ComplexOperationSet: - complexOpSet = p.Source.(*ComplexOperationSet) - opSet = complexOpSet.OperationSet + case *SpecOperationSet: + specOpSet = p.Source.(*SpecOperationSet) + opSet = specOpSet.OperationSet default: return nil, errors.New("source does not contain an OperationSet") } - expr, ok := p.Args["expr"].(string) + project, ok := p.Args["project"].(string) if !ok { - return nil, errors.New("transform without expr") - } - - ctx := context.Background() - smClient, err := sm.NewClient(ctx) - if err != nil { - log.Fatalf("failed to setup client: %v", err) + return nil, errors.New("project is not a string") } - defer smClient.Close() - for _, v := range opSet.values { - if v.Value.Status != "UNRESOLVED" { - v.Value.Status = "DELETED" - continue + return &ResolveOperationSet{ + OperationSet: opSet, + SpecOperationSet: specOpSet, + Project: project, + }, nil + }, + }, + "mapping": &graphql.Field{ + Type: graphql.NewList(graphql.String), + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if resolveOpSet, ok := p.Source.(*ResolveOperationSet); ok { + mapping := resolveOpSet.Mapping + items := make([]string, 0, len(mapping)) + for k, v := range mapping { + if k == "" || v == "" { + continue + } + items = append(items, fmt.Sprintf("%s=>%s", k, v)) } + return items, nil + } + return nil, nil + }, + }, + "done": &graphql.Field{ + Type: EnvironmentType, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + var opSet *OperationSet + var specOpSet *SpecOperationSet + var resolveOpSet *ResolveOperationSet - env := map[string]string{"key": v.Var.Key} + switch p.Source.(type) { + case *OperationSet: + opSet = p.Source.(*OperationSet) + case *SpecOperationSet: + specOpSet = p.Source.(*SpecOperationSet) + opSet = specOpSet.OperationSet + case *ResolveOperationSet: + resolveOpSet = p.Source.(*ResolveOperationSet) + specOpSet = resolveOpSet.SpecOperationSet + opSet = specOpSet.OperationSet + default: + return nil, errors.New("source does not contain an OperationSet") + } - program, err := exprlang.Compile(expr, exprlang.Env(env)) - if err != nil { - return nil, errors.Wrap(err, "failed to compile transform program") - } + credentials, ok := p.Context.Value(OwlGcpCredentialsKey).(*google.Credentials) + if !ok { + return nil, fmt.Errorf("missing GCP credentials in context") + } - output, err := exprlang.Run(program, env) - if err != nil { - return nil, errors.Wrap(err, "failed to run transform program") - } + gcpsm, err := sm.NewClient(p.Context, idtoken.WithCredentialsJSON(credentials.JSON)) + if err != nil { + log.Fatalf("failed to setup GCP client: %v", err) + } + defer gcpsm.Close() - res, ok := output.(string) - if !ok { - return nil, errors.New("transform output is not a string") - } + gcpProject := resolveOpSet.Project + if gcpProject == "" { + gcpProject = credentials.ProjectID + } - spec, ok := opSet.specs[v.Var.Key] + for _, v := range opSet.values { + k, ok := resolveOpSet.Mapping[v.Var.Key] if !ok { - return nil, fmt.Errorf("missing spec for %s", v.Var.Key) - } - - _, aitem, err := complexOpSet.GetAtomicItem(spec) - if err != nil { - return nil, err - } - - if aitem.Spec.Name != SpecNameSecret && aitem.Spec.Name != SpecNamePassword { - v.Value.Status = "DELETED" continue } - uri := fmt.Sprintf("projects/platform-staging-413816/secrets/%s", res) - // status := strings.ToLower(aitem.Value.Status) - // if aitem.Spec.Name == SpecNameSecret || aitem.Spec.Name == SpecNamePassword { - // _, _ = fmt.Println(status, aitem.Spec.Name, aitem.Var.Key, "via", uri) - // } else { - // _, _ = fmt.Println(status, aitem.Spec.Name, aitem.Var.Key) - // continue - // } - + smuri := fmt.Sprintf("projects/%s/secrets/%s", gcpProject, k) accessRequest := &smpb.AccessSecretVersionRequest{ - Name: fmt.Sprintf("%s/versions/latest", uri), + Name: fmt.Sprintf("%s/versions/latest", smuri), } - result, err := smClient.AccessSecretVersion(ctx, accessRequest) + result, err := gcpsm.AccessSecretVersion(p.Context, accessRequest) if err != nil { - return nil, errors.Errorf("failed to access secret version: %v", err) + verr := NewResolutionFailedError( + &SetVarItem{Var: v.Var, Value: v.Value, Spec: specOpSet.specs[v.Var.Key].Spec}, + v.Var.Key, + err, + ) + opSet.specs[v.Var.Key].Spec.Error = verr + continue } if err := opSet.resolveValue(v.Var.Key, string(result.Payload.Data)); err != nil { @@ -808,19 +930,9 @@ func init() { } } - if complexOpSet != nil { - return complexOpSet, nil - } - return opSet, nil }, }, - "done": &graphql.Field{ - Type: EnvironmentType, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - return p.Source, nil - }, - }, } return fields }), @@ -1154,13 +1266,13 @@ func init() { }), }) - SpecsListType := graphql.NewObject(graphql.ObjectConfig{ - Name: "SpecsListType", + AtomicsListType := graphql.NewObject(graphql.ObjectConfig{ + Name: "AtomisListType", Fields: (graphql.FieldsThunk)(func() graphql.Fields { return graphql.Fields{ "list": &graphql.Field{ Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: "SpecListType", + Name: "AtomicListType", Fields: graphql.Fields{ "name": &graphql.Field{ Type: graphql.String, @@ -1176,7 +1288,7 @@ func init() { })), Resolve: func(p graphql.ResolveParams) (interface{}, error) { var keys []map[string]string - for k := range SpecTypes { + for k := range AtomicTypes { keys = append(keys, map[string]string{"name": k}) } @@ -1187,20 +1299,107 @@ func init() { }), }) + EnvSpecInputType := graphql.NewInputObject(graphql.InputObjectConfig{ + Name: "EnvSpecInputType", + Fields: graphql.InputObjectConfigFieldMap{ + "name": &graphql.InputObjectFieldConfig{ + Type: graphql.String, + }, + "breaker": &graphql.InputObjectFieldConfig{ + Type: graphql.String, + }, + "atomics": &graphql.InputObjectFieldConfig{ + Type: graphql.NewList(graphql.NewInputObject(graphql.InputObjectConfig{ + Name: "AtomicEnvSpecInputType", + Fields: graphql.InputObjectConfigFieldMap{ + "key": &graphql.InputObjectFieldConfig{ + Type: graphql.String, + }, + "atomic": &graphql.InputObjectFieldConfig{ + Type: graphql.String, + }, + "rules": &graphql.InputObjectFieldConfig{ + Type: graphql.String, + }, + "required": &graphql.InputObjectFieldConfig{ + Type: graphql.Boolean, + }, + }, + })), + }, + }, + }) + var err error Schema, err = graphql.NewSchema(graphql.SchemaConfig{ Query: graphql.NewObject( graphql.ObjectConfig{ Name: "Query", Fields: graphql.Fields{ - "environment": &graphql.Field{ + "Environment": &graphql.Field{ Type: EnvironmentType, Resolve: func(p graphql.ResolveParams) (interface{}, error) { return p.Info.FieldName, nil }, }, - "specs": &graphql.Field{ - Type: SpecsListType, + "EnvSpecs": &graphql.Field{ + Type: graphql.NewObject(graphql.ObjectConfig{ + Name: "EnvSpecsType", + Fields: graphql.Fields{ + "definitions": &graphql.Field{ + Type: graphql.NewList( + graphql.NewObject(graphql.ObjectConfig{ + Name: "EnvSpecsDefType", + Fields: graphql.Fields{ + "name": &graphql.Field{ + Type: graphql.String, + }, + "breaker": &graphql.Field{ + Type: graphql.String, + }, + "atomics": &graphql.Field{ + Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ + Name: "AtomicEnvSpecsDefType", + Fields: graphql.Fields{ + "key": &graphql.Field{ + Type: graphql.String, + }, + "atomic": &graphql.Field{ + Type: graphql.String, + }, + "rules": &graphql.Field{ + Type: graphql.String, + }, + "required": &graphql.Field{ + Type: graphql.Boolean, + }, + }, + })), + }, + }, + }), + ), + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + return p.Source, nil + }, + }, + }, + }), + Args: graphql.FieldConfigArgument{ + "definitions": &graphql.ArgumentConfig{ + Type: graphql.NewList(EnvSpecInputType), + }, + }, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + defs, ok := p.Args["definitions"].([]interface{}) + if !ok { + return nil, errors.New("definitions not found") + } + return defs, nil + }, + }, + "Atomics": &graphql.Field{ + Type: AtomicsListType, Resolve: func(p graphql.ResolveParams) (interface{}, error) { return p.Info.FieldName, nil }, diff --git a/internal/owl/graph_test.go b/internal/owl/graph_test.go index 05652015..1334f8c3 100644 --- a/internal/owl/graph_test.go +++ b/internal/owl/graph_test.go @@ -34,10 +34,10 @@ func TestGraph(t *testing.T) { } func TestQuerySpecs(t *testing.T) { - t.Run("query list of specs", func(t *testing.T) { + t.Run("query list of atomics", func(t *testing.T) { result := graphql.Do(graphql.Params{ Schema: Schema, - RequestString: `query { specs { list { name } } }`, + RequestString: `query { Atomics { list { name } } }`, }) require.False(t, result.HasErrors()) @@ -133,7 +133,7 @@ func TestResolveEnv(t *testing.T) { if v.Var.Key != "NAME" { continue } - require.EqualValues(t, ".env", v.Var.Origin) + require.EqualValues(t, "[execution]", v.Var.Origin) require.EqualValues(t, "Loon", v.Value.Resolved) require.EqualValues(t, "LITERAL", v.Value.Status) require.EqualValues(t, "Plain", v.Spec.Name) diff --git a/internal/owl/parse.go b/internal/owl/parse.go index 240cffa9..e092c447 100644 --- a/internal/owl/parse.go +++ b/internal/owl/parse.go @@ -18,10 +18,10 @@ type Specs map[string]Spec // Define the mapping between flags and their corresponding specifications. var allowedSpecs = map[string]func(*Spec, string, map[string]interface{}){ - SpecNameOpaque: handleParams, - SpecNamePlain: handleParams, - SpecNameSecret: handleParams, - SpecNamePassword: handleParams, + AtomicNameOpaque: handleParams, + AtomicNamePlain: handleParams, + AtomicNameSecret: handleParams, + AtomicNamePassword: handleParams, } // Handler function to validate various types of input @@ -44,7 +44,7 @@ func ParseRawSpec(values map[string]string, comments map[string]string) Specs { // Iterate through each key-value pair in the comments map. for key, value := range values { // Initialize a new Spec instance. - spec := Spec{Name: SpecNameDefault} + spec := Spec{Name: AtomicNameDefault} comment := comments[key] // Skip empty comments. diff --git a/internal/owl/parse_test.go b/internal/owl/parse_test.go index a768667d..17d17682 100644 --- a/internal/owl/parse_test.go +++ b/internal/owl/parse_test.go @@ -35,11 +35,11 @@ func TestMapSpec(t *testing.T) { "KEY5": "Plain", }, Expected: Specs{ - "KEY1": {Name: SpecNameOpaque, Valid: false}, - "KEY2": {Name: SpecNamePlain, Valid: true}, - "KEY3": {Name: SpecNamePassword, Valid: true}, - "KEY4": {Name: SpecNameSecret, Valid: true}, - "KEY5": {Name: SpecNamePlain}, + "KEY1": {Name: AtomicNameOpaque, Valid: false}, + "KEY2": {Name: AtomicNamePlain, Valid: true}, + "KEY3": {Name: AtomicNamePassword, Valid: true}, + "KEY4": {Name: AtomicNameSecret, Valid: true}, + "KEY5": {Name: AtomicNamePlain}, }, }, "WithRequiredSpecs": { @@ -56,10 +56,10 @@ func TestMapSpec(t *testing.T) { "KEY4": "Secret!", }, Expected: Specs{ - "KEY1": {Name: SpecNameOpaque, Valid: true, Required: true}, - "KEY2": {Name: SpecNamePlain, Valid: true, Required: true}, - "KEY3": {Name: SpecNamePassword, Valid: true, Required: true}, - "KEY4": {Name: SpecNameSecret, Valid: true, Required: true}, + "KEY1": {Name: AtomicNameOpaque, Valid: true, Required: true}, + "KEY2": {Name: AtomicNamePlain, Valid: true, Required: true}, + "KEY3": {Name: AtomicNamePassword, Valid: true, Required: true}, + "KEY4": {Name: AtomicNameSecret, Valid: true, Required: true}, }, }, "WithParams": { @@ -72,8 +72,8 @@ func TestMapSpec(t *testing.T) { "KEY2": `Password!:{"length":9}`, }, Expected: Specs{ - "KEY1": {Name: SpecNamePassword, Required: true, Valid: true}, - "KEY2": {Name: SpecNamePassword, Required: true}, + "KEY1": {Name: AtomicNamePassword, Required: true, Valid: true}, + "KEY2": {Name: AtomicNamePassword, Required: true}, }, }, } diff --git a/internal/owl/query.go b/internal/owl/query.go index 4f9d8afa..35ab4d1b 100644 --- a/internal/owl/query.go +++ b/internal/owl/query.go @@ -56,8 +56,8 @@ func (s *Store) snapshotQuery(query, vars io.StringWriter, resolve bool) error { reconcileAsymmetry(s), reduceSetOperations(vars), reduceWrapValidate(), - reduceSpecsAtomic("", nil), - reduceSepcsComplex(), + reduceAtomic("", nil), + reduceSepcs(s), reduceWrapDone(), } @@ -66,8 +66,8 @@ func (s *Store) snapshotQuery(query, vars io.StringWriter, resolve bool) error { reduceWrapResolve(), reduceWrapDone(), reduceWrapValidate(), - reduceSpecsAtomic("", nil), - reduceSepcsComplex(), + reduceAtomic("", nil), + reduceSepcs(s), reduceWrapDone(), }...) } @@ -78,7 +78,7 @@ func (s *Store) snapshotQuery(query, vars io.StringWriter, resolve bool) error { if resolve { queryName = "Resolve" } - q, err := s.NewQuery(queryName, varDefs, + q, err := s.NewEnvironmentQuery(queryName, varDefs, reducers, ) if err != nil { @@ -98,6 +98,96 @@ func (s *Store) snapshotQuery(query, vars io.StringWriter, resolve bool) error { return nil } +func (s *Store) defineEnvSpecDefsQuery(query io.StringWriter) error { + varDefs := []*ast.VariableDefinition{ + ast.NewVariableDefinition(&ast.VariableDefinition{ + Variable: ast.NewVariable(&ast.Variable{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + }), + Type: ast.NewNamed(&ast.Named{ + Name: ast.NewName(&ast.Name{ + Value: "[EnvSpecInputType]!", + }), + }), + }), + } + q, err := s.NewEnvSpecsQuery( + "EnvSpecsDef", + varDefs, + []QueryNodeReducer{ + func(opSets []*OperationSet, opDef *ast.OperationDefinition, selSet *ast.SelectionSet) (*ast.SelectionSet, error) { + nextSelSet := ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "name", + }), + }), + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "breaker", + }), + }), + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "atomics", + }), + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "key", + }), + }), + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "atomic", + }), + }), + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "rules", + }), + }), + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "required", + }), + }), + }, + }), + }), + }, + }) + selSet.Selections = append(selSet.Selections, ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + SelectionSet: nextSelSet, + })) + return nil, nil + }, + }, + ) + if err != nil { + return err + } + + text, err := q.Print() + if err != nil { + return err + } + + _, err = query.WriteString(text) + if err != nil { + return err + } + + return nil +} + func (s *Store) sensitiveKeysQuery(query, vars io.StringWriter) error { varDefs := []*ast.VariableDefinition{ ast.NewVariableDefinition(&ast.VariableDefinition{ @@ -117,13 +207,13 @@ func (s *Store) sensitiveKeysQuery(query, vars io.StringWriter) error { }), } - q, err := s.NewQuery("Sensitive", varDefs, + q, err := s.NewEnvironmentQuery("Sensitive", varDefs, []QueryNodeReducer{ reconcileAsymmetry(s), reduceSetOperations(vars), reduceWrapValidate(), - reduceSpecsAtomic("", nil), - reduceSepcsComplex(), + reduceAtomic("", nil), + reduceSepcs(s), reduceWrapDone(), reduceSensitive(), }, @@ -196,13 +286,13 @@ func (s *Store) getterQuery(query, vars io.StringWriter) error { } s.logger.Debug("getter opSets breakdown", zap.Int("loaded", loaded), zap.Int("updated", updated), zap.Int("deleted", deleted), zap.Int("total", len(s.opSets))) - q, err := s.NewQuery("Get", varDefs, + q, err := s.NewEnvironmentQuery("Get", varDefs, []QueryNodeReducer{ reconcileAsymmetry(s), reduceSetOperations(vars), reduceWrapValidate(), - reduceSpecsAtomic("", nil), - reduceSepcsComplex(), + reduceAtomic("", nil), + reduceSepcs(s), reduceWrapDone(), reduceGetter(), }, @@ -226,32 +316,68 @@ func (s *Store) getterQuery(query, vars io.StringWriter) error { func reduceWrapResolve() QueryNodeReducer { return func(opSets []*OperationSet, opDef *ast.OperationDefinition, selSet *ast.SelectionSet) (*ast.SelectionSet, error) { - resolveSelSet := ast.NewSelectionSet(&ast.SelectionSet{}) - selSet.Selections = append(selSet.Selections, ast.NewField(&ast.Field{ - Name: ast.NewName(&ast.Name{ - Value: "resolve", - }), - SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ - Selections: []ast.Selection{ - ast.NewField(&ast.Field{ - Name: ast.NewName(&ast.Name{ - Value: "transform", - }), - Arguments: []*ast.Argument{ - ast.NewArgument(&ast.Argument{ - Name: ast.NewName(&ast.Name{ - Value: "expr", + resolveSelSet := ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "mapping", + }), + }), + }, + }) + selSet.Selections = append(selSet.Selections, + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "resolve", + }), + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "GcpProvider", + }), + Arguments: []*ast.Argument{ + ast.NewArgument(&ast.Argument{ + Name: ast.NewName(&ast.Name{ + Value: "api", + }), + Value: ast.NewStringValue(&ast.StringValue{ + Value: "secretmanager.apiv1", + }), }), - Value: ast.NewStringValue(&ast.StringValue{ - Value: `key | trimPrefix("REDWOOD_ENV_") | replace("SLACK_REDIRECT_URL", "SLACK_REDIRECT") | lower()`, + ast.NewArgument(&ast.Argument{ + Name: ast.NewName(&ast.Name{ + Value: "project", + }), + Value: ast.NewStringValue(&ast.StringValue{ + Value: "platform-staging-413816", + }), }), + }, + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "transform", + }), + Arguments: []*ast.Argument{ + ast.NewArgument(&ast.Argument{ + Name: ast.NewName(&ast.Name{ + Value: "expr", + }), + Value: ast.NewStringValue(&ast.StringValue{ + Value: `key | trimPrefix("REDWOOD_ENV_") | replace("SLACK_REDIRECT_URL", "SLACK_REDIRECT") | lower()`, + }), + }), + }, + SelectionSet: resolveSelSet, + }), + }, }), - }, - SelectionSet: resolveSelSet, - }), - }, - }), - })) + }), + }, + }), + })) return resolveSelSet, nil } } @@ -859,7 +985,7 @@ func reconcileAsymmetry(store *Store) QueryNodeReducer { Created: &created, }, Spec: &varSpec{ - Name: SpecNameDefault, + Name: AtomicNameDefault, Required: false, Checked: false, }, @@ -896,7 +1022,7 @@ func reconcileAsymmetry(store *Store) QueryNodeReducer { } } -func reduceSpecsAtomic(ns string, parent *ComplexDef) QueryNodeReducer { +func reduceAtomic(ns string, parent *SpecDef) QueryNodeReducer { return func(opSets []*OperationSet, opDef *ast.OperationDefinition, selSet *ast.SelectionSet) (*ast.SelectionSet, error) { var specKeys []string varSpecs := make(map[string]*SetVarItem) @@ -907,7 +1033,7 @@ func reduceSpecsAtomic(ns string, parent *ComplexDef) QueryNodeReducer { for _, s := range opSet.specs { isTransient := opSet.operation.kind == TransientSetOperation if isTransient && parent != nil { - for k, v := range parent.Items { + for k, v := range parent.Atomics { if fmt.Sprintf("%s_%s", ns, k) != s.Var.Key { continue } @@ -915,7 +1041,7 @@ func reduceSpecsAtomic(ns string, parent *ComplexDef) QueryNodeReducer { } } - if _, ok := SpecTypes[s.Spec.Name]; !ok { + if _, ok := AtomicTypes[s.Spec.Name]; !ok { // return nil, fmt.Errorf("unknown spec type: %s on %s", s.Spec.Name, s.Var.Key) continue } @@ -1018,7 +1144,7 @@ func reduceSpecsAtomic(ns string, parent *ComplexDef) QueryNodeReducer { } } -func reduceSepcsComplex() QueryNodeReducer { +func reduceSepcs(store *Store) QueryNodeReducer { return func(opSets []*OperationSet, opDef *ast.OperationDefinition, selSet *ast.SelectionSet) (*ast.SelectionSet, error) { var varKeys []string varSpecs := make(map[string]*SetVarItem) @@ -1027,7 +1153,7 @@ func reduceSepcsComplex() QueryNodeReducer { continue } for _, s := range opSet.specs { - if _, ok := SpecTypes[s.Spec.Name]; ok { + if _, ok := AtomicTypes[s.Spec.Name]; ok { continue } varSpecs[s.Var.Key] = &SetVarItem{ @@ -1086,7 +1212,7 @@ func reduceSepcsComplex() QueryNodeReducer { prevSelSet.Selections = append(prevSelSet.Selections, ast.NewField(&ast.Field{ Name: ast.NewName(&ast.Name{ - Value: ComplexSpecType, + Value: SpecTypeKey, }), Arguments: []*ast.Argument{ ast.NewArgument(&ast.Argument{ @@ -1115,10 +1241,10 @@ func reduceSepcsComplex() QueryNodeReducer { SelectionSet: nextSelSet, })) - specDef := ComplexDefTypes[spec] + specDef := store.specDefs[spec] transientOpSet, _ := NewOperationSet(WithOperation(TransientSetOperation), WithItems(items)) - atomicSelSet, err := reduceSpecsAtomic(ns, specDef)([]*OperationSet{transientOpSet}, opDef, nextSelSet) + atomicSelSet, err := reduceAtomic(ns, specDef)([]*OperationSet{transientOpSet}, opDef, nextSelSet) // todo: handle error if err == nil { return atomicSelSet @@ -1137,9 +1263,9 @@ func reduceSepcsComplex() QueryNodeReducer { ns := "" item := varSpecs[specKey] - specDef, ok := ComplexDefTypes[item.Spec.Name] + specDef, ok := store.specDefs[item.Spec.Name] if !ok { - return nil, fmt.Errorf("unknown complex spec type: %s on %s", item.Spec.Name, item.Var.Key) + return nil, fmt.Errorf("unknown spec type: %s on %s", item.Spec.Name, item.Var.Key) } breaker = specDef.Breaker @@ -1156,7 +1282,7 @@ func reduceSepcsComplex() QueryNodeReducer { } // if item.Spec.Name != prevNs && prevNs != "" { - // return nil, fmt.Errorf("complex spec type mismatch in namespace %q: %s != %s", ns, item.Spec.Name, prevNs) + // return nil, fmt.Errorf("spec type mismatch in namespace %q: %s != %s", ns, item.Spec.Name, prevNs) // } // prevNs = ns @@ -1175,7 +1301,53 @@ type Query struct { doc *ast.Document } -func (s *Store) NewQuery(name string, varDefs []*ast.VariableDefinition, reducers []QueryNodeReducer) (*Query, error) { +func (s *Store) NewEnvSpecsQuery(name string, varDefs []*ast.VariableDefinition, reducers []QueryNodeReducer) (*Query, error) { + selSet := ast.NewSelectionSet(&ast.SelectionSet{}) + opDef := ast.NewOperationDefinition(&ast.OperationDefinition{ + Operation: "query", + Name: ast.NewName(&ast.Name{ + Value: fmt.Sprintf("Owl%s", name), + }), + Directives: []*ast.Directive{}, + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "EnvSpecs", + }), + Arguments: []*ast.Argument{ + ast.NewArgument(&ast.Argument{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + Value: ast.NewVariable(&ast.Variable{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + }), + }), + }, + Directives: []*ast.Directive{}, + SelectionSet: selSet, + }), + }, + }), + VariableDefinitions: varDefs, + }) + + var err error + for _, reducer := range reducers { + if selSet, err = reducer(s.opSets, opDef, selSet); err != nil { + return nil, err + } + } + + doc := ast.NewDocument(&ast.Document{Definitions: []ast.Node{opDef}}) + + return &Query{doc: doc}, nil +} + +func (s *Store) NewEnvironmentQuery(name string, varDefs []*ast.VariableDefinition, reducers []QueryNodeReducer) (*Query, error) { selSet := ast.NewSelectionSet(&ast.SelectionSet{}) opDef := ast.NewOperationDefinition(&ast.OperationDefinition{ Operation: "query", @@ -1187,7 +1359,7 @@ func (s *Store) NewQuery(name string, varDefs []*ast.VariableDefinition, reducer Selections: []ast.Selection{ ast.NewField(&ast.Field{ Name: ast.NewName(&ast.Name{ - Value: "environment", + Value: "Environment", }), Arguments: []*ast.Argument{}, Directives: []*ast.Directive{}, diff --git a/internal/owl/store.go b/internal/owl/store.go index 4418c951..ac8acdb9 100644 --- a/internal/owl/store.go +++ b/internal/owl/store.go @@ -3,6 +3,7 @@ package owl import ( "bytes" "context" + _ "embed" "encoding/json" "fmt" "slices" @@ -11,6 +12,8 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/oauth2/google" + "gopkg.in/yaml.v3" "github.com/graphql-go/graphql" "github.com/stateful/godotenv" @@ -19,6 +22,16 @@ import ( "go.uber.org/zap" ) +type owlContextKey int + +const ( + OwlEnvSpecDefsKey owlContextKey = iota + OwlGcpCredentialsKey +) + +//go:embed envSpecDefs.defaults.yaml +var envSpecsCrdYaml []byte + type setOperationKind int const ( @@ -36,19 +49,29 @@ type Operation struct { } type OperationSet struct { + SpecDef operation Operation hasSpecs bool specs map[string]*SetVarSpec values map[string]*SetVarValue } -type ComplexOperationSet struct { +type SpecOperationSet struct { *OperationSet Name string Namespace string Keys []string } +type ResolveOperationSet struct { + *OperationSet + *SpecOperationSet + + Project string + Mapping map[string]string +} + +// todo(sebastian): once final, move to embedded structs for non-serializable fields type setVarOperation struct { Order uint `json:"-"` Kind setOperationKind `json:"kind"` @@ -58,15 +81,17 @@ type setVarOperation struct { type varValue struct { Original string `json:"original,omitempty"` Resolved string `json:"resolved,omitempty"` - Status string `json:"status,omitempty"` + Status string `json:"status,omitempty"` // todo(sebastian): enum Operation *setVarOperation `json:"operation"` } type varSpec struct { + Key string `json:"-" yaml:"key"` Name string `json:"name"` + Atomic string `json:"-" yaml:"atomic"` Required bool `json:"required"` Description string `json:"description"` - Complex string `json:"-"` + Spec string `json:"-"` Namespace string `json:"-"` Rules string `json:"validator,omitempty"` Operation *setVarOperation `json:"operation"` @@ -326,9 +351,12 @@ func (s *OperationSet) resolveValue(key, plainText string) error { return nil } +type SpecDefs map[string]*SpecDef + type Store struct { - mu sync.RWMutex - opSets []*OperationSet + mu sync.RWMutex + opSets []*OperationSet + specDefs SpecDefs logger *zap.Logger } @@ -336,7 +364,13 @@ type Store struct { type StoreOption func(*Store) error func NewStore(opts ...StoreOption) (*Store, error) { - s := &Store{logger: zap.NewNop()} + s := &Store{ + logger: zap.NewNop(), + specDefs: make(map[string]*SpecDef), + } + + // load ENV spec definitions from CRD + opts = append([]StoreOption{withSpecDefsCRD(envSpecsCrdYaml)}, opts...) for _, opt := range opts { if err := opt(s); err != nil { @@ -388,6 +422,23 @@ func WithEnvs(source string, envs ...string) StoreOption { } } +func withSpecDefsCRD(raw []byte) StoreOption { + return func(s *Store) error { + var crd map[string]interface{} + err := yaml.Unmarshal(raw, &crd) + if err != nil { + return err + } + + envSpecs, err := extractDataKey(crd, "envSpecs") + if err != nil { + return err + } + + return s.defineEnvSpecs(envSpecs) + } +} + func WithLogger(logger *zap.Logger) StoreOption { return func(s *Store) error { s.logger = logger @@ -420,6 +471,26 @@ func (s *Store) InsecureResolve() (SetVarItems, error) { return items, nil } +func (s *Store) DoQuery(query string, vars map[string]interface{}, resolve bool) (*graphql.Result, error) { + ctx := context.WithValue(context.Background(), OwlEnvSpecDefsKey, s.specDefs) + + if resolve { + // todo(sebastian): short-circuiting what should really happen at query construction + credentials, err := google.FindDefaultCredentials(ctx) + if err != nil { + return nil, fmt.Errorf("failed to find GCP default credentials: %w", err) + } + ctx = context.WithValue(ctx, OwlGcpCredentialsKey, credentials) + } + + return graphql.Do(graphql.Params{ + Schema: Schema, + RequestString: query, + VariableValues: vars, + Context: ctx, + }), nil +} + func (s *Store) InsecureGet(k string) (string, bool, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -448,11 +519,10 @@ func (s *Store) InsecureGet(k string) (string, bool, error) { // fmt.Println(string(j)) // s.logger.Debug("insecure getter", zap.String("vars", string(j))) - result := graphql.Do(graphql.Params{ - Schema: Schema, - RequestString: query.String(), - VariableValues: varValues, - }) + result, err := s.DoQuery(query.String(), varValues, false) + if err != nil { + return "", false, err + } if result.HasErrors() { return "", false, fmt.Errorf("graphql errors %s", result.Errors) @@ -524,11 +594,10 @@ func (s *Store) SensitiveKeys() ([]string, error) { // fmt.Println(string(j)) // s.logger.Debug("sensitiveKeys vars", zap.String("vars", string(j))) - result := graphql.Do(graphql.Params{ - Schema: Schema, - RequestString: query.String(), - VariableValues: varValues, - }) + result, err := s.DoQuery(query.String(), varValues, false) + if err != nil { + return nil, err + } if result.HasErrors() { return nil, fmt.Errorf("graphql errors %s", result.Errors) @@ -625,6 +694,68 @@ func (s *Store) Update(context context.Context, newOrUpdated, deleted []string) return nil } +func (s *Store) defineEnvSpecs(envSpecs interface{}) error { + s.mu.Lock() + defer s.mu.Unlock() + + var query bytes.Buffer + + varValues := make(map[string]interface{}) + varValues["definitions"] = envSpecs + + err := s.defineEnvSpecDefsQuery(&query) + if err != nil { + return err + } + + result, err := s.DoQuery(query.String(), varValues, false) + if err != nil { + return err + } + + definitions, err := extractDataKey(result.Data, "definitions") + if err != nil { + return err + } + + for _, def := range definitions.([]interface{}) { + ydef, err := yaml.Marshal(def) + if err != nil { + return err + } + + var specDef *SpecDef + if err = yaml.Unmarshal(ydef, &specDef); err != nil { + return err + } + + atomicsMap := make(map[string]*varSpec) + if raw, err := extractDataKey(def, "atomics"); err == nil { + yatomics, err := yaml.Marshal(raw) + if err != nil { + return err + } + + var atomics []*varSpec + if err := yaml.Unmarshal(yatomics, &atomics); err == nil { + for _, a := range atomics { + a.Name = a.Atomic + atomicsMap[a.Key] = a + } + } + } + specDef.Atomics = atomicsMap + specDef.Validator = TagValidator + s.specDefs[specDef.Name] = specDef + } + + if result.HasErrors() { + return fmt.Errorf("graphql errors %s", result.Errors) + } + + return err +} + func (s *Store) snapshot(insecure, resolve bool) (SetVarItems, error) { var query, vars bytes.Buffer err := s.snapshotQuery(&query, &vars, resolve) @@ -649,11 +780,10 @@ func (s *Store) snapshot(insecure, resolve bool) (SetVarItems, error) { // fmt.Println(string(j)) // s.logger.Debug("snapshot vars", zap.String("vars", string(j))) - result := graphql.Do(graphql.Params{ - Schema: Schema, - RequestString: query.String(), - VariableValues: varValues, - }) + result, err := s.DoQuery(query.String(), varValues, resolve) + if err != nil { + return nil, err + } if result.HasErrors() { return nil, fmt.Errorf("graphql errors %s", result.Errors) @@ -667,6 +797,11 @@ func (s *Store) snapshot(insecure, resolve bool) (SetVarItems, error) { return nil, err } + if resolve { + resolving, _ := extractDataKey(result.Data, "mapping") + s.logger.Debug("resolving", zap.Any("resolving", resolving)) + } + j, err := json.Marshal(val) if err != nil { return nil, err diff --git a/internal/owl/store_test.go b/internal/owl/store_test.go index a4649819..f1009943 100644 --- a/internal/owl/store_test.go +++ b/internal/owl/store_test.go @@ -4,6 +4,7 @@ package owl import ( "bytes" + _ "embed" "fmt" "os" "testing" @@ -86,7 +87,7 @@ func Test_Store(t *testing.T) { err = store.sensitiveKeysQuery(&query, &vars) require.NoError(t, err) - fmt.Println(query.String()) + // fmt.Println(query.String()) }) t.Run("Validate with process envs", func(t *testing.T) { @@ -607,30 +608,94 @@ func TestStore_Get(t *testing.T) { assert.EqualValues(t, "secret-fake-password", val) } +//go:embed testdata/resolve/.env.example +var resolveSpecsRaw []byte + +//go:embed testdata/resolve/.env.local +var resolveValuesRaw []byte + func TestStore_Resolve(t *testing.T) { t.Skip("Skip since it requires GCP's secret manager") - rawSpecs, err := os.ReadFile("testdata/resolve/.env.example") - require.NoError(t, err) + t.Run("Valid", func(t *testing.T) { + store, err := NewStore( + WithSpecFile(".env.example", resolveSpecsRaw), + WithEnvFile(".env.local", resolveValuesRaw), + ) + require.NoError(t, err) - rawValues, err := os.ReadFile("testdata/resolve/.env.local") - require.NoError(t, err) + snapshot, err := store.InsecureResolve() + require.NoError(t, err) + require.Len(t, snapshot, 10) + snapshot.sortbyKey() - store, err := NewStore(WithSpecFile(".env.example", rawSpecs), WithEnvFile(".env.local", rawValues)) - require.NoError(t, err) + errors := 0 + for _, item := range snapshot { + require.EqualValues(t, "LITERAL", item.Value.Status) + require.NotEmpty(t, item.Value.Original) + require.NotEmpty(t, item.Value.Resolved) + errors += len(item.Errors) + } + + require.Equal(t, 0, errors) + }) - snapshot, err := store.InsecureResolve() + t.Run("Snapshot", func(t *testing.T) { + store, err := NewStore( + WithSpecFile(".env.example", resolveSpecsRaw), + WithEnvFile(".env.local", resolveValuesRaw), + ) + require.NoError(t, err) + + snapshot, err := store.Snapshot() + require.NoError(t, err) + require.Len(t, snapshot, 17) + snapshot.sortbyKey() + + var ksa []string + for _, item := range snapshot { + ksa = append(ksa, fmt.Sprintf("Key: %q, Spec: %q, Atomic: %q", item.Var.Key, item.Spec.Spec, item.Spec.Name)) + } + + require.EqualValues(t, []string{ + `Key: "API_URL", Spec: "", Atomic: "Plain"`, + `Key: "AUTH0_AUDIENCE", Spec: "", Atomic: "Auth0"`, + `Key: "AUTH0_CLIENT_ID", Spec: "", Atomic: "Auth0"`, + `Key: "AUTH0_DEV_ID", Spec: "", Atomic: "Opaque"`, + `Key: "AUTH0_DOMAIN", Spec: "", Atomic: "Auth0"`, + `Key: "FRONTEND_URL", Spec: "", Atomic: "Plain"`, + `Key: "MIXPANEL_TOKEN", Spec: "", Atomic: "Secret"`, + `Key: "REDWOOD_ENV_DEBUG_IDE", Spec: "", Atomic: "Plain"`, + `Key: "REDWOOD_ENV_GITHUB_APP", Spec: "", Atomic: "Plain"`, + `Key: "REDWOOD_ENV_INSIGHT_ENABLED", Spec: "", Atomic: "Plain"`, + `Key: "REDWOOD_ENV_INSTRUMENTATION_KEY", Spec: "", Atomic: "Secret"`, + `Key: "REDWOOD_ENV_POLL_INTERVAL_MS", Spec: "", Atomic: "Opaque"`, + `Key: "REDWOOD_ENV_SHOW_EXTRA_LINKS", Spec: "", Atomic: "Opaque"`, + `Key: "REDWOOD_ENV_THEME", Spec: "", Atomic: "Opaque"`, + `Key: "SLACK_CLIENT_ID", Spec: "", Atomic: "Slack"`, + `Key: "SLACK_CLIENT_SECRET", Spec: "", Atomic: "Slack"`, + `Key: "SLACK_REDIRECT_URL", Spec: "", Atomic: "Slack"`, + }, ksa) + }) + + t.Run("Invalid", func(t *testing.T) { + invalidEnvSpec := `VECTOR_DB_COLLECTION="Collection for the vector DB" # VectorDB +VECTOR_DB_URL="URL for the vector DB" # VectorDB` + + store, err := NewStore( + WithSpecFile(".env.example", []byte(invalidEnvSpec)), + ) + require.NoError(t, err) + + _, err = store.InsecureResolve() + require.Error(t, err, "") + }) +} + +func TestStore_LoadEnvSpecDefs(t *testing.T) { + store, err := NewStore() require.NoError(t, err) - require.Len(t, snapshot, 4) - snapshot.sortbyKey() - - errors := 0 - for _, item := range snapshot { - require.EqualValues(t, "LITERAL", item.Value.Status) - require.NotEmpty(t, item.Value.Original) - require.NotEmpty(t, item.Value.Resolved) - errors += len(item.Errors) - } + require.NotNil(t, store) - require.Equal(t, 0, errors) + require.Len(t, store.specDefs, 7) } diff --git a/internal/owl/testdata/graph/dotenv.graphql b/internal/owl/testdata/graph/dotenv.graphql index 9f797e30..ebb1d763 100644 --- a/internal/owl/testdata/graph/dotenv.graphql +++ b/internal/owl/testdata/graph/dotenv.graphql @@ -1,5 +1,5 @@ query ResolveOwlDotEnv($insecure: Boolean = false, $prefix: String = "", $load_0: [VariableInput]!, $load_1: [VariableInput]!, $reconcile_3: [VariableInput]!, $update_4: [VariableInput]!, $reconcile_6: [VariableInput]!, $update_7: [VariableInput]!, $reconcile_9: [VariableInput]!, $update_10: [VariableInput]!, $reconcile_12: [VariableInput]!, $update_13: [VariableInput]!, $update_17: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { load(vars: $load_1, hasSpecs: true) { reconcile(vars: $reconcile_3, hasSpecs: true) { diff --git a/internal/owl/testdata/graph/env_without_specs.graphql b/internal/owl/testdata/graph/env_without_specs.graphql index ffc5e5e4..e4a9f595 100644 --- a/internal/owl/testdata/graph/env_without_specs.graphql +++ b/internal/owl/testdata/graph/env_without_specs.graphql @@ -2,7 +2,7 @@ query ResolveOwlSnapshot( $insecure: Boolean = false $load_0: [VariableInput]! ) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { validate { Opaque(keys: ["GOPATH"]) { diff --git a/internal/owl/testdata/graph/insecureget.graphql b/internal/owl/testdata/graph/insecureget.graphql index 0bec849c..2c9b8c1b 100644 --- a/internal/owl/testdata/graph/insecureget.graphql +++ b/internal/owl/testdata/graph/insecureget.graphql @@ -1,5 +1,5 @@ query ResolveOwlInsecureGet($insecure: Boolean = false, $load_0: [VariableInput]!, $load_1: [VariableInput]!, $load_2: [VariableInput]!, $reconcile_3: [VariableInput]!, $update_4: [VariableInput]!, $reconcile_6: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { load(vars: $load_1, hasSpecs: true) { load(vars: $load_2, hasSpecs: false) { diff --git a/internal/owl/testdata/graph/query_complex_env.graphql b/internal/owl/testdata/graph/query_complex_env.graphql index 0edb23d2..8205334e 100644 --- a/internal/owl/testdata/graph/query_complex_env.graphql +++ b/internal/owl/testdata/graph/query_complex_env.graphql @@ -1,5 +1,5 @@ query ResolveOwlSnapshot($insecure: Boolean = false, $load_0: [VariableInput]!, $load_1: [VariableInput]!, $load_2: [VariableInput]!, $reconcile_3: [VariableInput]!, $update_4: [VariableInput]!, $reconcile_6: [VariableInput]!, $update_7: [VariableInput]!, $reconcile_9: [VariableInput]!, $update_10: [VariableInput]!, $reconcile_12: [VariableInput]!, $update_13: [VariableInput]!, $update_15: [VariableInput]!, $update_17: [VariableInput]!, $update_19: [VariableInput]!, $update_21: [VariableInput]!, $update_23: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { load(vars: $load_1, hasSpecs: true) { load(vars: $load_2, hasSpecs: false) { diff --git a/internal/owl/testdata/graph/query_simple_env.graphql b/internal/owl/testdata/graph/query_simple_env.graphql index 722389f0..d305729e 100644 --- a/internal/owl/testdata/graph/query_simple_env.graphql +++ b/internal/owl/testdata/graph/query_simple_env.graphql @@ -1,5 +1,5 @@ query ResolveOwlSnapshot($insecure: Boolean = false, $load_0: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { render { snapshot(insecure: $insecure) { diff --git a/internal/owl/testdata/graph/reconcile_operationless.graphql b/internal/owl/testdata/graph/reconcile_operationless.graphql index f3536b29..01211455 100644 --- a/internal/owl/testdata/graph/reconcile_operationless.graphql +++ b/internal/owl/testdata/graph/reconcile_operationless.graphql @@ -1,5 +1,5 @@ query ResolveOwlSnapshot($insecure: Boolean = false, $load_0: [VariableInput]!, $load_1: [VariableInput]!, $load_2: [VariableInput]!, $reconcile_3: [VariableInput]!, $update_4: [VariableInput]!, $reconcile_6: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { load(vars: $load_1, hasSpecs: true) { load(vars: $load_2, hasSpecs: false) { diff --git a/internal/owl/testdata/graph/sensitive_keys.graphql b/internal/owl/testdata/graph/sensitive_keys.graphql index b59dc5a0..0b148393 100644 --- a/internal/owl/testdata/graph/sensitive_keys.graphql +++ b/internal/owl/testdata/graph/sensitive_keys.graphql @@ -1,5 +1,5 @@ query ResolveOwlSnapshot($insecure: Boolean = false, $load_0: [VariableInput]!, $load_1: [VariableInput]!, $load_2: [VariableInput]!, $reconcile_3: [VariableInput]!, $update_4: [VariableInput]!, $reconcile_6: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { load(vars: $load_1, hasSpecs: true) { load(vars: $load_2, hasSpecs: false) { diff --git a/internal/owl/testdata/graph/store_update.graphql b/internal/owl/testdata/graph/store_update.graphql index a21643fe..ad7910dd 100644 --- a/internal/owl/testdata/graph/store_update.graphql +++ b/internal/owl/testdata/graph/store_update.graphql @@ -1,5 +1,5 @@ query ResolveOwlSnapshot($insecure: Boolean = false, $load_0: [VariableInput]!, $load_1: [VariableInput]!, $reconcile_3: [VariableInput]!, $update_4: [VariableInput]!, $reconcile_6: [VariableInput]!, $update_7: [VariableInput]!, $reconcile_9: [VariableInput]!, $update_10: [VariableInput]!, $reconcile_12: [VariableInput]!, $update_13: [VariableInput]!, $update_17: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: false) { load(vars: $load_1, hasSpecs: true) { reconcile(vars: $reconcile_3, hasSpecs: true) { diff --git a/internal/owl/testdata/graph/validate_simple_env.graphql b/internal/owl/testdata/graph/validate_simple_env.graphql index 652519c6..b699fbe3 100644 --- a/internal/owl/testdata/graph/validate_simple_env.graphql +++ b/internal/owl/testdata/graph/validate_simple_env.graphql @@ -1,5 +1,5 @@ query ResolveOwlSnapshot($insecure: Boolean = false, $load_0: [VariableInput]!) { - environment { + Environment { load(vars: $load_0, hasSpecs: true) { validate { Opaque(insecure: $insecure, keys: ["GOPATH", "HOME", "HOMEBREW_REPOSITORY"]) { diff --git a/internal/owl/testdata/resolve/.env.example b/internal/owl/testdata/resolve/.env.example index 5ffac6f2..a95514b6 100644 --- a/internal/owl/testdata/resolve/.env.example +++ b/internal/owl/testdata/resolve/.env.example @@ -1,10 +1,11 @@ -API_URL="URL for the backend API" # Plain +API_URL="URL for the backend API" # Plain! AUTH0_AUDIENCE="Audience for Auth0" # Auth0 AUTH0_CLIENT_ID="Client ID for Auth0" # Auth0 AUTH0_DOMAIN="Domain for Auth0" # Auth0 -FRONTEND_URL="URL of the frontend" # Plain +FRONTEND_URL="URL of the frontend" # Plain! MIXPANEL_TOKEN="Token for Mixpanel" # Secret! -REDWOOD_ENV_GITHUB_APP="ID of the GitHub App" # Plain +REDWOOD_ENV_DEBUG_IDE="Flag to enable debug mode for the IDE" # Plain! +REDWOOD_ENV_GITHUB_APP="ID of the GitHub App" # Plain! REDWOOD_ENV_INSIGHT_ENABLED="Flag to enable insights" # Plain REDWOOD_ENV_INSTRUMENTATION_KEY="Instrumentation Key for Application Insights" # Secret! SLACK_CLIENT_ID="Client ID for Slack" # Slack diff --git a/internal/owl/validate.go b/internal/owl/validate.go index 69c28df7..dfe4c05a 100644 --- a/internal/owl/validate.go +++ b/internal/owl/validate.go @@ -4,10 +4,12 @@ import ( "fmt" "strings" - valid "github.com/go-playground/validator/v10" "github.com/xo/dburl" + + valid "github.com/go-playground/validator/v10" ) +// todo(sebastian): perhaps this should be ValueError instead? type ValidationError interface { fmt.Stringer VarItem() *SetVarItem @@ -27,67 +29,70 @@ type ValidateErrorType uint8 const ( ValidateErrorVarRequired ValidateErrorType = iota ValidateErrorTagFailed - ValidateErrorDatabaseUrl + ValidateErrorResolutionFailed + // ValidateErrorDatabaseUrl // ValidateErrorJwtFailed ) -type DatabaseUrlError struct { +type TagFailedError struct { varItem *SetVarItem code ValidateErrorType + tag string item string - error error } -func NewDatabaseUrlError(varItem *SetVarItem, err error, item string) *DatabaseUrlError { - return &DatabaseUrlError{ +func NewTagFailedError(varItem *SetVarItem, tag string, item string) *TagFailedError { + return &TagFailedError{ varItem: varItem, - code: ValidateErrorDatabaseUrl, + code: ValidateErrorTagFailed, + tag: tag, item: item, - error: err, } } -//revive:enable:var-naming - -func (e DatabaseUrlError) VarItem() *SetVarItem { +func (e TagFailedError) VarItem() *SetVarItem { return e.varItem } -func (e DatabaseUrlError) Error() string { - return fmt.Sprintf("Error %v: The value of variable \"%s\" failed DatabaseUrl validation \"%s\" required by \"%s->%s\" declared in \"%s\"", +func (e TagFailedError) Error() string { + return fmt.Sprintf("Error %v: The value of variable \"%s\" failed tag validation \"%s\" required by \"%s->%s\" declared in \"%s\"", e.Code(), e.Key(), - e.error.Error(), + e.Tag(), e.SpecName(), e.Item(), e.Source()) } -func (e DatabaseUrlError) Message() string { +func (e TagFailedError) Message() string { return e.Error() } -func (e DatabaseUrlError) String() string { +func (e TagFailedError) String() string { return e.Error() } -func (e DatabaseUrlError) Code() ValidateErrorType { +func (e TagFailedError) Code() ValidateErrorType { return e.code } -func (e DatabaseUrlError) Key() string { +func (e TagFailedError) Key() string { return e.varItem.Var.Key } -func (e DatabaseUrlError) SpecName() string { - return e.varItem.Spec.Complex +func (e TagFailedError) Tag() string { + return e.tag } -func (e DatabaseUrlError) Item() string { +func (e TagFailedError) SpecName() string { + return e.varItem.Spec.Spec +} + +func (e TagFailedError) Item() string { return e.item } -func (e DatabaseUrlError) Source() string { +func (e TagFailedError) Source() string { if e.varItem.Spec.Operation == nil { return "-" } @@ -97,71 +102,61 @@ func (e DatabaseUrlError) Source() string { return e.varItem.Spec.Operation.Source } -// make sure interfaces are satisfied -var ( - _ ValidationError = new(DatabaseUrlError) - _ error = new(DatabaseUrlError) -) - -type TagFailedError struct { +type ResolutionFailedError struct { varItem *SetVarItem + err error code ValidateErrorType - tag string item string } -func NewTagFailedError(varItem *SetVarItem, tag string, item string) *TagFailedError { - return &TagFailedError{ - varItem: varItem, - code: ValidateErrorTagFailed, - tag: tag, +func NewResolutionFailedError(varItem *SetVarItem, item string, err error) *ResolutionFailedError { + return &ResolutionFailedError{ + code: ValidateErrorResolutionFailed, + err: err, item: item, + varItem: varItem, } } -func (e TagFailedError) VarItem() *SetVarItem { +func (e ResolutionFailedError) VarItem() *SetVarItem { return e.varItem } -func (e TagFailedError) Error() string { - return fmt.Sprintf("Error %v: The value of variable \"%s\" failed tag validation \"%s\" required by \"%s->%s\" declared in \"%s\"", +func (e ResolutionFailedError) Error() string { + return fmt.Sprintf("Error %v: The value of variable \"%s\" failed resolution \"%s\" required by \"%s->%s\" declared in \"%s\"", e.Code(), e.Key(), - e.Tag(), + e.err.Error(), e.SpecName(), e.Item(), e.Source()) } -func (e TagFailedError) Message() string { +func (e ResolutionFailedError) Message() string { return e.Error() } -func (e TagFailedError) String() string { +func (e ResolutionFailedError) String() string { return e.Error() } -func (e TagFailedError) Code() ValidateErrorType { +func (e ResolutionFailedError) Code() ValidateErrorType { return e.code } -func (e TagFailedError) Key() string { +func (e ResolutionFailedError) Key() string { return e.varItem.Var.Key } -func (e TagFailedError) Tag() string { - return e.tag -} - -func (e TagFailedError) SpecName() string { - return e.varItem.Spec.Complex +func (e ResolutionFailedError) SpecName() string { + return e.varItem.Spec.Spec } -func (e TagFailedError) Item() string { +func (e ResolutionFailedError) Item() string { return e.item } -func (e TagFailedError) Source() string { +func (e ResolutionFailedError) Source() string { if e.varItem.Spec.Operation == nil { return "-" } @@ -171,108 +166,52 @@ func (e TagFailedError) Source() string { return e.varItem.Spec.Operation.Source } +// make sure interfaces are satisfied +var ( + _ ValidationError = new(ResolutionFailedError) + _ error = new(ResolutionFailedError) +) + // make sure interfaces are satisfied var ( _ ValidationError = new(TagFailedError) _ error = new(TagFailedError) ) -// type JwtFailedError struct { -// varItem *SetVarItem -// code ValidateErrorType -// item string -// reason string -// } - -// func NewJwtFailedError(varItem *SetVarItem, item string, reason string) *JwtFailedError { -// return &JwtFailedError{ -// varItem: varItem, -// code: ValidateErrorJwtFailed, -// item: item, -// reason: reason, -// } -// } - -// func (e JwtFailedError) VarItem() *SetVarItem { -// return e.varItem -// } - -// func (e JwtFailedError) Error() string { -// return fmt.Sprintf("Error %v: The value of variable \"%s\" failed JWT validation (%s) required by \"%s->%s\" declared in \"%s\"", -// e.Code(), -// e.Key(), -// e.Reason(), -// e.SpecName(), -// e.Item(), -// e.Source()) -// } - -// func (e JwtFailedError) Message() string { -// return e.Error() -// } - -// func (e JwtFailedError) String() string { -// return e.Error() -// } - -// func (e JwtFailedError) Code() ValidateErrorType { -// return e.code -// } - -// func (e JwtFailedError) Key() string { -// return e.varItem.Var.Key -// } - -// func (e JwtFailedError) Reason() string { -// return e.reason -// } - -// func (e JwtFailedError) SpecName() string { -// return e.varItem.Spec.Complex -// } - -// func (e JwtFailedError) Item() string { -// return e.item -// } - -// func (e JwtFailedError) Source() string { -// if e.varItem.Spec.Operation == nil { -// return "-" -// } -// if e.varItem.Spec.Operation.Source == "" { -// return "-" -// } -// return e.varItem.Spec.Operation.Source -// } - -// // make sure interfaces are satisfied -// var ( -// _ ValidationError = new(JwtFailedError) -// _ error = new(JwtFailedError) -// ) - -const ComplexSpecType string = "Complex" - -var validator = valid.New() - -type ComplexDef struct { - Name string - Breaker string - Items map[string]*varSpec +const SpecTypeKey string = "Spec" + +var validator *valid.Validate + +func init() { + validator = valid.New() + if err := validator.RegisterValidation("database_url", func(fl valid.FieldLevel) bool { + if _, err := dburl.Parse(fl.Field().String()); err != nil { + return false + } + return true + }); err != nil { + panic(err) + } +} + +type SpecDef struct { + Name string `json:"name"` + Breaker string `json:"breaker"` + Atomics map[string]*varSpec `json:"atomics" yaml:"-"` Validator func(item *varSpec, itemKey string, varItem *SetVarItem) (ValidationErrors, error) } -func (cd *ComplexDef) Validate(itemKey string, varItem *SetVarItem) (ValidationErrors, error) { - complexItem, ok := cd.Items[itemKey] +func (cd *SpecDef) Validate(itemKey string, varItem *SetVarItem) (ValidationErrors, error) { + itemAtomic, ok := cd.Atomics[itemKey] if !ok { - return nil, fmt.Errorf("complex item not found: %s", itemKey) + return nil, fmt.Errorf("spec item not found: %s", itemKey) } - if varItem.Value.Resolved == "" && !complexItem.Required { + if varItem.Value.Resolved == "" && !itemAtomic.Required { return nil, nil } - return cd.Validator(complexItem, itemKey, varItem) + return cd.Validator(itemAtomic, itemKey, varItem) } func TagValidator(item *varSpec, itemKey string, varItem *SetVarItem) (ValidationErrors, error) { @@ -308,147 +247,7 @@ func TagValidator(item *varSpec, itemKey string, varItem *SetVarItem) (Validatio return validationErrs, nil } -func DatabaseValidator(item *varSpec, itemKey string, varItem *SetVarItem) (ValidationErrors, error) { - var validationErrs ValidationErrors - - _, err := dburl.Parse(varItem.Value.Resolved) - if err != nil { - validationErrs = append(validationErrs, - NewDatabaseUrlError( - &SetVarItem{ - Var: varItem.Var, - Value: varItem.Value, - Spec: varItem.Spec, - }, - err, - itemKey, - )) - } - - return validationErrs, nil -} - -var ComplexDefTypes = map[string]*ComplexDef{ - "Auth0": { - Name: "Auth0", - Breaker: "AUTH0", - Items: map[string]*varSpec{ - "AUDIENCE": { - Name: SpecNamePlain, - Rules: "url", - Required: true, - }, - "CLIENT_ID": { - Name: SpecNamePlain, - Rules: "alphanum,min=32,max=32", - Required: true, - }, - "DOMAIN": { - Name: SpecNamePlain, - Rules: "fqdn", - Required: true, - }, - }, - Validator: TagValidator, - }, - "Auth0Mgmt": { - Name: "Auth0Mgmt", - Breaker: "AUTH0_MANAGEMENT", - Items: map[string]*varSpec{ - "CLIENT_ID": { - Name: SpecNamePlain, - Rules: "alphanum,min=32,max=32", - Required: true, - }, - "CLIENT_SECRET": { - Name: SpecNameSecret, - Rules: "ascii,min=64,max=64", - Required: true, - }, - "AUDIENCE": { - Name: SpecNamePlain, - Rules: "url", - Required: true, - }, - }, - Validator: TagValidator, - }, - "DatabaseUrl": { - Name: "DatabaseUrl", - Breaker: "DATABASE", - Items: map[string]*varSpec{ - "URL": { - Name: SpecNameSecret, - Rules: "url", - Required: true, - }, - }, - Validator: DatabaseValidator, - }, - "OpenAI": { - Name: "OpenAI", - Breaker: "OPENAI", - Items: map[string]*varSpec{ - "ORG_ID": { - Name: SpecNamePlain, - Rules: "ascii,min=28,max=28,startswith=org-", - Required: true, - }, - "API_KEY": { - Name: SpecNameSecret, - Rules: "ascii,min=34,startswith=sk-", - Required: true, - }, - }, - Validator: TagValidator, - }, - "Redis": { - Name: "Redis", - Breaker: "REDIS", - Items: map[string]*varSpec{ - "HOST": { - Name: SpecNamePlain, - Rules: "ip|hostname", - Required: true, - }, - "PORT": { - Name: SpecNamePlain, - Rules: "number", - Required: true, - }, - "PASSWORD": { - Name: SpecNamePassword, - Rules: "min=18,max=32", - Required: false, - }, - }, - Validator: TagValidator, - }, - "Slack": { - Name: "Slack", - Breaker: "SLACK", - Items: map[string]*varSpec{ - "CLIENT_ID": { - Name: SpecNamePlain, - Rules: "min=24,max=24", - Required: true, - }, - "CLIENT_SECRET": { - Name: SpecNameSecret, - Rules: "min=32,max=32", - Required: true, - }, - "REDIRECT_URL": { - Name: SpecNameSecret, - Rules: "url", - Required: true, - }, - }, - Validator: TagValidator, - }, -} - -func (s *ComplexOperationSet) validate() (ValidationErrors, error) { +func (s *SpecOperationSet) validate(specDefs SpecDefs) (ValidationErrors, error) { var validationErrs ValidationErrors for _, k := range s.Keys { @@ -461,17 +260,17 @@ func (s *ComplexOperationSet) validate() (ValidationErrors, error) { continue } - complexType, ok := ComplexDefTypes[spec.Spec.Name] + specType, ok := specDefs[spec.Spec.Name] if !ok { - return nil, fmt.Errorf("complex type not found: %s", spec.Spec.Name) + return nil, fmt.Errorf("spec type not found: %s", spec.Spec.Name) } - akey, aitem, err := s.GetAtomicItem(spec) + akey, aitem, err := s.GetAtomic(spec, specDefs) if err != nil { return nil, err } - verrs, err := complexType.Validate( + verrs, err := specType.Validate( akey, aitem) if err != nil { @@ -483,13 +282,13 @@ func (s *ComplexOperationSet) validate() (ValidationErrors, error) { return validationErrs, nil } -func (s *ComplexOperationSet) GetAtomicItem(spec *SetVarSpec) (string, *SetVarItem, error) { +func (s *SpecOperationSet) GetAtomic(spec *SetVarSpec, specDefs SpecDefs) (string, *SetVarItem, error) { val, ok := s.values[spec.Var.Key] if !ok { return "", nil, fmt.Errorf("value not found for key: %s", spec.Var.Key) } - complexType, ok := ComplexDefTypes[spec.Spec.Name] + specType, ok := specDefs[spec.Spec.Name] if !ok { return spec.Var.Key, &SetVarItem{ Var: spec.Var, @@ -498,19 +297,24 @@ func (s *ComplexOperationSet) GetAtomicItem(spec *SetVarSpec) (string, *SetVarIt }, nil } - varKeyParts := strings.Split(val.Var.Key, complexType.Breaker+"_") + varKeyParts := strings.Split(val.Var.Key, specType.Breaker+"_") if len(varKeyParts) < 2 { - return "", nil, fmt.Errorf("invalid key not matching complex item: %s", val.Var.Key) + return "", nil, fmt.Errorf("invalid key not matching spec item: %s", val.Var.Key) } varKey := (varKeyParts[len(varKeyParts)-1]) varNS := (varKeyParts[0]) - item := complexType.Items[varKey] + item, ok := specType.Atomics[varKey] + if !ok { + return "", nil, fmt.Errorf("spec missing atomic for %s", varKey) + } - aspec := *spec.Spec - aspec.Complex = aspec.Name - aspec.Name = item.Name + aspec := *item + // aspec := *spec.Spec + aspec.Spec = spec.Spec.Name + aspec.Description = spec.Spec.Description + aspec.Operation = spec.Spec.Operation aspec.Namespace = varNS return varKey, &SetVarItem{ diff --git a/internal/owl/validate_test.go b/internal/owl/validate_test.go index 28cefd5c..d22dd346 100644 --- a/internal/owl/validate_test.go +++ b/internal/owl/validate_test.go @@ -81,8 +81,8 @@ func TestStore_Specs(t *testing.T) { require.NoError(t, err) require.EqualValues(t, "https://staging.us-central1.stateful.com/", snapshot[0].Value.Resolved) - require.EqualValues(t, "WVKBKL2b8asAb8e3gRGDK0tGlTGQjkEV", snapshot[1].Value.Resolved) - require.EqualValues(t, "stateful-staging.us.auth0.com", snapshot[2].Value.Resolved) + require.EqualValues(t, "WVK...kEV", snapshot[1].Value.Resolved) + require.EqualValues(t, "sta...com", snapshot[2].Value.Resolved) for _, item := range snapshot { assert.EqualValues(t, []*SetVarError{}, item.Errors) @@ -122,7 +122,7 @@ func TestStore_Specs(t *testing.T) { require.NoError(t, err) require.EqualValues(t, "sk-...ake", snapshot[0].Value.Resolved) - require.EqualValues(t, "org-tmfakeynfake9fakeHfakek0", snapshot[1].Value.Resolved) + require.EqualValues(t, "", snapshot[1].Value.Resolved) for _, item := range snapshot { assert.EqualValues(t, []*SetVarError{}, item.Errors) @@ -247,7 +247,7 @@ func TestStore_ComplexWithDbUrlValidation(t *testing.T) { assert.EqualValues(t, "DATABASE_URL", snapshot[0].Var.Key) assert.EqualValues(t, "abc...orm", snapshot[0].Value.Resolved) assert.EqualValues(t, - `Error 2: The value of variable "DATABASE_URL" failed DatabaseUrl validation "unknown database scheme" required by "DatabaseUrl->URL" declared in ".env.example"`, + "Error 1: The value of variable \"DATABASE_URL\" failed tag validation \"database_url\" required by \"DatabaseUrl->URL\" declared in \".env.example\"", snapshot[0].Errors[0].Message, ) }) @@ -266,7 +266,7 @@ func TestStore_ComplexWithDbUrlValidation(t *testing.T) { assert.EqualValues(t, "DATABASE_URL", snapshot[0].Var.Key) assert.EqualValues(t, "thi...url", snapshot[0].Value.Resolved) assert.EqualValues(t, - `Error 2: The value of variable "DATABASE_URL" failed DatabaseUrl validation "invalid database scheme" required by "DatabaseUrl->URL" declared in ".env.example"`, + "Error 1: The value of variable \"DATABASE_URL\" failed tag validation \"database_url\" required by \"DatabaseUrl->URL\" declared in \".env.example\"", snapshot[0].Errors[0].Message, ) }) From f86c9a2687aaa9302b11dc0aaa92368b73c4d823 Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Mon, 4 Nov 2024 10:34:42 -0800 Subject: [PATCH 3/3] Use different atomic --- internal/owl/envSpecDefs.defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/owl/envSpecDefs.defaults.yaml b/internal/owl/envSpecDefs.defaults.yaml index a76641e7..254c30e0 100644 --- a/internal/owl/envSpecDefs.defaults.yaml +++ b/internal/owl/envSpecDefs.defaults.yaml @@ -98,7 +98,7 @@ spec: rules: min=8,max=64 required: true - key: CONN_PROVIDER_ID - atomic: Plain + atomic: Opaque rules: required: true - key: SKIP_CONNECTION