From f7fcae57bbf5cbb42ba59ce7e202edd93ddf4361 Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 08:16:21 -0800 Subject: [PATCH 1/7] Move args under defintions --- internal/cmd/environment.go | 2 +- internal/owl/graph.go | 22 +++++++++++----------- internal/owl/query.go | 24 ++++++++++++------------ internal/owl/store.go | 4 ++-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/internal/cmd/environment.go b/internal/cmd/environment.go index 44a43769..18e74f0d 100644 --- a/internal/cmd/environment.go +++ b/internal/cmd/environment.go @@ -337,7 +337,7 @@ func printStore(cmd *cobra.Command, msgData *runnerv1.MonitorEnvStoreResponse_Sn table.EndRow() specless := true - for i, _ := range msgData.Snapshot.Envs { + for i := range msgData.Snapshot.Envs { backwards := msgData.Snapshot.Envs[len(msgData.Snapshot.Envs)-i-1] if backwards.Spec != owl.AtomicNameOpaque { specless = false diff --git a/internal/owl/graph.go b/internal/owl/graph.go index a8bd7f45..ab5e4e1c 100644 --- a/internal/owl/graph.go +++ b/internal/owl/graph.go @@ -1399,23 +1399,23 @@ func init() { }, }), ), + Args: graphql.FieldConfigArgument{ + "definitions": &graphql.ArgumentConfig{ + Type: graphql.NewList(EnvSpecInputType), + }, + }, Resolve: func(p graphql.ResolveParams) (interface{}, error) { - return p.Source, nil + defs, ok := p.Args["definitions"].([]interface{}) + if !ok { + return nil, errors.New("definitions not found") + } + return defs, nil }, }, }, }), - Args: graphql.FieldConfigArgument{ - "definitions": &graphql.ArgumentConfig{ - Type: graphql.NewList(EnvSpecInputType), - }, - }, Resolve: func(p graphql.ResolveParams) (interface{}, error) { - defs, ok := p.Args["definitions"].([]interface{}) - if !ok { - return nil, errors.New("definitions not found") - } - return defs, nil + return p.Info.FieldName, nil }, }, "Atomics": &graphql.Field{ diff --git a/internal/owl/query.go b/internal/owl/query.go index 35ab4d1b..b4056070 100644 --- a/internal/owl/query.go +++ b/internal/owl/query.go @@ -165,6 +165,18 @@ func (s *Store) defineEnvSpecDefsQuery(query io.StringWriter) error { Name: ast.NewName(&ast.Name{ Value: "definitions", }), + Arguments: []*ast.Argument{ + ast.NewArgument(&ast.Argument{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + Value: ast.NewVariable(&ast.Variable{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + }), + }), + }, SelectionSet: nextSelSet, })) return nil, nil @@ -1315,18 +1327,6 @@ func (s *Store) NewEnvSpecsQuery(name string, varDefs []*ast.VariableDefinition, Name: ast.NewName(&ast.Name{ Value: "EnvSpecs", }), - Arguments: []*ast.Argument{ - ast.NewArgument(&ast.Argument{ - Name: ast.NewName(&ast.Name{ - Value: "definitions", - }), - Value: ast.NewVariable(&ast.Variable{ - Name: ast.NewName(&ast.Name{ - Value: "definitions", - }), - }), - }), - }, Directives: []*ast.Directive{}, SelectionSet: selSet, }), diff --git a/internal/owl/store.go b/internal/owl/store.go index 5bf74088..e6c8743c 100644 --- a/internal/owl/store.go +++ b/internal/owl/store.go @@ -30,7 +30,7 @@ const ( ) //go:embed envSpecDefs.defaults.yaml -var envSpecsCrdYaml []byte +var envSpecsDefaultsCRD []byte type setOperationKind int @@ -379,7 +379,7 @@ func NewStore(opts ...StoreOption) (*Store, error) { } // load ENV spec definitions from CRD - opts = append([]StoreOption{withSpecDefsCRD(envSpecsCrdYaml)}, opts...) + opts = append([]StoreOption{withSpecDefsCRD(envSpecsDefaultsCRD)}, opts...) for _, opt := range opts { if err := opt(s); err != nil { From e28a68a73c7697bb228e7d547f4858a1434483f9 Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 09:13:05 -0800 Subject: [PATCH 2/7] Nestable defintions --- internal/owl/graph.go | 113 +++++++++++++++++++++++------------------- internal/owl/query.go | 13 ++++- 2 files changed, 73 insertions(+), 53 deletions(-) diff --git a/internal/owl/graph.go b/internal/owl/graph.go index ab5e4e1c..0f0e3e24 100644 --- a/internal/owl/graph.go +++ b/internal/owl/graph.go @@ -39,6 +39,7 @@ var ( ) var EnvironmentType, + EnvSpecsType, ValidateType, ResolveType, RenderType, @@ -1350,6 +1351,66 @@ func init() { }, }) + EnvSpecsType = graphql.NewObject(graphql.ObjectConfig{ + Name: "EnvSpecsType", + Fields: (graphql.FieldsThunk)(func() graphql.Fields { + return graphql.Fields{ + "definitions": &graphql.Field{ + Type: graphql.NewList( + graphql.NewObject(graphql.ObjectConfig{ + Name: "EnvSpecsDefType", + Fields: graphql.Fields{ + "name": &graphql.Field{ + Type: graphql.String, + }, + "breaker": &graphql.Field{ + Type: graphql.String, + }, + "atomics": &graphql.Field{ + Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ + Name: "AtomicEnvSpecsDefType", + Fields: graphql.Fields{ + "key": &graphql.Field{ + Type: graphql.String, + }, + "atomic": &graphql.Field{ + Type: graphql.String, + }, + "rules": &graphql.Field{ + Type: graphql.String, + }, + "required": &graphql.Field{ + Type: graphql.Boolean, + }, + }, + })), + }, + }, + }), + ), + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + return p.Source, nil + }, + }, + "load": &graphql.Field{ + Type: EnvSpecsType, + Args: graphql.FieldConfigArgument{ + "definitions": &graphql.ArgumentConfig{ + Type: graphql.NewList(EnvSpecInputType), + }, + }, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + defs, ok := p.Args["definitions"].([]interface{}) + if !ok { + return nil, errors.New("definitions not found") + } + return defs, nil + }, + }, + } + }), + }) + var err error Schema, err = graphql.NewSchema(graphql.SchemaConfig{ Query: graphql.NewObject( @@ -1363,57 +1424,7 @@ func init() { }, }, "EnvSpecs": &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: "EnvSpecsType", - Fields: graphql.Fields{ - "definitions": &graphql.Field{ - Type: graphql.NewList( - graphql.NewObject(graphql.ObjectConfig{ - Name: "EnvSpecsDefType", - Fields: graphql.Fields{ - "name": &graphql.Field{ - Type: graphql.String, - }, - "breaker": &graphql.Field{ - Type: graphql.String, - }, - "atomics": &graphql.Field{ - Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: "AtomicEnvSpecsDefType", - Fields: graphql.Fields{ - "key": &graphql.Field{ - Type: graphql.String, - }, - "atomic": &graphql.Field{ - Type: graphql.String, - }, - "rules": &graphql.Field{ - Type: graphql.String, - }, - "required": &graphql.Field{ - Type: graphql.Boolean, - }, - }, - })), - }, - }, - }), - ), - Args: graphql.FieldConfigArgument{ - "definitions": &graphql.ArgumentConfig{ - Type: graphql.NewList(EnvSpecInputType), - }, - }, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - defs, ok := p.Args["definitions"].([]interface{}) - if !ok { - return nil, errors.New("definitions not found") - } - return defs, nil - }, - }, - }, - }), + Type: EnvSpecsType, Resolve: func(p graphql.ResolveParams) (interface{}, error) { return p.Info.FieldName, nil }, diff --git a/internal/owl/query.go b/internal/owl/query.go index b4056070..f8994f34 100644 --- a/internal/owl/query.go +++ b/internal/owl/query.go @@ -163,7 +163,7 @@ func (s *Store) defineEnvSpecDefsQuery(query io.StringWriter) error { }) selSet.Selections = append(selSet.Selections, ast.NewField(&ast.Field{ Name: ast.NewName(&ast.Name{ - Value: "definitions", + Value: "load", }), Arguments: []*ast.Argument{ ast.NewArgument(&ast.Argument{ @@ -177,7 +177,16 @@ func (s *Store) defineEnvSpecDefsQuery(query io.StringWriter) error { }), }), }, - SelectionSet: nextSelSet, + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "definitions", + }), + SelectionSet: nextSelSet, + }), + }, + }), })) return nil, nil }, From c687419cd5f1dccf3e5c369f344e2705e5bb3c7a Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 09:20:02 -0800 Subject: [PATCH 3/7] Consolidate NewQuery --- internal/owl/query.go | 49 +++++++------------------------------------ 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/internal/owl/query.go b/internal/owl/query.go index f8994f34..18a2b448 100644 --- a/internal/owl/query.go +++ b/internal/owl/query.go @@ -78,7 +78,7 @@ func (s *Store) snapshotQuery(query, vars io.StringWriter, resolve bool) error { if resolve { queryName = "Resolve" } - q, err := s.NewEnvironmentQuery(queryName, varDefs, + q, err := s.NewQuery(queryName, "Environment", varDefs, reducers, ) if err != nil { @@ -113,8 +113,9 @@ func (s *Store) defineEnvSpecDefsQuery(query io.StringWriter) error { }), }), } - q, err := s.NewEnvSpecsQuery( + q, err := s.NewQuery( "EnvSpecsDef", + "EnvSpecs", varDefs, []QueryNodeReducer{ func(opSets []*OperationSet, opDef *ast.OperationDefinition, selSet *ast.SelectionSet) (*ast.SelectionSet, error) { @@ -228,7 +229,7 @@ func (s *Store) sensitiveKeysQuery(query, vars io.StringWriter) error { }), } - q, err := s.NewEnvironmentQuery("Sensitive", varDefs, + q, err := s.NewQuery("Sensitive", "Environment", varDefs, []QueryNodeReducer{ reconcileAsymmetry(s), reduceSetOperations(vars), @@ -307,7 +308,7 @@ func (s *Store) getterQuery(query, vars io.StringWriter) error { } s.logger.Debug("getter opSets breakdown", zap.Int("loaded", loaded), zap.Int("updated", updated), zap.Int("deleted", deleted), zap.Int("total", len(s.opSets))) - q, err := s.NewEnvironmentQuery("Get", varDefs, + q, err := s.NewQuery("Get", "Environment", varDefs, []QueryNodeReducer{ reconcileAsymmetry(s), reduceSetOperations(vars), @@ -1322,53 +1323,19 @@ type Query struct { doc *ast.Document } -func (s *Store) NewEnvSpecsQuery(name string, varDefs []*ast.VariableDefinition, reducers []QueryNodeReducer) (*Query, error) { +func (s *Store) NewQuery(queryName, rootSelelection string, varDefs []*ast.VariableDefinition, reducers []QueryNodeReducer) (*Query, error) { selSet := ast.NewSelectionSet(&ast.SelectionSet{}) opDef := ast.NewOperationDefinition(&ast.OperationDefinition{ Operation: "query", Name: ast.NewName(&ast.Name{ - Value: fmt.Sprintf("Owl%s", name), + Value: fmt.Sprintf("Owl%s", queryName), }), Directives: []*ast.Directive{}, SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ Selections: []ast.Selection{ ast.NewField(&ast.Field{ Name: ast.NewName(&ast.Name{ - Value: "EnvSpecs", - }), - Directives: []*ast.Directive{}, - SelectionSet: selSet, - }), - }, - }), - VariableDefinitions: varDefs, - }) - - var err error - for _, reducer := range reducers { - if selSet, err = reducer(s.opSets, opDef, selSet); err != nil { - return nil, err - } - } - - doc := ast.NewDocument(&ast.Document{Definitions: []ast.Node{opDef}}) - - return &Query{doc: doc}, nil -} - -func (s *Store) NewEnvironmentQuery(name string, varDefs []*ast.VariableDefinition, reducers []QueryNodeReducer) (*Query, error) { - selSet := ast.NewSelectionSet(&ast.SelectionSet{}) - opDef := ast.NewOperationDefinition(&ast.OperationDefinition{ - Operation: "query", - Name: ast.NewName(&ast.Name{ - Value: fmt.Sprintf("Owl%s", name), - }), - Directives: []*ast.Directive{}, - SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ - Selections: []ast.Selection{ - ast.NewField(&ast.Field{ - Name: ast.NewName(&ast.Name{ - Value: "Environment", + Value: rootSelelection, }), Arguments: []*ast.Argument{}, Directives: []*ast.Directive{}, From 7b0587fcb2db9c39b111b7abab0c9a4bd3e8625e Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 11:23:59 -0800 Subject: [PATCH 4/7] Use flimsy CRDs for now --- internal/owl/graph.go | 6 ++++ internal/owl/query.go | 34 +++++++++++++++++--- internal/owl/store.go | 58 ++++++++++++++++++++++++++++----- internal/owl/store_test.go | 66 +++++++++++++++++++++++++++++++++++--- internal/runner/session.go | 48 +++++++++++++++------------ pkg/project/project.go | 2 +- 6 files changed, 177 insertions(+), 37 deletions(-) diff --git a/internal/owl/graph.go b/internal/owl/graph.go index 0f0e3e24..b8b11801 100644 --- a/internal/owl/graph.go +++ b/internal/owl/graph.go @@ -1329,6 +1329,9 @@ func init() { "breaker": &graphql.InputObjectFieldConfig{ Type: graphql.String, }, + "origin": &graphql.InputObjectFieldConfig{ + Type: graphql.String, + }, "atomics": &graphql.InputObjectFieldConfig{ Type: graphql.NewList(graphql.NewInputObject(graphql.InputObjectConfig{ Name: "AtomicEnvSpecInputType", @@ -1366,6 +1369,9 @@ func init() { "breaker": &graphql.Field{ Type: graphql.String, }, + "origin": &graphql.Field{ + Type: graphql.String, + }, "atomics": &graphql.Field{ Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ Name: "AtomicEnvSpecsDefType", diff --git a/internal/owl/query.go b/internal/owl/query.go index 18a2b448..1f1f5068 100644 --- a/internal/owl/query.go +++ b/internal/owl/query.go @@ -63,7 +63,7 @@ func (s *Store) snapshotQuery(query, vars io.StringWriter, resolve bool) error { if resolve { reducers = append(reducers, []QueryNodeReducer{ - reduceWrapResolve(), + reduceWrapResolve(s), reduceWrapDone(), reduceWrapValidate(), reduceAtomic("", nil), @@ -131,6 +131,11 @@ func (s *Store) defineEnvSpecDefsQuery(query io.StringWriter) error { Value: "breaker", }), }), + ast.NewField(&ast.Field{ + Name: ast.NewName(&ast.Name{ + Value: "origin", + }), + }), ast.NewField(&ast.Field{ Name: ast.NewName(&ast.Name{ Value: "atomics", @@ -336,7 +341,28 @@ func (s *Store) getterQuery(query, vars io.StringWriter) error { return nil } -func reduceWrapResolve() QueryNodeReducer { +func reduceWrapResolve(store *Store) QueryNodeReducer { + exprVal := `key | lower()` + projectVal := "dev" + + // todo(sebastian): we should traverse the path and gen the query + if store.resolvePath != nil { + if t, err := extractDataKey(store.resolvePath, "transform"); err == nil { + if expr, err := extractDataKey(t, "expr"); err == nil { + exprVal = expr.(string) + } + if gcp, err := extractDataKey(t, "gcp"); err == nil { + if auth, err := extractDataKey(gcp, "auth"); err == nil { + if v, ok := auth.(string); !ok || v != "ADC" { + return nil + } + } + if project, err := extractDataKey(gcp, "project"); err == nil { + projectVal = project.(string) + } + } + } + } return func(opSets []*OperationSet, opDef *ast.OperationDefinition, selSet *ast.SelectionSet) (*ast.SelectionSet, error) { resolveSelSet := ast.NewSelectionSet(&ast.SelectionSet{ Selections: []ast.Selection{ @@ -372,7 +398,7 @@ func reduceWrapResolve() QueryNodeReducer { Value: "project", }), Value: ast.NewStringValue(&ast.StringValue{ - Value: "platform-staging-413816", + Value: projectVal, }), }), }, @@ -388,7 +414,7 @@ func reduceWrapResolve() QueryNodeReducer { Value: "expr", }), Value: ast.NewStringValue(&ast.StringValue{ - Value: `key | trimPrefix("REDWOOD_ENV_") | replace("SLACK_REDIRECT_URL", "SLACK_REDIRECT") | lower()`, + Value: exprVal, }), }), }, diff --git a/internal/owl/store.go b/internal/owl/store.go index e6c8743c..1edb7e6e 100644 --- a/internal/owl/store.go +++ b/internal/owl/store.go @@ -6,6 +6,7 @@ import ( _ "embed" "encoding/json" "fmt" + "io" "slices" "strings" "sync" @@ -367,6 +368,8 @@ type Store struct { opSets []*OperationSet specDefs SpecDefs + resolvePath interface{} + logger *zap.Logger } @@ -379,7 +382,7 @@ func NewStore(opts ...StoreOption) (*Store, error) { } // load ENV spec definitions from CRD - opts = append([]StoreOption{withSpecDefsCRD(envSpecsDefaultsCRD)}, opts...) + opts = append([]StoreOption{WithSpecDefsCRD(envSpecsDefaultsCRD)}, opts...) for _, opt := range opts { if err := opt(s); err != nil { @@ -431,10 +434,26 @@ func WithEnvs(source string, envs ...string) StoreOption { } } -func withSpecDefsCRD(raw []byte) StoreOption { +func WithResolutionCRD(raw []byte) StoreOption { return func(s *Store) error { - var crd map[string]interface{} - err := yaml.Unmarshal(raw, &crd) + crd, err := extractCrdKind(raw, "EnvResolution") + if err != nil { + return nil + } + + envResPath, err := extractDataKey(crd, "path") + if err != nil { + return err + } + s.resolvePath = envResPath + + return nil + } +} + +func WithSpecDefsCRD(raw []byte) StoreOption { + return func(s *Store) error { + crd, err := extractCrdKind(raw, "EnvSpecDefinitions") if err != nil { return err } @@ -448,6 +467,29 @@ func withSpecDefsCRD(raw []byte) StoreOption { } } +func extractCrdKind(raw []byte, targetKind string) (map[string]interface{}, error) { + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + + var crd map[string]interface{} + for { + err := decoder.Decode(&crd) + + if err != nil { + return nil, err + } + + kind, ok := crd["kind"].(string) + if ok && kind == targetKind { + break + } + + if err == io.EOF { + return nil, fmt.Errorf("failed to find kind %q in CRD", targetKind) + } + } + return crd, nil +} + func WithLogger(logger *zap.Logger) StoreOption { return func(s *Store) error { s.logger = logger @@ -722,6 +764,10 @@ func (s *Store) defineEnvSpecs(envSpecs interface{}) error { return err } + if result.HasErrors() { + return fmt.Errorf("graphql errors %s", result.Errors) + } + definitions, err := extractDataKey(result.Data, "definitions") if err != nil { return err @@ -758,10 +804,6 @@ func (s *Store) defineEnvSpecs(envSpecs interface{}) error { s.specDefs[specDef.Name] = specDef } - if result.HasErrors() { - return fmt.Errorf("graphql errors %s", result.Errors) - } - return err } diff --git a/internal/owl/store_test.go b/internal/owl/store_test.go index f1009943..4cd508c9 100644 --- a/internal/owl/store_test.go +++ b/internal/owl/store_test.go @@ -614,11 +614,15 @@ var resolveSpecsRaw []byte //go:embed testdata/resolve/.env.local var resolveValuesRaw []byte +//go:embed testdata/resolve/envResolution.yaml +var envResolveCRD []byte + func TestStore_Resolve(t *testing.T) { t.Skip("Skip since it requires GCP's secret manager") t.Run("Valid", func(t *testing.T) { store, err := NewStore( + WithResolutionCRD(envResolveCRD), WithSpecFile(".env.example", resolveSpecsRaw), WithEnvFile(".env.local", resolveValuesRaw), ) @@ -692,10 +696,64 @@ VECTOR_DB_URL="URL for the vector DB" # VectorDB` }) } +//go:embed testdata/custom/.env.example +var customSpecsRaw []byte + +//go:embed testdata/custom/.env.local +var customValuesRaw []byte + +//go:embed testdata/custom/envSpecDefs.custom.yaml +var envSpecsCustomCRD []byte + func TestStore_LoadEnvSpecDefs(t *testing.T) { - store, err := NewStore() - require.NoError(t, err) - require.NotNil(t, store) + t.Skip("Skip since it requires GCP's secret manager") + + t.Run("Defaults", func(t *testing.T) { + store, err := NewStore() + require.NoError(t, err) + require.NotNil(t, store) + + require.Len(t, store.specDefs, 7) + }) + + t.Run("Custom", func(t *testing.T) { + store, err := NewStore( + WithSpecDefsCRD(envSpecsCustomCRD), + WithSpecFile(".env.example", customSpecsRaw), + WithEnvFile(".env.local", customValuesRaw), + ) + + require.NoError(t, err) + require.NotNil(t, store) + + require.Len(t, store.specDefs, 8) + }) - require.Len(t, store.specDefs, 7) + t.Run("ResolvePath", func(t *testing.T) { + store, err := NewStore( + WithSpecDefsCRD(envSpecsCustomCRD), + WithResolutionCRD(envSpecsCustomCRD), + WithSpecFile(".env.example", customSpecsRaw), + WithEnvFile(".env.local", customValuesRaw), + ) + + require.NoError(t, err) + require.NotNil(t, store) + + require.Len(t, store.specDefs, 8) + + snapshot, err := store.InsecureResolve() + require.NoError(t, err) + require.Len(t, snapshot, 2) + + errors := 0 + for _, item := range snapshot { + require.EqualValues(t, "LITERAL", item.Value.Status) + require.NotEmpty(t, item.Value.Original) + require.NotEmpty(t, item.Value.Resolved) + errors += len(item.Errors) + } + + require.Equal(t, 0, errors) + }) } diff --git a/internal/runner/session.go b/internal/runner/session.go index ac5e42d0..775f81ca 100644 --- a/internal/runner/session.go +++ b/internal/runner/session.go @@ -188,7 +188,7 @@ func newOwlStorer(envs []string, proj *project.Project, logger *zap.Logger) (*ow } for _, specFile := range envSpecFiles { - raw, _ := proj.LoadRawEnv(specFile) + raw, _ := proj.LoadRawFile(specFile) if raw == nil { continue } @@ -216,25 +216,33 @@ func newOwlStorer(envs []string, proj *project.Project, logger *zap.Logger) (*ow opts = append(opts, owl.WithEnvs(envSource, envs...)) } - // todo(sebastian): code's not stale, disabled since it's unclear how this will work - // resolverOwlStore, err := owl.NewStore(opts...) - // if err != nil { - // return nil, err - // } - - // logger.Debug("Resolving env external to the graph") - // if snapshot, err := resolverOwlStore.InsecureResolve(); err == nil { - // resolved := []string{} - // for _, item := range snapshot { - // if item.Value.Status != "LITERAL" { - // continue - // } - // resolved = append(resolved, fmt.Sprintf("%s=%s", item.Var.Key, item.Value.Resolved)) - // } - // opts = append(opts, owl.WithEnvs("[gcp:secrets]", resolved...)) - // } else { - // logger.Error("failed to resolve owl store", zap.Error(err)) - // } + owlYAML, err := proj.LoadRawFile(".runme/owl.yaml") + if err != nil { + return nil, err + } else if owlYAML != nil { + opts = append([]owl.StoreOption{owl.WithSpecDefsCRD(owlYAML)}, opts...) + } + + if owlYAML != nil { + resolverOwlStore, err := owl.NewStore(opts...) + if err != nil { + return nil, err + } + + logger.Debug("Resolving env external to the graph") + if snapshot, err := resolverOwlStore.InsecureResolve(); err == nil { + resolved := []string{} + for _, item := range snapshot { + if item.Value.Status != "LITERAL" { + continue + } + resolved = append(resolved, fmt.Sprintf("%s=%s", item.Var.Key, item.Value.Resolved)) + } + opts = append(opts, owl.WithEnvs("[gcp:secrets]", resolved...)) + } else { + logger.Error("failed to resolve owl store", zap.Error(err)) + } + } owlStore, err := owl.NewStore(opts...) if err != nil { diff --git a/pkg/project/project.go b/pkg/project/project.go index 95582327..52de7bca 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -595,7 +595,7 @@ func (p *Project) LoadEnvAsMap() (map[string]string, error) { return env, nil } -func (p *Project) LoadRawEnv(file string) ([]byte, error) { +func (p *Project) LoadRawFile(file string) ([]byte, error) { raw, err := util.ReadFile(p.fs, file) if err != nil && errors.Is(err, os.ErrNotExist) { // not an error if file does not exist From 7c4fb5c87164767f7cd9922194307155635c59ea Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 11:28:50 -0800 Subject: [PATCH 5/7] Add testdata --- internal/owl/store.go | 1 - internal/owl/store_test.go | 2 +- internal/owl/testdata/custom/.env.example | 4 ++++ internal/owl/testdata/custom/.env.local | 2 ++ internal/owl/testdata/resolve/envResolution.yaml | 15 +++++++++++++++ 5 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 internal/owl/testdata/custom/.env.example create mode 100644 internal/owl/testdata/custom/.env.local create mode 100644 internal/owl/testdata/resolve/envResolution.yaml diff --git a/internal/owl/store.go b/internal/owl/store.go index 1edb7e6e..6adf6d8f 100644 --- a/internal/owl/store.go +++ b/internal/owl/store.go @@ -473,7 +473,6 @@ func extractCrdKind(raw []byte, targetKind string) (map[string]interface{}, erro var crd map[string]interface{} for { err := decoder.Decode(&crd) - if err != nil { return nil, err } diff --git a/internal/owl/store_test.go b/internal/owl/store_test.go index 4cd508c9..5bb507d2 100644 --- a/internal/owl/store_test.go +++ b/internal/owl/store_test.go @@ -702,7 +702,7 @@ var customSpecsRaw []byte //go:embed testdata/custom/.env.local var customValuesRaw []byte -//go:embed testdata/custom/envSpecDefs.custom.yaml +//go:embed testdata/custom/envSpecDefs.yaml var envSpecsCustomCRD []byte func TestStore_LoadEnvSpecDefs(t *testing.T) { diff --git a/internal/owl/testdata/custom/.env.example b/internal/owl/testdata/custom/.env.example new file mode 100644 index 00000000..99175f6a --- /dev/null +++ b/internal/owl/testdata/custom/.env.example @@ -0,0 +1,4 @@ +DOCS_GCP_ARTIFACTS_REPO="Artifacts Docker repo in GCP" # Docs +DOCS_GCP_CREDENTIALS="Credentials for auth with GCP" # Docs +DOCS_GCP_PROJECT_ID="Project ID for deploy into GCP" # Docs +DOCS_GCP_REGION="Region for deploy into GCP" # Docs diff --git a/internal/owl/testdata/custom/.env.local b/internal/owl/testdata/custom/.env.local new file mode 100644 index 00000000..3d8a64f6 --- /dev/null +++ b/internal/owl/testdata/custom/.env.local @@ -0,0 +1,2 @@ +DOCS_GCP_PROJECT_ID="runme-ci" +DOCS_GCP_REGION="us-central1" diff --git a/internal/owl/testdata/resolve/envResolution.yaml b/internal/owl/testdata/resolve/envResolution.yaml new file mode 100644 index 00000000..f2d0700c --- /dev/null +++ b/internal/owl/testdata/resolve/envResolution.yaml @@ -0,0 +1,15 @@ +apiVersion: runme.stateful.com/v1beta1 +kind: EnvResolution +metadata: + name: docs.runme.dev + namespace: stateful + annotations: + github.com/repo-url: https://github.com/stateful/docs.runme.dev +spec: + type: owl + path: + transform: + expr: key | trimPrefix("REDWOOD_ENV_") | replace("SLACK_REDIRECT_URL", "SLACK_REDIRECT") | lower() + gcp: + auth: ADC + project: platform-staging-413816 From a2880b533924bd8e5c141fcb59087f1917dc0e12 Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 11:50:43 -0800 Subject: [PATCH 6/7] Use v1alpha1 --- internal/owl/envSpecDefs.defaults.yaml | 2 +- internal/owl/query.go | 3 +++ internal/owl/store.go | 2 +- internal/owl/testdata/resolve/envResolution.yaml | 2 +- internal/runner/session.go | 5 ++++- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/internal/owl/envSpecDefs.defaults.yaml b/internal/owl/envSpecDefs.defaults.yaml index 254c30e0..7579c1f0 100644 --- a/internal/owl/envSpecDefs.defaults.yaml +++ b/internal/owl/envSpecDefs.defaults.yaml @@ -1,4 +1,4 @@ -apiVersion: runme.stateful.com/v1beta1 +apiVersion: runme.stateful.com/v1alpha1 kind: EnvSpecDefinitions metadata: name: runme diff --git a/internal/owl/query.go b/internal/owl/query.go index 1f1f5068..d6ab9f54 100644 --- a/internal/owl/query.go +++ b/internal/owl/query.go @@ -1374,6 +1374,9 @@ func (s *Store) NewQuery(queryName, rootSelelection string, varDefs []*ast.Varia var err error for _, reducer := range reducers { + if reducer == nil { + continue + } if selSet, err = reducer(s.opSets, opDef, selSet); err != nil { return nil, err } diff --git a/internal/owl/store.go b/internal/owl/store.go index 6adf6d8f..e872b2dd 100644 --- a/internal/owl/store.go +++ b/internal/owl/store.go @@ -455,7 +455,7 @@ func WithSpecDefsCRD(raw []byte) StoreOption { return func(s *Store) error { crd, err := extractCrdKind(raw, "EnvSpecDefinitions") if err != nil { - return err + return nil } envSpecs, err := extractDataKey(crd, "envSpecs") diff --git a/internal/owl/testdata/resolve/envResolution.yaml b/internal/owl/testdata/resolve/envResolution.yaml index f2d0700c..f6799dd0 100644 --- a/internal/owl/testdata/resolve/envResolution.yaml +++ b/internal/owl/testdata/resolve/envResolution.yaml @@ -1,4 +1,4 @@ -apiVersion: runme.stateful.com/v1beta1 +apiVersion: runme.stateful.com/v1alpha1 kind: EnvResolution metadata: name: docs.runme.dev diff --git a/internal/runner/session.go b/internal/runner/session.go index 775f81ca..9ed5655d 100644 --- a/internal/runner/session.go +++ b/internal/runner/session.go @@ -220,7 +220,10 @@ func newOwlStorer(envs []string, proj *project.Project, logger *zap.Logger) (*ow if err != nil { return nil, err } else if owlYAML != nil { - opts = append([]owl.StoreOption{owl.WithSpecDefsCRD(owlYAML)}, opts...) + opts = append([]owl.StoreOption{ + owl.WithSpecDefsCRD(owlYAML), + owl.WithResolutionCRD(owlYAML), + }, opts...) } if owlYAML != nil { From 2490c6205ffa6cef9d278d6dd1ff743cd1ebda95 Mon Sep 17 00:00:00 2001 From: Sebastian Tiedtke Date: Tue, 5 Nov 2024 12:46:35 -0800 Subject: [PATCH 7/7] Multi-doc is intentional --- .pre-commit-config.yaml | 2 - internal/owl/testdata/custom/envSpecDefs.yaml | 45 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 internal/owl/testdata/custom/envSpecDefs.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7827bc5..0488fc52 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,8 +2,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - - id: check-yaml - stages: [pre-commit] - id: check-json exclude: "^.vscode/" stages: [pre-commit] diff --git a/internal/owl/testdata/custom/envSpecDefs.yaml b/internal/owl/testdata/custom/envSpecDefs.yaml new file mode 100644 index 00000000..b27ba289 --- /dev/null +++ b/internal/owl/testdata/custom/envSpecDefs.yaml @@ -0,0 +1,45 @@ +apiVersion: runme.stateful.com/v1alpha1 +kind: EnvSpecDefinitions +metadata: + name: docs.runme.dev + namespace: stateful + annotations: + github.com/repo-url: https://github.com/stateful/docs.runme.dev +spec: + type: owl + envSpecs: + - name: Docs + breaker: DOCS + atomics: + - key: GCP_ARTIFACTS_REPO + atomic: Opaque + rules: printascii + required: true + - key: GCP_CREDENTIALS + atomic: Secret + rules: json + required: true + - key: GCP_PROJECT_ID + atomic: Plain + rules: printascii + required: true + - key: GCP_REGION + atomic: Plain + rules: printascii + required: true +--- +apiVersion: runme.stateful.com/v1alpha1 +kind: EnvResolution +metadata: + name: docs.runme.dev + namespace: stateful + annotations: + github.com/repo-url: https://github.com/stateful/docs.runme.dev +spec: + type: owl + path: + transform: + expr: key | trimPrefix("DOCS_GCP_") | lower() + gcp: + auth: ADC + project: runme-ci