diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 2c7f9ee2c..6e55bcf42 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -901,8 +901,38 @@ bar: "bar1" } func TestVisitDesiredStatesWithReleasesFiltered_StateValueOverrides(t *testing.T) { - files := map[string]string{ - "/path/to/helmfile.yaml": ` + envTmplExpr := "{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \"b\" | first | first | pluck \"c\" | first }}" + relTmplExpr := "\"{{`{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \\\"b\\\" | first | first | pluck \\\"c\\\" | first }}`}}\"" + + testcases := []struct { + expr, env, expected string + }{ + { + expr: envTmplExpr, + env: "default", + expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C", + }, + { + expr: envTmplExpr, + env: "production", + expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C", + }, + { + expr: relTmplExpr, + env: "default", + expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C", + }, + { + expr: relTmplExpr, + env: "production", + expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C", + }, + } + for i := range testcases { + testcase := testcases[i] + t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.yaml": fmt.Sprintf(` # The top-level "values" are "base" values has inherited to state values with the lowest priority. # The lowest priority results in environment-specific values to override values defined in the base. values: @@ -917,67 +947,80 @@ environments: - production.yaml --- releases: -- name: {{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }} - chart: stable/zipkin -`, - "/path/to/values.yaml": ` +- name: %s + chart: %s + namespace: %s +`, testcase.expr, testcase.expr, testcase.expr), + "/path/to/values.yaml": ` foo: foo bar: bar baz: baz hoge: hoge -fuga; fuga +fuga: fuga + +a: [] `, - "/path/to/default.yaml": ` + "/path/to/default.yaml": ` bar: "bar_default" baz: "baz_default" + +a: +- b: [] `, - "/path/to/production.yaml": ` + "/path/to/production.yaml": ` bar: "bar_production" baz: "baz_production" + +a: +- b: [] `, - "/path/to/overrides.yaml": ` + "/path/to/overrides.yaml": ` baz: baz_override hoge: hoge_override + +a: +- b: + - c: C `, - } + } - testcases := []struct { - env, expected string - }{ - {env: "default", expected: "foo-bar_default-baz_override-hoge_set-fuga_set"}, - {env: "production", expected: "foo-bar_production-baz_override-hoge_set-fuga_set"}, - } - for _, testcase := range testcases { - actual := []string{} + actual := []state.ReleaseSpec{} - collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error { - for _, r := range st.Releases { - actual = append(actual, r.Name) + collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error { + for _, r := range st.Releases { + actual = append(actual, r) + } + return []error{} } - return []error{} - } - app := appWithFs(&App{ - KubeContext: "default", - Logger: helmexec.NewLogger(os.Stderr, "debug"), - Reverse: false, - Namespace: "", - Selectors: []string{}, - Env: testcase.env, - ValuesFiles: []string{"overrides.yaml"}, - Set: map[string]interface{}{"hoge": "hoge_set", "fuga": "fuga_set"}, - }, files) - err := app.VisitDesiredStatesWithReleasesFiltered( - "helmfile.yaml", collectReleases, - ) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(actual) != 1 { - t.Errorf("unexpected number of processed releases: expected=1, got=%d", len(actual)) - } - if actual[0] != testcase.expected { - t.Errorf("unexpected result: expected=%s, got=%s", testcase.expected, actual[0]) - } + app := appWithFs(&App{ + KubeContext: "default", + Logger: helmexec.NewLogger(os.Stderr, "debug"), + Reverse: false, + Namespace: "", + Selectors: []string{}, + Env: testcase.env, + ValuesFiles: []string{"overrides.yaml"}, + Set: map[string]interface{}{"hoge": "hoge_set", "fuga": "fuga_set"}, + }, files) + err := app.VisitDesiredStatesWithReleasesFiltered( + "helmfile.yaml", collectReleases, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(actual) != 1 { + t.Errorf("unexpected number of processed releases: expected=1, got=%d", len(actual)) + } + if actual[0].Name != testcase.expected { + t.Errorf("unexpected name: expected=%s, got=%s", testcase.expected, actual[0].Name) + } + if actual[0].Chart != testcase.expected { + t.Errorf("unexpected chart: expected=%s, got=%s", testcase.expected, actual[0].Chart) + } + if actual[0].Namespace != testcase.expected { + t.Errorf("unexpected namespace: expected=%s, got=%s", testcase.expected, actual[0].Namespace) + } + }) } } diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index 19820c34f..51af19e5c 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -83,6 +83,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases) } + func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) { var f string if filepath.IsAbs(file) { diff --git a/pkg/app/two_pass_renderer.go b/pkg/app/two_pass_renderer.go index 51d6b0616..583b921c1 100644 --- a/pkg/app/two_pass_renderer.go +++ b/pkg/app/two_pass_renderer.go @@ -19,8 +19,8 @@ func prependLineNumbers(text string) string { return buf.String() } -func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment { - tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace} +func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) (*environment.Environment, *state.HelmState) { + tmplData := state.EnvironmentTemplateData{*firstPassEnv, r.namespace, map[string]interface{}{}} firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData) // parse as much as we can, tolerate errors, this is a preparse @@ -29,7 +29,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content))) if yamlBuf == nil { // we have a template syntax error, let the second parse report r.logger.Debugf("template syntax error: %v", err) - return firstPassEnv + return firstPassEnv, nil } } yamlData := yamlBuf.String() @@ -57,7 +57,8 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ if prestate != nil { firstPassEnv = &prestate.Env } - return firstPassEnv + + return firstPassEnv, prestate } type RenderOpts struct { @@ -88,13 +89,18 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en r.logger.Debugf("first-pass uses: %v", initEnv) } - renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content) + renderedEnv, prestate := r.renderPrestate(initEnv, baseDir, filename, content) if r.logger != nil { r.logger.Debugf("first-pass produced: %v", renderedEnv) } - finalEnv, err := renderedEnv.Merge(overrode) + finalEnv, err := inherited.Merge(renderedEnv) + if err != nil { + return nil, err + } + + finalEnv, err = finalEnv.Merge(overrode) if err != nil { return nil, err } @@ -103,7 +109,19 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv) } - tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace} + vals := map[string]interface{}{} + if prestate != nil { + prestate.Env = *finalEnv + vals, err = prestate.Values() + if err != nil { + return nil, err + } + } + if prestate != nil { + r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues) + } + + tmplData := state.EnvironmentTemplateData{*finalEnv, r.namespace, vals} secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData) yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content) if err != nil { diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 69bc59d74..f125b2e9e 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -7,28 +7,44 @@ import ( ) type Environment struct { - Name string - Values map[string]interface{} + Name string + Values map[string]interface{} + Defaults map[string]interface{} } var EmptyEnvironment Environment func (e Environment) DeepCopy() Environment { - bytes, err := yaml.Marshal(e.Values) + valuesBytes, err := yaml.Marshal(e.Values) if err != nil { panic(err) } var values map[string]interface{} - if err := yaml.Unmarshal(bytes, &values); err != nil { + if err := yaml.Unmarshal(valuesBytes, &values); err != nil { panic(err) } values, err = maputil.CastKeysToStrings(values) if err != nil { panic(err) } + + defaultsBytes, err := yaml.Marshal(e.Defaults) + if err != nil { + panic(err) + } + var defaults map[string]interface{} + if err := yaml.Unmarshal(defaultsBytes, &defaults); err != nil { + panic(err) + } + defaults, err = maputil.CastKeysToStrings(defaults) + if err != nil { + panic(err) + } + return Environment{ - Name: e.Name, - Values: values, + Name: e.Name, + Values: values, + Defaults: defaults, } } diff --git a/pkg/maputil/maputil_test.go b/pkg/maputil/maputil_test.go new file mode 100644 index 000000000..dd43b9c5f --- /dev/null +++ b/pkg/maputil/maputil_test.go @@ -0,0 +1,61 @@ +package maputil + +import "testing" + +func TestMapUtil_StrKeys(t *testing.T) { + m := map[string]interface{}{ + "a": []interface{}{ + map[string]interface{}{ + "b": []interface{}{ + map[string]interface{}{ + "c": "C", + }, + }, + }, + }, + } + + r, err := CastKeysToStrings(m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + a := r["a"].([]interface{}) + a0 := a[0].(map[string]interface{}) + b := a0["b"].([]interface{}) + b0 := b[0].(map[string]interface{}) + c := b0["c"] + + if c != "C" { + t.Errorf("unexpected c: expected=C, got=%s", c) + } +} + +func TestMapUtil_IFKeys(t *testing.T) { + m := map[interface{}]interface{}{ + "a": []interface{}{ + map[interface{}]interface{}{ + "b": []interface{}{ + map[interface{}]interface{}{ + "c": "C", + }, + }, + }, + }, + } + + r, err := CastKeysToStrings(m) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + a := r["a"].([]interface{}) + a0 := a[0].(map[string]interface{}) + b := a0["b"].([]interface{}) + b0 := b[0].(map[string]interface{}) + c := b0["c"] + + if c != "C" { + t.Errorf("unexpected c: expected=C, got=%s", c) + } +} diff --git a/pkg/state/create.go b/pkg/state/create.go index 9c4d42fa9..7c3acf3d8 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -114,6 +114,12 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi if err != nil { return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err} } + + e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues) + if err != nil { + return nil, err + } + state.Env = *e return &state, nil @@ -137,7 +143,12 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam return nil, err } - return c.LoadEnvValues(state, envName, envValues) + state, err = c.LoadEnvValues(state, envName, envValues) + if err != nil { + return nil, err + } + + return state, nil } func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmState, baseDir string) (*HelmState, error) { @@ -164,13 +175,8 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, envVals := map[string]interface{}{} envSpec, ok := st.Environments[name] if ok { - envValues := append([]interface{}{}, envSpec.Values...) - ld := &EnvironmentValuesLoader{ - storage: st.storage(), - readFile: st.readFile, - } var err error - envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues) + envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values) if err != nil { return nil, err } @@ -237,3 +243,20 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, return newEnv, nil } + +func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) { + envVals := map[string]interface{}{} + + valuesEntries := append([]interface{}{}, entries...) + ld := &EnvironmentValuesLoader{ + storage: st.storage(), + readFile: st.readFile, + } + var err error + envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries) + if err != nil { + return nil, err + } + + return envVals, nil +} diff --git a/pkg/state/environment_values_loader.go b/pkg/state/environment_values_loader.go index c02507f0d..3e81bd1c9 100644 --- a/pkg/state/environment_values_loader.go +++ b/pkg/state/environment_values_loader.go @@ -39,7 +39,7 @@ func (ld *EnvironmentValuesLoader) LoadEnvironmentValues(missingFileHandler *str } for _, envvalFullPath := range resolved { - tmplData := EnvironmentTemplateData{Environment: environment.EmptyEnvironment, Namespace: ""} + tmplData := EnvironmentTemplateData{environment.EmptyEnvironment, "", map[string]interface{}{}} r := tmpl.NewFileRenderer(ld.readFile, filepath.Dir(envvalFullPath), tmplData) bytes, err := r.RenderToBytes(envvalFullPath) if err != nil { diff --git a/pkg/state/release.go b/pkg/state/release.go index 6c9c2b610..f3ee680f6 100644 --- a/pkg/state/release.go +++ b/pkg/state/release.go @@ -15,6 +15,14 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R return nil, fmt.Errorf("failed executing template expressions in release \"%s\": %v", r.Name, err) } + { + ts := result.Name + result.Name, err = renderer.RenderTemplateContentToString([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".name = \"%s\": %v", r.Name, ts, err) + } + } + { ts := result.Chart result.Chart, err = renderer.RenderTemplateContentToString([]byte(ts)) diff --git a/pkg/state/state.go b/pkg/state/state.go index 62524bb84..8c2cf0ed4 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -27,6 +27,9 @@ type HelmState struct { basePath string FilePath string + // DefaultValues is the default values to be overrode by environment values and command-line overrides + DefaultValues []interface{} `yaml:"values"` + Environments map[string]EnvironmentSpec `yaml:"environments"` Bases []string `yaml:"bases"` @@ -1243,7 +1246,7 @@ func (st *HelmState) flagsForLint(helm helmexec.Interface, release *ReleaseSpec, } func (st *HelmState) RenderValuesFileToBytes(path string) ([]byte, error) { - r := tmpl.NewFileRenderer(st.readFile, filepath.Dir(path), st.envTemplateData()) + r := tmpl.NewFileRenderer(st.readFile, filepath.Dir(path), st.valuesFileTemplateData()) return r.RenderToBytes(path) } diff --git a/pkg/state/state_exec_tmpl.go b/pkg/state/state_exec_tmpl.go index 38164199d..b16488d47 100644 --- a/pkg/state/state_exec_tmpl.go +++ b/pkg/state/state_exec_tmpl.go @@ -2,23 +2,58 @@ package state import ( "fmt" + "github.com/imdario/mergo" + "github.com/roboll/helmfile/pkg/maputil" "github.com/roboll/helmfile/pkg/tmpl" ) -func (st *HelmState) envTemplateData() EnvironmentTemplateData { +func (st *HelmState) Values() (map[string]interface{}, error) { + vals := map[string]interface{}{} + + if err := mergo.Merge(&vals, st.Env.Defaults, mergo.WithOverride); err != nil { + return nil, err + } + if err := mergo.Merge(&vals, st.Env.Values, mergo.WithOverride); err != nil { + return nil, err + } + + vals, err := maputil.CastKeysToStrings(vals) + if err != nil { + return nil, err + } + + return vals, nil +} + +func (st *HelmState) mustLoadVals() map[string]interface{} { + vals, err := st.Values() + if err != nil { + panic(err) + } + return vals +} + +func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData { return EnvironmentTemplateData{ st.Env, st.Namespace, + st.mustLoadVals(), } } func (st *HelmState) ExecuteTemplates() (*HelmState, error) { r := *st + vals, err := st.Values() + if err != nil { + return nil, err + } + for i, rt := range st.Releases { - tmplData := ReleaseTemplateData{ - Environment: st.Env, - Release: rt, + tmplData := releaseTemplateData{ + st.Env, + rt, + vals, } renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData) r, err := rt.ExecuteTemplateExpressions(renderer) diff --git a/pkg/state/types.go b/pkg/state/types.go index e998a7ec7..fed10683e 100644 --- a/pkg/state/types.go +++ b/pkg/state/types.go @@ -15,12 +15,16 @@ type EnvironmentTemplateData struct { Environment environment.Environment // Namespace is accessible as `.Namespace` from any non-values template executed by the renderer Namespace string + // Values is accessible as `.Values` and it contains default state values overrode by environment values and override values. + Values map[string]interface{} } -// ReleaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file -type ReleaseTemplateData struct { +// releaseTemplateData provides variables accessible while executing golang text/template expressions in releases of a helmfile YAML file +type releaseTemplateData struct { // Environment is accessible as `.Environment` from any template expression executed by the renderer Environment environment.Environment // Release is accessible as `.Release` from any template expression executed by the renderer Release ReleaseSpec + // Values is accessible as `.Values` and it contains default state values overrode by environment values and override values. + Values map[string]interface{} }