diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9511ddfd28ab..97c957042da0 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -36,6 +36,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* - Drain response buffers when pipelining is used by redis output. {pull}1353[1353] +- Unterminated environment variable expressions in config files will now cause an error {pull}1389[1389] *Packetbeat* @@ -52,6 +53,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d - Add SOCKS5 proxy support to redis output. {pull}1353[1353] - Failover and load balancing support in redis output. {pull}1353[1353] - Multiple-worker per host support for redis output. {pull}1353[1353] +- Added ability to escape `${x}` in config files to avoid environment variable expansion {pull}1389[1389] *Packetbeat* diff --git a/libbeat/cfgfile/cfgfile.go b/libbeat/cfgfile/cfgfile.go index 1e8c31f7292d..3fb893a3f48b 100644 --- a/libbeat/cfgfile/cfgfile.go +++ b/libbeat/cfgfile/cfgfile.go @@ -6,10 +6,8 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" ) // Command line flags. @@ -67,7 +65,10 @@ func Load(path string) (*common.Config, error) { if err != nil { return nil, fmt.Errorf("failed to read %s: %v", path, err) } - fileContent = expandEnv(fileContent) + fileContent, err = expandEnv(filepath.Base(path), fileContent) + if err != nil { + return nil, err + } config, err := common.NewConfigWithYAML(fileContent, path) if err != nil { @@ -81,89 +82,3 @@ func Load(path string) (*common.Config, error) { func IsTestConfig() bool { return *testConfig } - -// expandEnv replaces ${var} in config according to the values of the current -// environment variables. The replacement is case-sensitive. References to -// undefined variables are replaced by the empty string. A default value can be -// given by using the form ${var:default value}. -func expandEnv(config []byte) []byte { - return []byte(expand(string(config), func(key string) string { - keyAndDefault := strings.SplitN(key, ":", 2) - key = keyAndDefault[0] - - v := os.Getenv(key) - if v == "" && len(keyAndDefault) == 2 { - // Set value to the default. - v = keyAndDefault[1] - logp.Info("Replacing config environment variable '${%s}' with "+ - "default '%s'", key, keyAndDefault[1]) - } else { - logp.Info("Replacing config environment variable '${%s}' with '%s'", - key, v) - } - - return v - })) -} - -// The following methods were copied from the os package of the stdlib. The -// expand method was modified to only expand variables defined with braces and -// ignore $var. - -// Expand replaces ${var} in the string based on the mapping function. -func expand(s string, mapping func(string) string) string { - buf := make([]byte, 0, 2*len(s)) - // ${} is all ASCII, so bytes are fine for this operation. - i := 0 - for j := 0; j < len(s); j++ { - if s[j] == '$' && j+2 < len(s) && s[j+1] == '{' { - buf = append(buf, s[i:j]...) - name, w := getShellName(s[j+1:]) - buf = append(buf, mapping(name)...) - j += w - i = j + 1 - } - } - return string(buf) + s[i:] -} - -// isShellSpecialVar reports whether the character identifies a special -// shell variable such as $*. -func isShellSpecialVar(c uint8) bool { - switch c { - case '*', '#', '$', '@', '!', '?', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - return true - } - return false -} - -// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore -func isAlphaNum(c uint8) bool { - return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' -} - -// getShellName returns the name that begins the string and the number of bytes -// consumed to extract it. If the name is enclosed in {}, it's part of a ${} -// expansion and two more bytes are needed than the length of the name. -func getShellName(s string) (string, int) { - switch { - case s[0] == '{': - if len(s) > 2 && isShellSpecialVar(s[1]) && s[2] == '}' { - return s[1:2], 3 - } - // Scan to closing brace - for i := 1; i < len(s); i++ { - if s[i] == '}' { - return s[1:i], i + 1 - } - } - return "", 1 // Bad syntax; just eat the brace. - case isShellSpecialVar(s[0]): - return s[0:1], 1 - } - // Scan alphanumerics. - var i int - for i = 0; i < len(s) && isAlphaNum(s[i]); i++ { - } - return s[:i], i -} diff --git a/libbeat/cfgfile/cfgfile_test.go b/libbeat/cfgfile/cfgfile_test.go index 05e75fdcc3c4..29b5e143cb3d 100644 --- a/libbeat/cfgfile/cfgfile_test.go +++ b/libbeat/cfgfile/cfgfile_test.go @@ -45,37 +45,61 @@ func TestExpandEnv(t *testing.T) { var tests = []struct { in string out string + err string }{ // Environment variables can be specified as ${env} only. - {"${y}", "y"}, - {"$y", "$y"}, + {"${y}", "y", ""}, + {"$y", "$y", ""}, // Environment variables are case-sensitive. - {"${Y}", ""}, + {"${Y}", "", ""}, // Defaults can be specified. - {"x${Z:D}", "xD"}, - {"x${Z:A B C D}", "xA B C D"}, // Spaces are allowed in the default. - {"x${Z:}", "x"}, + {"x${Z:D}", "xD", ""}, + {"x${Z:A B C D}", "xA B C D", ""}, // Spaces are allowed in the default. + {"x${Z:}", "x", ""}, - // Un-matched braces are swallowed by the Go os.Expand function. - {"x${Y ${Z:Z}", "xZ"}, + // Un-matched braces cause an error. + {"x${Y ${Z:Z}", "", "unexpected character in variable expression: " + + "U+0020 ' ', expected a default value or closing brace"}, // Special environment variables are not replaced. - {"$*", "$*"}, - {"${*}", ""}, - {"$@", "$@"}, - {"${@}", ""}, - {"$1", "$1"}, - {"${1}", ""}, - - {"", ""}, - {"$$", "$$"}, + {"$*", "$*", ""}, + {"${*}", "", "shell variable cannot start with U+002A '*'"}, + {"$@", "$@", ""}, + {"${@}", "", "shell variable cannot start with U+0040 '@'"}, + {"$1", "$1", ""}, + {"${1}", "", "shell variable cannot start with U+0031 '1'"}, + + {"", "", ""}, + {"$$", "$$", ""}, + + {"${a_b}", "", ""}, // Underscores are allowed in variable names. + + // ${} cannot be split across newlines. + {"hello ${name: world\n}", "", "unterminated brace"}, + + // To use a literal '${' you write '$${'. + {`password: "abc$${!"`, `password: "abc${!"`, ""}, + + // The full error contains the line number. + {"shipper:\n name: ${var", "", "failure while expanding environment " + + "variables in config.yml at line=2, unterminated brace"}, } for _, test := range tests { os.Setenv("y", "y") - output := expandEnv([]byte(test.in)) - assert.Equal(t, test.out, string(output), "Input: %s", test.in) + output, err := expandEnv("config.yml", []byte(test.in)) + + switch { + case test.err != "" && err == nil: + t.Errorf("Expected an error for test case %+v", test) + case test.err == "" && err != nil: + t.Errorf("Unexpected error for test case %+v, %v", test, err) + case err != nil: + assert.Contains(t, err.Error(), test.err) + default: + assert.Equal(t, test.out, string(output), "Input: %s", test.in) + } } } diff --git a/libbeat/cfgfile/env.go b/libbeat/cfgfile/env.go new file mode 100644 index 000000000000..07af365789a5 --- /dev/null +++ b/libbeat/cfgfile/env.go @@ -0,0 +1,352 @@ +package cfgfile + +import ( + "bytes" + "fmt" + "os" + "strings" + "unicode" + "unicode/utf8" +) + +// Inspired by: https://cuddle.googlecode.com/hg/talk/lex.html + +const ( + errUnterminatedBrace = "unterminated brace" +) + +// item represents a token returned from the scanner. +type item struct { + typ itemType // Token type, such as itemVariable. + pos int // The starting position, in bytes, of this item in the input string. + val string // Value, such as "${". +} + +func (i item) String() string { + switch { + case i.typ == itemEOF: + return "EOF" + default: + return i.val + } +} + +// itemType identifies the type of lex items. +type itemType int + +// lex tokens. +const ( + itemError itemType = iota + 1 + itemEscapedLeftDelim + itemLeftDelim + itemVariable + itemDefaultValue + itemRightDelim + itemText + itemEOF +) + +const eof = -1 + +// stateFn represents the state of the scanner as a function that returns the +// next state. +type stateFn func(*lexer) stateFn + +// lexer holds the state of the scanner. +type lexer struct { + name string // used only for error reports. + input string // the string being scanned. + start int // start position of this item. + pos int // current position in the input. + width int // width of last rune read from input. + lastPos int // position of most recent item returned by nextItem + items chan item // channel of scanned items. +} + +// next returns the next rune in the input. +func (l *lexer) next() rune { + if int(l.pos) >= len(l.input) { + l.width = 0 + return eof + } + r, w := utf8.DecodeRuneInString(l.input[l.pos:]) + l.width = w + l.pos += l.width + return r +} + +// peek returns but does not consume the next rune in the input. +func (l *lexer) peek() rune { + r := l.next() + l.backup() + return r +} + +// backup steps back one rune. Can only be called once per call of next. +func (l *lexer) backup() { + l.pos -= l.width +} + +// emit passes an item back to the client. +func (l *lexer) emit(t itemType) { + l.items <- item{t, l.start, l.input[l.start:l.pos]} + l.start = l.pos +} + +// ignore skips over the pending input before this point. +func (l *lexer) ignore() { + l.start = l.pos +} + +// lineNumber reports which line we're on, based on the position of +// the previous item returned by nextItem. Doing it this way +// means we don't have to worry about peek double counting. +func (l *lexer) lineNumber() int { + return 1 + strings.Count(l.input[:l.lastPos], "\n") +} + +// errorf returns an error token and terminates the scan by passing +// back a nil pointer that will be the next state, terminating l.nextItem. +func (l *lexer) errorf(format string, args ...interface{}) stateFn { + l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)} + return nil +} + +// nextItem returns the next item from the input. +// Called by the parser, not in the lexing goroutine. +func (l *lexer) nextItem() item { + item := <-l.items + l.lastPos = item.pos + return item +} + +// run lexes the input by executing state functions until the state is nil. +func (l *lexer) run() { + for state := lexText; state != nil; { + state = state(l) + } + close(l.items) // No more tokens will be delivered. +} + +// state functions + +// token values. +const ( + leftDelim = "${" + rightDelim = '}' + defaultValueSeperator = ':' + escapedLeftDelim = "$${" +) + +// lexText scans until an opening action delimiter, "${". +func lexText(l *lexer) stateFn { + for { + switch { + case strings.HasPrefix(l.input[l.pos:], escapedLeftDelim): + if l.pos > l.start { + l.emit(itemText) + } + return lexEscapedLeftDelim + case strings.HasPrefix(l.input[l.pos:], leftDelim): + if l.pos > l.start { + l.emit(itemText) + } + return lexLeftDelim + } + + if l.next() == eof { + break + } + } + // Correctly reached EOF. + if l.pos > l.start { + l.emit(itemText) + } + l.emit(itemEOF) + return nil +} + +// lexEscapedLeftDelim scans the escaped left delimiter, which is known to be +// present. +func lexEscapedLeftDelim(l *lexer) stateFn { + l.pos += len(escapedLeftDelim) + l.emit(itemEscapedLeftDelim) + return lexText +} + +// lexLeftDelim scans the left delimiter, which is known to be present. +func lexLeftDelim(l *lexer) stateFn { + l.pos += len(leftDelim) + l.emit(itemLeftDelim) + return lexVariable +} + +// lexVariable scans a shell variable name which is alphanumeric and does not +// start with a number or other special shell variable character. +// The ${ has already been scanned. +func lexVariable(l *lexer) stateFn { + var r rune = l.peek() + if isShellSpecialVar(r) { + return l.errorf("shell variable cannot start with %#U", r) + } + for { + r = l.next() + if !isAlphaNumeric(r) { + l.backup() + break + } + } + l.emit(itemVariable) + return lexDefaultValueOrRightDelim +} + +// lexDefaultValueOrRightDelim scans for a default value for the variable +// expansion or for the '}' to close the variable definition. +func lexDefaultValueOrRightDelim(l *lexer) stateFn { + switch r := l.next(); { + case r == eof || isEndOfLine(r): + return l.errorf(errUnterminatedBrace) + case r == ':': + l.ignore() + return lexDefaultValue + case r == '}': + l.backup() + return lexRightDelim + default: + return l.errorf("unexpected character in variable expression: %#U, "+ + "expected a default value or closing brace", r) + } +} + +// lexRightDelim scans the right delimiter, which is known to be present. +func lexRightDelim(l *lexer) stateFn { + l.pos += 1 + l.emit(itemRightDelim) + return lexText +} + +// lexDefaultValue scans the default value for a variable expansion. It scans +// until a '}' is encountered. If EOF or EOL occur before the '}' then this +// is an error. +func lexDefaultValue(l *lexer) stateFn { +loop: + for { + r := l.next() + switch { + case r == eof || isEndOfLine(r): + return l.errorf(errUnterminatedBrace) + case r == rightDelim: + l.backup() + break loop + } + } + l.emit(itemDefaultValue) + return lexRightDelim +} + +// isSpace reports whether r is a space character. +func isSpace(r rune) bool { + return r == ' ' || r == '\t' +} + +// isEndOfLine reports whether r is an end-of-line character. +func isEndOfLine(r rune) bool { + return r == '\r' || r == '\n' +} + +// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore. +func isAlphaNumeric(r rune) bool { + return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + +// isShellSpecialVar reports whether r identifies a special shell variable +// such as $*. +func isShellSpecialVar(r rune) bool { + switch r { + case '*', '#', '$', '@', '!', '?', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return true + } + return false +} + +// Functions for using the lexer to parse. + +// lex creates a new scanner for the input string. +func lex(name, input string) *lexer { + l := &lexer{ + name: name, + input: input, + items: make(chan item), + } + go l.run() // Concurrently run state machine. + return l +} + +// parseLexer parses the tokens from the lexer. It expands the environment +// variables that it encounters. +func parseLexer(l *lexer) ([]byte, error) { + var peekItem *item + next := func() item { + if peekItem != nil { + rtn := *peekItem + peekItem = nil + return rtn + } + return l.nextItem() + } + peek := func() item { + if peekItem != nil { + return *peekItem + } + rtn := l.nextItem() + peekItem = &rtn + return rtn + } + + var buf bytes.Buffer +loop: + for { + item := next() + + switch item.typ { + case itemText: + buf.WriteString(item.val) + case itemVariable: + variable := item.val + value := os.Getenv(variable) + if peek().typ == itemDefaultValue { + item = next() + if value == "" { + value = item.val + } + } + buf.WriteString(value) + case itemEscapedLeftDelim: + buf.WriteString(leftDelim) + case itemLeftDelim, itemRightDelim: + case itemError: + return nil, fmt.Errorf("failure while expanding environment "+ + "variables in %s at line=%d, %v", l.name, l.lineNumber(), + item.val) + case itemEOF: + break loop + default: + return nil, fmt.Errorf("unexpected token type %d", item.typ) + } + } + return buf.Bytes(), nil +} + +// expandEnv replaces ${var} in config according to the values of the current +// environment variables. The replacement is case-sensitive. References to +// undefined variables are replaced by the empty string. A default value can be +// given by using the form ${var:default value}. +// +// Valid variable names consist of letters, numbers, and underscores and do not +// begin with numbers. Variable blocks cannot be split across lines. Unmatched +// braces will causes a parse error. To use a literal '${' in config write +// '$${'. +func expandEnv(filename string, contents []byte) ([]byte, error) { + l := lex(filename, string(contents)) + return parseLexer(l) +} diff --git a/libbeat/docs/shared-env-vars.asciidoc b/libbeat/docs/shared-env-vars.asciidoc index 7798e8bbd6ff..f95bd6af7cd7 100644 --- a/libbeat/docs/shared-env-vars.asciidoc +++ b/libbeat/docs/shared-env-vars.asciidoc @@ -14,47 +14,39 @@ experimental[] -You can use environment variable references in the +{beatname_lc}.yml+ file to set values -that need to be configurable during deployment. To do this, use: +You can use environment variable references in the +{beatname_lc}.yml+ file to +set values that need to be configurable during deployment. To do this, use: `${VAR}` Where `VAR` is the name of the environment variable. -Each variable reference is replaced at startup by the value of the environment variable. -The replacement is case-sensitive and occurs before the YAML file is parsed. References -to undefined variables are replaced by empty strings unless you specify a default value. -To specify a default value, use: +Each variable reference is replaced at startup by the value of the environment +variable. The replacement is case-sensitive and occurs before the YAML file is +parsed. References to undefined variables are replaced by empty strings unless +you specify a default value. To specify a default value, use: `${VAR:default_value}` -Where `default_value` is the value to use if the environment variable is undefined. +Where `default_value` is the value to use if the environment variable is +undefined. -After changing the value of an environment variable, you need to restart {beatname_uc} to -pick up the new value. +If you need to use a literal `${` in your configuration file then you can write +`$${` to escape the expansion. + +After changing the value of an environment variable, you need to restart +{beatname_uc} to pick up the new value. ==== Examples Here are some examples of configurations that use environment variables -and what each configuration looks like after replacement: +and what each configuration looks like after replacement: [options="header"] |================================== |Config source |Environment setting |Config after replacement |`name: ${NAME}` |`export NAME=elastic` |`name: elastic` +|`name: ${NAME}` |no setting |`name: ` |`name: ${NAME:beats}` |no setting |`name: beats` |`name: ${NAME:beats}` |`export NAME=elastic` |`name: elastic` -|`hosts: [$HOSTS]` |`export HOSTS="'localhost:9200', 'localhost:9202'"` |`hosts: ['localhost:9200', 'localhost:9202']` |================================== - -==== Troubleshooting - -When <> is enabled at the `info` level or higher, a message is logged for each -environment variable replacement. For example: - -["source","sh"] -------------------------------------------------------------------------------------- -2016/01/13 23:00:27.925523 cfgfile.go:87: INFO Replacing config environment variable '${NAME}' with 'elastic' -2016/01/13 23:00:27.925786 cfgfile.go:87: INFO Replacing config environment variable '${HOSTS}' with ''localhost:9200', 'localhost:9202'' -------------------------------------------------------------------------------------- -