Gracefully handle encountered regular expression when running jsonnet…

…fmt (google#724)

* Gracefully handle encountered regular expression when running jsonnetfmt, adding tests.

* Do not validate verbatim strings.

* Also do not validate string blocks.

* Change golden prefix for formatter tests to fmt.golden to be consistant with cpp version.

formatter/formatter_test.go

@@ -0,0 +1,126 @@
+package formatter
+
+import (
+	"flag"
+	"fmt"
+	"github.com/google/go-jsonnet/internal/testutils"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+var update = flag.Bool("update", false, "update .golden files")
+
+// ErrorWriter encapsulates a writer and an error state indicating when at least
+// one error has been written to the writer.
+type ErrorWriter struct {
+	ErrorsFound bool
+	Writer      io.Writer
+}
+
+type formatterTest struct {
+	name   string
+	input  string
+	output string
+}
+
+type ChangedGoldensList struct {
+	changedGoldens []string
+}
+
+func runTest(t *testing.T, test *formatterTest, changedGoldensList *ChangedGoldensList) {
+	read := func(file string) []byte {
+		bytz, err := ioutil.ReadFile(file)
+		if err != nil {
+			t.Fatalf("reading file: %s: %v", file, err)
+		}
+		return bytz
+	}
+
+	input := read(test.input)
+	var outBuilder strings.Builder
+	output, err := Format(test.name, string(input), Options{})
+	if err != nil {
+		errWriter := ErrorWriter{
+			Writer:      &outBuilder,
+			ErrorsFound: false,
+		}
+
+		_, writeErr := errWriter.Writer.Write([]byte(err.Error()))
+		if writeErr != nil {
+			panic(writeErr)
+		}
+	} else {
+		outBuilder.Write([]byte(output))
+	}
+
+	outData := outBuilder.String()
+
+	if *update {
+		changed, err := testutils.UpdateGoldenFile(test.output, []byte(outData), 0666)
+		if err != nil {
+			t.Error(err)
+		}
+		if changed {
+			changedGoldensList.changedGoldens = append(changedGoldensList.changedGoldens, test.output)
+		}
+	} else {
+		golden, err := ioutil.ReadFile(test.output)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if diff, hasDiff := testutils.CompareWithGolden(outData, golden); hasDiff {
+			t.Error(fmt.Errorf("golden file %v has diff:\n%v", test.input, diff))
+		}
+	}
+}
+
+func TestFormatter(t *testing.T) {
+	flag.Parse()
+
+	var tests []*formatterTest
+
+	match, err := filepath.Glob("testdata/*.jsonnet")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jsonnetExtRE := regexp.MustCompile(`\.jsonnet$`)
+
+	for _, input := range match {
+		// Skip escaped filenames.
+		if strings.ContainsRune(input, '%') {
+			continue
+		}
+		name := jsonnetExtRE.ReplaceAllString(input, "")
+		golden := jsonnetExtRE.ReplaceAllString(input, ".fmt.golden")
+		tests = append(tests, &formatterTest{
+			name:   name,
+			input:  input,
+			output: golden,
+		})
+	}
+
+	changedGoldensList := ChangedGoldensList{}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			runTest(t, test, &changedGoldensList)
+		})
+	}
+
+	if *update {
+		// Little hack: a failed test which prints update stats.
+		t.Run("Goldens Updated", func(t *testing.T) {
+			t.Logf("Expected failure, for printing update stats. Does not appear without `-update`.")
+			t.Logf("%d formatter goldens updated:\n", len(changedGoldensList.changedGoldens))
+			for _, golden := range changedGoldensList.changedGoldens {
+				t.Log(golden)
+			}
+			t.Fail()
+		})
+	}
+}

formatter/testdata/regular_expression.fmt.golden

@@ -0,0 +1 @@
+testdata/regular_expression:3:11-29 testdata/regular_expression:3:11-29 Unknown escape sequence in string literal: \d

formatter/testdata/regular_expression.jsonnet

@@ -0,0 +1,5 @@
+{
+  x: {
+    data: '([^:]+)(?::\d+)?',
+  },
+}

internal/parser/parser.go

@@ -420,6 +420,8 @@ func (p *parser) parseObjectRemainderField(literalFields *LiteralFieldSet, tok *
 	var expr1 ast.Node
 	var id *ast.Identifier
 	var fodder2 ast.Fodder
+	var err errors.StaticError
+
 	switch next.kind {
 	case tokenIdentifier:
 		kind = ast.ObjectFieldID
@@ -428,7 +430,10 @@ func (p *parser) parseObjectRemainderField(literalFields *LiteralFieldSet, tok *
 	case tokenStringDouble, tokenStringSingle,
 		tokenStringBlock, tokenVerbatimStringDouble, tokenVerbatimStringSingle:
 		kind = ast.ObjectFieldStr
-		expr1 = tokenStringToAst(next)
+		expr1, err = tokenStringToAst(next)
+		if err != nil {
+			return nil, err
+		}
 	default:
 		fodder1 = next.fodder
 		kind = ast.ObjectFieldExpr
@@ -827,43 +832,58 @@ func (p *parser) parseArray(tok *token) (ast.Node, errors.StaticError) {
 	}, nil
 }
 
-func tokenStringToAst(tok *token) *ast.LiteralString {
+func tokenStringToAst(tok *token) (*ast.LiteralString, errors.StaticError) {
+	var node *ast.LiteralString
+	var validate bool = true
+
 	switch tok.kind {
 	case tokenStringSingle:
-		return &ast.LiteralString{
+		node = &ast.LiteralString{
 			NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
 			Value:    tok.data,
 			Kind:     ast.StringSingle,
 		}
 	case tokenStringDouble:
-		return &ast.LiteralString{
+		node = &ast.LiteralString{
 			NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
 			Value:    tok.data,
 			Kind:     ast.StringDouble,
 		}
 	case tokenStringBlock:
-		return &ast.LiteralString{
+		node = &ast.LiteralString{
 			NodeBase:        ast.NewNodeBaseLoc(tok.loc, tok.fodder),
 			Value:           tok.data,
 			Kind:            ast.StringBlock,
 			BlockIndent:     tok.stringBlockIndent,
 			BlockTermIndent: tok.stringBlockTermIndent,
 		}
+		validate = false
 	case tokenVerbatimStringDouble:
-		return &ast.LiteralString{
+		node = &ast.LiteralString{
 			NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
 			Value:    tok.data,
 			Kind:     ast.VerbatimStringDouble,
 		}
+		validate = false
 	case tokenVerbatimStringSingle:
-		return &ast.LiteralString{
+		node = &ast.LiteralString{
 			NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),
 			Value:    tok.data,
 			Kind:     ast.VerbatimStringSingle,
 		}
+		validate = false
 	default:
 		panic(fmt.Sprintf("Not a string token %#+v", tok))
 	}
+
+	if validate {
+		_, err := StringUnescape((*node).Loc(), (*node).Value)
+		if err != nil {
+			return node, errors.MakeStaticError(err.Error(), tok.loc)
+		}
+	}
+
+	return node, nil
 }
 
 func (p *parser) parseTerminal() (ast.Node, errors.StaticError) {
@@ -907,7 +927,7 @@ func (p *parser) parseTerminal() (ast.Node, errors.StaticError) {
 		}, nil
 	case tokenStringDouble, tokenStringSingle,
 		tokenStringBlock, tokenVerbatimStringDouble, tokenVerbatimStringSingle:
-		return tokenStringToAst(tok), nil
+		return tokenStringToAst(tok)
 	case tokenFalse:
 		return &ast.LiteralBoolean{
 			NodeBase: ast.NewNodeBaseLoc(tok.loc, tok.fodder),