returns InvalidToken if scanner encounters error (#486)

lexer/lexer_test.go

@@ -2393,3 +2393,30 @@ b: 1`,
 		})
 	}
 }
+
+func TestInvalid(t *testing.T) {
+	tests := []struct {
+		name string
+		src  string
+	}{
+		{
+			name: "literal opt",
+			src: `
+a: |invalid
+  foo`,
+		},
+		{
+			name: "literal opt",
+			src: `
+a: |invalid`,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := lexer.Tokenize(test.src)
+			if got.InvalidToken() == nil {
+				t.Fatal("expected contains invalid token")
+			}
+		})
+	}
+}

parser/parser.go

@@ -719,6 +719,9 @@ func ParseBytes(bytes []byte, mode Mode) (*ast.File, error) {
 
 // Parse parse from token instances, and returns ast.File
 func Parse(tokens token.Tokens, mode Mode) (*ast.File, error) {
+	if tk := tokens.InvalidToken(); tk != nil {
+		return nil, errors.ErrSyntax("found invalid token", tk)
+	}
 	var p parser
 	f, err := p.parse(tokens, mode)
 	if err != nil {

parser/parser_test.go

@@ -712,6 +712,17 @@ a:
        ^
 `,
 		},
+		{
+			`
+a: |invalidopt
+  foo
+`,
+			`
+[2:4] found invalid token
+>  2 | a:|invalidopt
+          ^
+   3 |   foo`,
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.source, func(t *testing.T) {

scanner/context.go

@@ -21,7 +21,6 @@ type Context struct {
 	isRawFolded        bool
 	isLiteral          bool
 	isFolded           bool
-	isSingleLine       bool
 	literalOpt         string
 }
 
@@ -35,9 +34,8 @@ var (
 
 func createContext() *Context {
 	return &Context{
-		idx:          0,
-		tokens:       token.Tokens{},
-		isSingleLine: true,
+		idx:    0,
+		tokens: token.Tokens{},
 	}
 }
 
@@ -58,7 +56,6 @@ func (c *Context) reset(src []rune) {
 	c.tokens = c.tokens[:0]
 	c.resetBuffer()
 	c.isRawFolded = false
-	c.isSingleLine = true
 	c.isLiteral = false
 	c.isFolded = false
 	c.literalOpt = ""
@@ -71,10 +68,6 @@ func (c *Context) resetBuffer() {
 	c.notSpaceOrgCharPos = 0
 }
 
-func (c *Context) isSaveIndentMode() bool {
-	return c.isLiteral || c.isFolded || c.isRawFolded
-}
-
 func (c *Context) breakLiteral() {
 	c.isLiteral = false
 	c.isRawFolded = false
@@ -186,10 +179,6 @@ func (c *Context) progress(num int) {
 	c.idx += num
 }
 
-func (c *Context) nextPos() int {
-	return c.idx + 1
-}
-
 func (c *Context) existsBuffer() bool {
 	return len(c.bufferedSrc()) != 0
 }

scanner/error.go

@@ -0,0 +1,19 @@
+package scanner
+
+import "github.com/goccy/go-yaml/token"
+
+type InvalidTokenError struct {
+	Message string
+	Token   *token.Token
+}
+
+func (e *InvalidTokenError) Error() string {
+	return e.Message
+}
+
+func ErrInvalidToken(msg string, tk *token.Token) *InvalidTokenError {
+	return &InvalidTokenError{
+		Message: msg,
+		Token:   tk,
+	}
+}