refactor: 💡 parser.nextIdent concat multiple lit into one token

const.go

@@ -44,8 +44,8 @@ func (r *Const) parse(p *Parser) (err error) {
 		return
 	}
 	p.peekNonWhitespace()
-	ident, _, _ := p.nextIdent(false)
-	r.Ident = ident
+	identTok := p.nextIdent(false)
+	r.Ident = identTok.Raw
 	ru := p.peekNonWhitespace()
 	if toToken(string(ru)) != T_EQUALS {
 		return p.unexpected(string(ru), "=")
@@ -141,14 +141,14 @@ func (r *ConstValue) parse(p *Parser) (err error) {
 		}
 		r.EndToken = r.Map.EndToken
 	} else {
-		fullLit, startTok, endTok := p.nextIdent(false)
-		if startTok.Type != T_IDENT || endTok.Type != T_IDENT {
-			return p.unexpected("identifier", fullLit)
+		identTok := p.nextIdent(false)
+		if identTok.Type != T_IDENT {
+			return p.unexpected("identifier", identTok.Raw)
 		}
 		r.Type = CONST_VALUE_IDENT
-		r.StartToken = startTok
-		r.EndToken = endTok
-		r.Value = fullLit
+		r.StartToken = identTok
+		r.EndToken = identTok
+		r.Value = identTok.Raw
 	}
 
 	return

enum.go

@@ -34,8 +34,8 @@ func (r *Enum) String() string {
 
 func (r *Enum) parse(p *Parser) (err error) {
 	p.peekNonWhitespace()
-	fullLit, _, _ := p.nextIdent(false)
-	r.Ident = fullLit
+	identTok := p.nextIdent(false)
+	r.Ident = identTok.Raw
 	ru := p.peekNonWhitespace()
 	if toToken(string(ru)) != T_LEFTCURLY {
 		return p.unexpected(string(ru), "{")
@@ -103,16 +103,16 @@ func (r *EnumElement) patchToParentMap() {
 
 func (r *EnumElement) parse(p *Parser) (err error) {
 	p.peekNonWhitespace()
-	fullLit, startTok, endTok := p.nextIdent(false)
-	r.StartToken = startTok
-	r.Ident = fullLit
+	identTok := p.nextIdent(false)
+	r.StartToken = identTok
+	r.Ident = identTok.Raw
 	ru := p.peekNonWhitespace()
 	// if there is no = after enum field identifier, then directly parse EndToken
 	if toToken(string(ru)) != T_EQUALS {
 		// parse options
 		ru = p.peekNonWhitespace()
 		if toToken(string(ru)) != T_LEFTPAREN {
-			r.EndToken = endTok
+			r.EndToken = identTok
 			// parse separator
 			if err = r.parseSeparator(p); err != nil {
 				return err

field.go

@@ -57,7 +57,7 @@ func (r *Field) parse(p *Parser) (err error) {
 
 	// parse requiredness
 	p.peekNonWhitespace()
-	tok := p.next()
+	tok := p.nextIdent(true)
 	if tok.Value == "required" || tok.Value == "optional" {
 		r.Requiredness = tok.Value
 	} else {
@@ -72,8 +72,8 @@ func (r *Field) parse(p *Parser) (err error) {
 
 	// parse identifier
 	p.peekNonWhitespace()
-	ident, _, endTok := p.nextIdent(false)
-	r.Ident = ident
+	identTok := p.nextIdent(false)
+	r.Ident = identTok.Raw
 
 	// parse DefaultValue/Options
 	ru = p.peekNonWhitespace()
@@ -112,7 +112,7 @@ func (r *Field) parse(p *Parser) (err error) {
 			return err
 		}
 	} else {
-		if err = r.parseEnd(endTok, p); err != nil {
+		if err = r.parseEnd(identTok, p); err != nil {
 			return err
 		}
 	}

field_type.go

@@ -42,37 +42,37 @@ func (r *FieldType) String() string {
 
 func (r *FieldType) parse(p *Parser) (err error) {
 	p.peekNonWhitespace()
-	fullLit, startTok, endTok := p.nextIdent(true)
-	r.StartToken = startTok
-	if isBaseTypeToken(fullLit) {
+	identTok := p.nextIdent(true)
+	r.StartToken = identTok
+	if isBaseTypeToken(identTok.Raw) {
 		r.Type = FIELD_TYPE_BASE
-		r.BaseType = fullLit
-		r.EndToken = endTok
-	} else if fullLit == "map" {
+		r.BaseType = identTok.Raw
+		r.EndToken = identTok
+	} else if identTok.Raw == "map" {
 		r.Type = FIELD_TYPE_MAP
-		r.Map = NewMapType(startTok, r)
+		r.Map = NewMapType(identTok, r)
 		if err = r.Map.parse(p); err != nil {
 			return
 		}
 		r.EndToken = r.Map.EndToken
-	} else if fullLit == "set" {
+	} else if identTok.Raw == "set" {
 		r.Type = FIELD_TYPE_SET
-		r.Set = NewSetType(startTok, r)
+		r.Set = NewSetType(identTok, r)
 		if err = r.Set.parse(p); err != nil {
 			return
 		}
 		r.EndToken = r.Set.EndToken
-	} else if fullLit == "list" {
+	} else if identTok.Raw == "list" {
 		r.Type = FIELD_TYPE_LIST
-		r.List = NewListType(startTok, r)
+		r.List = NewListType(identTok, r)
 		if err = r.List.parse(p); err != nil {
 			return
 		}
 		r.EndToken = r.List.EndToken
 	} else {
 		r.Type = FIELD_TYPE_IDENT
-		r.Ident = fullLit
-		r.EndToken = endTok
+		r.Ident = identTok.Raw
+		r.EndToken = identTok
 	}
 
 	// parse options

namespace.go

@@ -29,9 +29,11 @@ func (r *Namespace) String() string {
 }
 
 func (r *Namespace) parse(p *Parser) (err error) {
-	r.Name, _, _ = p.nextIdent(true)
+	identTok := p.nextIdent(true)
+	r.Name = identTok.Raw
 	var endIdent *Token
-	r.Value, _, endIdent = p.nextIdent(true)
+	identTok = p.nextIdent(true)
+	r.Value = identTok.Raw
 	ru := p.peekNonWhitespace()
 	if toToken(string(ru)) != T_LEFTPAREN {
 		r.EndToken = endIdent

option.go

@@ -29,9 +29,9 @@ func (r *Option) String() string {
 
 func (r *Option) parse(p *Parser) (err error) {
 	// can't use keyword as option name
-	name, start, _ := p.nextIdent(false)
-	if start == nil || start.Type != T_IDENT {
-		return p.unexpected(name, "identifier")
+	identTok := p.nextIdent(false)
+	if identTok == nil || identTok.Type != T_IDENT {
+		return p.unexpected(identTok.Raw, "identifier")
 	}
 	// if there is no = token
 	tok := p.nextNonWhitespace()
@@ -50,10 +50,10 @@ func (r *Option) parse(p *Parser) (err error) {
 		return err
 	}
 
-	r.Name = name
+	r.Name = identTok.Raw
 	r.Value = tok.Raw
 	r.Parent = r
-	r.StartToken = start
+	r.StartToken = identTok
 	r.EndToken = tok
 	// since Options are always gathered in a slice during parent node parsing, we not need to link each Option with these pointers
 	r.Next = nil

parser.go

@@ -152,36 +152,44 @@ func (p *Parser) nextComment(commentType int) (res *Token, err error) {
 	return
 }
 
-// TODO: concat dot-separated ident into one token
 // Find next identifier, it will consume white spaces during scanning.
 // 1. Allow leading && trailing dot.
 // 2. If keywordAllowed == true, it will allow keyword inside an identifier, e.g. enum.aaa.struct. In this case, the token for keyword will be replace to T_IDENT, since the meaning for it is no more a keyword.
-// 3. For dot-separated identifier, it will automatically connected to a single string.
-func (p *Parser) nextIdent(keywordAllowed bool) (res string, startToken *Token, endToken *Token) {
+// 3. For dot-separated identifier, it will automatically connected to a single string, and return one single token.
+func (p *Parser) nextIdent(keywordAllowed bool) (res *Token) {
 	var fullLit string
 	var skipDot bool
 	// if buffer containers a token, consume buffer first
-	if p.buf != nil && p.buf.Type == T_IDENT {
-		startToken, endToken = p.buf, p.buf
-		fullLit = p.buf.Value
+	if p.buf != nil {
+		res = p.buf
 		p.buf = nil
+		return
 	} else {
-		t := p.nextNonWhitespace()
-		tok, lit := t.Type, t.Value
+		p.peekNonWhitespace()
+		p.scanner.Scan()
+		lit := p.scanner.TokenText()
+		tok := toToken(lit)
 		if T_IDENT != tok && T_DOT != tok {
 			// can be keyword, change its token.Type
 			if IsKeyword(tok) && keywordAllowed {
-				t.Type = T_IDENT
+				tok = T_IDENT
 			} else {
+				// if its not valid ident or keyword or dot, save it to buffer until next scan
+				p.buf = &Token{
+					Type:  tok,
+					Raw:   lit,
+					Value: lit,
+					Prev:  p.currToken,
+				}
+				p.chainToken(p.buf)
 				return
 			}
 			// proceed with keyword as first literal
 		}
-		startToken, endToken = t, t
 		fullLit = lit
 		// if we have a leading dot, we need to skip dot handling in first iteration
 		skipDot = false
-		if t.Type == T_DOT {
+		if tok == T_DOT {
 			skipDot = true
 		}
 	}
@@ -195,22 +203,46 @@ func (p *Parser) nextIdent(keywordAllowed bool) (res string, startToken *Token,
 			if '.' != r {
 				break
 			}
-			endToken = p.next() // consume dot
+			p.scanner.Next() // consume dot
 		}
 		// scan next token, see if it's a identifier or keyword, if not, save it to p.buf until next p.next() calling
-		tok := p.next()
-		if IsKeyword(tok.Type) && keywordAllowed {
-			tok.Type = T_IDENT
+		p.scanner.Scan()
+		lit := p.scanner.TokenText()
+		tok := toToken(lit)
+		if IsKeyword(tok) && keywordAllowed {
+			tok = T_IDENT
 		}
-		if T_IDENT != tok.Type {
+		if T_IDENT != tok {
 			fullLit = fmt.Sprintf("%s.", fullLit)
-			p.buf = tok
-			break
+			res = &Token{
+				Type:  T_IDENT,
+				Raw:   fullLit,
+				Value: fullLit,
+				Pos:   p.scanner.Position,
+				Prev:  p.currToken,
+			}
+			p.chainToken(res)
+			p.buf = &Token{
+				Type:  tok,
+				Raw:   lit,
+				Value: lit,
+				Prev:  res,
+			}
+			p.chainToken(p.buf)
+			return
 		}
-		fullLit = fmt.Sprintf("%s.%s", fullLit, tok.Value)
-		endToken = tok
+		fullLit = fmt.Sprintf("%s.%s", fullLit, lit)
+	}
+
+	res = &Token{
+		Type:  T_IDENT,
+		Raw:   fullLit,
+		Value: fullLit,
+		Pos:   p.scanner.Position,
+		Prev:  p.currToken,
 	}
-	return fullLit, startToken, endToken
+	p.chainToken(res)
+	return
 }
 
 func (p *Parser) peek() rune {