From 41a0fd5bdcd7b0d8ab9c16123e146acfd1e53f44 Mon Sep 17 00:00:00 2001 From: Markey Date: Thu, 18 Nov 2021 17:33:07 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20parser.nextIdent=20c?= =?UTF-8?q?oncat=20multiple=20lit=20into=20one=20token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- const.go | 16 +++++------ enum.go | 12 ++++---- field.go | 8 +++--- field_type.go | 26 ++++++++--------- namespace.go | 6 ++-- option.go | 10 +++---- parser.go | 74 +++++++++++++++++++++++++++++++++++-------------- parser_test.go | 72 ++++++++++++++++------------------------------- service.go | 28 +++++++++---------- service_test.go | 34 +++++++++++++++++++++++ struct.go | 4 +-- typedef.go | 6 ++-- 12 files changed, 170 insertions(+), 126 deletions(-) diff --git a/const.go b/const.go index 88fe673..76d7ae8 100644 --- a/const.go +++ b/const.go @@ -44,8 +44,8 @@ func (r *Const) parse(p *Parser) (err error) { return } p.peekNonWhitespace() - ident, _, _ := p.nextIdent(false) - r.Ident = ident + identTok := p.nextIdent(false) + r.Ident = identTok.Raw ru := p.peekNonWhitespace() if toToken(string(ru)) != T_EQUALS { return p.unexpected(string(ru), "=") @@ -141,14 +141,14 @@ func (r *ConstValue) parse(p *Parser) (err error) { } r.EndToken = r.Map.EndToken } else { - fullLit, startTok, endTok := p.nextIdent(false) - if startTok.Type != T_IDENT || endTok.Type != T_IDENT { - return p.unexpected("identifier", fullLit) + identTok := p.nextIdent(false) + if identTok.Type != T_IDENT { + return p.unexpected("identifier", identTok.Raw) } r.Type = CONST_VALUE_IDENT - r.StartToken = startTok - r.EndToken = endTok - r.Value = fullLit + r.StartToken = identTok + r.EndToken = identTok + r.Value = identTok.Raw } return diff --git a/enum.go b/enum.go index c15e93f..35540df 100644 --- a/enum.go +++ b/enum.go @@ -34,8 +34,8 @@ func (r *Enum) String() string { func (r *Enum) parse(p *Parser) (err error) { p.peekNonWhitespace() - fullLit, _, _ := p.nextIdent(false) - r.Ident = fullLit + identTok := p.nextIdent(false) + r.Ident = identTok.Raw ru := p.peekNonWhitespace() if toToken(string(ru)) != T_LEFTCURLY { return p.unexpected(string(ru), "{") @@ -103,16 +103,16 @@ func (r *EnumElement) patchToParentMap() { func (r *EnumElement) parse(p *Parser) (err error) { p.peekNonWhitespace() - fullLit, startTok, endTok := p.nextIdent(false) - r.StartToken = startTok - r.Ident = fullLit + identTok := p.nextIdent(false) + r.StartToken = identTok + r.Ident = identTok.Raw ru := p.peekNonWhitespace() // if there is no = after enum field identifier, then directly parse EndToken if toToken(string(ru)) != T_EQUALS { // parse options ru = p.peekNonWhitespace() if toToken(string(ru)) != T_LEFTPAREN { - r.EndToken = endTok + r.EndToken = identTok // parse separator if err = r.parseSeparator(p); err != nil { return err diff --git a/field.go b/field.go index d59e751..4c314fa 100644 --- a/field.go +++ b/field.go @@ -57,7 +57,7 @@ func (r *Field) parse(p *Parser) (err error) { // parse requiredness p.peekNonWhitespace() - tok := p.next() + tok := p.nextIdent(true) if tok.Value == "required" || tok.Value == "optional" { r.Requiredness = tok.Value } else { @@ -72,8 +72,8 @@ func (r *Field) parse(p *Parser) (err error) { // parse identifier p.peekNonWhitespace() - ident, _, endTok := p.nextIdent(false) - r.Ident = ident + identTok := p.nextIdent(false) + r.Ident = identTok.Raw // parse DefaultValue/Options ru = p.peekNonWhitespace() @@ -112,7 +112,7 @@ func (r *Field) parse(p *Parser) (err error) { return err } } else { - if err = r.parseEnd(endTok, p); err != nil { + if err = r.parseEnd(identTok, p); err != nil { return err } } diff --git a/field_type.go b/field_type.go index cc2a18b..ce8d3e8 100644 --- a/field_type.go +++ b/field_type.go @@ -42,37 +42,37 @@ func (r *FieldType) String() string { func (r *FieldType) parse(p *Parser) (err error) { p.peekNonWhitespace() - fullLit, startTok, endTok := p.nextIdent(true) - r.StartToken = startTok - if isBaseTypeToken(fullLit) { + identTok := p.nextIdent(true) + r.StartToken = identTok + if isBaseTypeToken(identTok.Raw) { r.Type = FIELD_TYPE_BASE - r.BaseType = fullLit - r.EndToken = endTok - } else if fullLit == "map" { + r.BaseType = identTok.Raw + r.EndToken = identTok + } else if identTok.Raw == "map" { r.Type = FIELD_TYPE_MAP - r.Map = NewMapType(startTok, r) + r.Map = NewMapType(identTok, r) if err = r.Map.parse(p); err != nil { return } r.EndToken = r.Map.EndToken - } else if fullLit == "set" { + } else if identTok.Raw == "set" { r.Type = FIELD_TYPE_SET - r.Set = NewSetType(startTok, r) + r.Set = NewSetType(identTok, r) if err = r.Set.parse(p); err != nil { return } r.EndToken = r.Set.EndToken - } else if fullLit == "list" { + } else if identTok.Raw == "list" { r.Type = FIELD_TYPE_LIST - r.List = NewListType(startTok, r) + r.List = NewListType(identTok, r) if err = r.List.parse(p); err != nil { return } r.EndToken = r.List.EndToken } else { r.Type = FIELD_TYPE_IDENT - r.Ident = fullLit - r.EndToken = endTok + r.Ident = identTok.Raw + r.EndToken = identTok } // parse options diff --git a/namespace.go b/namespace.go index bd95b5c..f7926c5 100644 --- a/namespace.go +++ b/namespace.go @@ -29,9 +29,11 @@ func (r *Namespace) String() string { } func (r *Namespace) parse(p *Parser) (err error) { - r.Name, _, _ = p.nextIdent(true) + identTok := p.nextIdent(true) + r.Name = identTok.Raw var endIdent *Token - r.Value, _, endIdent = p.nextIdent(true) + identTok = p.nextIdent(true) + r.Value = identTok.Raw ru := p.peekNonWhitespace() if toToken(string(ru)) != T_LEFTPAREN { r.EndToken = endIdent diff --git a/option.go b/option.go index 9c0bdf1..63754f7 100644 --- a/option.go +++ b/option.go @@ -29,9 +29,9 @@ func (r *Option) String() string { func (r *Option) parse(p *Parser) (err error) { // can't use keyword as option name - name, start, _ := p.nextIdent(false) - if start == nil || start.Type != T_IDENT { - return p.unexpected(name, "identifier") + identTok := p.nextIdent(false) + if identTok == nil || identTok.Type != T_IDENT { + return p.unexpected(identTok.Raw, "identifier") } // if there is no = token tok := p.nextNonWhitespace() @@ -50,10 +50,10 @@ func (r *Option) parse(p *Parser) (err error) { return err } - r.Name = name + r.Name = identTok.Raw r.Value = tok.Raw r.Parent = r - r.StartToken = start + r.StartToken = identTok r.EndToken = tok // since Options are always gathered in a slice during parent node parsing, we not need to link each Option with these pointers r.Next = nil diff --git a/parser.go b/parser.go index a449ff2..5ff85f9 100644 --- a/parser.go +++ b/parser.go @@ -152,36 +152,44 @@ func (p *Parser) nextComment(commentType int) (res *Token, err error) { return } -// TODO: concat dot-separated ident into one token // Find next identifier, it will consume white spaces during scanning. // 1. Allow leading && trailing dot. // 2. If keywordAllowed == true, it will allow keyword inside an identifier, e.g. enum.aaa.struct. In this case, the token for keyword will be replace to T_IDENT, since the meaning for it is no more a keyword. -// 3. For dot-separated identifier, it will automatically connected to a single string. -func (p *Parser) nextIdent(keywordAllowed bool) (res string, startToken *Token, endToken *Token) { +// 3. For dot-separated identifier, it will automatically connected to a single string, and return one single token. +func (p *Parser) nextIdent(keywordAllowed bool) (res *Token) { var fullLit string var skipDot bool // if buffer containers a token, consume buffer first - if p.buf != nil && p.buf.Type == T_IDENT { - startToken, endToken = p.buf, p.buf - fullLit = p.buf.Value + if p.buf != nil { + res = p.buf p.buf = nil + return } else { - t := p.nextNonWhitespace() - tok, lit := t.Type, t.Value + p.peekNonWhitespace() + p.scanner.Scan() + lit := p.scanner.TokenText() + tok := toToken(lit) if T_IDENT != tok && T_DOT != tok { // can be keyword, change its token.Type if IsKeyword(tok) && keywordAllowed { - t.Type = T_IDENT + tok = T_IDENT } else { + // if its not valid ident or keyword or dot, save it to buffer until next scan + p.buf = &Token{ + Type: tok, + Raw: lit, + Value: lit, + Prev: p.currToken, + } + p.chainToken(p.buf) return } // proceed with keyword as first literal } - startToken, endToken = t, t fullLit = lit // if we have a leading dot, we need to skip dot handling in first iteration skipDot = false - if t.Type == T_DOT { + if tok == T_DOT { skipDot = true } } @@ -195,22 +203,46 @@ func (p *Parser) nextIdent(keywordAllowed bool) (res string, startToken *Token, if '.' != r { break } - endToken = p.next() // consume dot + p.scanner.Next() // consume dot } // scan next token, see if it's a identifier or keyword, if not, save it to p.buf until next p.next() calling - tok := p.next() - if IsKeyword(tok.Type) && keywordAllowed { - tok.Type = T_IDENT + p.scanner.Scan() + lit := p.scanner.TokenText() + tok := toToken(lit) + if IsKeyword(tok) && keywordAllowed { + tok = T_IDENT } - if T_IDENT != tok.Type { + if T_IDENT != tok { fullLit = fmt.Sprintf("%s.", fullLit) - p.buf = tok - break + res = &Token{ + Type: T_IDENT, + Raw: fullLit, + Value: fullLit, + Pos: p.scanner.Position, + Prev: p.currToken, + } + p.chainToken(res) + p.buf = &Token{ + Type: tok, + Raw: lit, + Value: lit, + Prev: res, + } + p.chainToken(p.buf) + return } - fullLit = fmt.Sprintf("%s.%s", fullLit, tok.Value) - endToken = tok + fullLit = fmt.Sprintf("%s.%s", fullLit, lit) + } + + res = &Token{ + Type: T_IDENT, + Raw: fullLit, + Value: fullLit, + Pos: p.scanner.Position, + Prev: p.currToken, } - return fullLit, startToken, endToken + p.chainToken(res) + return } func (p *Parser) peek() rune { diff --git a/parser_test.go b/parser_test.go index 1f8a541..7493559 100644 --- a/parser_test.go +++ b/parser_test.go @@ -12,15 +12,12 @@ func newParserOn(def string) *Parser { func TestNextIdent_singleIdent(t *testing.T) { parser := newParserOn(` ab2 `) - lit, start, end := parser.nextIdent(false) + tok := parser.nextIdent(false) - if got, want := start.Type, T_IDENT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := end.Type, T_IDENT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := lit, "ab2"; got != want { + if got, want := tok.Raw, "ab2"; got != want { t.Errorf("got [%v] want [%v]", got, want) } } @@ -28,105 +25,84 @@ func TestNextIdent_singleIdent(t *testing.T) { func TestNextIdent_mulitpleWhitespace(t *testing.T) { parser := newParserOn(` ab2 `) - lit, start, end := parser.nextIdent(false) + tok := parser.nextIdent(false) - if got, want := start.Type, T_IDENT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := end.Type, T_IDENT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := lit, "ab2"; got != want { + if got, want := tok.Raw, "ab2"; got != want { t.Errorf("got [%v] want [%v]", got, want) } } func TestNextIdent_trailingDot(t *testing.T) { parser := newParserOn(` abc.def. `) - lit, start, end := parser.nextIdent(false) + tok := parser.nextIdent(false) - if got, want := start.Type, T_IDENT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := end.Type, T_DOT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := lit, "abc.def."; got != want { + if got, want := tok.Raw, "abc.def."; got != want { t.Errorf("got [%v] want [%v]", got, want) } } func TestNextIdent_leadingDot(t *testing.T) { parser := newParserOn(` .abc.def `) - lit, start, end := parser.nextIdent(false) + tok := parser.nextIdent(false) - if got, want := start.Type, T_DOT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := end.Type, T_IDENT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := lit, ".abc.def"; got != want { + if got, want := tok.Raw, ".abc.def"; got != want { t.Errorf("got [%v] want [%v]", got, want) } } func TestNextIdent_leadingAndTrailingDot(t *testing.T) { parser := newParserOn(` .abc.def. `) - lit, start, end := parser.nextIdent(false) + tok := parser.nextIdent(false) - if got, want := start.Type, T_DOT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := end.Type, T_DOT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := lit, ".abc.def."; got != want { + if got, want := tok.Raw, ".abc.def."; got != want { t.Errorf("got [%v] want [%v]", got, want) } } func TestNextIdent_underscore(t *testing.T) { parser := newParserOn(`abc_def_123 `) - lit, start, end := parser.nextIdent(false) + tok := parser.nextIdent(false) - if got, want := start.Type, T_IDENT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := start, end; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := lit, "abc_def_123"; got != want { + if got, want := tok.Raw, "abc_def_123"; got != want { t.Errorf("got [%v] want [%v]", got, want) } } func TestNextIdent_keyword(t *testing.T) { parser := newParserOn(`enum.def.struct `) - lit, start, end := parser.nextIdent(true) + tok := parser.nextIdent(true) - if got, want := start.Type, T_IDENT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := end.Type, T_IDENT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := lit, "enum.def.struct"; got != want { + if got, want := tok.Raw, "enum.def.struct"; got != want { t.Errorf("got [%v] want [%v]", got, want) } } func TestNextIdent_star(t *testing.T) { parser := newParserOn(` * `) - lit, start, end := parser.nextIdent(true) + tok := parser.nextIdent(true) - if got, want := start.Type, T_IDENT; got != want { - t.Errorf("got [%v] want [%v]", got, want) - } - if got, want := end.Type, T_IDENT; got != want { + if got, want := tok.Type, T_IDENT; got != want { t.Errorf("got [%v] want [%v]", got, want) } - if got, want := lit, "*"; got != want { + if got, want := tok.Raw, "*"; got != want { t.Errorf("got [%v] want [%v]", got, want) } } diff --git a/service.go b/service.go index dd62c9e..0c96ef7 100644 --- a/service.go +++ b/service.go @@ -33,8 +33,8 @@ func (r *Service) String() string { func (r *Service) parse(p *Parser) (err error) { p.peekNonWhitespace() - fullLit, _, _ := p.nextIdent(false) - r.Ident = fullLit + identTok := p.nextIdent(false) + r.Ident = identTok.Raw tok := p.nextNonWhitespace() if tok.Type == T_LEFTCURLY { r.Elems, err = r.parseFunctions(p) @@ -52,8 +52,8 @@ func (r *Service) parse(p *Parser) (err error) { return err } } else if tok.Value == "extends" { - fullLit, _, _ := p.nextIdent(false) - r.Extends = fullLit + identTok := p.nextIdent(false) + r.Extends = identTok.Raw tok := p.nextNonWhitespace() if tok.Type == T_LEFTCURLY { r.Elems, err = r.parseFunctions(p) @@ -139,13 +139,13 @@ func (r *Function) patchToParentMap() { func (r *Function) parse(p *Parser) (err error) { p.peekNonWhitespace() - ident, startTok, _ := p.nextIdent(true) - if ident == "oneway" { - r.StartToken = startTok + identTok := p.nextIdent(true) + if identTok.Raw == "oneway" { + r.StartToken = identTok r.Oneway = true p.peekNonWhitespace() - ident, _, _ := p.nextIdent(true) - if ident == "void" { + identTok := p.nextIdent(true) + if identTok.Raw == "void" { r.Void = true } else { r.FunctionType = NewFieldType(r) @@ -153,11 +153,11 @@ func (r *Function) parse(p *Parser) (err error) { return err } } - } else if ident == "void" { - r.StartToken = startTok + } else if identTok.Raw == "void" { + r.StartToken = identTok r.Void = true } else { - p.buf = startTok + p.buf = identTok r.FunctionType = NewFieldType(r) if err = r.FunctionType.parse(p); err != nil { return err @@ -165,8 +165,8 @@ func (r *Function) parse(p *Parser) (err error) { r.StartToken = r.FunctionType.StartToken } p.peekNonWhitespace() - ident, _, _ = p.nextIdent(false) - r.Ident = ident + identTok = p.nextIdent(false) + r.Ident = identTok.Raw // parse argument fields var rightParenTok *Token diff --git a/service_test.go b/service_test.go index d6edb3f..f06814c 100644 --- a/service_test.go +++ b/service_test.go @@ -250,6 +250,40 @@ func TestFunction_throwsMap(t *testing.T) { } } +func TestFunction_identWithDotArg(t *testing.T) { + parser := newParserOn(`applet.ChannelListResponse ChannelList(1: applet.ChannelListRequest req) (api.get = "/ky/feed/channel_list", api.serializer = "json"), //获取首页频道列表`) + n := NewFunction(nil) + if err := n.parse(parser); err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if got, want := n.Ident, "ChannelList"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := n.FunctionType.Type, FIELD_TYPE_IDENT; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := n.FunctionType.Ident, "applet.ChannelListResponse"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := len(n.Args), 1; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := n.Args[0].FieldType.Type, FIELD_TYPE_IDENT; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := n.Args[0].FieldType.Ident, "applet.ChannelListRequest"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := n.StartToken.Value, "applet.ChannelListResponse"; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } + if got, want := n.EndToken.Value, ","; got != want { + t.Errorf("got [%v] want [%v]", got, want) + } +} + func TestFunction_basic(t *testing.T) { parser := newParserOn(`double testDouble(1: double thing) // test double`) n := NewFunction(nil) diff --git a/struct.go b/struct.go index eb4c761..b7d0aa1 100644 --- a/struct.go +++ b/struct.go @@ -62,8 +62,8 @@ func (r *Struct) patchFieldToMap(node *Field) { func (r *Struct) parse(p *Parser) (err error) { p.peekNonWhitespace() - fullLit, _, _ := p.nextIdent(false) - r.Ident = fullLit + identTok := p.nextIdent(false) + r.Ident = identTok.Raw ru := p.peekNonWhitespace() if toToken(string(ru)) != T_LEFTCURLY { return p.unexpected(string(ru), "{") diff --git a/typedef.go b/typedef.go index e73042f..5cf6ec3 100644 --- a/typedef.go +++ b/typedef.go @@ -38,9 +38,9 @@ func (r *TypeDef) parse(p *Parser) (err error) { return p.unexpected(r.Type.Ident, "base type or map or list or set") } - fullLit, _, endTok := p.nextIdent(true) - r.Ident = fullLit - r.EndToken = endTok + identTok := p.nextIdent(true) + r.Ident = identTok.Raw + r.EndToken = identTok // parse options ru := p.peekNonWhitespace()