From 80b251a0e1f7769267c54904050ba8d20bb04812 Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 21:56:56 +0900 Subject: [PATCH 1/7] Bump go version 1.8.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 654b496..8f32b13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.7.5 +FROM golang:1.8.0 RUN go get github.com/laher/goxc From 90ffabcb158d3320acc5f46ca88c16332a6a93ae Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 22:03:00 +0900 Subject: [PATCH 2/7] Make skipComments normal function --- parser/lexer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index 6bf5403..5449e1d 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -213,7 +213,7 @@ func (l *Lexer) scanNextToken() (int, string) { s := l.scanner.TokenText() for s == "!" || s == "#" { - l.skipComments() + skipComments(&l.scanner) token = int(l.scanner.Scan()) s = l.scanner.TokenText() @@ -224,10 +224,10 @@ func (l *Lexer) scanNextToken() (int, string) { return token, s } -func (l *Lexer) skipComments() { - ch := l.scanner.Next() +func skipComments(scanner *scanner.Scanner) { + ch := scanner.Next() for ch != '\n' && ch >= 0 { - ch = l.scanner.Next() + ch = scanner.Next() } } From 1f123939bbe8d658fb5ebf8e0c6473b68c026570 Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 22:07:45 +0900 Subject: [PATCH 3/7] Bring Lexer.curFilename and Lexer.filename together as Lexer.filename --- parser/lexer.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index 5449e1d..6170f23 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -174,12 +174,11 @@ var SYMBOL_TABLES = map[string]int{ } type Lexer struct { - scanner scanner.Scanner - tokens []int - pos int - filename string - curFilename string - e error + scanner scanner.Scanner + tokens []int + pos int + filename string + e error } type Error struct { @@ -200,7 +199,6 @@ func NewLexer(src io.Reader, filename string) *Lexer { lex.scanner.IsIdentRune = isIdentRune lex.tokens = []int{} lex.filename = filename - lex.curFilename = filename return &lex } @@ -237,7 +235,7 @@ func (l *Lexer) scanInclude(rawfilename string) error { return err } - baseDir := filepath.Dir(l.curFilename) + baseDir := filepath.Dir(l.filename) os.Chdir(baseDir) defer os.Chdir(curDir) @@ -252,11 +250,11 @@ func (l *Lexer) scanInclude(rawfilename string) error { prevScanner := l.scanner defer func() { l.scanner = prevScanner }() - prevFilename := l.curFilename - defer func() { l.curFilename = prevFilename }() + prevFilename := l.filename + defer func() { l.filename = prevFilename }() for _, rawpath := range rawpaths { - l.curFilename = rawpath + l.filename = rawpath log.Verbosef("--> Parsing ... %s\n", rawpath) f, err := os.Open(rawpath) @@ -355,7 +353,7 @@ func (l *Lexer) tokenize() { func (l *Lexer) Error(msg string) { l.e = &Error{ - Filename: l.curFilename, + Filename: l.filename, Line: l.scanner.Line, Column: l.scanner.Column, Message: msg, From 0256da95876d233b8a22720f2e10eb2907483f5c Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 22:48:52 +0900 Subject: [PATCH 4/7] Split Lexer into Tokenizer and Lexer --- parser/lexer.go | 259 +++++++++++++++++++++++++++--------------------- 1 file changed, 145 insertions(+), 114 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index 6170f23..1e60b98 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -173,127 +173,34 @@ var SYMBOL_TABLES = map[string]int{ "quorum_down": QUORUM_DOWN, } -type Lexer struct { +type Tokenizer struct { scanner scanner.Scanner - tokens []int - pos int filename string - e error -} - -type Error struct { - Message string - Filename string - Line int - Column int -} - -func (e *Error) Error() string { - return e.Message -} - -func NewLexer(src io.Reader, filename string) *Lexer { - var lex Lexer - lex.scanner.Init(src) - lex.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments - lex.scanner.IsIdentRune = isIdentRune - lex.tokens = []int{} - lex.filename = filename - return &lex -} - -func isIdentRune(ch rune, i int) bool { - return ch == '_' || ch == '.' || ch == '/' || ch == ':' || ch == '-' || ch == '+' || ch == '*' || ch == '?' || ch == '=' || ch == '&' || ch == '@' || unicode.IsLetter(ch) || unicode.IsDigit(ch) -} - -func (l *Lexer) scanNextToken() (int, string) { - token := int(l.scanner.Scan()) - s := l.scanner.TokenText() - - for s == "!" || s == "#" { - skipComments(&l.scanner) - - token = int(l.scanner.Scan()) - s = l.scanner.TokenText() - } - - log.Debugf("token text: %s\n", s) - - return token, s -} - -func skipComments(scanner *scanner.Scanner) { - ch := scanner.Next() - for ch != '\n' && ch >= 0 { - ch = scanner.Next() - } } -func (l *Lexer) scanInclude(rawfilename string) error { - curDir, err := filepath.Abs(".") - if err != nil { - return err - } - - baseDir := filepath.Dir(l.filename) - os.Chdir(baseDir) - defer os.Chdir(curDir) - - rawpaths, err := filepath.Glob(rawfilename) - if err != nil { - return err - } - - if len(rawpaths) < 1 { - return fmt.Errorf("warning: %s: No such file or directory", rawfilename) - } - - prevScanner := l.scanner - defer func() { l.scanner = prevScanner }() - prevFilename := l.filename - defer func() { l.filename = prevFilename }() - - for _, rawpath := range rawpaths { - l.filename = rawpath - log.Verbosef("--> Parsing ... %s\n", rawpath) - - f, err := os.Open(rawpath) - if err != nil { - return err - } - - l.scanner.Init(f) - l.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments - l.scanner.IsIdentRune = isIdentRune - l.tokenize() - - f.Close() - } - - return nil +func NewTokenizer(src io.Reader, filename string) *Tokenizer { + var t Tokenizer + t.scanner.Init(src) + t.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments + t.scanner.IsIdentRune = isIdentRune + t.filename = filename + return &t } -func (l *Lexer) Lex(lval *yySymType) int { - if len(l.tokens) == l.pos { - return EOF - } - token := l.tokens[l.pos] - l.pos++ - return token -} +func (t *Tokenizer) NextAll() ([]*Token, error) { + var tokens []*Token -func (l *Lexer) tokenize() { for { - token, s := l.scanNextToken() + token, s := t.scanNextToken() for s == "include" { - token, s = l.scanNextToken() + token, s = t.scanNextToken() - if err := l.scanInclude(s); err != nil { - l.Error(err.Error()) + if err := t.scanInclude(s); err != nil { + return nil, err } - token, s = l.scanNextToken() + token, s = t.scanNextToken() } if token == scanner.EOF { @@ -347,23 +254,147 @@ func (l *Lexer) tokenize() { token = SYMBOL_TABLES[s] } - l.tokens = append(l.tokens, token) + tokens = append(tokens, &Token{ + value: token, + filename: t.filename, + line: t.scanner.Line, + column: t.scanner.Column, + }) + } + + return tokens, nil +} + +func skipComments(scanner *scanner.Scanner) { + ch := scanner.Next() + for ch != '\n' && ch >= 0 { + ch = scanner.Next() + } +} + +func (t *Tokenizer) scanNextToken() (int, string) { + token := int(t.scanner.Scan()) + s := t.scanner.TokenText() + + for s == "!" || s == "#" { + skipComments(&t.scanner) + + token = int(t.scanner.Scan()) + s = t.scanner.TokenText() + } + + log.Debugf("token text: %s\n", s) + + return token, s +} + +func (t *Tokenizer) scanInclude(rawfilename string) error { + curDir, err := filepath.Abs(".") + if err != nil { + return err + } + + baseDir := filepath.Dir(t.filename) + os.Chdir(baseDir) + defer os.Chdir(curDir) + + rawpaths, err := filepath.Glob(rawfilename) + if err != nil { + return err + } + + if len(rawpaths) < 1 { + return fmt.Errorf("warning: %s: No such file or directory", rawfilename) + } + + prevScanner := t.scanner + defer func() { t.scanner = prevScanner }() + prevFilename := t.filename + defer func() { t.filename = prevFilename }() + + for _, rawpath := range rawpaths { + t.filename = rawpath + log.Verbosef("--> Parsing ... %s\n", rawpath) + + f, err := os.Open(rawpath) + if err != nil { + return err + } + + t.scanner.Init(f) + t.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments + t.scanner.IsIdentRune = isIdentRune + t.NextAll() + + f.Close() + } + + return nil +} + +type Token struct { + value int + filename string + line int + column int +} + +type Lexer struct { + tokens []*Token + pos int + e error +} + +type Error struct { + Message string + Filename string + Line int + Column int +} + +func (e *Error) Error() string { + return e.Message +} + +func NewLexer(tokens []*Token) *Lexer { + return &Lexer{tokens: tokens, pos: 0} +} + +func isIdentRune(ch rune, i int) bool { + return ch == '_' || ch == '.' || ch == '/' || ch == ':' || ch == '-' || ch == '+' || ch == '*' || ch == '?' || ch == '=' || ch == '&' || ch == '@' || unicode.IsLetter(ch) || unicode.IsDigit(ch) +} + +func (l *Lexer) curToken() *Token { + return l.tokens[l.pos] +} + +func (l *Lexer) Lex(lval *yySymType) int { + if len(l.tokens) == l.pos { + return EOF } + token := l.tokens[l.pos] + l.pos++ + return token.value } func (l *Lexer) Error(msg string) { + token := l.curToken() l.e = &Error{ - Filename: l.filename, - Line: l.scanner.Line, - Column: l.scanner.Column, + Filename: token.filename, + Line: token.line, + Column: token.column, Message: msg, } } func Parse(src io.Reader, filename string) error { yyErrorVerbose = true - l := NewLexer(src, filename) - l.tokenize() + t := NewTokenizer(src, filename) + tokens, err := t.NextAll() + if err != nil { + return err + } + l := NewLexer(tokens) if ret := yyParse(l); ret != 0 { return l.e } From 22ae7227b71845e00edb37a7bfcc54bb709903fc Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 22:53:13 +0900 Subject: [PATCH 5/7] Cut getting next token into nextToken() --- parser/lexer.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index 1e60b98..f4f54da 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -357,7 +357,7 @@ func (e *Error) Error() string { } func NewLexer(tokens []*Token) *Lexer { - return &Lexer{tokens: tokens, pos: 0} + return &Lexer{tokens: tokens, pos: -1} } func isIdentRune(ch rune, i int) bool { @@ -368,12 +368,16 @@ func (l *Lexer) curToken() *Token { return l.tokens[l.pos] } +func (l *Lexer) nextToken() *Token { + l.pos++ + return l.tokens[l.pos] +} + func (l *Lexer) Lex(lval *yySymType) int { - if len(l.tokens) == l.pos { + if (len(l.tokens) - 1) == l.pos { return EOF } - token := l.tokens[l.pos] - l.pos++ + token := l.nextToken() return token.value } From fc24485c215d47394d2b9778e4fe0384cefb8ac9 Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 23:02:07 +0900 Subject: [PATCH 6/7] Forget to append tokens of 'include' files --- parser/lexer.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index f4f54da..249da8e 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -188,7 +188,7 @@ func NewTokenizer(src io.Reader, filename string) *Tokenizer { } func (t *Tokenizer) NextAll() ([]*Token, error) { - var tokens []*Token + var result []*Token for { token, s := t.scanNextToken() @@ -196,9 +196,11 @@ func (t *Tokenizer) NextAll() ([]*Token, error) { for s == "include" { token, s = t.scanNextToken() - if err := t.scanInclude(s); err != nil { + tokens, err := t.scanInclude(s) + if err != nil { return nil, err } + result = append(result, tokens...) token, s = t.scanNextToken() } @@ -254,7 +256,7 @@ func (t *Tokenizer) NextAll() ([]*Token, error) { token = SYMBOL_TABLES[s] } - tokens = append(tokens, &Token{ + result = append(result, &Token{ value: token, filename: t.filename, line: t.scanner.Line, @@ -262,7 +264,7 @@ func (t *Tokenizer) NextAll() ([]*Token, error) { }) } - return tokens, nil + return result, nil } func skipComments(scanner *scanner.Scanner) { @@ -288,10 +290,10 @@ func (t *Tokenizer) scanNextToken() (int, string) { return token, s } -func (t *Tokenizer) scanInclude(rawfilename string) error { +func (t *Tokenizer) scanInclude(rawfilename string) ([]*Token, error) { curDir, err := filepath.Abs(".") if err != nil { - return err + return nil, err } baseDir := filepath.Dir(t.filename) @@ -300,11 +302,11 @@ func (t *Tokenizer) scanInclude(rawfilename string) error { rawpaths, err := filepath.Glob(rawfilename) if err != nil { - return err + return nil, err } if len(rawpaths) < 1 { - return fmt.Errorf("warning: %s: No such file or directory", rawfilename) + return nil, fmt.Errorf("warning: %s: No such file or directory", rawfilename) } prevScanner := t.scanner @@ -312,24 +314,29 @@ func (t *Tokenizer) scanInclude(rawfilename string) error { prevFilename := t.filename defer func() { t.filename = prevFilename }() + var result []*Token for _, rawpath := range rawpaths { t.filename = rawpath log.Verbosef("--> Parsing ... %s\n", rawpath) f, err := os.Open(rawpath) if err != nil { - return err + return nil, err } t.scanner.Init(f) t.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments t.scanner.IsIdentRune = isIdentRune - t.NextAll() + tokens, err := t.NextAll() + if err != nil { + return nil, err + } + result = append(result, tokens...) f.Close() } - return nil + return result, nil } type Token struct { From e635fd88241a4bab47736eac38c90c7b7e633330 Mon Sep 17 00:00:00 2001 From: y_uuki Date: Thu, 30 Mar 2017 23:08:24 +0900 Subject: [PATCH 7/7] Use new tokenizer for 'include' file instead of new scanner --- parser/lexer.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/parser/lexer.go b/parser/lexer.go index 249da8e..2784883 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -309,14 +309,8 @@ func (t *Tokenizer) scanInclude(rawfilename string) ([]*Token, error) { return nil, fmt.Errorf("warning: %s: No such file or directory", rawfilename) } - prevScanner := t.scanner - defer func() { t.scanner = prevScanner }() - prevFilename := t.filename - defer func() { t.filename = prevFilename }() - var result []*Token for _, rawpath := range rawpaths { - t.filename = rawpath log.Verbosef("--> Parsing ... %s\n", rawpath) f, err := os.Open(rawpath) @@ -324,10 +318,8 @@ func (t *Tokenizer) scanInclude(rawfilename string) ([]*Token, error) { return nil, err } - t.scanner.Init(f) - t.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments - t.scanner.IsIdentRune = isIdentRune - tokens, err := t.NextAll() + child := NewTokenizer(f, rawpath) + tokens, err := child.NextAll() if err != nil { return nil, err }