-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokenize.go
104 lines (84 loc) · 1.37 KB
/
tokenize.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
package evmasm
import (
"github.com/reiver/go-whitespace"
"bufio"
"io"
"unicode/utf8"
)
type tokenize struct {
reader io.Reader
bufreader *bufio.Reader
eof bool
}
func (receiver *tokenize) SetReader(reader io.Reader) error {
if nil == receiver {
return errNilReceiver
}
if nil != receiver.reader {
return errReaderFound
}
if nil == reader {
return errNilReader
}
receiver.reader = reader
bufreader := bufio.NewReader(reader)
if nil == bufreader {
return errInternalError
}
receiver.bufreader = bufreader
return nil
}
func (receiver *tokenize) Token(writer io.Writer) error {
if nil == receiver {
return errNilReceiver
}
if nil == writer {
return errNilWriter
}
bufreader := receiver.bufreader
if nil == bufreader {
return errNilReader
}
if receiver.eof {
return io.EOF
}
var n uint64
for {
r, size, err := bufreader.ReadRune()
if 0 > size {
return errInternalError
}
if io.EOF == err {
receiver.eof = true
if 0 >= n {
return io.EOF
} else {
return nil
}
}
if nil != err {
return err
}
if whitespace.IsWhitespace(r) {
if 0 < n {
return nil
} else {
continue
}
}
var b [utf8.UTFMax]byte
var p []byte
{
p = b[:]
m := utf8.EncodeRune(p, r)
p = p[:m]
}
{
_, err := writer.Write(p)
if nil != err {
return err
}
}
n += uint64(size)
}
}