-
Notifications
You must be signed in to change notification settings - Fork 116
/
write.go
216 lines (191 loc) · 5.46 KB
/
write.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
// Copyright 2020 InfluxData, Inc. All rights reserved.
// Use of this source code is governed by MIT
// license that can be found in the LICENSE file.
package api
import (
"context"
"strings"
"time"
http2 "github.com/influxdata/influxdb-client-go/v2/api/http"
"github.com/influxdata/influxdb-client-go/v2/api/write"
"github.com/influxdata/influxdb-client-go/v2/internal/log"
iwrite "github.com/influxdata/influxdb-client-go/v2/internal/write"
)
// WriteAPI is Write client interface with non-blocking methods for writing time series data asynchronously in batches into an InfluxDB server.
type WriteAPI interface {
// WriteRecord writes asynchronously line protocol record into bucket.
// WriteRecord adds record into the buffer which is sent on the background when it reaches the batch size.
// Blocking alternative is available in the WriteAPIBlocking interface
WriteRecord(line string)
// WritePoint writes asynchronously Point into bucket.
// WritePoint adds Point into the buffer which is sent on the background when it reaches the batch size.
// Blocking alternative is available in the WriteAPIBlocking interface
WritePoint(point *write.Point)
// Flush forces all pending writes from the buffer to be sent
Flush()
// Errors returns a channel for reading errors which occurs during async writes.
// Must be called before performing any writes for errors to be collected.
// The chan is unbuffered and must be drained or the writer will block.
Errors() <-chan error
}
// WriteAPIImpl provides main implementation for WriteAPI
type WriteAPIImpl struct {
service *iwrite.Service
writeBuffer []string
errCh chan error
writeCh chan *iwrite.Batch
bufferCh chan string
writeStop chan struct{}
bufferStop chan struct{}
bufferFlush chan struct{}
doneCh chan struct{}
bufferInfoCh chan writeBuffInfoReq
writeInfoCh chan writeBuffInfoReq
writeOptions *write.Options
}
type writeBuffInfoReq struct {
writeBuffLen int
}
func NewWriteAPI(org string, bucket string, service http2.Service, writeOptions *write.Options) *WriteAPIImpl {
w := &WriteAPIImpl{
service: iwrite.NewService(org, bucket, service, writeOptions),
writeBuffer: make([]string, 0, writeOptions.BatchSize()+1),
writeCh: make(chan *iwrite.Batch),
bufferCh: make(chan string),
bufferStop: make(chan struct{}),
writeStop: make(chan struct{}),
bufferFlush: make(chan struct{}),
doneCh: make(chan struct{}),
bufferInfoCh: make(chan writeBuffInfoReq),
writeInfoCh: make(chan writeBuffInfoReq),
writeOptions: writeOptions,
}
go w.bufferProc()
go w.writeProc()
return w
}
func (w *WriteAPIImpl) Errors() <-chan error {
if w.errCh == nil {
w.errCh = make(chan error)
}
return w.errCh
}
func (w *WriteAPIImpl) Flush() {
w.bufferFlush <- struct{}{}
w.waitForFlushing()
}
func (w *WriteAPIImpl) waitForFlushing() {
for {
w.bufferInfoCh <- writeBuffInfoReq{}
writeBuffInfo := <-w.bufferInfoCh
if writeBuffInfo.writeBuffLen == 0 {
break
}
log.Info("Waiting buffer is flushed")
time.Sleep(time.Millisecond)
}
for {
w.writeInfoCh <- writeBuffInfoReq{}
writeBuffInfo := <-w.writeInfoCh
if writeBuffInfo.writeBuffLen == 0 {
break
}
log.Info("Waiting buffer is flushed")
time.Sleep(time.Millisecond)
}
}
func (w *WriteAPIImpl) bufferProc() {
log.Info("Buffer proc started")
ticker := time.NewTicker(time.Duration(w.writeOptions.FlushInterval()) * time.Millisecond)
x:
for {
select {
case line := <-w.bufferCh:
w.writeBuffer = append(w.writeBuffer, line)
if len(w.writeBuffer) == int(w.writeOptions.BatchSize()) {
w.flushBuffer()
}
case <-ticker.C:
w.flushBuffer()
case <-w.bufferFlush:
w.flushBuffer()
case <-w.bufferStop:
ticker.Stop()
w.flushBuffer()
break x
case buffInfo := <-w.bufferInfoCh:
buffInfo.writeBuffLen = len(w.bufferInfoCh)
w.bufferInfoCh <- buffInfo
}
}
log.Info("Buffer proc finished")
w.doneCh <- struct{}{}
}
func (w *WriteAPIImpl) flushBuffer() {
if len(w.writeBuffer) > 0 {
log.Info("sending batch")
batch := iwrite.NewBatch(buffer(w.writeBuffer), w.writeOptions.RetryInterval())
w.writeCh <- batch
w.writeBuffer = w.writeBuffer[:0]
}
}
func (w *WriteAPIImpl) writeProc() {
log.Info("Write proc started")
x:
for {
select {
case batch := <-w.writeCh:
err := w.service.HandleWrite(context.Background(), batch)
if err != nil && w.errCh != nil {
w.errCh <- err
}
case <-w.writeStop:
log.Info("Write proc: received stop")
break x
case buffInfo := <-w.writeInfoCh:
buffInfo.writeBuffLen = len(w.writeCh)
w.writeInfoCh <- buffInfo
}
}
log.Info("Write proc finished")
w.doneCh <- struct{}{}
}
func (w *WriteAPIImpl) Close() {
if w.writeCh != nil {
// Flush outstanding metrics
w.Flush()
// stop and wait for buffer proc
close(w.bufferStop)
<-w.doneCh
close(w.bufferFlush)
close(w.bufferCh)
// stop and wait for write proc
close(w.writeStop)
<-w.doneCh
close(w.writeCh)
close(w.writeInfoCh)
close(w.bufferInfoCh)
w.writeCh = nil
// close errors if open
if w.errCh != nil {
close(w.errCh)
w.errCh = nil
}
}
}
func (w *WriteAPIImpl) WriteRecord(line string) {
b := []byte(line)
b = append(b, 0xa)
w.bufferCh <- string(b)
}
func (w *WriteAPIImpl) WritePoint(point *write.Point) {
line, err := w.service.EncodePoints(point)
if err != nil {
log.Errorf("point encoding error: %s\n", err.Error())
} else {
w.bufferCh <- line
}
}
func buffer(lines []string) string {
return strings.Join(lines, "")
}