From 1cfa9481f07659d2ca2368fc2a040c408f6780ae Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 5 Jul 2016 18:04:34 +0800 Subject: [PATCH 01/30] Add protocol: Cassandra --- glide.yaml | 5 + packetbeat/docs/fields.asciidoc | 39 + packetbeat/etc/beat.full.yml | 19 + packetbeat/etc/fields.yml | 26 + packetbeat/main.go | 1 + packetbeat/packetbeat.full.yml | 19 + packetbeat/protos/cassandra/cassandra.go | 200 ++++ packetbeat/protos/cassandra/compressor.go | 43 + packetbeat/protos/cassandra/config.go | 27 + packetbeat/protos/cassandra/frame.go | 900 ++++++++++++++++++ packetbeat/protos/cassandra/marshal.go | 587 ++++++++++++ packetbeat/protos/cassandra/parser.go | 173 ++++ packetbeat/protos/cassandra/pub.go | 88 ++ packetbeat/protos/cassandra/trans.go | 205 ++++ .../tests/system/config/packetbeat.yml.j2 | 7 + .../cassandra/v4/cassandra_create_index.pcap | Bin 0 -> 8765 bytes .../v4/cassandra_create_keyspace.pcap | Bin 0 -> 3346 bytes .../cassandra/v4/cassandra_create_table.pcap | Bin 0 -> 8502 bytes .../pcaps/cassandra/v4/cassandra_insert.pcap | Bin 0 -> 431 bytes .../pcaps/cassandra/v4/cassandra_select.pcap | Bin 0 -> 460 bytes .../v4/cassandra_select_via_index.pcap | Bin 0 -> 482 bytes .../tests/system/test_0062_cassandra.py | 231 +++++ vendor/gopkg.in/inf.v0/LICENSE | 28 + vendor/gopkg.in/inf.v0/benchmark_test.go | 210 ++++ vendor/gopkg.in/inf.v0/dec.go | 615 ++++++++++++ vendor/gopkg.in/inf.v0/dec_go1_2_test.go | 33 + vendor/gopkg.in/inf.v0/dec_internal_test.go | 40 + vendor/gopkg.in/inf.v0/dec_test.go | 379 ++++++++ vendor/gopkg.in/inf.v0/example_test.go | 62 ++ vendor/gopkg.in/inf.v0/rounder.go | 145 +++ .../gopkg.in/inf.v0/rounder_example_test.go | 72 ++ vendor/gopkg.in/inf.v0/rounder_test.go | 109 +++ 32 files changed, 4263 insertions(+) create mode 100644 packetbeat/protos/cassandra/cassandra.go create mode 100644 packetbeat/protos/cassandra/compressor.go create mode 100644 packetbeat/protos/cassandra/config.go create mode 100644 packetbeat/protos/cassandra/frame.go create mode 100644 packetbeat/protos/cassandra/marshal.go create mode 100644 packetbeat/protos/cassandra/parser.go create mode 100644 packetbeat/protos/cassandra/pub.go create mode 100644 packetbeat/protos/cassandra/trans.go create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_index.pcap create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_keyspace.pcap create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_table.pcap create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_insert.pcap create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_select.pcap create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_select_via_index.pcap create mode 100644 packetbeat/tests/system/test_0062_cassandra.py create mode 100644 vendor/gopkg.in/inf.v0/LICENSE create mode 100644 vendor/gopkg.in/inf.v0/benchmark_test.go create mode 100644 vendor/gopkg.in/inf.v0/dec.go create mode 100644 vendor/gopkg.in/inf.v0/dec_go1_2_test.go create mode 100644 vendor/gopkg.in/inf.v0/dec_internal_test.go create mode 100644 vendor/gopkg.in/inf.v0/dec_test.go create mode 100644 vendor/gopkg.in/inf.v0/example_test.go create mode 100644 vendor/gopkg.in/inf.v0/rounder.go create mode 100644 vendor/gopkg.in/inf.v0/rounder_example_test.go create mode 100644 vendor/gopkg.in/inf.v0/rounder_test.go diff --git a/glide.yaml b/glide.yaml index 4f0385b0784..b2451e75bd7 100644 --- a/glide.yaml +++ b/glide.yaml @@ -97,3 +97,8 @@ import: - zlib - package: github.com/klauspost/cpuid version: v1.0 +- package: https://github.com/go-inf/inf + version: v0.9.0 +- package: github.com/golang/snappy + version: d9eb7a3d35ec988b8585d4a0068e462c27d28380 + diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 5df23b3b7b7..d94c92d442d 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -14,6 +14,7 @@ grouped in the following categories: * <> * <> +* <> * <> * <> * <> @@ -401,6 +402,44 @@ type: dict Contains user configurable fields. +[[exported-fields-cassandra]] +== Cassandra Fields + +Cassandra v4/3 specific event fields. + + +[float] +=== cassandra_request + +type: dict + +Cassandra request field table. + + +[float] +=== cassandra_request.request_headers + +type: dict + +Cassandra request header field table. + + +[float] +=== cassandra_response + +type: dict + +Cassandra response field table. + + +[float] +=== cassandra_response.response_headers + +type: dict + +Cassandra response header field table. + + [[exported-fields-common]] == Common Fields diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 56a3c60bb50..9938d8d76b6 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -416,3 +416,22 @@ packetbeat.protocols.nfs: # # - process: app # cmdline_grep: gunicorn + +packetbeat.protocols.cassandra: + #Cassandra port for traffic monitoring. + ports: [9042] + # If this option is enabled, the raw message of the request (`cassandra_request` field) + # is sent to Elasticsearch. The default is false. + send_request: true + # If this option is enabled, the raw message of the response (`cassandra_response` field) + # is sent to Elasticsearch. The default is false. + send_response: true + # If this option is enabled, the raw message of the response (`cassandra_request_header` field) + # is sent to Elasticsearch. The default is false. + send_request_header: true + # If this option is enabled, the raw message of the response (`cassandra_response_header` field) + # is sent to Elasticsearch. The default is false. + send_response_header: true + # If this option is enabled, and also the flag indicates that the frame body was compressed, and + # the stream will decode by the compressor, currently support: `snappy` , The default is `nil`. + compressor: "snappy" diff --git a/packetbeat/etc/fields.yml b/packetbeat/etc/fields.yml index 8e9d7ee3148..32b805cafa7 100644 --- a/packetbeat/etc/fields.yml +++ b/packetbeat/etc/fields.yml @@ -1390,6 +1390,32 @@ description: NFS operation reply status. +- key: cassandra + title: "Cassandra" + description: Cassandra v4/3 specific event fields. + fields: + - name: cassandra_request + type: dict + dict-type: keyword + description: > + Cassandra request field table. + - name: cassandra_request.request_headers + type: dict + dict-type: keyword + description: > + Cassandra request header field table. + - name: cassandra_response + type: dict + dict-type: keyword + description: > + Cassandra response field table. + - name: cassandra_response.response_headers + type: dict + dict-type: keyword + description: > + Cassandra response header field table. + + - key: raw title: Raw description: These fields contain the raw transaction data. diff --git a/packetbeat/main.go b/packetbeat/main.go index 742d9d97295..8f7a2b22f09 100644 --- a/packetbeat/main.go +++ b/packetbeat/main.go @@ -8,6 +8,7 @@ import ( // import support protocol modules _ "github.com/elastic/beats/packetbeat/protos/amqp" + _ "github.com/elastic/beats/packetbeat/protos/cassandra" _ "github.com/elastic/beats/packetbeat/protos/dns" _ "github.com/elastic/beats/packetbeat/protos/http" _ "github.com/elastic/beats/packetbeat/protos/memcache" diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index b96c0acb2c5..20cff228e49 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -417,6 +417,25 @@ packetbeat.protocols.nfs: # - process: app # cmdline_grep: gunicorn +packetbeat.protocols.cassandra: + #Cassandra port for traffic monitoring. + ports: [9042] + # If this option is enabled, the raw message of the request (`cassandra_request` field) + # is sent to Elasticsearch. The default is false. + send_request: true + # If this option is enabled, the raw message of the response (`cassandra_response` field) + # is sent to Elasticsearch. The default is false. + send_response: true + # If this option is enabled, the raw message of the response (`cassandra_request_header` field) + # is sent to Elasticsearch. The default is false. + send_request_header: true + # If this option is enabled, the raw message of the response (`cassandra_response_header` field) + # is sent to Elasticsearch. The default is false. + send_response_header: true + # If this option is enabled, and also the flag indicates that the frame body was compressed, and + # the stream will decode by the compressor, currently support: `snappy` , The default is `nil`. + compressor: "snappy" + #================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go new file mode 100644 index 00000000000..e5285b36729 --- /dev/null +++ b/packetbeat/protos/cassandra/cassandra.go @@ -0,0 +1,200 @@ +package cassandra + +import ( + "time" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + + "github.com/elastic/beats/packetbeat/protos" + "github.com/elastic/beats/packetbeat/protos/tcp" + "github.com/elastic/beats/packetbeat/publish" +) + +// cassandra application level protocol analyzer plugin +type cassandra struct { + ports protos.PortsConfig + parserConfig parserConfig + transConfig transactionConfig + pub transPub +} + +// Application Layer tcp stream data to be stored on tcp connection context. +type connection struct { + streams [2]*stream + trans transactions +} + +// Uni-directioal tcp stream state for parsing messages. +type stream struct { + parser parser +} + +var ( + debugf = logp.MakeDebug("cassandra") + + // use isDebug/isDetailed to guard debugf/detailedf to minimize allocations + // (garbage collection) when debug log is disabled. + isDebug = false +) + +func init() { + protos.Register("cassandra", New) +} + +// New create and initializes a new cassandra protocol analyzer instance. +func New( + testMode bool, + results publish.Transactions, + cfg *common.Config, +) (protos.Plugin, error) { + p := &cassandra{} + config := defaultConfig + if !testMode { + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + } + + if err := p.init(results, &config); err != nil { + return nil, err + } + return p, nil +} + +func (cassandra *cassandra) init(results publish.Transactions, config *cassandraConfig) error { + if err := cassandra.setFromConfig(config); err != nil { + return err + } + cassandra.pub.results = results + + isDebug = logp.IsDebug("http") + return nil +} + +func (cassandra *cassandra) setFromConfig(config *cassandraConfig) error { + + // set module configuration + if err := cassandra.ports.Set(config.Ports); err != nil { + return err + } + + // set parser configuration + parser := &cassandra.parserConfig + parser.maxBytes = tcp.TCP_MAX_DATA_IN_STREAM + + // set parser's compressor, only `snappy` supported right now + if config.Compressor == Snappy { + parser.compressor = SnappyCompressor{} + } else { + parser.compressor = nil + } + + // set transaction correlator configuration + trans := &cassandra.transConfig + trans.transactionTimeout = config.TransactionTimeout + + // set transaction publisher configuration + pub := &cassandra.pub + pub.sendRequest = config.SendRequest + pub.sendResponse = config.SendResponse + pub.sendRequestHeader = config.SendRequestHeader + pub.sendResponseHeader = config.SendResponseHeader + + return nil +} + +// ConnectionTimeout returns the per stream connection timeout. +// Return <=0 to set default tcp module transaction timeout. +func (cassandra *cassandra) ConnectionTimeout() time.Duration { + return cassandra.transConfig.transactionTimeout +} + +// GetPorts returns the ports numbers packets shall be processed for. +func (cassandra *cassandra) GetPorts() []int { + return cassandra.ports.Ports +} + +// Parse processes a TCP packet. Return nil if connection +// state shall be dropped (e.g. parser not in sync with tcp stream) +func (cassandra *cassandra) Parse( + pkt *protos.Packet, + tcptuple *common.TcpTuple, dir uint8, + private protos.ProtocolData, +) protos.ProtocolData { + defer logp.Recover("Parse cassandra exception") + + conn := cassandra.ensureConnection(private) + st := conn.streams[dir] + if st == nil { + st = &stream{} + st.parser.init(&cassandra.parserConfig, func(msg *message) error { + return conn.trans.onMessage(tcptuple.IpPort(), dir, msg) + }) + conn.streams[dir] = st + } + + if err := st.parser.feed(pkt.Ts, pkt.Payload); err != nil { + debugf("%v, dropping TCP stream for error in direction %v.", err, dir) + cassandra.onDropConnection(conn) + return nil + } + return conn +} + +// ReceivedFin handles TCP-FIN packet. +func (cassandra *cassandra) ReceivedFin( + tcptuple *common.TcpTuple, dir uint8, + private protos.ProtocolData, +) protos.ProtocolData { + return private +} + +// GapInStream handles lost packets in tcp-stream. +func (cassandra *cassandra) GapInStream(tcptuple *common.TcpTuple, dir uint8, + nbytes int, + private protos.ProtocolData, +) (protos.ProtocolData, bool) { + conn := getConnection(private) + if conn != nil { + cassandra.onDropConnection(conn) + } + + return nil, true +} + +// onDropConnection processes and optionally sends incomplete +// transaction in case of connection being dropped due to error +func (cassandra *cassandra) onDropConnection(conn *connection) { +} + +func (cassandra *cassandra) ensureConnection(private protos.ProtocolData) *connection { + conn := getConnection(private) + if conn == nil { + conn = &connection{} + conn.trans.init(&cassandra.transConfig, cassandra.pub.onTransaction) + } + return conn +} + +func (conn *connection) dropStreams() { + conn.streams[0] = nil + conn.streams[1] = nil +} + +func getConnection(private protos.ProtocolData) *connection { + if private == nil { + return nil + } + + priv, ok := private.(*connection) + if !ok { + logp.Warn("cassandra connection type error") + return nil + } + if priv == nil { + logp.Warn("Unexpected: cassandra connection data not set") + return nil + } + return priv +} diff --git a/packetbeat/protos/cassandra/compressor.go b/packetbeat/protos/cassandra/compressor.go new file mode 100644 index 00000000000..fc3ddcb45f6 --- /dev/null +++ b/packetbeat/protos/cassandra/compressor.go @@ -0,0 +1,43 @@ +package cassandra + +import ( + "github.com/golang/snappy" +) + +type Compressor interface { + Name() string + Encode(data []byte) ([]byte, error) + Decode(data []byte) ([]byte, error) +} + +const Snappy string = "snappy" + +// SnappyCompressor implements the Compressor interface and can be used to +// compress incoming and outgoing frames. The snappy compression algorithm +// aims for very high speeds and reasonable compression. +type SnappyCompressor struct{} + + +func (s SnappyCompressor) Name() string { + return Snappy +} + +func (s SnappyCompressor) Encode(data []byte) ([]byte, error) { + return snappy.Encode(nil, data), nil +} + +func (s SnappyCompressor) Decode(data []byte) ([]byte, error) { + return snappy.Decode(nil, data) +} + +const LZ4 string = "lz4" + +type LZ4Compressor struct{ + //TODO +} + +const Deflate string = "deflate" + +type DeflateCompressor struct{ + //TODO +} diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go new file mode 100644 index 00000000000..721dcd425c7 --- /dev/null +++ b/packetbeat/protos/cassandra/config.go @@ -0,0 +1,27 @@ +package cassandra + +import ( + "github.com/elastic/beats/packetbeat/config" + "github.com/elastic/beats/packetbeat/protos" +) + +type cassandraConfig struct { + config.ProtocolCommon `config:",inline"` + SendRequestHeader bool `config:"send_request_header"` + SendResponseHeader bool `config:"send_response_header"` + Compressor string `config:"compressor"` +} + +var ( + defaultConfig = cassandraConfig{ + ProtocolCommon: config.ProtocolCommon{ + TransactionTimeout: protos.DefaultTransactionExpiration, + }, + SendRequestHeader: false, + SendResponseHeader: false, + } +) + +func (c *cassandraConfig) Validate() error { + return nil +} diff --git a/packetbeat/protos/cassandra/frame.go b/packetbeat/protos/cassandra/frame.go new file mode 100644 index 00000000000..669443f24d6 --- /dev/null +++ b/packetbeat/protos/cassandra/frame.go @@ -0,0 +1,900 @@ +/** +https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v4.spec + + The CQL binary protocol is a frame based protocol. Frames are defined as: + + 0 8 16 24 32 40 + +---------+---------+---------+---------+---------+ + | version | flags | stream | opcode | + +---------+---------+---------+---------+---------+ + | length | + +---------+---------+---------+---------+ + | | + . ... body ... . + . . + . . + +---------------------------------------- + + The protocol is big-endian (network byte order). + + some code derived from https://github.com/gocql/gocql + +*/ +package cassandra + +import ( + "errors" + "fmt" + "github.com/elastic/beats/libbeat/logp" + "io" + "io/ioutil" + "net" + "runtime" + "sync" +) + +var ( + ErrFrameTooBig = errors.New("frame length is bigger than the maximum allowed") +) + +type frameHeader struct { + version protoVersion + flags byte + stream int + op frameOp + length int + customPayload map[string][]byte +} + +func (f frameHeader) toMap() map[string]interface{} { + data := make(map[string]interface{}) + data["version"] = fmt.Sprintf("%d", f.version.version()) + data["flags"] = f.flags + data["stream"] = f.stream + data["op"] = f.op.String() + data["length"] = f.length + return data +} + +func (f frameHeader) String() string { + return fmt.Sprintf("[header version=%s flags=0x%x stream=%d op=%s length=%d]", f.version.String(), f.flags, f.stream, f.op.String(), f.length) +} + +func (f frameHeader) Header() frameHeader { + return f +} + +const defaultBufSize = 128 + +var framerPool = sync.Pool{ + New: func() interface{} { + return &framer{ + readBuffer: make([]byte, defaultBufSize), + } + }, +} + +// a framer is responsible for reading, writing and parsing frames on a single stream +type framer struct { + r io.Reader + + proto byte + // flags are for outgoing flags, enabling compression and tracing etc + flags byte + compres Compressor + headSize int + // if this frame was read then the header will be here + header *frameHeader + + // if tracing flag is set this is not nil + traceID []byte + + // holds a ref to the whole byte slice for rbuf so that it can be reset to + // 0 after a read. + readBuffer []byte + + rbuf []byte +} + +func newFramer(r io.Reader, compressor Compressor, version byte) *framer { + f := framerPool.Get().(*framer) + var flags byte + if compressor != nil { + flags |= flagCompress + } + + version &= protoVersionMask + + headSize := 8 + if version > protoVersion2 { + headSize = 9 + } + + f.compres = compressor + f.proto = version + f.flags = flags + f.headSize = headSize + + f.r = r + f.rbuf = f.readBuffer[:0] + + f.header = nil + f.traceID = nil + + return f +} + +type frame interface { + Header() frameHeader +} + +func readHeader(r io.Reader, p []byte) (head frameHeader, err error) { + _, err = io.ReadFull(r, p[:1]) + if err != nil { + return frameHeader{}, err + } + + version := p[0] & protoVersionMask + + if version < protoVersion1 || version > protoVersion4 { + return frameHeader{}, fmt.Errorf("unsupported response version: %d", version) + } + + headSize := 9 + if version < protoVersion3 { + headSize = 8 + } + + _, err = io.ReadFull(r, p[1:headSize]) + if err != nil { + return frameHeader{}, err + } + + p = p[:headSize] + + v := p[0] + + head.version = protoVersion(v) + + head.flags = p[1] + + if version > protoVersion2 { + if len(p) != 9 { + return frameHeader{}, fmt.Errorf("not enough bytes to read header require 9 got: %d", len(p)) + } + + head.stream = int(int16(p[2])<<8 | int16(p[3])) + head.op = frameOp(p[4]) + head.length = int(readInt(p[5:])) + } else { + if len(p) != 8 { + return frameHeader{}, fmt.Errorf("not enough bytes to read header require 8 got: %d", len(p)) + } + + head.stream = int(int8(p[2])) + head.op = frameOp(p[3]) + head.length = int(readInt(p[4:])) + } + + return head, nil +} + +// reads a frame form the wire into the framers buffer +func (f *framer) readFrame(head *frameHeader) error { + if head.length < 0 { + return fmt.Errorf("frame body length can not be less than 0: %d", head.length) + } else if head.length > maxFrameSize { + // need to free up the connection to be used again + _, err := io.CopyN(ioutil.Discard, f.r, int64(head.length)) + if err != nil { + return fmt.Errorf("error whilst trying to discard frame with invalid length: %v", err) + } + return ErrFrameTooBig + } + + if cap(f.readBuffer) >= head.length { + f.rbuf = f.readBuffer[:head.length] + } else { + f.readBuffer = make([]byte, head.length) + f.rbuf = f.readBuffer + } + + // assume the underlying reader takes care of timeouts and retries + n, err := io.ReadFull(f.r, f.rbuf) + if err != nil { + return fmt.Errorf("unable to read frame body: read %d/%d bytes: %v", n, head.length, err) + } + + // dealing with compressed frame body + if head.flags&flagCompress == flagCompress { + if f.compres == nil { + return errors.New("no compressor available with compressed frame body") + } + + f.rbuf, err = f.compres.Decode(f.rbuf) + if err != nil { + return err + } + } + + f.header = head + return nil +} + +func (f *framer) parseFrame(msg *message) (data map[string]interface{}, err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + + data = make(map[string]interface{}) + + data["request_type"] = f.header.op.String() + + if f.header.flags&flagTracing == flagTracing { + uuid, err := f.readUUID() + if err != nil { + f.traceID = uuid.Bytes() + // dealing with traceId + data["trace_id"] = uuid.String() + } + + } + + if f.header.flags&flagWarning == flagWarning { + warnings := f.readStringList() + // dealing with warnings + data["warnings"] = warnings + } + + if f.header.flags&flagCustomPayload == flagCustomPayload { + f.header.customPayload = f.readBytesMap() + } + + // assumes that the frame body has been read into rbuf + switch f.header.op { + case opError: + data = f.parseErrorFrame() + case opReady: + // the body should be empty + case opQuery: + data = f.parseQueryFrame() + case opResult: + data = f.parseResultFrame() + case opSupported: + data = f.parseSupportedFrame() + case opAuthenticate: + data = f.parseAuthenticateFrame() + case opAuthChallenge: + data = f.parseAuthChallengeFrame() + case opAuthSuccess: + data = f.parseAuthSuccessFrame() + case opEvent: + data = f.parseEventFrame() + case opOptions: + case opStartup: + default: + return nil, errors.New(fmt.Sprintf("unknown op and not parsed,%s", f.header)) + } + + return data, nil +} + +func (f *framer) parseErrorFrame() (data map[string]interface{}) { + + code := f.readInt() + msg := f.readString() + + errT := ErrType(code) + + data = make(map[string]interface{}) + data["err_code"] = code + data["err_msg"] = msg + data["err_type"] = errT.String() + + switch errT { + case errUnavailable: + cl := f.readConsistency() + required := f.readInt() + alive := f.readInt() + data["read_consistency"] = cl.String() + data["required"] = required + data["alive"] = alive + + case errWriteTimeout: + cl := f.readConsistency() + received := f.readInt() + blockfor := f.readInt() + writeType := f.readString() + + data["read_consistency"] = cl.String() + data["received"] = received + data["blockfor"] = blockfor + data["write_type"] = writeType + + case errReadTimeout: + cl := f.readConsistency() + received := f.readInt() + blockfor := f.readInt() + dataPresent := f.readByte() + + data["read_consistency"] = cl.String() + data["received"] = received + data["blockfor"] = blockfor + data["data_present"] = dataPresent + + case errAlreadyExists: + ks := f.readString() + table := f.readString() + + data["keyspace"] = ks + data["table"] = table + + case errUnprepared: + stmtId := f.readShortBytes() + + data["stmt_id"] = copyBytes(stmtId) + + case errReadFailure: + data["read_consistency"] = f.readConsistency().String() + data["received"] = f.readInt() + data["blockfor"] = f.readInt() + data["data_present"] = f.readByte() != 0 + + case errWriteFailure: + data["read_consistency"] = f.readConsistency().String() + data["received"] = f.readInt() + data["blockfor"] = f.readInt() + data["num_failures"] = f.readInt() + data["write_type"] = f.readString() + + case errFunctionFailure: + data["keyspace"] = f.readString() + data["function"] = f.readString() + data["arg_types"] = f.readStringList() + + case errInvalid, errBootstrapping, errConfig, errCredentials, errOverloaded, + errProtocol, errServer, errSyntax, errTruncate, errUnauthorized: + default: + logp.Err("unknown error code: 0x%x", code) + } + return data +} + +func (f *framer) parseSupportedFrame() (data map[string]interface{}) { + + data = make(map[string]interface{}) + data["supported"] = f.readStringMultiMap() + return data +} + +func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { + + meta := make(map[string]interface{}) + flags := f.readInt() + meta["flags"] = flags + colCount := f.readInt() + meta["col_count"] = colCount + + if getPKinfo { + + //only for prepared result + if f.proto >= protoVersion4 { + pkeyCount := f.readInt() + pkeys := make([]int, pkeyCount) + for i := 0; i < pkeyCount; i++ { + pkeys[i] = int(f.readShort()) + } + meta["pkey_columns"] = pkeys + } + } + + if flags&flagHasMorePages == flagHasMorePages { + meta["paging_state"] = fmt.Sprintf("%X", f.readBytes()) + } + + if flags&flagNoMetaData == flagNoMetaData { + return meta + } + + var keyspace, table string + globalSpec := flags&flagGlobalTableSpec == flagGlobalTableSpec + if globalSpec { + keyspace = f.readString() + table = f.readString() + meta["keyspace"] = keyspace + meta["table"] = table + } + + var cols []ColumnInfo + if colCount < 1000 { + // preallocate columninfo to avoid excess copying + cols = make([]ColumnInfo, colCount) + for i := 0; i < colCount; i++ { + f.readCol(&cols[i], globalSpec, keyspace, table) + } + + } else { + // use append, huge number of columns usually indicates a corrupt frame or + // just a huge row. + for i := 0; i < colCount; i++ { + var col ColumnInfo + f.readCol(&col, globalSpec, keyspace, table) + cols = append(cols, col) + } + } + + return meta +} + +func (f *framer) parseQueryFrame() (data map[string]interface{}) { + data = make(map[string]interface{}) + data["query"] = string(f.readBytes()) + return data +} + +func (f *framer) parseResultFrame() (data map[string]interface{}) { + + kind := f.readInt() + + data = make(map[string]interface{}) + switch kind { + case resultKindVoid: + data["result_type"] = "void" + case resultKindRows: + data["result_type"] = "rows" + data["rows"] = f.parseResultRows() + case resultKindSetKeyspace: + data["result_type"] = "set_keyspace" + data["keyspace"] = f.readString() + case resultKindPrepared: + data["result_type"] = "prepared" + data["result"] = f.parseResultPrepared() + case resultKindSchemaChanged: + data["result_type"] = "schemaChanged" + data["result"] = f.parseResultSchemaChange() + } + + return data +} + +func (f *framer) parseResultRows() map[string]interface{} { + + result := make(map[string]interface{}) + result["meta"] = f.parseResultMetadata(false) + result["num_rows"] = f.readInt() + + return result +} + +func (f *framer) parseResultPrepared() map[string]interface{} { + + result := make(map[string]interface{}) + + result["prepared_id"] = string(f.readShortBytes()) + result["req_meta"] = f.parseResultMetadata(true) + + if f.proto < protoVersion2 { + return result + } + + result["resp_meta"] = f.parseResultMetadata(false) + + return result +} + +func (f *framer) parseResultSchemaChange() (data map[string]interface{}) { + data = make(map[string]interface{}) + + if f.proto <= protoVersion2 { + change := f.readString() + keyspace := f.readString() + table := f.readString() + + data["change"] = change + data["keyspace"] = keyspace + data["table"] = table + } else { + change := f.readString() + target := f.readString() + + data["change"] = change + data["type"] = target + + switch target { + case "KEYSPACE": + data["keyspace"] = f.readString() + + case "TABLE", "TYPE": + data["keyspace"] = f.readString() + data["object"] = f.readString() + + case "FUNCTION", "AGGREGATE": + data["keyspace"] = f.readString() + data["name"] = f.readString() + data["args"] = f.readStringList() + + default: + logp.Warn("unknown SCHEMA_CHANGE target: %q change: %q", target, change) + } + } + return data +} + +func (f *framer) parseAuthenticateFrame() (data map[string]interface{}) { + data = make(map[string]interface{}) + data["class"] = f.readString() + return data +} + +func (f *framer) parseAuthSuccessFrame() (data map[string]interface{}) { + data = make((map[string]interface{})) + data["data"] = fmt.Sprintf("%q", f.readBytes()) + return data +} + +func (f *framer) parseAuthChallengeFrame() (data map[string]interface{}) { + data = make((map[string]interface{})) + data["data"] = fmt.Sprintf("%q", f.readBytes()) + return data +} + +func (f *framer) parseEventFrame() (data map[string]interface{}) { + + data = make((map[string]interface{})) + + eventType := f.readString() + data["event_type"] = eventType + + switch eventType { + case "TOPOLOGY_CHANGE": + data["change"] = f.readString() + host, port := f.readInet() + data["host"] = host + data["port"] = port + + case "STATUS_CHANGE": + data["change"] = f.readString() + host, port := f.readInet() + data["host"] = host + data["port"] = port + + case "SCHEMA_CHANGE": + // this should work for all versions + data = f.parseResultSchemaChange() + default: + logp.Err("unknown event type: %q", eventType) + } + + return data +} + +// explicitly enables tracing for the framers outgoing requests +func (f *framer) trace() { + f.flags |= flagTracing +} + +type UUID [16]byte + +// Bytes returns the raw byte slice for this UUID. A UUID is always 128 bits +// (16 bytes) long. +func (u UUID) Bytes() []byte { + return u[:] +} + +// String returns the UUID in it's canonical form, a 32 digit hexadecimal +// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. +func (u UUID) String() string { + var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} + const hexString = "0123456789abcdef" + r := make([]byte, 36) + for i, b := range u { + r[offsets[i]] = hexString[b>>4] + r[offsets[i]+1] = hexString[b&0xF] + } + r[8] = '-' + r[13] = '-' + r[18] = '-' + r[23] = '-' + return string(r) + +} + +// UUIDFromBytes converts a raw byte slice to an UUID. +func UUIDFromBytes(input []byte) (UUID, error) { + var u UUID + if len(input) != 16 { + return u, errors.New("UUIDs must be exactly 16 bytes long") + } + + copy(u[:], input) + return u, nil +} + +func copyBytes(p []byte) []byte { + b := make([]byte, len(p)) + copy(b, p) + return b +} + +type ColumnInfo struct { + Keyspace string + Table string + Name string + TypeInfo TypeInfo +} + +func (f *framer) readCol(col *ColumnInfo, globalSpec bool, keyspace, table string) { + if !globalSpec { + col.Keyspace = f.readString() + col.Table = f.readString() + } else { + col.Keyspace = keyspace + col.Table = table + } + + col.Name = f.readString() + col.TypeInfo = f.readTypeInfo() +} + +func (f *framer) readTypeInfo() TypeInfo { + + id := f.readShort() + + simple := NativeType{ + proto: f.proto, + typ: Type(id), + } + + if simple.typ == TypeCustom { + simple.custom = f.readString() + if cassType := getApacheCassandraType(simple.custom); cassType != TypeCustom { + simple.typ = cassType + } + } + + switch simple.typ { + case TypeTuple: + n := f.readShort() + tuple := TupleTypeInfo{ + NativeType: simple, + Elems: make([]TypeInfo, n), + } + + for i := 0; i < int(n); i++ { + tuple.Elems[i] = f.readTypeInfo() + } + + return tuple + + case TypeUDT: + udt := UDTTypeInfo{ + NativeType: simple, + } + udt.KeySpace = f.readString() + udt.Name = f.readString() + + n := f.readShort() + udt.Elements = make([]UDTField, n) + for i := 0; i < int(n); i++ { + field := &udt.Elements[i] + field.Name = f.readString() + field.Type = f.readTypeInfo() + } + + return udt + case TypeMap, TypeList, TypeSet: + collection := CollectionType{ + NativeType: simple, + } + + if simple.typ == TypeMap { + collection.Key = f.readTypeInfo() + } + + collection.Elem = f.readTypeInfo() + + return collection + } + + return simple +} + +func (f *framer) readByte() byte { + if len(f.rbuf) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to read byte require 1 got: %d", len(f.rbuf))) + } + + b := f.rbuf[0] + f.rbuf = f.rbuf[1:] + return b +} + +func (f *framer) readInt() (n int) { + if len(f.rbuf) < 4 { + panic(fmt.Errorf("not enough bytes in buffer to read int require 4 got: %d", len(f.rbuf))) + } + + n = int(int32(f.rbuf[0])<<24 | int32(f.rbuf[1])<<16 | int32(f.rbuf[2])<<8 | int32(f.rbuf[3])) + f.rbuf = f.rbuf[4:] + return +} + +func (f *framer) readShort() (n uint16) { + if len(f.rbuf) < 2 { + panic(fmt.Errorf("not enough bytes in buffer to read short require 2 got: %d", len(f.rbuf))) + } + n = uint16(f.rbuf[0])<<8 | uint16(f.rbuf[1]) + f.rbuf = f.rbuf[2:] + return +} + +func (f *framer) readLong() (n int64) { + if len(f.rbuf) < 8 { + panic(fmt.Errorf("not enough bytes in buffer to read long require 8 got: %d", len(f.rbuf))) + } + n = int64(f.rbuf[0])<<56 | int64(f.rbuf[1])<<48 | int64(f.rbuf[2])<<40 | int64(f.rbuf[3])<<32 | + int64(f.rbuf[4])<<24 | int64(f.rbuf[5])<<16 | int64(f.rbuf[6])<<8 | int64(f.rbuf[7]) + f.rbuf = f.rbuf[8:] + return +} + +func (f *framer) readString() (s string) { + size := f.readShort() + + if len(f.rbuf) < int(size) { + panic(fmt.Errorf("not enough bytes in buffer to read string require %d got: %d", size, len(f.rbuf))) + } + + s = string(f.rbuf[:size]) + f.rbuf = f.rbuf[size:] + return +} + +func (f *framer) readLongString() (s string) { + size := f.readInt() + + if len(f.rbuf) < size { + panic(fmt.Errorf("not enough bytes in buffer to read long string require %d got: %d", size, len(f.rbuf))) + } + + s = string(f.rbuf[:size]) + f.rbuf = f.rbuf[size:] + return +} + +func (f *framer) readUUID() (*UUID, error) { + if len(f.rbuf) < 16 { + return nil, fmt.Errorf("not enough bytes in buffer to read uuid require %d got: %d", 16, len(f.rbuf)) + } + + u, _ := UUIDFromBytes(f.rbuf[:16]) + f.rbuf = f.rbuf[16:] + return &u, nil +} + +func (f *framer) readStringList() []string { + size := f.readShort() + + l := make([]string, size) + for i := 0; i < int(size); i++ { + l[i] = f.readString() + } + + return l +} + +func (f *framer) readBytesInternal() ([]byte, error) { + size := f.readInt() + if size < 0 { + return nil, nil + } + + if len(f.rbuf) < size { + return nil, fmt.Errorf("not enough bytes in buffer to read bytes require %d got: %d", size, len(f.rbuf)) + } + + l := f.rbuf[:size] + f.rbuf = f.rbuf[size:] + + return l, nil +} + +func (f *framer) readBytes() []byte { + l, err := f.readBytesInternal() + if err != nil { + panic(err) + } + + return l +} + +func (f *framer) readShortBytes() []byte { + size := f.readShort() + if len(f.rbuf) < int(size) { + panic(fmt.Errorf("not enough bytes in buffer to read short bytes: require %d got %d", size, len(f.rbuf))) + } + + l := f.rbuf[:size] + f.rbuf = f.rbuf[size:] + + return l +} + +func (f *framer) readInet() (net.IP, int) { + if len(f.rbuf) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to read inet size require %d got: %d", 1, len(f.rbuf))) + } + + size := f.rbuf[0] + f.rbuf = f.rbuf[1:] + + if !(size == 4 || size == 16) { + panic(fmt.Errorf("invalid IP size: %d", size)) + } + + if len(f.rbuf) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to read inet require %d got: %d", size, len(f.rbuf))) + } + + ip := make([]byte, size) + copy(ip, f.rbuf[:size]) + f.rbuf = f.rbuf[size:] + + port := f.readInt() + return net.IP(ip), port +} + +func (f *framer) readConsistency() Consistency { + return Consistency(f.readShort()) +} + +func (f *framer) readStringMap() map[string]string { + size := f.readShort() + m := make(map[string]string) + + for i := 0; i < int(size); i++ { + k := f.readString() + v := f.readString() + m[k] = v + } + + return m +} + +func (f *framer) readBytesMap() map[string][]byte { + size := f.readShort() + m := make(map[string][]byte) + + for i := 0; i < int(size); i++ { + k := f.readString() + v := f.readBytes() + m[k] = v + } + + return m +} + +func (f *framer) readStringMultiMap() map[string][]string { + size := f.readShort() + m := make(map[string][]string) + + for i := 0; i < int(size); i++ { + k := f.readString() + v := f.readStringList() + m[k] = v + } + return m +} + +func readInt(p []byte) int32 { + return int32(p[0])<<24 | int32(p[1])<<16 | int32(p[2])<<8 | int32(p[3]) +} + +func readShort(p []byte) uint16 { + return uint16(p[0])<<8 | uint16(p[1]) +} diff --git a/packetbeat/protos/cassandra/marshal.go b/packetbeat/protos/cassandra/marshal.go new file mode 100644 index 00000000000..e54d406ccce --- /dev/null +++ b/packetbeat/protos/cassandra/marshal.go @@ -0,0 +1,587 @@ +package cassandra + +// Copyright (c) 2012 The gocql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +import ( + "bytes" + "fmt" + "math/big" + "reflect" + "time" + + "gopkg.in/inf.v0" + "strings" + "github.com/elastic/beats/libbeat/logp" +) + +// TypeInfo describes a Cassandra specific data type. +type TypeInfo interface { + Type() Type + Version() byte + Custom() string + + // New creates a pointer to an empty version of whatever type + // is referenced by the TypeInfo receiver + New() interface{} +} + +type NativeType struct { + proto byte + typ Type + custom string // only used for TypeCustom +} + +func (t NativeType) New() interface{} { + return reflect.New(goType(t)).Interface() +} + +func (s NativeType) Type() Type { + return s.typ +} + +func (s NativeType) Version() byte { + return s.proto +} + +func (s NativeType) Custom() string { + return s.custom +} + +func (s NativeType) String() string { + switch s.typ { + case TypeCustom: + return fmt.Sprintf("%s(%s)", s.typ, s.custom) + default: + return s.typ.String() + } +} + +type CollectionType struct { + NativeType + Key TypeInfo // only used for TypeMap + Elem TypeInfo // only used for TypeMap, TypeList and TypeSet +} + +func goType(t TypeInfo) reflect.Type { + switch t.Type() { + case TypeVarchar, TypeAscii, TypeInet, TypeText: + return reflect.TypeOf(*new(string)) + case TypeBigInt, TypeCounter: + return reflect.TypeOf(*new(int64)) + case TypeTimestamp: + return reflect.TypeOf(*new(time.Time)) + case TypeBlob: + return reflect.TypeOf(*new([]byte)) + case TypeBoolean: + return reflect.TypeOf(*new(bool)) + case TypeFloat: + return reflect.TypeOf(*new(float32)) + case TypeDouble: + return reflect.TypeOf(*new(float64)) + case TypeInt: + return reflect.TypeOf(*new(int)) + case TypeDecimal: + return reflect.TypeOf(*new(*inf.Dec)) + case TypeUUID, TypeTimeUUID: + return reflect.TypeOf(*new(UUID)) + case TypeList, TypeSet: + return reflect.SliceOf(goType(t.(CollectionType).Elem)) + case TypeMap: + return reflect.MapOf(goType(t.(CollectionType).Key), goType(t.(CollectionType).Elem)) + case TypeVarint: + return reflect.TypeOf(*new(*big.Int)) + case TypeTuple: + // what can we do here? all there is to do is to make a list of interface{} + tuple := t.(TupleTypeInfo) + return reflect.TypeOf(make([]interface{}, len(tuple.Elems))) + case TypeUDT: + return reflect.TypeOf(make(map[string]interface{})) + default: + return nil + } +} + +func (t CollectionType) New() interface{} { + return reflect.New(goType(t)).Interface() +} + +func (c CollectionType) String() string { + switch c.typ { + case TypeMap: + return fmt.Sprintf("%s(%s, %s)", c.typ, c.Key, c.Elem) + case TypeList, TypeSet: + return fmt.Sprintf("%s(%s)", c.typ, c.Elem) + case TypeCustom: + return fmt.Sprintf("%s(%s)", c.typ, c.custom) + default: + return c.typ.String() + } +} + +type TupleTypeInfo struct { + NativeType + Elems []TypeInfo +} + +func (t TupleTypeInfo) New() interface{} { + return reflect.New(goType(t)).Interface() +} + +type UDTField struct { + Name string + Type TypeInfo +} + +type UDTTypeInfo struct { + NativeType + KeySpace string + Name string + Elements []UDTField +} + +func (u UDTTypeInfo) New() interface{} { + return reflect.New(goType(u)).Interface() +} + +func (u UDTTypeInfo) String() string { + buf := &bytes.Buffer{} + + fmt.Fprintf(buf, "%s.%s{", u.KeySpace, u.Name) + first := true + for _, e := range u.Elements { + if !first { + fmt.Fprint(buf, ",") + } else { + first = false + } + + fmt.Fprintf(buf, "%s=%v", e.Name, e.Type) + } + fmt.Fprint(buf, "}") + + return buf.String() +} + +// String returns a human readable name for the Cassandra datatype +// described by t. +// Type is the identifier of a Cassandra internal datatype. +type Type int + +const ( + TypeCustom Type = 0x0000 + TypeAscii Type = 0x0001 + TypeBigInt Type = 0x0002 + TypeBlob Type = 0x0003 + TypeBoolean Type = 0x0004 + TypeCounter Type = 0x0005 + TypeDecimal Type = 0x0006 + TypeDouble Type = 0x0007 + TypeFloat Type = 0x0008 + TypeInt Type = 0x0009 + TypeText Type = 0x000A + TypeTimestamp Type = 0x000B + TypeUUID Type = 0x000C + TypeVarchar Type = 0x000D + TypeVarint Type = 0x000E + TypeTimeUUID Type = 0x000F + TypeInet Type = 0x0010 + TypeDate Type = 0x0011 + TypeTime Type = 0x0012 + TypeSmallInt Type = 0x0013 + TypeTinyInt Type = 0x0014 + TypeList Type = 0x0020 + TypeMap Type = 0x0021 + TypeSet Type = 0x0022 + TypeUDT Type = 0x0030 + TypeTuple Type = 0x0031 +) + +// String returns the name of the identifier. +func (t Type) String() string { + switch t { + case TypeCustom: + return "custom" + case TypeAscii: + return "ascii" + case TypeBigInt: + return "bigint" + case TypeBlob: + return "blob" + case TypeBoolean: + return "boolean" + case TypeCounter: + return "counter" + case TypeDecimal: + return "decimal" + case TypeDouble: + return "double" + case TypeFloat: + return "float" + case TypeInt: + return "int" + case TypeText: + return "text" + case TypeTimestamp: + return "timestamp" + case TypeUUID: + return "uuid" + case TypeVarchar: + return "varchar" + case TypeTimeUUID: + return "timeuuid" + case TypeInet: + return "inet" + case TypeDate: + return "date" + case TypeTime: + return "time" + case TypeSmallInt: + return "smallint" + case TypeTinyInt: + return "tinyint" + case TypeList: + return "list" + case TypeMap: + return "map" + case TypeSet: + return "set" + case TypeVarint: + return "varint" + case TypeTuple: + return "tuple" + default: + return fmt.Sprintf("unknown_type_%d", t) + } +} + +const ( + apacheCassandraTypePrefix = "org.apache.cassandra.db.marshal." +) + +// get Apache Cassandra types +func getApacheCassandraType(class string) Type { + switch strings.TrimPrefix(class, apacheCassandraTypePrefix) { + case "AsciiType": + return TypeAscii + case "LongType": + return TypeBigInt + case "BytesType": + return TypeBlob + case "BooleanType": + return TypeBoolean + case "CounterColumnType": + return TypeCounter + case "DecimalType": + return TypeDecimal + case "DoubleType": + return TypeDouble + case "FloatType": + return TypeFloat + case "Int32Type": + return TypeInt + case "ShortType": + return TypeSmallInt + case "ByteType": + return TypeTinyInt + case "DateType", "TimestampType": + return TypeTimestamp + case "UUIDType", "LexicalUUIDType": + return TypeUUID + case "UTF8Type": + return TypeVarchar + case "IntegerType": + return TypeVarint + case "TimeUUIDType": + return TypeTimeUUID + case "InetAddressType": + return TypeInet + case "MapType": + return TypeMap + case "ListType": + return TypeList + case "SetType": + return TypeSet + case "TupleType": + return TypeTuple + default: + return TypeCustom + } +} + +// error Types +type ErrType int + +const ( + errServer ErrType = 0x0000 + errProtocol ErrType = 0x000A + errCredentials ErrType = 0x0100 + errUnavailable ErrType = 0x1000 + errOverloaded ErrType = 0x1001 + errBootstrapping ErrType = 0x1002 + errTruncate ErrType = 0x1003 + errWriteTimeout ErrType = 0x1100 + errReadTimeout ErrType = 0x1200 + errReadFailure ErrType = 0x1300 + errFunctionFailure ErrType = 0x1400 + errWriteFailure ErrType = 0x1500 + errSyntax ErrType = 0x2000 + errUnauthorized ErrType = 0x2100 + errInvalid ErrType = 0x2200 + errConfig ErrType = 0x2300 + errAlreadyExists ErrType = 0x2400 + errUnprepared ErrType = 0x2500 +) + +func (this ErrType) String() string { + switch this { + case errUnavailable: + return "errUnavailable" + case errWriteTimeout: + return "errWriteTimeout" + case errReadTimeout: + return "errReadTimeout" + case errAlreadyExists: + return "errAlreadyExists" + case errUnprepared: + return "errUnprepared" + case errReadFailure: + return "errReadFailure" + case errWriteFailure: + return "errWriteFailure" + case errFunctionFailure: + return "errFunctionFailure" + case errInvalid: + return "errInvalid" + case errBootstrapping: + return "errBootstrapping" + case errConfig: + return "errConfig" + case errCredentials: + return "errCredentials" + case errOverloaded: + return "errOverloaded" + case errProtocol: + return "errProtocol" + case errServer: + return "errServer" + case errSyntax: + return "errSyntax" + case errTruncate: + return "errTruncate" + case errUnauthorized: + return "errUnauthorized" + } + + return fmt.Sprintf("ErrUnknown: 0x%x", this) +} + +const ( + protoDirectionMask = 0x80 + protoVersionMask = 0x7F + protoVersion1 = 0x01 + protoVersion2 = 0x02 + protoVersion3 = 0x03 + protoVersion4 = 0x04 + + maxFrameSize = 256 * 1024 * 1024 +) + +type protoVersion byte + +func (p protoVersion) request() bool { + v := p.version() + + if v < protoVersion1 || v > protoVersion4 { + logp.Err("unsupported request version: %x", v) + } + + if v == protoVersion4 { + return p == 0x04 + } + + if v == protoVersion3 { + return p == 0x03 + } + + return p == 0x00 +} + +func (p protoVersion) response() bool { + v := p.version() + + if v < protoVersion1 || v > protoVersion4 { + logp.Err("unsupported response version: %x", v) + } + + if v == protoVersion4 { + return p == 0x84 + } + + if v == protoVersion3 { + return p == 0x83 + } + + return p == 0x80 +} + +func (p protoVersion) version() byte { + return byte(p) & protoVersionMask +} + +func (p protoVersion) String() string { + dir := "REQ" + if p.response() { + dir = "RESP" + } + + return fmt.Sprintf("[version=%d direction=%s]", p.version(), dir) +} + +type frameOp byte + +const ( + // header ops + opError frameOp = 0x00 + opStartup frameOp = 0x01 + opReady frameOp = 0x02 + opAuthenticate frameOp = 0x03 + opOptions frameOp = 0x05 + opSupported frameOp = 0x06 + opQuery frameOp = 0x07 + opResult frameOp = 0x08 + opPrepare frameOp = 0x09 + opExecute frameOp = 0x0A + opRegister frameOp = 0x0B + opEvent frameOp = 0x0C + opBatch frameOp = 0x0D + opAuthChallenge frameOp = 0x0E + opAuthResponse frameOp = 0x0F + opAuthSuccess frameOp = 0x10 +) + +func (f frameOp) String() string { + switch f { + case opError: + return "ERROR" + case opStartup: + return "STARTUP" + case opReady: + return "READY" + case opAuthenticate: + return "AUTHENTICATE" + case opOptions: + return "OPTIONS" + case opSupported: + return "SUPPORTED" + case opQuery: + return "QUERY" + case opResult: + return "RESULT" + case opPrepare: + return "PREPARE" + case opExecute: + return "EXECUTE" + case opRegister: + return "REGISTER" + case opEvent: + return "EVENT" + case opBatch: + return "BATCH" + case opAuthChallenge: + return "AUTH_CHALLENGE" + case opAuthResponse: + return "AUTH_RESPONSE" + case opAuthSuccess: + return "AUTH_SUCCESS" + default: + return fmt.Sprintf("UNKNOWN_OP_%d", f) + } +} + +const ( + // result kind + resultKindVoid = 1 + resultKindRows = 2 + resultKindSetKeyspace = 3 + resultKindPrepared = 4 + resultKindSchemaChanged = 5 + + // rows flags + flagGlobalTableSpec int = 0x01 + flagHasMorePages int = 0x02 + flagNoMetaData int = 0x04 + + // query flags + flagValues byte = 0x01 + flagSkipMetaData byte = 0x02 + flagPageSize byte = 0x04 + flagWithPagingState byte = 0x08 + flagWithSerialConsistency byte = 0x10 + flagDefaultTimestamp byte = 0x20 + flagWithNameValues byte = 0x40 + + // header flags + flagCompress byte = 0x01 + flagTracing byte = 0x02 + flagCustomPayload byte = 0x04 + flagWarning byte = 0x08 +) + +type Consistency uint16 + +const ( + Any Consistency = 0x00 + One Consistency = 0x01 + Two Consistency = 0x02 + Three Consistency = 0x03 + Quorum Consistency = 0x04 + All Consistency = 0x05 + LocalQuorum Consistency = 0x06 + EachQuorum Consistency = 0x07 + LocalOne Consistency = 0x0A +) + +func (c Consistency) String() string { + switch c { + case Any: + return "ANY" + case One: + return "ONE" + case Two: + return "TWO" + case Three: + return "THREE" + case Quorum: + return "QUORUM" + case All: + return "ALL" + case LocalQuorum: + return "LOCAL_QUORUM" + case EachQuorum: + return "EACH_QUORUM" + case LocalOne: + return "LOCAL_ONE" + default: + return fmt.Sprintf("UNKNOWN_CONS_0x%x", uint16(c)) + } +} + +type SerialConsistency uint16 + +const ( + Serial SerialConsistency = 0x08 + LocalSerial SerialConsistency = 0x09 +) + +func (s SerialConsistency) String() string { + switch s { + case Serial: + return "SERIAL" + case LocalSerial: + return "LOCAL_SERIAL" + default: + return fmt.Sprintf("UNKNOWN_SERIAL_CONS_0x%x", uint16(s)) + } +} diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go new file mode 100644 index 00000000000..f656ad026c2 --- /dev/null +++ b/packetbeat/protos/cassandra/parser.go @@ -0,0 +1,173 @@ +package cassandra + +import ( + "errors" + "time" + + "bytes" + "fmt" + "github.com/elastic/beats/libbeat/common/streambuf" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/packetbeat/protos/applayer" +) + +type parser struct { + buf streambuf.Buffer + config *parserConfig + message *message + onMessage func(m *message) error +} + +type parserConfig struct { + maxBytes int + compressor Compressor +} + +type message struct { + applayer.Message + + // indicator for parsed message being complete or requires more messages + // (if false) to be merged to generate full message. + isComplete bool + + failed bool + data map[string]interface{} + header frameHeader + // list element use by 'transactions' for correlation + next *message + + transactionTimeout time.Duration + + results transactions +} + +// Error code if stream exceeds max allowed size on append. +var ( + ErrStreamTooLarge = errors.New("Stream data too large") +) + +func (p *parser) init( + cfg *parserConfig, + onMessage func(*message) error, +) { + *p = parser{ + buf: streambuf.Buffer{}, + config: cfg, + onMessage: onMessage, + } +} + +func (p *parser) append(data []byte) error { + _, err := p.buf.Write(data) + if err != nil { + return err + } + + if p.config.maxBytes > 0 && p.buf.Total() > p.config.maxBytes { + return ErrStreamTooLarge + } + return nil +} + +func (p *parser) feed(ts time.Time, data []byte) error { + if err := p.append(data); err != nil { + return err + } + + for p.buf.Total() > 0 { + if p.message == nil { + // allocate new message object to be used by parser with current timestamp + p.message = p.newMessage(ts) + } + + msg, err := p.parse() + if err != nil { + return err + } + if msg == nil { + break // wait for more data + } + + // reset buffer and message -> handle next message in buffer + p.buf.Reset() + p.message = nil + + // call message handler callback + if err := p.onMessage(msg); err != nil { + return err + } + } + + return nil +} + +func (p *parser) newMessage(ts time.Time) *message { + return &message{ + Message: applayer.Message{ + Ts: ts, + }, + } +} + +func (p *parser) parse() (*message, error) { + + r := bytes.NewReader(p.buf.Bytes()) + head, err := readHeader(r, make([]byte, 9)) + if err != nil { + logp.Err(err.Error()) + return nil, nil + } + + if logp.IsDebug("cassandra") { + logp.Debug("cassandra", fmt.Sprint(head)) + } + + framer := newFramer(r, p.config.compressor, byte(head.version)) + err = framer.readFrame(&head) + if err != nil { + logp.Err(err.Error()) + return nil, nil + } + msg := p.message + + data, err := framer.parseFrame(msg) + + if err != nil { + logp.Err(err.Error()) + return nil, nil + } + + dir := applayer.NetOriginalDirection + + isRequest := true + if head.version.response() { + dir = applayer.NetReverseDirection + isRequest = false + } + + //collect and wait for enough stream + _, err = p.buf.Collect(head.length + 9) + + if err == streambuf.ErrNoMoreBytes { + return nil, nil + } + + msg.Size = uint64(p.buf.BufferConsumed()) + msg.IsRequest = isRequest + msg.Direction = dir + + msg.data = data + msg.header = head + + if msg.IsRequest { + p.message.results.requests.append(msg) + } else { + p.message.results.responses.append(msg) + } + + if logp.IsDebug("cassandra") { + logp.Debug("cassandra", fmt.Sprint(msg)) + } + + return msg, nil +} diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go new file mode 100644 index 00000000000..c763c318fd5 --- /dev/null +++ b/packetbeat/protos/cassandra/pub.go @@ -0,0 +1,88 @@ +package cassandra + +import ( + "fmt" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/packetbeat/publish" +) + +// Transaction Publisher. +type transPub struct { + sendRequest bool + sendResponse bool + sendRequestHeader bool + sendResponseHeader bool + + results publish.Transactions +} + +func (pub *transPub) onTransaction(requ, resp *message) error { + if pub.results == nil { + return nil + } + + event := pub.createEvent(requ, resp) + pub.results.PublishTransaction(event) + return nil +} + +func (pub *transPub) createEvent(requ, resp *message) common.MapStr { + status := common.OK_STATUS + + if resp.failed { + status = common.ERROR_STATUS + } + + // resp_time in milliseconds + responseTime := int32(resp.Ts.Sub(requ.Ts).Nanoseconds() / 1e6) + + src := &common.Endpoint{ + Ip: requ.Tuple.Src_ip.String(), + Port: requ.Tuple.Src_port, + Proc: string(requ.CmdlineTuple.Src), + } + dst := &common.Endpoint{ + Ip: requ.Tuple.Dst_ip.String(), + Port: requ.Tuple.Dst_port, + Proc: string(requ.CmdlineTuple.Dst), + } + + event := common.MapStr{ + "@timestamp": common.Time(requ.Ts), + "type": "cassandra", + "status": status, + "responsetime": responseTime, + "bytes_in": requ.Size, + "bytes_out": resp.Size, + "src": src, + "dst": dst, + } + + // add processing notes/errors to event + if len(requ.Notes)+len(resp.Notes) > 0 { + event["notes"] = append(requ.Notes, resp.Notes...) + } + + if pub.sendRequest { + if pub.sendRequestHeader { + requ.data["request_headers"] = requ.header.toMap() + } + + event["cassandra_request"] = requ.data + } + + if pub.sendResponse { + if pub.sendResponseHeader { + resp.data["response_headers"] = resp.header.toMap() + } + + event["cassandra_response"] = resp.data + } + + if logp.IsDebug("cassandra") { + logp.Debug("cassandra", fmt.Sprint(event)) + } + + return event +} diff --git a/packetbeat/protos/cassandra/trans.go b/packetbeat/protos/cassandra/trans.go new file mode 100644 index 00000000000..dde21a7e1ac --- /dev/null +++ b/packetbeat/protos/cassandra/trans.go @@ -0,0 +1,205 @@ +package cassandra + +import ( + "time" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/packetbeat/procs" + "github.com/elastic/beats/packetbeat/protos/applayer" +) + +type transactions struct { + config *transactionConfig + + requests messageList + responses messageList + + onTransaction transactionHandler +} + +type transactionConfig struct { + transactionTimeout time.Duration +} + +type transactionHandler func(requ, resp *message) error + +// List of messages available for correlation +type messageList struct { + head, tail *message +} + +func (trans *transactions) init(c *transactionConfig, cb transactionHandler) { + trans.config = c + trans.onTransaction = cb +} + +func (trans *transactions) onMessage( + tuple *common.IpPortTuple, + dir uint8, + msg *message, +) error { + var err error + msg.Tuple = *tuple + msg.Transport = applayer.TransportTcp + msg.CmdlineTuple = procs.ProcWatcher.FindProcessesTuple(&msg.Tuple) + + if msg.IsRequest { + if isDebug { + debugf("Received request with tuple: %s", tuple) + } + err = trans.onRequest(tuple, dir, msg) + } else { + if isDebug { + debugf("Received response with tuple: %s", tuple) + } + err = trans.onResponse(tuple, dir, msg) + } + + return err +} + +// onRequest handles request messages, merging with incomplete requests +// and adding non-merged requests into the correlation list. +func (trans *transactions) onRequest( + tuple *common.IpPortTuple, + dir uint8, + msg *message, +) error { + prev := trans.requests.last() + merged, err := trans.tryMergeRequests(prev, msg) + if err != nil { + return err + } + if merged { + if isDebug { + debugf("request message got merged") + } + msg = prev + } else { + trans.requests.append(msg) + } + + if !msg.isComplete { + return nil + } + + if isDebug { + debugf("request message complete") + } + + return trans.correlate() +} + +// onRequest handles response messages, merging with incomplete requests +// and adding non-merged responses into the correlation list. +func (trans *transactions) onResponse( + tuple *common.IpPortTuple, + dir uint8, + msg *message, +) error { + prev := trans.responses.last() + merged, err := trans.tryMergeResponses(prev, msg) + if err != nil { + return err + } + if merged { + if isDebug { + debugf("response message got merged") + } + msg = prev + } else { + trans.responses.append(msg) + } + + if !msg.isComplete { + return nil + } + + if isDebug { + debugf("response message complete") + } + + return trans.correlate() +} + +func (trans *transactions) tryMergeRequests( + prev, msg *message, +) (merged bool, err error) { + + msg.isComplete = true + return false, nil +} + +func (trans *transactions) tryMergeResponses(prev, msg *message) (merged bool, err error) { + + msg.isComplete = true + return false, nil +} + +func (trans *transactions) correlate() error { + + requests := &trans.requests + responses := &trans.responses + + // drop responses with missing requests + if requests.empty() { + for !responses.empty() { + logp.Warn("Response from unknown transaction. Ignoring.") + responses.pop() + } + return nil + } + + // merge requests with responses into transactions + for !responses.empty() && !requests.empty() { + resp := responses.first() + if !resp.isComplete { + break + } + + requ := requests.pop() + responses.pop() + + if err := trans.onTransaction(requ, resp); err != nil { + return err + } + } + + return nil +} + +func (ml *messageList) append(msg *message) { + if ml.tail == nil { + ml.head = msg + } else { + ml.tail.next = msg + } + msg.next = nil + ml.tail = msg +} + +func (ml *messageList) empty() bool { + return ml.head == nil +} + +func (ml *messageList) pop() *message { + if ml.head == nil { + return nil + } + + msg := ml.head + ml.head = ml.head.next + if ml.head == nil { + ml.tail = nil + } + return msg +} + +func (ml *messageList) first() *message { + return ml.head +} + +func (ml *messageList) last() *message { + return ml.tail +} diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 4f284f015da..9542792dfb2 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -118,6 +118,13 @@ packetbeat.protocols.mongodb: {% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} {% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} +packetbeat.protocols.cassandra: + ports: [{{ cassandra_ports|default([9042])|join(", ") }}] +{% if cassandra_send_request %} send_request: true{%endif %} +{% if cassandra_send_response %} send_response: true{% endif %} +{% if cassandra_send_request_header %} send_request_header: true{% endif %} +{% if cassandra_send_response_header %} send_response_header: true{% endif %} + {% if procs_enabled %} #=========================== Monitored processes ============================== diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_index.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_index.pcap new file mode 100644 index 0000000000000000000000000000000000000000..39906151574cf8c0d52c8ad5e9d08ed9fabe3447 GIT binary patch literal 8765 zcmeHLYitzP6~61;wHMOTh*uDS{tBe}x|eC6%f+s@kGPsNjBQW_EXG zceBt~Bud3dXJ_`#o$sFSp2t1+UV8U;&sQ)X%U{-Ws$Cm+`#TlPzzdQJNR+Y6Tb(Ve zrIN9|jCn}T2N(6szqsMi8};vA7;jj%;FgR=PoTR0rc&2`v+0jO+gycjS3(|Y>xs7Z zMZ@c3?a__lu2^_j=b9dVIGGkx{E^kn(}Yek3o!qx-oAtV%SnfMjTP)MG?UmmG(EG< zt;3?8^FbOPdg$Y!<-@lgcIu$i5l+)Dd@KPB5*tT;a~dG&Cw?wm@`iWLst$mN^B>=niJ2 z(FwwN4mcCgh{R^LC;e|Xjy*%C%o!qPJ6+69fZ0}J)&gm^nU$fDR3$MP9_WbnM8mMz znntss(I)n*62!VrxfRM2i&cWy&Qin{e&}Ndpb&{o>_?~HDudGioUt6S(=>! z9X5sscQ327%)$JYJ$!UyAvW7#M*N-Qo&i>aZ%xbfL$N~3J?(WXX3co4@<#pHZusN_ z4uYiEX|mj#LpYEA)5nfs z5E7f+5#dT1oCe_hGRJO;aNw5$`6Y*Jwf%CjGol2`MiE0COmaQ%l4cqaC@+TBgd6jj zr7_$ZYY&^fb8pRDlXYGRS{K@O<(qD;ZJQi|O$<11%E6wro+9>Rz}b&ECb8Ii_Q`wP zIQA?W@R38v{qv^twl#6UejBkLf(YfBFaV+b0oyl7F z*3VtOJq7*}@s}W*%kdvk_^vx9Pn}D9_OTM&U-*Vw=REiB2)GNSxF4AIu_#gwiN!t8 zy82idoQCB><*FR_a}*(yppDNxm+bkYWOP3ZT4~viY34VHvtYI=Fln(Mo0a2;Kfys! z=Ykc`ii{k6pQC}|AARm0OsZ;17*di3*My;rASJ{!s*EaDC5ci(Nyphjg2G=lmEvgw zkJWfYDPmv+w_nbt7Fyx+O(98>0LG;qdB|_dl9_Y`jf*mVGNPj8-`2}~NE}WYf}y0K zwkjl*5zc0@YCdZ4G(qd;a9$RQOF~>jF(l|*Qq!`IUKJ+8IwdV7hh;8^$)sS1J}S`K zlsGDwp^0A*9TAiGsry_#!=>S*Xh3BU zGqf?xQ_Zdf*SH`hMWj*u#|7gGFs^s2!?<;Y#;vq6cCmG<^T`=-UrW>ZhmvAkXWxSU zpKryl#$1RwSjAatqZBt6Kr{!|P42$>#Ow7>;d!}jJRe0Lp1rM|ow(I%>Mos}NXUqq zp_t*pm@n29qXLAkJrFO%50}vC%XO+2kBEq+1dm8)64SCKM&!XrKEfis%5L7Lz>(y( zT-fy*nrQGi3OW`f*ESLezJ!{T(NM8+h1m?}bvBzDpdZqENUf9@PVW$sJRLU@CGzIB;}NglvGYpP30)qBb`q+kxuliVn2XM`a6EDE3oblzozRT{`%~;>^+44G}i&5 zsSc+}Z66CRrEa0mc<nHz5Em&M99{O?#*6np~m zdLGTu6wF1LfHyL#96MSJmY9)!L}>eu3&=GSj_rgC`-MMYuR%?pkxtb(dLa-;r( zSI0w3|Kuo6sHiwcIG=R+86_POniFIt_0y&n*O3c~oSMw#e_g+}Y0X(|spMOz!0aQJ zEB!Z@zA(wAr)jUZf(3iqI-(m|g|?2?*b~t^Jh9l;?JuHY{MfC+`I<#NG<@dr^=0Fs z#pfMVpn`Hm1(mnCr|87u%B(Ut4Zz8rSY%!#9Jq^2HD)VhW+xV;k!Q$L#Oz45TO(_7 zuP?i{Y3&?TkYb)0X_R@9aO!|F0gXs(BklR_3O9~5Lu==lGepdCCl(eCm~mp(0-4c? z1u3+!1Tno}tLh*&;4Y2x#D13Y6rET+wZ%=$g@e*Ke`1k&i6#h9EmKr?K(bLiqZ11v zIG8TB+{LSI#q1d$z>H78CqHmljud-|aNgYIDLSz@wb6}Z6P$nt3?RIO{E0>86~X~Q zP6Vl7$P)bGCl;jD#Aq?Qx_-A-4t5v7?wo@iY4wWf2H30@o>+7w%iuHsCwF3zIZin6 z%OUbhT4L`0i3MrB30fDX&ehx9TH7|c0GqrA=iTgJPg)-*oSne=2l6wC6_uV(=O1<6(We z>n8-{0eee$(sQTW9+3B>mc8(#SSe3>@Kq06=4Yl@p5$LL^^-C<4a;NRg*i_;Ngg=~ zCqkCEw~4&BiM+RoysMkYIKu4Srq14rAk%)en>xzFe|@cZMc>`#&Y#vw+_E=w`NsRG zXP$TDValH;31|1~9<~8{ViGG4d)JG9cH>z2{Kos&HX)xQ4|~?S^6*LIVTGzSTJs14 z#lhV^Wk}F?U-wNd=50F#*1Z$%|HBh*3NG(&i-UB*x81>&57L_-B1jK7g7o(kr2hru CfMqlQ literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_keyspace.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_keyspace.pcap new file mode 100644 index 0000000000000000000000000000000000000000..8cf49f633438264fe621d59259bc9235d84beebb GIT binary patch literal 3346 zcmds(Z%i9?7{`AFN?}lB1V)qb;sjicU>#)9F(uNJgD$S^Y$1fiq~6L2WQEqdiXn>Y z%z`m2DxewJrqLH_!XSz-G#E7tzQAZgVi;k;3<)H@(D?6-5u&Z%r+01dTGE(ozLDff z%W?PpJ)iIId(V#>Uw-sX9xGt!ALaXKtJHJk<3jd1{y>$7;u%|f;P*DxR>;^n#tc*| z<~nc9D!;T>dF@)%a&$8`kBf$)1Kr!M>)qRjtDshC!n4n#`W;^0-ogymbFJKSoHY;>WZBA{;HR<2MI<32_K#W{ zIjbawgM$GfG8h{6KP3brA&G8m;>I3khGST*j}}u zQ|DbV6G+W!M`yIW>4Vt4S{PSC=kYg8ECdZIRp+cPo~2V7Uc2?gPVey}QA^Wg%%9TP zg(({sSUg^e=WWdFb2xdI-S2SP-JLu$DGs%>qNGRcK-wh2cnQmAV&uK9Uw);u&Pz+9 zh3RN{!-NI{%e!_-=kTlv4v|YKj3fT z_kkV#no6EJI=2Q) zp;}b392mh+KnQX@PTtFNcpF=-i3?U0EN>wjtgKq!kOpi6V6*#yO)Qz%IvS!}wNy4GmGLc3qsB$)RI`PBdkgk#vxQYRZ8J@!DV&Cq zN2YSoxq?b>q*;lD%E*dSkj$L?n&4w*R@V27B%BF~{%57Zh$!PQ zEYX}T>~7p*hBgI6GmPm9UFt#6vc|+&g)a>usoy5xP#X|!0UTe#aGxaDD4jT>Z2EHy zw^wM^QqM75H8hbu+*-EH^S|lsQR39uyjMBzLY%(U*i81ANGG&o&Kj5##ta?pp<%(H zks6quX}o>Ah*K@a$&R}Hf%zUf%6w7opVfDS0MRKrjiW4#0KJkXqmDe&M79N;^XQ05 zeNo2#VtO6*I1DF_Ln2EJOhpG|bwt*Ny6u4(+Xrm5M&FPIY!P4=_XC^WHD?XX@7iVWDwqr+B-VpAXrC&cV+_9{(S|^%p5E)>oL{+TMVylVvsb9YwRM-8Pa*R zpe$=(@^9wQu|Ox9vgOTZg Bd)@#5 literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_table.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_table.pcap new file mode 100644 index 0000000000000000000000000000000000000000..f5caae7d4f3eea3ad27b8f6be38bdd9c6185dc38 GIT binary patch literal 8502 zcmeHLacmRk75_|P?1Z?OBx2TvG`C0#%`!tEgeH(CCb&%?Nh>E&n{MN{++A!dw$I#M z0)fF22&j|RRa#hS%G8F2P1_$`)^wwtSVgr>n)qW(T8gIrH_F=oKy9Z<9e*VIy}Pr0 zXFD!yR9GcWdOqKGcklcC-h1Es-n)1IIDdW>D`nZk_|LZ~PyJe3!)9@VWECXD*el~d zZDVb#8T&qC4w9>$jo}Ro9bLDp=Wmbuwp>`8!l1)l-hXqZuK(tNGtgG=!n=cz?)Gpf z*cbBm1$TFa{3EKQsQyjme*GU86aBHI=6^EW*&PfI`1ged>Tqi)$tNVgCLPx3DxSG2 ze|RTzG(e}w+|1e7yZ**=PZQ*i0QnINlbDbjg74TNZwz1c5M021bKF~hZgGVTV1;sat}TC&q@QF*U^t5_9TiAqf8ggj_0ZDLaI#6&u4eJz!iO_kJ^M zp#lgqv!utNdRP=NNK7i?h-lYiGl>r-c{)SoHS**nyy(XJH4x_=;%~#ex%eM_#y+nZ zJ#XHwexr8WyX932ewz0+9sl<{>;&c|G4UTi@#7+Te9)t0@XwJRMB!G7!XTu%H`Ens z@AIc5NvZQIDS_ifQIS-&&aVnlDZz822+WwA^gkAUqT8>IshX4s(9a<;!=i80ZdmAaAjqaI%m~pO={W*D6F@{_GCMXsVAo?znkj9{>=EN$k{z z?-kMGgPySrvDe8!2)V5kavi1)2hJsIwc!rtH2hIYC+1~qEN;Z*19l5AT7|mjo&&I4 z&n36pfteQH3yU8`OwL#c60q0FbH91k!%jmdiAnIpbl9%PG}KWvx}n{g7c&Y z1h)}EWK~@U5p-k;>W(SMgdX&M!$0g3!u5{xI|Ht-Og$(Qs`r5^m))_Ac2pL2SHSLN z3p+wJPkMeo?O{L1BqS!gSLLr4(c^=j7c%TFk{-lm1I48T*>A>Wt#w6lhK~XoKNy$P zY)I;9kkb==i@zqD8EgE(o;`m3>C9a{SEhsa7EtH1uj;jc@k4;PN4 zwTOV+ND^!85s;0feF`G!RG~=v?i_Y504Fgb$#wi+&ll0-t3TyL#bMS@mnb49v5t@> zO7??0>7d15#XLqDG}jPYZ7d?@NtY2iFXmz8+4X{}^@DL)PH;o9xF#vwP>K_xd=fQ> zi@5||h{lo;_8=bf;8IQ`q@+d~@E3)NhW*gIxrRn)zswGK1x0`|Nf^$m{HiGEDU4Gj zUc^s|k15%^)uJ@SkHj@jizOsZleu_oRAObU9F?AwB-N@uT?umz#rcTJ9s(9`MBpL{ zsw7U81UV_HcwtpcM0&4G@P|1)xbX{eqkJ6qzLt;@h5?4+BWjdONlHu>@dTHkS1S|v zm-kX-LA>kKlq8JAc?~c@MpMRs$MA-=dP8-j=Oo;6w@x0p{`jw|kK%tlgq&AO{=vB< z*wuwor6M0v2^E*(6)mRQ7!UP4(L;Lz9mgOg5T_95ReG~kRw4l&4v9(u0YiLJRQP~6 z7|6OQ&>K4<^~EGb65BJr>(vxqlOot-nTKqpMd87y92YULYTH(Q7}l^3>*e*g={}@V zh>j$Oxww>!Xi>7@@F3l3q!K&>E>I9$OxD3s#00vYX&_AWUq*=rBYlKl<8ZhBZkmpl zaETSKAL+9fR}}efqA8Cg33e&-Y^I02pGv(mE#7q>UOw40GjaOUzv}nU78qwb>H5ZT z)(2?-M@3W9lc8{XsHd;9D>R7|gPV?baU;FV*5@uWr=DN(S6Nvml^`fQgS?7}rdHogiX!wmEEudreUUOacRM-v zS&MVsxz0t-)3@S$_u7*v(jPsS&ahrEM}wdsd2+HMq8jzEb(TlC!7)uzv4~5Dk!o<$ zosv~!sh6=*Z7gMAq(#0=B5~?HFz%No^O^n z*hN)c5VEqKoDjVOM7hlR&)5;cBlN-M@{NIJd~K+o$TA~Dvj<6!v#czyCvTjx>oLB% zG}r21U5HtxCpYwfSqm|1gRH10CqV50luK+%wF9ycQ$g%_A!2J@D)p=|LPkN^w9*`-|^^2{n5- zpZ#Mwox2e%AMD2}xQ<{sYGF^P$4JjZ&~qBzB{71=OVdF}S;K?fGpH zJ-+&c^G1gI1g(%stZQVg`#m|z+2y-}dsF_yQ;5oC%HR7vx%t81%hHp#Vwv4<+n6)m zrQP$kt&OU+yV|xP*#>D++Gx`y;uqN_VZPlIb@#snE&EA!`JDg%{w0ViiHgqr8u`5~ Y_6oqPe$J(?-SnYDSZk>yr~;VyFFOPoBme*a literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_insert.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_insert.pcap new file mode 100644 index 0000000000000000000000000000000000000000..95efed52e418899c8bdd8effa46983e66dce357f GIT binary patch literal 431 zcmca|c+)~A1{MYcfUs}e$PYiVl7nFpPy&RRfY_CRVIK1|2L=Z=28MbD21XD*^HDj7 zDWmE*=h5gZ4HDYB|4jl4GIDSQ7u-w?F7V?6n)Zhss5cRaeLVewU4udtJpDra6-tX! zi;5LAKvaBYijIOpT3%vqs*XYqh|=UzPzZDM33Uxt&@ePNG1XB}&&topQwP$;xtS#y z>YCOJj3z)MQy4fHSk3g(`g)Z>E*4zH!C(Ot24Sd+O}-kVx>z~r%tw%GnKG&k8w3nu z|7jq*_+}!=9H#FaAm4&9)Eq~nB0T0u0L?Lln^OYveGAa%96&Qb;mHUx)EH zWmFq-9*ut3AfdhcUldS~k%KF^(Edws!Oaq&X@A*)dNqMqB-qu*)j32#OTjJ3-&diu zIJKzQnt{;-D3!v%!N6*!mmz=PJIH1aHC_e_pfCtSZ8n*2hibEO(3y`QTY4KT#zyR_^3(yA~K>d+WGZ=w1GXqy{ zWp-+1aY15oDg!InCk*T$Dn2uXfs=tXEiW-Qm4TOmH3!N7nhDeg3=Ou6AUZ2QBM(Tk o7UyP`WPn_{9_UhZA9s8}btyEglR1yZI5!Ar??Ct%B>(^b literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_select_via_index.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_select_via_index.pcap new file mode 100644 index 0000000000000000000000000000000000000000..240d9d2f075d2e4e97eaa6e368bf7f8333f01917 GIT binary patch literal 482 zcmca|c+)~A1{MYcfUu=*=7%r6%E-_FlmKBSAa-S7s9FBmfx&@|fuWv(ff0nyd{ho% z%BYLtJen%sAfa9OuK*~>$iWp{cvCXCaFQzv10x$dP_I1@YX!UdxH^X@Xeqb_`THuA z7N-^!D};Nv2DvKaLjuH4G()XL(5#N<>4R_RBZ~XuO literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/test_0062_cassandra.py b/packetbeat/tests/system/test_0062_cassandra.py new file mode 100644 index 00000000000..51448fc8f11 --- /dev/null +++ b/packetbeat/tests/system/test_0062_cassandra.py @@ -0,0 +1,231 @@ +from packetbeat import BaseTest + +""" +Tests for the Cassandra +""" + + +class Test(BaseTest): + + def test_create_keyspace(self): + """ + Should correctly create a keyspace in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_create_keyspace.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert q["query"] == "CREATE KEYSPACE mykeyspace WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };" + + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 124 + assert h["flags"] == 0 + assert h["stream"] == 20 + + + r=o["cassandra_response"] + assert r["result_type"]=="schemaChanged" + + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 35 + assert h2["op"] == "RESULT" + assert h2["flags"] == 0 + assert h2["stream"] == 20 + + def test_create_table(self): + """ + Should correctly create a table in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_create_table.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert q["query"] == "CREATE TABLE users (\n user_id int PRIMARY KEY,\n fname text,\n lname text\n);" + + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 98 + assert h["flags"] == 0 + assert h["stream"] == 49 + + + r=o["cassandra_response"] + assert r["result_type"]=="schemaChanged" + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 39 + assert h2["op"] == "RESULT" + assert h2["flags"] == 0 + assert h2["stream"] == 49 + + + def test_insert_data(self): + """ + Should correctly insert record into table in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_insert.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert q["query"] == "INSERT INTO users (user_id, fname, lname)\n VALUES (1745, 'john', 'smith');" + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 97 + assert h["flags"] == 0 + assert h["stream"] == 252 + + r=o["cassandra_response"] + assert r["result_type"]=="void" + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 4 + assert h2["op"] == "RESULT" + assert h2["flags"] == 0 + assert h2["stream"] == 252 + + + def test_select_data(self): + """ + Should correctly select record from table in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_select.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert q["query"] == "SELECT * FROM users;" + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 41 + assert h["flags"] == 0 + assert h["stream"] == 253 + + + r=o["cassandra_response"] + assert r["result_type"]=="rows" + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 89 + assert h2["op"] == "RESULT" + assert h2["flags"] == 0 + assert h2["stream"] == 253 + + def test_create_index(self): + """ + Should correctly create index of table in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_create_index.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert q["query"] == "CREATE INDEX ON users (lname);" + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 51 + assert h["flags"] == 0 + assert h["stream"] == 92 + + r=o["cassandra_response"] + assert r["result_type"]=="schemaChanged" + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 39 + assert h2["op"] == "RESULT" + assert h2["flags"] == 0 + assert h2["stream"] == 92 + + def test_select_use_index(self): + """ + Should correctly select record from table (use index) in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_select_via_index.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert q["query"] == "SELECT * FROM users WHERE lname = 'smith';" + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 63 + assert h["flags"] == 0 + assert h["stream"] == 262 + + + r=o["cassandra_response"] + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 89 + assert h2["op"] == "RESULT" + assert h2["flags"] == 0 + assert h2["stream"] == 262 + assert r["result_type"]=="rows" + diff --git a/vendor/gopkg.in/inf.v0/LICENSE b/vendor/gopkg.in/inf.v0/LICENSE new file mode 100644 index 00000000000..87a5cede339 --- /dev/null +++ b/vendor/gopkg.in/inf.v0/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 Péter Surányi. Portions Copyright (c) 2009 The Go +Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/gopkg.in/inf.v0/benchmark_test.go b/vendor/gopkg.in/inf.v0/benchmark_test.go new file mode 100644 index 00000000000..27071da0e8a --- /dev/null +++ b/vendor/gopkg.in/inf.v0/benchmark_test.go @@ -0,0 +1,210 @@ +package inf + +import ( + "fmt" + "math/big" + "math/rand" + "sync" + "testing" +) + +const maxcap = 1024 * 1024 +const bits = 256 +const maxscale = 32 + +var once sync.Once + +var decInput [][2]Dec +var intInput [][2]big.Int + +var initBench = func() { + decInput = make([][2]Dec, maxcap) + intInput = make([][2]big.Int, maxcap) + max := new(big.Int).Lsh(big.NewInt(1), bits) + r := rand.New(rand.NewSource(0)) + for i := 0; i < cap(decInput); i++ { + decInput[i][0].SetUnscaledBig(new(big.Int).Rand(r, max)). + SetScale(Scale(r.Int31n(int32(2*maxscale-1)) - int32(maxscale))) + decInput[i][1].SetUnscaledBig(new(big.Int).Rand(r, max)). + SetScale(Scale(r.Int31n(int32(2*maxscale-1)) - int32(maxscale))) + } + for i := 0; i < cap(intInput); i++ { + intInput[i][0].Rand(r, max) + intInput[i][1].Rand(r, max) + } +} + +func doBenchmarkDec1(b *testing.B, f func(z *Dec)) { + once.Do(initBench) + b.ResetTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + f(&decInput[i%maxcap][0]) + } +} + +func doBenchmarkDec2(b *testing.B, f func(x, y *Dec)) { + once.Do(initBench) + b.ResetTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + f(&decInput[i%maxcap][0], &decInput[i%maxcap][1]) + } +} + +func doBenchmarkInt1(b *testing.B, f func(z *big.Int)) { + once.Do(initBench) + b.ResetTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + f(&intInput[i%maxcap][0]) + } +} + +func doBenchmarkInt2(b *testing.B, f func(x, y *big.Int)) { + once.Do(initBench) + b.ResetTimer() + b.StartTimer() + for i := 0; i < b.N; i++ { + f(&intInput[i%maxcap][0], &intInput[i%maxcap][1]) + } +} + +func Benchmark_Dec_String(b *testing.B) { + doBenchmarkDec1(b, func(x *Dec) { + x.String() + }) +} + +func Benchmark_Dec_StringScan(b *testing.B) { + doBenchmarkDec1(b, func(x *Dec) { + s := x.String() + d := new(Dec) + fmt.Sscan(s, d) + }) +} + +func Benchmark_Dec_GobEncode(b *testing.B) { + doBenchmarkDec1(b, func(x *Dec) { + x.GobEncode() + }) +} + +func Benchmark_Dec_GobEnDecode(b *testing.B) { + doBenchmarkDec1(b, func(x *Dec) { + g, _ := x.GobEncode() + new(Dec).GobDecode(g) + }) +} + +func Benchmark_Dec_Add(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + ys := y.Scale() + y.SetScale(x.Scale()) + _ = new(Dec).Add(x, y) + y.SetScale(ys) + }) +} + +func Benchmark_Dec_AddMixed(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + _ = new(Dec).Add(x, y) + }) +} + +func Benchmark_Dec_Sub(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + ys := y.Scale() + y.SetScale(x.Scale()) + _ = new(Dec).Sub(x, y) + y.SetScale(ys) + }) +} + +func Benchmark_Dec_SubMixed(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + _ = new(Dec).Sub(x, y) + }) +} + +func Benchmark_Dec_Mul(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + _ = new(Dec).Mul(x, y) + }) +} + +func Benchmark_Dec_Mul_QuoExact(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + v := new(Dec).Mul(x, y) + _ = new(Dec).QuoExact(v, y) + }) +} + +func Benchmark_Dec_QuoRound_Fixed_Down(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + _ = new(Dec).QuoRound(x, y, 0, RoundDown) + }) +} + +func Benchmark_Dec_QuoRound_Fixed_HalfUp(b *testing.B) { + doBenchmarkDec2(b, func(x, y *Dec) { + _ = new(Dec).QuoRound(x, y, 0, RoundHalfUp) + }) +} + +func Benchmark_Int_String(b *testing.B) { + doBenchmarkInt1(b, func(x *big.Int) { + x.String() + }) +} + +func Benchmark_Int_StringScan(b *testing.B) { + doBenchmarkInt1(b, func(x *big.Int) { + s := x.String() + d := new(big.Int) + fmt.Sscan(s, d) + }) +} + +func Benchmark_Int_GobEncode(b *testing.B) { + doBenchmarkInt1(b, func(x *big.Int) { + x.GobEncode() + }) +} + +func Benchmark_Int_GobEnDecode(b *testing.B) { + doBenchmarkInt1(b, func(x *big.Int) { + g, _ := x.GobEncode() + new(big.Int).GobDecode(g) + }) +} + +func Benchmark_Int_Add(b *testing.B) { + doBenchmarkInt2(b, func(x, y *big.Int) { + _ = new(big.Int).Add(x, y) + }) +} + +func Benchmark_Int_Sub(b *testing.B) { + doBenchmarkInt2(b, func(x, y *big.Int) { + _ = new(big.Int).Sub(x, y) + }) +} + +func Benchmark_Int_Mul(b *testing.B) { + doBenchmarkInt2(b, func(x, y *big.Int) { + _ = new(big.Int).Mul(x, y) + }) +} + +func Benchmark_Int_Quo(b *testing.B) { + doBenchmarkInt2(b, func(x, y *big.Int) { + _ = new(big.Int).Quo(x, y) + }) +} + +func Benchmark_Int_QuoRem(b *testing.B) { + doBenchmarkInt2(b, func(x, y *big.Int) { + _, _ = new(big.Int).QuoRem(x, y, new(big.Int)) + }) +} diff --git a/vendor/gopkg.in/inf.v0/dec.go b/vendor/gopkg.in/inf.v0/dec.go new file mode 100644 index 00000000000..3b4afedf1a3 --- /dev/null +++ b/vendor/gopkg.in/inf.v0/dec.go @@ -0,0 +1,615 @@ +// Package inf (type inf.Dec) implements "infinite-precision" decimal +// arithmetic. +// "Infinite precision" describes two characteristics: practically unlimited +// precision for decimal number representation and no support for calculating +// with any specific fixed precision. +// (Although there is no practical limit on precision, inf.Dec can only +// represent finite decimals.) +// +// This package is currently in experimental stage and the API may change. +// +// This package does NOT support: +// - rounding to specific precisions (as opposed to specific decimal positions) +// - the notion of context (each rounding must be explicit) +// - NaN and Inf values, and distinguishing between positive and negative zero +// - conversions to and from float32/64 types +// +// Features considered for possible addition: +// + formatting options +// + Exp method +// + combined operations such as AddRound/MulAdd etc +// + exchanging data in decimal32/64/128 formats +// +package inf // import "gopkg.in/inf.v0" + +// TODO: +// - avoid excessive deep copying (quo and rounders) + +import ( + "fmt" + "io" + "math/big" + "strings" +) + +// A Dec represents a signed arbitrary-precision decimal. +// It is a combination of a sign, an arbitrary-precision integer coefficient +// value, and a signed fixed-precision exponent value. +// The sign and the coefficient value are handled together as a signed value +// and referred to as the unscaled value. +// (Positive and negative zero values are not distinguished.) +// Since the exponent is most commonly non-positive, it is handled in negated +// form and referred to as scale. +// +// The mathematical value of a Dec equals: +// +// unscaled * 10**(-scale) +// +// Note that different Dec representations may have equal mathematical values. +// +// unscaled scale String() +// ------------------------- +// 0 0 "0" +// 0 2 "0.00" +// 0 -2 "0" +// 1 0 "1" +// 100 2 "1.00" +// 10 0 "10" +// 1 -1 "10" +// +// The zero value for a Dec represents the value 0 with scale 0. +// +// Operations are typically performed through the *Dec type. +// The semantics of the assignment operation "=" for "bare" Dec values is +// undefined and should not be relied on. +// +// Methods are typically of the form: +// +// func (z *Dec) Op(x, y *Dec) *Dec +// +// and implement operations z = x Op y with the result as receiver; if it +// is one of the operands it may be overwritten (and its memory reused). +// To enable chaining of operations, the result is also returned. Methods +// returning a result other than *Dec take one of the operands as the receiver. +// +// A "bare" Quo method (quotient / division operation) is not provided, as the +// result is not always a finite decimal and thus in general cannot be +// represented as a Dec. +// Instead, in the common case when rounding is (potentially) necessary, +// QuoRound should be used with a Scale and a Rounder. +// QuoExact or QuoRound with RoundExact can be used in the special cases when it +// is known that the result is always a finite decimal. +// +type Dec struct { + unscaled big.Int + scale Scale +} + +// Scale represents the type used for the scale of a Dec. +type Scale int32 + +const scaleSize = 4 // bytes in a Scale value + +// Scaler represents a method for obtaining the scale to use for the result of +// an operation on x and y. +type scaler interface { + Scale(x *Dec, y *Dec) Scale +} + +var bigInt = [...]*big.Int{ + big.NewInt(0), big.NewInt(1), big.NewInt(2), big.NewInt(3), big.NewInt(4), + big.NewInt(5), big.NewInt(6), big.NewInt(7), big.NewInt(8), big.NewInt(9), + big.NewInt(10), +} + +var exp10cache [64]big.Int = func() [64]big.Int { + e10, e10i := [64]big.Int{}, bigInt[1] + for i, _ := range e10 { + e10[i].Set(e10i) + e10i = new(big.Int).Mul(e10i, bigInt[10]) + } + return e10 +}() + +// NewDec allocates and returns a new Dec set to the given int64 unscaled value +// and scale. +func NewDec(unscaled int64, scale Scale) *Dec { + return new(Dec).SetUnscaled(unscaled).SetScale(scale) +} + +// NewDecBig allocates and returns a new Dec set to the given *big.Int unscaled +// value and scale. +func NewDecBig(unscaled *big.Int, scale Scale) *Dec { + return new(Dec).SetUnscaledBig(unscaled).SetScale(scale) +} + +// Scale returns the scale of x. +func (x *Dec) Scale() Scale { + return x.scale +} + +// Unscaled returns the unscaled value of x for u and true for ok when the +// unscaled value can be represented as int64; otherwise it returns an undefined +// int64 value for u and false for ok. Use x.UnscaledBig().Int64() to avoid +// checking the validity of the value when the check is known to be redundant. +func (x *Dec) Unscaled() (u int64, ok bool) { + u = x.unscaled.Int64() + var i big.Int + ok = i.SetInt64(u).Cmp(&x.unscaled) == 0 + return +} + +// UnscaledBig returns the unscaled value of x as *big.Int. +func (x *Dec) UnscaledBig() *big.Int { + return &x.unscaled +} + +// SetScale sets the scale of z, with the unscaled value unchanged, and returns +// z. +// The mathematical value of the Dec changes as if it was multiplied by +// 10**(oldscale-scale). +func (z *Dec) SetScale(scale Scale) *Dec { + z.scale = scale + return z +} + +// SetUnscaled sets the unscaled value of z, with the scale unchanged, and +// returns z. +func (z *Dec) SetUnscaled(unscaled int64) *Dec { + z.unscaled.SetInt64(unscaled) + return z +} + +// SetUnscaledBig sets the unscaled value of z, with the scale unchanged, and +// returns z. +func (z *Dec) SetUnscaledBig(unscaled *big.Int) *Dec { + z.unscaled.Set(unscaled) + return z +} + +// Set sets z to the value of x and returns z. +// It does nothing if z == x. +func (z *Dec) Set(x *Dec) *Dec { + if z != x { + z.SetUnscaledBig(x.UnscaledBig()) + z.SetScale(x.Scale()) + } + return z +} + +// Sign returns: +// +// -1 if x < 0 +// 0 if x == 0 +// +1 if x > 0 +// +func (x *Dec) Sign() int { + return x.UnscaledBig().Sign() +} + +// Neg sets z to -x and returns z. +func (z *Dec) Neg(x *Dec) *Dec { + z.SetScale(x.Scale()) + z.UnscaledBig().Neg(x.UnscaledBig()) + return z +} + +// Cmp compares x and y and returns: +// +// -1 if x < y +// 0 if x == y +// +1 if x > y +// +func (x *Dec) Cmp(y *Dec) int { + xx, yy := upscale(x, y) + return xx.UnscaledBig().Cmp(yy.UnscaledBig()) +} + +// Abs sets z to |x| (the absolute value of x) and returns z. +func (z *Dec) Abs(x *Dec) *Dec { + z.SetScale(x.Scale()) + z.UnscaledBig().Abs(x.UnscaledBig()) + return z +} + +// Add sets z to the sum x+y and returns z. +// The scale of z is the greater of the scales of x and y. +func (z *Dec) Add(x, y *Dec) *Dec { + xx, yy := upscale(x, y) + z.SetScale(xx.Scale()) + z.UnscaledBig().Add(xx.UnscaledBig(), yy.UnscaledBig()) + return z +} + +// Sub sets z to the difference x-y and returns z. +// The scale of z is the greater of the scales of x and y. +func (z *Dec) Sub(x, y *Dec) *Dec { + xx, yy := upscale(x, y) + z.SetScale(xx.Scale()) + z.UnscaledBig().Sub(xx.UnscaledBig(), yy.UnscaledBig()) + return z +} + +// Mul sets z to the product x*y and returns z. +// The scale of z is the sum of the scales of x and y. +func (z *Dec) Mul(x, y *Dec) *Dec { + z.SetScale(x.Scale() + y.Scale()) + z.UnscaledBig().Mul(x.UnscaledBig(), y.UnscaledBig()) + return z +} + +// Round sets z to the value of x rounded to Scale s using Rounder r, and +// returns z. +func (z *Dec) Round(x *Dec, s Scale, r Rounder) *Dec { + return z.QuoRound(x, NewDec(1, 0), s, r) +} + +// QuoRound sets z to the quotient x/y, rounded using the given Rounder to the +// specified scale. +// +// If the rounder is RoundExact but the result can not be expressed exactly at +// the specified scale, QuoRound returns nil, and the value of z is undefined. +// +// There is no corresponding Div method; the equivalent can be achieved through +// the choice of Rounder used. +// +func (z *Dec) QuoRound(x, y *Dec, s Scale, r Rounder) *Dec { + return z.quo(x, y, sclr{s}, r) +} + +func (z *Dec) quo(x, y *Dec, s scaler, r Rounder) *Dec { + scl := s.Scale(x, y) + var zzz *Dec + if r.UseRemainder() { + zz, rA, rB := new(Dec).quoRem(x, y, scl, true, new(big.Int), new(big.Int)) + zzz = r.Round(new(Dec), zz, rA, rB) + } else { + zz, _, _ := new(Dec).quoRem(x, y, scl, false, nil, nil) + zzz = r.Round(new(Dec), zz, nil, nil) + } + if zzz == nil { + return nil + } + return z.Set(zzz) +} + +// QuoExact sets z to the quotient x/y and returns z when x/y is a finite +// decimal. Otherwise it returns nil and the value of z is undefined. +// +// The scale of a non-nil result is "x.Scale() - y.Scale()" or greater; it is +// calculated so that the remainder will be zero whenever x/y is a finite +// decimal. +func (z *Dec) QuoExact(x, y *Dec) *Dec { + return z.quo(x, y, scaleQuoExact{}, RoundExact) +} + +// quoRem sets z to the quotient x/y with the scale s, and if useRem is true, +// it sets remNum and remDen to the numerator and denominator of the remainder. +// It returns z, remNum and remDen. +// +// The remainder is normalized to the range -1 < r < 1 to simplify rounding; +// that is, the results satisfy the following equation: +// +// x / y = z + (remNum/remDen) * 10**(-z.Scale()) +// +// See Rounder for more details about rounding. +// +func (z *Dec) quoRem(x, y *Dec, s Scale, useRem bool, + remNum, remDen *big.Int) (*Dec, *big.Int, *big.Int) { + // difference (required adjustment) compared to "canonical" result scale + shift := s - (x.Scale() - y.Scale()) + // pointers to adjusted unscaled dividend and divisor + var ix, iy *big.Int + switch { + case shift > 0: + // increased scale: decimal-shift dividend left + ix = new(big.Int).Mul(x.UnscaledBig(), exp10(shift)) + iy = y.UnscaledBig() + case shift < 0: + // decreased scale: decimal-shift divisor left + ix = x.UnscaledBig() + iy = new(big.Int).Mul(y.UnscaledBig(), exp10(-shift)) + default: + ix = x.UnscaledBig() + iy = y.UnscaledBig() + } + // save a copy of iy in case it to be overwritten with the result + iy2 := iy + if iy == z.UnscaledBig() { + iy2 = new(big.Int).Set(iy) + } + // set scale + z.SetScale(s) + // set unscaled + if useRem { + // Int division + _, intr := z.UnscaledBig().QuoRem(ix, iy, new(big.Int)) + // set remainder + remNum.Set(intr) + remDen.Set(iy2) + } else { + z.UnscaledBig().Quo(ix, iy) + } + return z, remNum, remDen +} + +type sclr struct{ s Scale } + +func (s sclr) Scale(x, y *Dec) Scale { + return s.s +} + +type scaleQuoExact struct{} + +func (sqe scaleQuoExact) Scale(x, y *Dec) Scale { + rem := new(big.Rat).SetFrac(x.UnscaledBig(), y.UnscaledBig()) + f2, f5 := factor2(rem.Denom()), factor(rem.Denom(), bigInt[5]) + var f10 Scale + if f2 > f5 { + f10 = Scale(f2) + } else { + f10 = Scale(f5) + } + return x.Scale() - y.Scale() + f10 +} + +func factor(n *big.Int, p *big.Int) int { + // could be improved for large factors + d, f := n, 0 + for { + dd, dm := new(big.Int).DivMod(d, p, new(big.Int)) + if dm.Sign() == 0 { + f++ + d = dd + } else { + break + } + } + return f +} + +func factor2(n *big.Int) int { + // could be improved for large factors + f := 0 + for ; n.Bit(f) == 0; f++ { + } + return f +} + +func upscale(a, b *Dec) (*Dec, *Dec) { + if a.Scale() == b.Scale() { + return a, b + } + if a.Scale() > b.Scale() { + bb := b.rescale(a.Scale()) + return a, bb + } + aa := a.rescale(b.Scale()) + return aa, b +} + +func exp10(x Scale) *big.Int { + if int(x) < len(exp10cache) { + return &exp10cache[int(x)] + } + return new(big.Int).Exp(bigInt[10], big.NewInt(int64(x)), nil) +} + +func (x *Dec) rescale(newScale Scale) *Dec { + shift := newScale - x.Scale() + switch { + case shift < 0: + e := exp10(-shift) + return NewDecBig(new(big.Int).Quo(x.UnscaledBig(), e), newScale) + case shift > 0: + e := exp10(shift) + return NewDecBig(new(big.Int).Mul(x.UnscaledBig(), e), newScale) + } + return x +} + +var zeros = []byte("00000000000000000000000000000000" + + "00000000000000000000000000000000") +var lzeros = Scale(len(zeros)) + +func appendZeros(s []byte, n Scale) []byte { + for i := Scale(0); i < n; i += lzeros { + if n > i+lzeros { + s = append(s, zeros...) + } else { + s = append(s, zeros[0:n-i]...) + } + } + return s +} + +func (x *Dec) String() string { + if x == nil { + return "" + } + scale := x.Scale() + s := []byte(x.UnscaledBig().String()) + if scale <= 0 { + if scale != 0 && x.unscaled.Sign() != 0 { + s = appendZeros(s, -scale) + } + return string(s) + } + negbit := Scale(-((x.Sign() - 1) / 2)) + // scale > 0 + lens := Scale(len(s)) + if lens-negbit <= scale { + ss := make([]byte, 0, scale+2) + if negbit == 1 { + ss = append(ss, '-') + } + ss = append(ss, '0', '.') + ss = appendZeros(ss, scale-lens+negbit) + ss = append(ss, s[negbit:]...) + return string(ss) + } + // lens > scale + ss := make([]byte, 0, lens+1) + ss = append(ss, s[:lens-scale]...) + ss = append(ss, '.') + ss = append(ss, s[lens-scale:]...) + return string(ss) +} + +// Format is a support routine for fmt.Formatter. It accepts the decimal +// formats 'd' and 'f', and handles both equivalently. +// Width, precision, flags and bases 2, 8, 16 are not supported. +func (x *Dec) Format(s fmt.State, ch rune) { + if ch != 'd' && ch != 'f' && ch != 'v' && ch != 's' { + fmt.Fprintf(s, "%%!%c(dec.Dec=%s)", ch, x.String()) + return + } + fmt.Fprintf(s, x.String()) +} + +func (z *Dec) scan(r io.RuneScanner) (*Dec, error) { + unscaled := make([]byte, 0, 256) // collects chars of unscaled as bytes + dp, dg := -1, -1 // indexes of decimal point, first digit +loop: + for { + ch, _, err := r.ReadRune() + if err == io.EOF { + break loop + } + if err != nil { + return nil, err + } + switch { + case ch == '+' || ch == '-': + if len(unscaled) > 0 || dp >= 0 { // must be first character + r.UnreadRune() + break loop + } + case ch == '.': + if dp >= 0 { + r.UnreadRune() + break loop + } + dp = len(unscaled) + continue // don't add to unscaled + case ch >= '0' && ch <= '9': + if dg == -1 { + dg = len(unscaled) + } + default: + r.UnreadRune() + break loop + } + unscaled = append(unscaled, byte(ch)) + } + if dg == -1 { + return nil, fmt.Errorf("no digits read") + } + if dp >= 0 { + z.SetScale(Scale(len(unscaled) - dp)) + } else { + z.SetScale(0) + } + _, ok := z.UnscaledBig().SetString(string(unscaled), 10) + if !ok { + return nil, fmt.Errorf("invalid decimal: %s", string(unscaled)) + } + return z, nil +} + +// SetString sets z to the value of s, interpreted as a decimal (base 10), +// and returns z and a boolean indicating success. The scale of z is the +// number of digits after the decimal point (including any trailing 0s), +// or 0 if there is no decimal point. If SetString fails, the value of z +// is undefined but the returned value is nil. +func (z *Dec) SetString(s string) (*Dec, bool) { + r := strings.NewReader(s) + _, err := z.scan(r) + if err != nil { + return nil, false + } + _, _, err = r.ReadRune() + if err != io.EOF { + return nil, false + } + // err == io.EOF => scan consumed all of s + return z, true +} + +// Scan is a support routine for fmt.Scanner; it sets z to the value of +// the scanned number. It accepts the decimal formats 'd' and 'f', and +// handles both equivalently. Bases 2, 8, 16 are not supported. +// The scale of z is the number of digits after the decimal point +// (including any trailing 0s), or 0 if there is no decimal point. +func (z *Dec) Scan(s fmt.ScanState, ch rune) error { + if ch != 'd' && ch != 'f' && ch != 's' && ch != 'v' { + return fmt.Errorf("Dec.Scan: invalid verb '%c'", ch) + } + s.SkipSpace() + _, err := z.scan(s) + return err +} + +// Gob encoding version +const decGobVersion byte = 1 + +func scaleBytes(s Scale) []byte { + buf := make([]byte, scaleSize) + i := scaleSize + for j := 0; j < scaleSize; j++ { + i-- + buf[i] = byte(s) + s >>= 8 + } + return buf +} + +func scale(b []byte) (s Scale) { + for j := 0; j < scaleSize; j++ { + s <<= 8 + s |= Scale(b[j]) + } + return +} + +// GobEncode implements the gob.GobEncoder interface. +func (x *Dec) GobEncode() ([]byte, error) { + buf, err := x.UnscaledBig().GobEncode() + if err != nil { + return nil, err + } + buf = append(append(buf, scaleBytes(x.Scale())...), decGobVersion) + return buf, nil +} + +// GobDecode implements the gob.GobDecoder interface. +func (z *Dec) GobDecode(buf []byte) error { + if len(buf) == 0 { + return fmt.Errorf("Dec.GobDecode: no data") + } + b := buf[len(buf)-1] + if b != decGobVersion { + return fmt.Errorf("Dec.GobDecode: encoding version %d not supported", b) + } + l := len(buf) - scaleSize - 1 + err := z.UnscaledBig().GobDecode(buf[:l]) + if err != nil { + return err + } + z.SetScale(scale(buf[l : l+scaleSize])) + return nil +} + +// MarshalText implements the encoding.TextMarshaler interface. +func (x *Dec) MarshalText() ([]byte, error) { + return []byte(x.String()), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +func (z *Dec) UnmarshalText(data []byte) error { + _, ok := z.SetString(string(data)) + if !ok { + return fmt.Errorf("invalid inf.Dec") + } + return nil +} diff --git a/vendor/gopkg.in/inf.v0/dec_go1_2_test.go b/vendor/gopkg.in/inf.v0/dec_go1_2_test.go new file mode 100644 index 00000000000..5df0f7b5537 --- /dev/null +++ b/vendor/gopkg.in/inf.v0/dec_go1_2_test.go @@ -0,0 +1,33 @@ +// +build go1.2 + +package inf + +import ( + "encoding" + "encoding/json" + "testing" +) + +var _ encoding.TextMarshaler = new(Dec) +var _ encoding.TextUnmarshaler = new(Dec) + +type Obj struct { + Val *Dec +} + +func TestDecJsonMarshalUnmarshal(t *testing.T) { + o := Obj{Val: NewDec(123, 2)} + js, err := json.Marshal(o) + if err != nil { + t.Fatalf("json.Marshal(%v): got %v, want ok", o, err) + } + o2 := &Obj{} + err = json.Unmarshal(js, o2) + if err != nil { + t.Fatalf("json.Unmarshal(%#q): got %v, want ok", js, err) + } + if o.Val.Scale() != o2.Val.Scale() || + o.Val.UnscaledBig().Cmp(o2.Val.UnscaledBig()) != 0 { + t.Fatalf("json.Unmarshal(json.Marshal(%v)): want %v, got %v", o, o, o2) + } +} diff --git a/vendor/gopkg.in/inf.v0/dec_internal_test.go b/vendor/gopkg.in/inf.v0/dec_internal_test.go new file mode 100644 index 00000000000..d4fbe3e5bcd --- /dev/null +++ b/vendor/gopkg.in/inf.v0/dec_internal_test.go @@ -0,0 +1,40 @@ +package inf + +import ( + "math/big" + "testing" +) + +var decQuoRemZZZ = []struct { + z, x, y *Dec + r *big.Rat + srA, srB int +}{ + // basic examples + {NewDec(1, 0), NewDec(2, 0), NewDec(2, 0), big.NewRat(0, 1), 0, 1}, + {NewDec(15, 1), NewDec(3, 0), NewDec(2, 0), big.NewRat(0, 1), 0, 1}, + {NewDec(1, 1), NewDec(1, 0), NewDec(10, 0), big.NewRat(0, 1), 0, 1}, + {NewDec(0, 0), NewDec(2, 0), NewDec(3, 0), big.NewRat(2, 3), 1, 1}, + {NewDec(0, 0), NewDec(2, 0), NewDec(6, 0), big.NewRat(1, 3), 1, 1}, + {NewDec(1, 1), NewDec(2, 0), NewDec(12, 0), big.NewRat(2, 3), 1, 1}, + + // examples from the Go Language Specification + {NewDec(1, 0), NewDec(5, 0), NewDec(3, 0), big.NewRat(2, 3), 1, 1}, + {NewDec(-1, 0), NewDec(-5, 0), NewDec(3, 0), big.NewRat(-2, 3), -1, 1}, + {NewDec(-1, 0), NewDec(5, 0), NewDec(-3, 0), big.NewRat(-2, 3), 1, -1}, + {NewDec(1, 0), NewDec(-5, 0), NewDec(-3, 0), big.NewRat(2, 3), -1, -1}, +} + +func TestDecQuoRem(t *testing.T) { + for i, a := range decQuoRemZZZ { + z, rA, rB := new(Dec), new(big.Int), new(big.Int) + s := scaleQuoExact{}.Scale(a.x, a.y) + z.quoRem(a.x, a.y, s, true, rA, rB) + if a.z.Cmp(z) != 0 || a.r.Cmp(new(big.Rat).SetFrac(rA, rB)) != 0 { + t.Errorf("#%d QuoRemZZZ got %v, %v, %v; expected %v, %v", i, z, rA, rB, a.z, a.r) + } + if a.srA != rA.Sign() || a.srB != rB.Sign() { + t.Errorf("#%d QuoRemZZZ wrong signs, got %v, %v; expected %v, %v", i, rA.Sign(), rB.Sign(), a.srA, a.srB) + } + } +} diff --git a/vendor/gopkg.in/inf.v0/dec_test.go b/vendor/gopkg.in/inf.v0/dec_test.go new file mode 100644 index 00000000000..e4b09b3fdcc --- /dev/null +++ b/vendor/gopkg.in/inf.v0/dec_test.go @@ -0,0 +1,379 @@ +package inf_test + +import ( + "bytes" + "encoding/gob" + "fmt" + "math/big" + "strings" + "testing" + + "gopkg.in/inf.v0" +) + +type decFunZZ func(z, x, y *inf.Dec) *inf.Dec +type decArgZZ struct { + z, x, y *inf.Dec +} + +var decSumZZ = []decArgZZ{ + {inf.NewDec(0, 0), inf.NewDec(0, 0), inf.NewDec(0, 0)}, + {inf.NewDec(1, 0), inf.NewDec(1, 0), inf.NewDec(0, 0)}, + {inf.NewDec(1111111110, 0), inf.NewDec(123456789, 0), inf.NewDec(987654321, 0)}, + {inf.NewDec(-1, 0), inf.NewDec(-1, 0), inf.NewDec(0, 0)}, + {inf.NewDec(864197532, 0), inf.NewDec(-123456789, 0), inf.NewDec(987654321, 0)}, + {inf.NewDec(-1111111110, 0), inf.NewDec(-123456789, 0), inf.NewDec(-987654321, 0)}, + {inf.NewDec(12, 2), inf.NewDec(1, 1), inf.NewDec(2, 2)}, +} + +var decProdZZ = []decArgZZ{ + {inf.NewDec(0, 0), inf.NewDec(0, 0), inf.NewDec(0, 0)}, + {inf.NewDec(0, 0), inf.NewDec(1, 0), inf.NewDec(0, 0)}, + {inf.NewDec(1, 0), inf.NewDec(1, 0), inf.NewDec(1, 0)}, + {inf.NewDec(-991*991, 0), inf.NewDec(991, 0), inf.NewDec(-991, 0)}, + {inf.NewDec(2, 3), inf.NewDec(1, 1), inf.NewDec(2, 2)}, + {inf.NewDec(2, -3), inf.NewDec(1, -1), inf.NewDec(2, -2)}, + {inf.NewDec(2, 3), inf.NewDec(1, 1), inf.NewDec(2, 2)}, +} + +func TestDecSignZ(t *testing.T) { + var zero inf.Dec + for _, a := range decSumZZ { + s := a.z.Sign() + e := a.z.Cmp(&zero) + if s != e { + t.Errorf("got %d; want %d for z = %v", s, e, a.z) + } + } +} + +func TestDecAbsZ(t *testing.T) { + var zero inf.Dec + for _, a := range decSumZZ { + var z inf.Dec + z.Abs(a.z) + var e inf.Dec + e.Set(a.z) + if e.Cmp(&zero) < 0 { + e.Sub(&zero, &e) + } + if z.Cmp(&e) != 0 { + t.Errorf("got z = %v; want %v", z, e) + } + } +} + +func testDecFunZZ(t *testing.T, msg string, f decFunZZ, a decArgZZ) { + var z inf.Dec + f(&z, a.x, a.y) + if (&z).Cmp(a.z) != 0 { + t.Errorf("%s%+v\n\tgot z = %v; want %v", msg, a, &z, a.z) + } +} + +func TestDecSumZZ(t *testing.T) { + AddZZ := func(z, x, y *inf.Dec) *inf.Dec { return z.Add(x, y) } + SubZZ := func(z, x, y *inf.Dec) *inf.Dec { return z.Sub(x, y) } + for _, a := range decSumZZ { + arg := a + testDecFunZZ(t, "AddZZ", AddZZ, arg) + + arg = decArgZZ{a.z, a.y, a.x} + testDecFunZZ(t, "AddZZ symmetric", AddZZ, arg) + + arg = decArgZZ{a.x, a.z, a.y} + testDecFunZZ(t, "SubZZ", SubZZ, arg) + + arg = decArgZZ{a.y, a.z, a.x} + testDecFunZZ(t, "SubZZ symmetric", SubZZ, arg) + } +} + +func TestDecProdZZ(t *testing.T) { + MulZZ := func(z, x, y *inf.Dec) *inf.Dec { return z.Mul(x, y) } + for _, a := range decProdZZ { + arg := a + testDecFunZZ(t, "MulZZ", MulZZ, arg) + + arg = decArgZZ{a.z, a.y, a.x} + testDecFunZZ(t, "MulZZ symmetric", MulZZ, arg) + } +} + +var decUnscaledTests = []struct { + d *inf.Dec + u int64 // ignored when ok == false + ok bool +}{ + {new(inf.Dec), 0, true}, + {inf.NewDec(-1<<63, 0), -1 << 63, true}, + {inf.NewDec(-(-1<<63 + 1), 0), -(-1<<63 + 1), true}, + {new(inf.Dec).Neg(inf.NewDec(-1<<63, 0)), 0, false}, + {new(inf.Dec).Sub(inf.NewDec(-1<<63, 0), inf.NewDec(1, 0)), 0, false}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), 0, false}, +} + +func TestDecUnscaled(t *testing.T) { + for i, tt := range decUnscaledTests { + u, ok := tt.d.Unscaled() + if ok != tt.ok { + t.Errorf("#%d Unscaled: got %v, expected %v", i, ok, tt.ok) + } else if ok && u != tt.u { + t.Errorf("#%d Unscaled: got %v, expected %v", i, u, tt.u) + } + } +} + +var decRoundTests = [...]struct { + in *inf.Dec + s inf.Scale + r inf.Rounder + exp *inf.Dec +}{ + {inf.NewDec(123424999999999993, 15), 2, inf.RoundHalfUp, inf.NewDec(12342, 2)}, + {inf.NewDec(123425000000000001, 15), 2, inf.RoundHalfUp, inf.NewDec(12343, 2)}, + {inf.NewDec(123424999999999993, 15), 15, inf.RoundHalfUp, inf.NewDec(123424999999999993, 15)}, + {inf.NewDec(123424999999999993, 15), 16, inf.RoundHalfUp, inf.NewDec(1234249999999999930, 16)}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), -1, inf.RoundHalfUp, inf.NewDec(1844674407370955162, -1)}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), -2, inf.RoundHalfUp, inf.NewDec(184467440737095516, -2)}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), -3, inf.RoundHalfUp, inf.NewDec(18446744073709552, -3)}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), -4, inf.RoundHalfUp, inf.NewDec(1844674407370955, -4)}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), -5, inf.RoundHalfUp, inf.NewDec(184467440737096, -5)}, + {inf.NewDecBig(new(big.Int).Lsh(big.NewInt(1), 64), 0), -6, inf.RoundHalfUp, inf.NewDec(18446744073710, -6)}, +} + +func TestDecRound(t *testing.T) { + for i, tt := range decRoundTests { + z := new(inf.Dec).Round(tt.in, tt.s, tt.r) + if tt.exp.Cmp(z) != 0 { + t.Errorf("#%d Round got %v; expected %v", i, z, tt.exp) + } + } +} + +var decStringTests = []struct { + in string + out string + val int64 + scale inf.Scale // skip SetString if negative + ok bool + scanOk bool +}{ + {in: "", ok: false, scanOk: false}, + {in: "a", ok: false, scanOk: false}, + {in: "z", ok: false, scanOk: false}, + {in: "+", ok: false, scanOk: false}, + {in: "-", ok: false, scanOk: false}, + {in: "g", ok: false, scanOk: false}, + {in: ".", ok: false, scanOk: false}, + {in: ".-0", ok: false, scanOk: false}, + {in: ".+0", ok: false, scanOk: false}, + // Scannable but not SetStringable + {"0b", "ignored", 0, 0, false, true}, + {"0x", "ignored", 0, 0, false, true}, + {"0xg", "ignored", 0, 0, false, true}, + {"0.0g", "ignored", 0, 1, false, true}, + // examples from godoc for Dec + {"0", "0", 0, 0, true, true}, + {"0.00", "0.00", 0, 2, true, true}, + {"ignored", "0", 0, -2, true, false}, + {"1", "1", 1, 0, true, true}, + {"1.00", "1.00", 100, 2, true, true}, + {"10", "10", 10, 0, true, true}, + {"ignored", "10", 1, -1, true, false}, + // other tests + {"+0", "0", 0, 0, true, true}, + {"-0", "0", 0, 0, true, true}, + {"0.0", "0.0", 0, 1, true, true}, + {"0.1", "0.1", 1, 1, true, true}, + {"0.", "0", 0, 0, true, true}, + {"-10", "-10", -1, -1, true, true}, + {"-1", "-1", -1, 0, true, true}, + {"-0.1", "-0.1", -1, 1, true, true}, + {"-0.01", "-0.01", -1, 2, true, true}, + {"+0.", "0", 0, 0, true, true}, + {"-0.", "0", 0, 0, true, true}, + {".0", "0.0", 0, 1, true, true}, + {"+.0", "0.0", 0, 1, true, true}, + {"-.0", "0.0", 0, 1, true, true}, + {"0.0000000000", "0.0000000000", 0, 10, true, true}, + {"0.0000000001", "0.0000000001", 1, 10, true, true}, + {"-0.0000000000", "0.0000000000", 0, 10, true, true}, + {"-0.0000000001", "-0.0000000001", -1, 10, true, true}, + {"-10", "-10", -10, 0, true, true}, + {"+10", "10", 10, 0, true, true}, + {"00", "0", 0, 0, true, true}, + {"023", "23", 23, 0, true, true}, // decimal, not octal + {"-02.3", "-2.3", -23, 1, true, true}, // decimal, not octal +} + +func TestDecGetString(t *testing.T) { + z := new(inf.Dec) + for i, test := range decStringTests { + if !test.ok { + continue + } + z.SetUnscaled(test.val) + z.SetScale(test.scale) + + s := z.String() + if s != test.out { + t.Errorf("#%da got %s; want %s", i, s, test.out) + } + + s = fmt.Sprintf("%d", z) + if s != test.out { + t.Errorf("#%db got %s; want %s", i, s, test.out) + } + } +} + +func TestDecSetString(t *testing.T) { + tmp := new(inf.Dec) + for i, test := range decStringTests { + if test.scale < 0 { + // SetString only supports scale >= 0 + continue + } + // initialize to a non-zero value so that issues with parsing + // 0 are detected + tmp.Set(inf.NewDec(1234567890, 123)) + n1, ok1 := new(inf.Dec).SetString(test.in) + n2, ok2 := tmp.SetString(test.in) + expected := inf.NewDec(test.val, test.scale) + if ok1 != test.ok || ok2 != test.ok { + t.Errorf("#%d (input '%s') ok incorrect (should be %t)", i, test.in, test.ok) + continue + } + if !ok1 { + if n1 != nil { + t.Errorf("#%d (input '%s') n1 != nil", i, test.in) + } + continue + } + if !ok2 { + if n2 != nil { + t.Errorf("#%d (input '%s') n2 != nil", i, test.in) + } + continue + } + + if n1.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n1, test.val) + } + if n2.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n2, test.val) + } + } +} + +func TestDecScan(t *testing.T) { + tmp := new(inf.Dec) + for i, test := range decStringTests { + if test.scale < 0 { + // SetString only supports scale >= 0 + continue + } + // initialize to a non-zero value so that issues with parsing + // 0 are detected + tmp.Set(inf.NewDec(1234567890, 123)) + n1, n2 := new(inf.Dec), tmp + nn1, err1 := fmt.Sscan(test.in, n1) + nn2, err2 := fmt.Sscan(test.in, n2) + if !test.scanOk { + if err1 == nil || err2 == nil { + t.Errorf("#%d (input '%s') ok incorrect, should be %t", i, test.in, test.scanOk) + } + continue + } + expected := inf.NewDec(test.val, test.scale) + if nn1 != 1 || err1 != nil || nn2 != 1 || err2 != nil { + t.Errorf("#%d (input '%s') error %d %v, %d %v", i, test.in, nn1, err1, nn2, err2) + continue + } + if n1.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n1, test.val) + } + if n2.Cmp(expected) != 0 { + t.Errorf("#%d (input '%s') got: %s want: %d", i, test.in, n2, test.val) + } + } +} + +var decScanNextTests = []struct { + in string + ok bool + next rune +}{ + {"", false, 0}, + {"a", false, 'a'}, + {"z", false, 'z'}, + {"+", false, 0}, + {"-", false, 0}, + {"g", false, 'g'}, + {".", false, 0}, + {".-0", false, '-'}, + {".+0", false, '+'}, + {"0b", true, 'b'}, + {"0x", true, 'x'}, + {"0xg", true, 'x'}, + {"0.0g", true, 'g'}, +} + +func TestDecScanNext(t *testing.T) { + for i, test := range decScanNextTests { + rdr := strings.NewReader(test.in) + n1 := new(inf.Dec) + nn1, _ := fmt.Fscan(rdr, n1) + if (test.ok && nn1 == 0) || (!test.ok && nn1 > 0) { + t.Errorf("#%d (input '%s') ok incorrect should be %t", i, test.in, test.ok) + continue + } + r := rune(0) + nn2, err := fmt.Fscanf(rdr, "%c", &r) + if test.next != r { + t.Errorf("#%d (input '%s') next incorrect, got %c should be %c, %d, %v", i, test.in, r, test.next, nn2, err) + } + } +} + +var decGobEncodingTests = []string{ + "0", + "1", + "2", + "10", + "42", + "1234567890", + "298472983472983471903246121093472394872319615612417471234712061", +} + +func TestDecGobEncoding(t *testing.T) { + var medium bytes.Buffer + enc := gob.NewEncoder(&medium) + dec := gob.NewDecoder(&medium) + for i, test := range decGobEncodingTests { + for j := 0; j < 2; j++ { + for k := inf.Scale(-5); k <= 5; k++ { + medium.Reset() // empty buffer for each test case (in case of failures) + stest := test + if j != 0 { + // negative numbers + stest = "-" + test + } + var tx inf.Dec + tx.SetString(stest) + tx.SetScale(k) // test with positive, negative, and zero scale + if err := enc.Encode(&tx); err != nil { + t.Errorf("#%d%c: encoding failed: %s", i, 'a'+j, err) + } + var rx inf.Dec + if err := dec.Decode(&rx); err != nil { + t.Errorf("#%d%c: decoding failed: %s", i, 'a'+j, err) + } + if rx.Cmp(&tx) != 0 { + t.Errorf("#%d%c: transmission failed: got %s want %s", i, 'a'+j, &rx, &tx) + } + } + } + } +} diff --git a/vendor/gopkg.in/inf.v0/example_test.go b/vendor/gopkg.in/inf.v0/example_test.go new file mode 100644 index 00000000000..fa1e54d16e9 --- /dev/null +++ b/vendor/gopkg.in/inf.v0/example_test.go @@ -0,0 +1,62 @@ +package inf_test + +import ( + "fmt" + "log" +) + +import "gopkg.in/inf.v0" + +func ExampleDec_SetString() { + d := new(inf.Dec) + d.SetString("012345.67890") // decimal; leading 0 ignored; trailing 0 kept + fmt.Println(d) + // Output: 12345.67890 +} + +func ExampleDec_Scan() { + // The Scan function is rarely used directly; + // the fmt package recognizes it as an implementation of fmt.Scanner. + d := new(inf.Dec) + _, err := fmt.Sscan("184467440.73709551617", d) + if err != nil { + log.Println("error scanning value:", err) + } else { + fmt.Println(d) + } + // Output: 184467440.73709551617 +} + +func ExampleDec_QuoRound_scale2RoundDown() { + // 10 / 3 is an infinite decimal; it has no exact Dec representation + x, y := inf.NewDec(10, 0), inf.NewDec(3, 0) + // use 2 digits beyond the decimal point, round towards 0 + z := new(inf.Dec).QuoRound(x, y, 2, inf.RoundDown) + fmt.Println(z) + // Output: 3.33 +} + +func ExampleDec_QuoRound_scale2RoundCeil() { + // -42 / 400 is an finite decimal with 3 digits beyond the decimal point + x, y := inf.NewDec(-42, 0), inf.NewDec(400, 0) + // use 2 digits beyond decimal point, round towards positive infinity + z := new(inf.Dec).QuoRound(x, y, 2, inf.RoundCeil) + fmt.Println(z) + // Output: -0.10 +} + +func ExampleDec_QuoExact_ok() { + // 1 / 25 is a finite decimal; it has exact Dec representation + x, y := inf.NewDec(1, 0), inf.NewDec(25, 0) + z := new(inf.Dec).QuoExact(x, y) + fmt.Println(z) + // Output: 0.04 +} + +func ExampleDec_QuoExact_fail() { + // 1 / 3 is an infinite decimal; it has no exact Dec representation + x, y := inf.NewDec(1, 0), inf.NewDec(3, 0) + z := new(inf.Dec).QuoExact(x, y) + fmt.Println(z) + // Output: +} diff --git a/vendor/gopkg.in/inf.v0/rounder.go b/vendor/gopkg.in/inf.v0/rounder.go new file mode 100644 index 00000000000..3a97ef529b9 --- /dev/null +++ b/vendor/gopkg.in/inf.v0/rounder.go @@ -0,0 +1,145 @@ +package inf + +import ( + "math/big" +) + +// Rounder represents a method for rounding the (possibly infinite decimal) +// result of a division to a finite Dec. It is used by Dec.Round() and +// Dec.Quo(). +// +// See the Example for results of using each Rounder with some sample values. +// +type Rounder rounder + +// See http://speleotrove.com/decimal/damodel.html#refround for more detailed +// definitions of these rounding modes. +var ( + RoundDown Rounder // towards 0 + RoundUp Rounder // away from 0 + RoundFloor Rounder // towards -infinity + RoundCeil Rounder // towards +infinity + RoundHalfDown Rounder // to nearest; towards 0 if same distance + RoundHalfUp Rounder // to nearest; away from 0 if same distance + RoundHalfEven Rounder // to nearest; even last digit if same distance +) + +// RoundExact is to be used in the case when rounding is not necessary. +// When used with Quo or Round, it returns the result verbatim when it can be +// expressed exactly with the given precision, and it returns nil otherwise. +// QuoExact is a shorthand for using Quo with RoundExact. +var RoundExact Rounder + +type rounder interface { + + // When UseRemainder() returns true, the Round() method is passed the + // remainder of the division, expressed as the numerator and denominator of + // a rational. + UseRemainder() bool + + // Round sets the rounded value of a quotient to z, and returns z. + // quo is rounded down (truncated towards zero) to the scale obtained from + // the Scaler in Quo(). + // + // When the remainder is not used, remNum and remDen are nil. + // When used, the remainder is normalized between -1 and 1; that is: + // + // -|remDen| < remNum < |remDen| + // + // remDen has the same sign as y, and remNum is zero or has the same sign + // as x. + Round(z, quo *Dec, remNum, remDen *big.Int) *Dec +} + +type rndr struct { + useRem bool + round func(z, quo *Dec, remNum, remDen *big.Int) *Dec +} + +func (r rndr) UseRemainder() bool { + return r.useRem +} + +func (r rndr) Round(z, quo *Dec, remNum, remDen *big.Int) *Dec { + return r.round(z, quo, remNum, remDen) +} + +var intSign = []*big.Int{big.NewInt(-1), big.NewInt(0), big.NewInt(1)} + +func roundHalf(f func(c int, odd uint) (roundUp bool)) func(z, q *Dec, rA, rB *big.Int) *Dec { + return func(z, q *Dec, rA, rB *big.Int) *Dec { + z.Set(q) + brA, brB := rA.BitLen(), rB.BitLen() + if brA < brB-1 { + // brA < brB-1 => |rA| < |rB/2| + return z + } + roundUp := false + srA, srB := rA.Sign(), rB.Sign() + s := srA * srB + if brA == brB-1 { + rA2 := new(big.Int).Lsh(rA, 1) + if s < 0 { + rA2.Neg(rA2) + } + roundUp = f(rA2.Cmp(rB)*srB, z.UnscaledBig().Bit(0)) + } else { + // brA > brB-1 => |rA| > |rB/2| + roundUp = true + } + if roundUp { + z.UnscaledBig().Add(z.UnscaledBig(), intSign[s+1]) + } + return z + } +} + +func init() { + RoundExact = rndr{true, + func(z, q *Dec, rA, rB *big.Int) *Dec { + if rA.Sign() != 0 { + return nil + } + return z.Set(q) + }} + RoundDown = rndr{false, + func(z, q *Dec, rA, rB *big.Int) *Dec { + return z.Set(q) + }} + RoundUp = rndr{true, + func(z, q *Dec, rA, rB *big.Int) *Dec { + z.Set(q) + if rA.Sign() != 0 { + z.UnscaledBig().Add(z.UnscaledBig(), intSign[rA.Sign()*rB.Sign()+1]) + } + return z + }} + RoundFloor = rndr{true, + func(z, q *Dec, rA, rB *big.Int) *Dec { + z.Set(q) + if rA.Sign()*rB.Sign() < 0 { + z.UnscaledBig().Add(z.UnscaledBig(), intSign[0]) + } + return z + }} + RoundCeil = rndr{true, + func(z, q *Dec, rA, rB *big.Int) *Dec { + z.Set(q) + if rA.Sign()*rB.Sign() > 0 { + z.UnscaledBig().Add(z.UnscaledBig(), intSign[2]) + } + return z + }} + RoundHalfDown = rndr{true, roundHalf( + func(c int, odd uint) bool { + return c > 0 + })} + RoundHalfUp = rndr{true, roundHalf( + func(c int, odd uint) bool { + return c >= 0 + })} + RoundHalfEven = rndr{true, roundHalf( + func(c int, odd uint) bool { + return c > 0 || c == 0 && odd == 1 + })} +} diff --git a/vendor/gopkg.in/inf.v0/rounder_example_test.go b/vendor/gopkg.in/inf.v0/rounder_example_test.go new file mode 100644 index 00000000000..803c1d7ee56 --- /dev/null +++ b/vendor/gopkg.in/inf.v0/rounder_example_test.go @@ -0,0 +1,72 @@ +package inf_test + +import ( + "fmt" + "os" + "text/tabwriter" + + "gopkg.in/inf.v0" +) + +// This example displays the results of Dec.Round with each of the Rounders. +// +func ExampleRounder() { + var vals = []struct { + x string + s inf.Scale + }{ + {"-0.18", 1}, {"-0.15", 1}, {"-0.12", 1}, {"-0.10", 1}, + {"-0.08", 1}, {"-0.05", 1}, {"-0.02", 1}, {"0.00", 1}, + {"0.02", 1}, {"0.05", 1}, {"0.08", 1}, {"0.10", 1}, + {"0.12", 1}, {"0.15", 1}, {"0.18", 1}, + } + + var rounders = []struct { + name string + rounder inf.Rounder + }{ + {"RoundDown", inf.RoundDown}, {"RoundUp", inf.RoundUp}, + {"RoundCeil", inf.RoundCeil}, {"RoundFloor", inf.RoundFloor}, + {"RoundHalfDown", inf.RoundHalfDown}, {"RoundHalfUp", inf.RoundHalfUp}, + {"RoundHalfEven", inf.RoundHalfEven}, {"RoundExact", inf.RoundExact}, + } + + fmt.Println("The results of new(inf.Dec).Round(x, s, inf.RoundXXX):\n") + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.AlignRight) + fmt.Fprint(w, "x\ts\t|\t") + for _, r := range rounders { + fmt.Fprintf(w, "%s\t", r.name[5:]) + } + fmt.Fprintln(w) + for _, v := range vals { + fmt.Fprintf(w, "%s\t%d\t|\t", v.x, v.s) + for _, r := range rounders { + x, _ := new(inf.Dec).SetString(v.x) + z := new(inf.Dec).Round(x, v.s, r.rounder) + fmt.Fprintf(w, "%d\t", z) + } + fmt.Fprintln(w) + } + w.Flush() + + // Output: + // The results of new(inf.Dec).Round(x, s, inf.RoundXXX): + // + // x s | Down Up Ceil Floor HalfDown HalfUp HalfEven Exact + // -0.18 1 | -0.1 -0.2 -0.1 -0.2 -0.2 -0.2 -0.2 + // -0.15 1 | -0.1 -0.2 -0.1 -0.2 -0.1 -0.2 -0.2 + // -0.12 1 | -0.1 -0.2 -0.1 -0.2 -0.1 -0.1 -0.1 + // -0.10 1 | -0.1 -0.1 -0.1 -0.1 -0.1 -0.1 -0.1 -0.1 + // -0.08 1 | 0.0 -0.1 0.0 -0.1 -0.1 -0.1 -0.1 + // -0.05 1 | 0.0 -0.1 0.0 -0.1 0.0 -0.1 0.0 + // -0.02 1 | 0.0 -0.1 0.0 -0.1 0.0 0.0 0.0 + // 0.00 1 | 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + // 0.02 1 | 0.0 0.1 0.1 0.0 0.0 0.0 0.0 + // 0.05 1 | 0.0 0.1 0.1 0.0 0.0 0.1 0.0 + // 0.08 1 | 0.0 0.1 0.1 0.0 0.1 0.1 0.1 + // 0.10 1 | 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 + // 0.12 1 | 0.1 0.2 0.2 0.1 0.1 0.1 0.1 + // 0.15 1 | 0.1 0.2 0.2 0.1 0.1 0.2 0.2 + // 0.18 1 | 0.1 0.2 0.2 0.1 0.2 0.2 0.2 + +} diff --git a/vendor/gopkg.in/inf.v0/rounder_test.go b/vendor/gopkg.in/inf.v0/rounder_test.go new file mode 100644 index 00000000000..d7e14c58c6e --- /dev/null +++ b/vendor/gopkg.in/inf.v0/rounder_test.go @@ -0,0 +1,109 @@ +package inf_test + +import ( + "math/big" + "testing" + + "gopkg.in/inf.v0" +) + +var decRounderInputs = [...]struct { + quo *inf.Dec + rA, rB *big.Int +}{ + // examples from go language spec + {inf.NewDec(1, 0), big.NewInt(2), big.NewInt(3)}, // 5 / 3 + {inf.NewDec(-1, 0), big.NewInt(-2), big.NewInt(3)}, // -5 / 3 + {inf.NewDec(-1, 0), big.NewInt(2), big.NewInt(-3)}, // 5 / -3 + {inf.NewDec(1, 0), big.NewInt(-2), big.NewInt(-3)}, // -5 / -3 + // examples from godoc + {inf.NewDec(-1, 1), big.NewInt(-8), big.NewInt(10)}, + {inf.NewDec(-1, 1), big.NewInt(-5), big.NewInt(10)}, + {inf.NewDec(-1, 1), big.NewInt(-2), big.NewInt(10)}, + {inf.NewDec(0, 1), big.NewInt(-8), big.NewInt(10)}, + {inf.NewDec(0, 1), big.NewInt(-5), big.NewInt(10)}, + {inf.NewDec(0, 1), big.NewInt(-2), big.NewInt(10)}, + {inf.NewDec(0, 1), big.NewInt(0), big.NewInt(1)}, + {inf.NewDec(0, 1), big.NewInt(2), big.NewInt(10)}, + {inf.NewDec(0, 1), big.NewInt(5), big.NewInt(10)}, + {inf.NewDec(0, 1), big.NewInt(8), big.NewInt(10)}, + {inf.NewDec(1, 1), big.NewInt(2), big.NewInt(10)}, + {inf.NewDec(1, 1), big.NewInt(5), big.NewInt(10)}, + {inf.NewDec(1, 1), big.NewInt(8), big.NewInt(10)}, +} + +var decRounderResults = [...]struct { + rounder inf.Rounder + results [len(decRounderInputs)]*inf.Dec +}{ + {inf.RoundExact, [...]*inf.Dec{nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, + inf.NewDec(0, 1), nil, nil, nil, nil, nil, nil}}, + {inf.RoundDown, [...]*inf.Dec{ + inf.NewDec(1, 0), inf.NewDec(-1, 0), inf.NewDec(-1, 0), inf.NewDec(1, 0), + inf.NewDec(-1, 1), inf.NewDec(-1, 1), inf.NewDec(-1, 1), + inf.NewDec(0, 1), inf.NewDec(0, 1), inf.NewDec(0, 1), + inf.NewDec(0, 1), + inf.NewDec(0, 1), inf.NewDec(0, 1), inf.NewDec(0, 1), + inf.NewDec(1, 1), inf.NewDec(1, 1), inf.NewDec(1, 1)}}, + {inf.RoundUp, [...]*inf.Dec{ + inf.NewDec(2, 0), inf.NewDec(-2, 0), inf.NewDec(-2, 0), inf.NewDec(2, 0), + inf.NewDec(-2, 1), inf.NewDec(-2, 1), inf.NewDec(-2, 1), + inf.NewDec(-1, 1), inf.NewDec(-1, 1), inf.NewDec(-1, 1), + inf.NewDec(0, 1), + inf.NewDec(1, 1), inf.NewDec(1, 1), inf.NewDec(1, 1), + inf.NewDec(2, 1), inf.NewDec(2, 1), inf.NewDec(2, 1)}}, + {inf.RoundHalfDown, [...]*inf.Dec{ + inf.NewDec(2, 0), inf.NewDec(-2, 0), inf.NewDec(-2, 0), inf.NewDec(2, 0), + inf.NewDec(-2, 1), inf.NewDec(-1, 1), inf.NewDec(-1, 1), + inf.NewDec(-1, 1), inf.NewDec(0, 1), inf.NewDec(0, 1), + inf.NewDec(0, 1), + inf.NewDec(0, 1), inf.NewDec(0, 1), inf.NewDec(1, 1), + inf.NewDec(1, 1), inf.NewDec(1, 1), inf.NewDec(2, 1)}}, + {inf.RoundHalfUp, [...]*inf.Dec{ + inf.NewDec(2, 0), inf.NewDec(-2, 0), inf.NewDec(-2, 0), inf.NewDec(2, 0), + inf.NewDec(-2, 1), inf.NewDec(-2, 1), inf.NewDec(-1, 1), + inf.NewDec(-1, 1), inf.NewDec(-1, 1), inf.NewDec(0, 1), + inf.NewDec(0, 1), + inf.NewDec(0, 1), inf.NewDec(1, 1), inf.NewDec(1, 1), + inf.NewDec(1, 1), inf.NewDec(2, 1), inf.NewDec(2, 1)}}, + {inf.RoundHalfEven, [...]*inf.Dec{ + inf.NewDec(2, 0), inf.NewDec(-2, 0), inf.NewDec(-2, 0), inf.NewDec(2, 0), + inf.NewDec(-2, 1), inf.NewDec(-2, 1), inf.NewDec(-1, 1), + inf.NewDec(-1, 1), inf.NewDec(0, 1), inf.NewDec(0, 1), + inf.NewDec(0, 1), + inf.NewDec(0, 1), inf.NewDec(0, 1), inf.NewDec(1, 1), + inf.NewDec(1, 1), inf.NewDec(2, 1), inf.NewDec(2, 1)}}, + {inf.RoundFloor, [...]*inf.Dec{ + inf.NewDec(1, 0), inf.NewDec(-2, 0), inf.NewDec(-2, 0), inf.NewDec(1, 0), + inf.NewDec(-2, 1), inf.NewDec(-2, 1), inf.NewDec(-2, 1), + inf.NewDec(-1, 1), inf.NewDec(-1, 1), inf.NewDec(-1, 1), + inf.NewDec(0, 1), + inf.NewDec(0, 1), inf.NewDec(0, 1), inf.NewDec(0, 1), + inf.NewDec(1, 1), inf.NewDec(1, 1), inf.NewDec(1, 1)}}, + {inf.RoundCeil, [...]*inf.Dec{ + inf.NewDec(2, 0), inf.NewDec(-1, 0), inf.NewDec(-1, 0), inf.NewDec(2, 0), + inf.NewDec(-1, 1), inf.NewDec(-1, 1), inf.NewDec(-1, 1), + inf.NewDec(0, 1), inf.NewDec(0, 1), inf.NewDec(0, 1), + inf.NewDec(0, 1), + inf.NewDec(1, 1), inf.NewDec(1, 1), inf.NewDec(1, 1), + inf.NewDec(2, 1), inf.NewDec(2, 1), inf.NewDec(2, 1)}}, +} + +func TestDecRounders(t *testing.T) { + for i, a := range decRounderResults { + for j, input := range decRounderInputs { + q := new(inf.Dec).Set(input.quo) + rA, rB := new(big.Int).Set(input.rA), new(big.Int).Set(input.rB) + res := a.rounder.Round(new(inf.Dec), q, rA, rB) + if a.results[j] == nil && res == nil { + continue + } + if (a.results[j] == nil && res != nil) || + (a.results[j] != nil && res == nil) || + a.results[j].Cmp(res) != 0 { + t.Errorf("#%d,%d Rounder got %v; expected %v", i, j, res, a.results[j]) + } + } + } +} From 323b1bb677d4ff05cbf1c8dc2efb509e8a64fb72 Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 5 Jul 2016 21:17:51 +0800 Subject: [PATCH 02/30] Fix gofmt style --- packetbeat/protos/cassandra/compressor.go | 11 +++++------ packetbeat/protos/cassandra/config.go | 4 ++-- packetbeat/protos/cassandra/marshal.go | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packetbeat/protos/cassandra/compressor.go b/packetbeat/protos/cassandra/compressor.go index fc3ddcb45f6..78c7e46a85b 100644 --- a/packetbeat/protos/cassandra/compressor.go +++ b/packetbeat/protos/cassandra/compressor.go @@ -10,14 +10,13 @@ type Compressor interface { Decode(data []byte) ([]byte, error) } -const Snappy string = "snappy" +const Snappy string = "snappy" // SnappyCompressor implements the Compressor interface and can be used to // compress incoming and outgoing frames. The snappy compression algorithm // aims for very high speeds and reasonable compression. type SnappyCompressor struct{} - func (s SnappyCompressor) Name() string { return Snappy } @@ -30,14 +29,14 @@ func (s SnappyCompressor) Decode(data []byte) ([]byte, error) { return snappy.Decode(nil, data) } -const LZ4 string = "lz4" +const LZ4 string = "lz4" -type LZ4Compressor struct{ +type LZ4Compressor struct { //TODO } -const Deflate string = "deflate" +const Deflate string = "deflate" -type DeflateCompressor struct{ +type DeflateCompressor struct { //TODO } diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go index 721dcd425c7..65fcd9a826f 100644 --- a/packetbeat/protos/cassandra/config.go +++ b/packetbeat/protos/cassandra/config.go @@ -7,8 +7,8 @@ import ( type cassandraConfig struct { config.ProtocolCommon `config:",inline"` - SendRequestHeader bool `config:"send_request_header"` - SendResponseHeader bool `config:"send_response_header"` + SendRequestHeader bool `config:"send_request_header"` + SendResponseHeader bool `config:"send_response_header"` Compressor string `config:"compressor"` } diff --git a/packetbeat/protos/cassandra/marshal.go b/packetbeat/protos/cassandra/marshal.go index e54d406ccce..516ee21195c 100644 --- a/packetbeat/protos/cassandra/marshal.go +++ b/packetbeat/protos/cassandra/marshal.go @@ -11,9 +11,9 @@ import ( "reflect" "time" + "github.com/elastic/beats/libbeat/logp" "gopkg.in/inf.v0" "strings" - "github.com/elastic/beats/libbeat/logp" ) // TypeInfo describes a Cassandra specific data type. From ecc5e375fc280b970f6e44b0937d1813f0bb321c Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 5 Jul 2016 21:51:19 +0800 Subject: [PATCH 03/30] Remove recursive call in errType --- packetbeat/protos/cassandra/marshal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/protos/cassandra/marshal.go b/packetbeat/protos/cassandra/marshal.go index 516ee21195c..809612ed04a 100644 --- a/packetbeat/protos/cassandra/marshal.go +++ b/packetbeat/protos/cassandra/marshal.go @@ -374,7 +374,7 @@ func (this ErrType) String() string { return "errUnauthorized" } - return fmt.Sprintf("ErrUnknown: 0x%x", this) + return "ErrUnknown" } const ( From 11a8d23dd9bfca4a8e897237081b7feba202df6d Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 18 Jul 2016 14:47:05 +0800 Subject: [PATCH 04/30] remove duplicated snappy in glide.yaml --- glide.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/glide.yaml b/glide.yaml index b2451e75bd7..7cc66ffffb0 100644 --- a/glide.yaml +++ b/glide.yaml @@ -99,6 +99,3 @@ import: version: v1.0 - package: https://github.com/go-inf/inf version: v0.9.0 -- package: github.com/golang/snappy - version: d9eb7a3d35ec988b8585d4a0068e462c27d28380 - From 589ae111da3da2fd2bdcf72c9b721645e05e9394 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 18 Jul 2016 15:02:35 +0800 Subject: [PATCH 05/30] update beats config --- packetbeat/etc/beat.full.yml | 23 ++++++++++++++--------- packetbeat/packetbeat.full.yml | 21 +++++++++++++-------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 9938d8d76b6..1d1c65b0d90 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -420,18 +420,23 @@ packetbeat.protocols.nfs: packetbeat.protocols.cassandra: #Cassandra port for traffic monitoring. ports: [9042] + # If this option is enabled, the raw message of the request (`cassandra_request` field) - # is sent to Elasticsearch. The default is false. - send_request: true + # is included in published events. The default is false. + #send_request: true + # If this option is enabled, the raw message of the response (`cassandra_response` field) - # is sent to Elasticsearch. The default is false. - send_response: true + # is included in published events. The default is false. + #send_response: true + # If this option is enabled, the raw message of the response (`cassandra_request_header` field) - # is sent to Elasticsearch. The default is false. - send_request_header: true + # is included in published events. The default is false. + #send_request_header: true + # If this option is enabled, the raw message of the response (`cassandra_response_header` field) - # is sent to Elasticsearch. The default is false. - send_response_header: true + # is included in published events. The default is false. + #send_response_header: true + # If this option is enabled, and also the flag indicates that the frame body was compressed, and # the stream will decode by the compressor, currently support: `snappy` , The default is `nil`. - compressor: "snappy" + #compressor: "snappy" diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 20cff228e49..d9251e56dcb 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -420,21 +420,26 @@ packetbeat.protocols.nfs: packetbeat.protocols.cassandra: #Cassandra port for traffic monitoring. ports: [9042] + # If this option is enabled, the raw message of the request (`cassandra_request` field) - # is sent to Elasticsearch. The default is false. - send_request: true + # is included in published events. The default is false. + #send_request: true + # If this option is enabled, the raw message of the response (`cassandra_response` field) - # is sent to Elasticsearch. The default is false. - send_response: true + # is included in published events. The default is false. + #send_response: true + # If this option is enabled, the raw message of the response (`cassandra_request_header` field) # is sent to Elasticsearch. The default is false. - send_request_header: true + #send_request_header: true + # If this option is enabled, the raw message of the response (`cassandra_response_header` field) - # is sent to Elasticsearch. The default is false. - send_response_header: true + # is included in published events. The default is false. + #send_response_header: true + # If this option is enabled, and also the flag indicates that the frame body was compressed, and # the stream will decode by the compressor, currently support: `snappy` , The default is `nil`. - compressor: "snappy" + #compressor: "snappy" #================================ General ===================================== From 26ddacadbf82ef4e5f88186fb9b7ed004a7956f2 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 18 Jul 2016 15:04:05 +0800 Subject: [PATCH 06/30] fix logging bug --- packetbeat/protos/cassandra/cassandra.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go index e5285b36729..3536d542d8a 100644 --- a/packetbeat/protos/cassandra/cassandra.go +++ b/packetbeat/protos/cassandra/cassandra.go @@ -68,7 +68,7 @@ func (cassandra *cassandra) init(results publish.Transactions, config *cassandra } cassandra.pub.results = results - isDebug = logp.IsDebug("http") + isDebug = logp.IsDebug("cassandra") return nil } From e3d01229eac84d80f69dabb96c4786cbad8d959a Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 20 Jul 2016 14:46:58 +0800 Subject: [PATCH 07/30] refactor and try to direct use streambuf.Buffer --- packetbeat/protos/cassandra/cassandra.go | 4 +- packetbeat/protos/cassandra/config.go | 5 + .../protos/cassandra/internal/gocql/LICENSE | 27 ++++ .../{ => internal/gocql}/compressor.go | 0 .../cassandra/{ => internal/gocql}/frame.go | 135 ++++++++++-------- .../cassandra/{ => internal/gocql}/marshal.go | 7 +- packetbeat/protos/cassandra/parser.go | 44 +++--- packetbeat/protos/cassandra/pub.go | 6 +- 8 files changed, 135 insertions(+), 93 deletions(-) create mode 100644 packetbeat/protos/cassandra/internal/gocql/LICENSE rename packetbeat/protos/cassandra/{ => internal/gocql}/compressor.go (100%) rename packetbeat/protos/cassandra/{ => internal/gocql}/frame.go (89%) rename packetbeat/protos/cassandra/{ => internal/gocql}/marshal.go (99%) diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go index 3536d542d8a..2e4c5b1fc46 100644 --- a/packetbeat/protos/cassandra/cassandra.go +++ b/packetbeat/protos/cassandra/cassandra.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/beats/packetbeat/protos" "github.com/elastic/beats/packetbeat/protos/tcp" "github.com/elastic/beats/packetbeat/publish" + . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" ) // cassandra application level protocol analyzer plugin @@ -40,6 +41,7 @@ var ( func init() { protos.Register("cassandra", New) + isDebug = logp.IsDebug("cassandra") } // New create and initializes a new cassandra protocol analyzer instance. @@ -67,8 +69,6 @@ func (cassandra *cassandra) init(results publish.Transactions, config *cassandra return err } cassandra.pub.results = results - - isDebug = logp.IsDebug("cassandra") return nil } diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go index 65fcd9a826f..1aa0741ea6f 100644 --- a/packetbeat/protos/cassandra/config.go +++ b/packetbeat/protos/cassandra/config.go @@ -1,8 +1,10 @@ package cassandra import ( + "fmt" "github.com/elastic/beats/packetbeat/config" "github.com/elastic/beats/packetbeat/protos" + "github.com/pkg/errors" ) type cassandraConfig struct { @@ -23,5 +25,8 @@ var ( ) func (c *cassandraConfig) Validate() error { + if !(c.Compressor == "" || c.Compressor == "snappy") { + return errors.New(fmt.Sprintf("invalid compressor config: %s, only snappy supported", c.Compressor)) + } return nil } diff --git a/packetbeat/protos/cassandra/internal/gocql/LICENSE b/packetbeat/protos/cassandra/internal/gocql/LICENSE new file mode 100644 index 00000000000..82380acef9c --- /dev/null +++ b/packetbeat/protos/cassandra/internal/gocql/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2016 The gocql Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packetbeat/protos/cassandra/compressor.go b/packetbeat/protos/cassandra/internal/gocql/compressor.go similarity index 100% rename from packetbeat/protos/cassandra/compressor.go rename to packetbeat/protos/cassandra/internal/gocql/compressor.go diff --git a/packetbeat/protos/cassandra/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go similarity index 89% rename from packetbeat/protos/cassandra/frame.go rename to packetbeat/protos/cassandra/internal/gocql/frame.go index 669443f24d6..13080dbd738 100644 --- a/packetbeat/protos/cassandra/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -31,6 +31,7 @@ import ( "net" "runtime" "sync" + "github.com/elastic/beats/libbeat/common/streambuf" ) var ( @@ -38,26 +39,26 @@ var ( ) type frameHeader struct { - version protoVersion - flags byte - stream int - op frameOp - length int - customPayload map[string][]byte + Version protoVersion + Flags byte + Stream int + Op frameOp + Length int + CustomPayload map[string][]byte } -func (f frameHeader) toMap() map[string]interface{} { +func (f frameHeader) ToMap() map[string]interface{} { data := make(map[string]interface{}) - data["version"] = fmt.Sprintf("%d", f.version.version()) - data["flags"] = f.flags - data["stream"] = f.stream - data["op"] = f.op.String() - data["length"] = f.length + data["version"] = fmt.Sprintf("%d", f.Version.version()) + data["flags"] = f.Flags + data["stream"] = f.Stream + data["op"] = f.Op.String() + data["length"] = f.Length return data } func (f frameHeader) String() string { - return fmt.Sprintf("[header version=%s flags=0x%x stream=%d op=%s length=%d]", f.version.String(), f.flags, f.stream, f.op.String(), f.length) + return fmt.Sprintf("[header version=%s flags=0x%x stream=%d op=%s length=%d]", f.Version.String(), f.Flags, f.Stream, f.Op.String(), f.Length) } func (f frameHeader) Header() frameHeader { @@ -96,7 +97,7 @@ type framer struct { rbuf []byte } -func newFramer(r io.Reader, compressor Compressor, version byte) *framer { +func NewFramer(r *streambuf.Buffer, compressor Compressor, version byte) *framer { f := framerPool.Get().(*framer) var flags byte if compressor != nil { @@ -128,85 +129,95 @@ type frame interface { Header() frameHeader } -func readHeader(r io.Reader, p []byte) (head frameHeader, err error) { - _, err = io.ReadFull(r, p[:1]) +func ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { + v,err:=r.ReadByte() if err != nil { - return frameHeader{}, err + return nil, err } - - version := p[0] & protoVersionMask + version := v & protoVersionMask if version < protoVersion1 || version > protoVersion4 { - return frameHeader{}, fmt.Errorf("unsupported response version: %d", version) - } - - headSize := 9 - if version < protoVersion3 { - headSize = 8 - } - - _, err = io.ReadFull(r, p[1:headSize]) - if err != nil { - return frameHeader{}, err + return nil, fmt.Errorf("unsupported response version: %d", version) } - p = p[:headSize] + head = &frameHeader{} - v := p[0] + head.Version = protoVersion(v) - head.version = protoVersion(v) - - head.flags = p[1] + head.Flags,err = r.ReadByte() if version > protoVersion2 { - if len(p) != 9 { - return frameHeader{}, fmt.Errorf("not enough bytes to read header require 9 got: %d", len(p)) + stream,err := r.ReadNetUint16() + if err != nil { + return nil, err + } + head.Stream=int(stream) + + b,err := r.ReadByte() + if err != nil { + return nil, err } - head.stream = int(int16(p[2])<<8 | int16(p[3])) - head.op = frameOp(p[4]) - head.length = int(readInt(p[5:])) + head.Op = frameOp(b) + l,err := r.ReadNetUint32() + if err != nil { + return nil, err + } + head.Length=int(l) } else { - if len(p) != 8 { - return frameHeader{}, fmt.Errorf("not enough bytes to read header require 8 got: %d", len(p)) + stream,err := r.ReadNetUint8() + if err != nil { + return nil, err + } + head.Stream=int(stream) + + + b,err := r.ReadByte() + if err != nil { + return nil, err } - head.stream = int(int8(p[2])) - head.op = frameOp(p[3]) - head.length = int(readInt(p[4:])) + head.Op = frameOp(b) + l,err := r.ReadNetUint32() + if err != nil { + return nil, err + } + head.Length=int(l) } + logp.Debug("cassandra","header: %v", head) + return head, nil } // reads a frame form the wire into the framers buffer -func (f *framer) readFrame(head *frameHeader) error { - if head.length < 0 { - return fmt.Errorf("frame body length can not be less than 0: %d", head.length) - } else if head.length > maxFrameSize { +func (f *framer) ReadFrame(head *frameHeader) error { + if head.Length < 0 { + return fmt.Errorf("frame body length can not be less than 0: %d", head.Length) + } else if head.Length > maxFrameSize { // need to free up the connection to be used again - _, err := io.CopyN(ioutil.Discard, f.r, int64(head.length)) + _, err := io.CopyN(ioutil.Discard, f.r, int64(head.Length)) if err != nil { return fmt.Errorf("error whilst trying to discard frame with invalid length: %v", err) } return ErrFrameTooBig } - if cap(f.readBuffer) >= head.length { - f.rbuf = f.readBuffer[:head.length] + if cap(f.readBuffer) >= head.Length { + f.rbuf = f.readBuffer[:head.Length] } else { - f.readBuffer = make([]byte, head.length) + f.readBuffer = make([]byte, head.Length) f.rbuf = f.readBuffer } // assume the underlying reader takes care of timeouts and retries n, err := io.ReadFull(f.r, f.rbuf) if err != nil { - return fmt.Errorf("unable to read frame body: read %d/%d bytes: %v", n, head.length, err) + return fmt.Errorf("unable to read frame body: read %d/%d bytes: %v", n, head.Length, err) } // dealing with compressed frame body - if head.flags&flagCompress == flagCompress { + if head.Flags &flagCompress == flagCompress { if f.compres == nil { return errors.New("no compressor available with compressed frame body") } @@ -221,7 +232,7 @@ func (f *framer) readFrame(head *frameHeader) error { return nil } -func (f *framer) parseFrame(msg *message) (data map[string]interface{}, err error) { +func (f *framer) ParseFrame() (data map[string]interface{}, err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { @@ -233,9 +244,9 @@ func (f *framer) parseFrame(msg *message) (data map[string]interface{}, err erro data = make(map[string]interface{}) - data["request_type"] = f.header.op.String() + data["request_type"] = f.header.Op.String() - if f.header.flags&flagTracing == flagTracing { + if f.header.Flags &flagTracing == flagTracing { uuid, err := f.readUUID() if err != nil { f.traceID = uuid.Bytes() @@ -245,18 +256,18 @@ func (f *framer) parseFrame(msg *message) (data map[string]interface{}, err erro } - if f.header.flags&flagWarning == flagWarning { + if f.header.Flags &flagWarning == flagWarning { warnings := f.readStringList() // dealing with warnings data["warnings"] = warnings } - if f.header.flags&flagCustomPayload == flagCustomPayload { - f.header.customPayload = f.readBytesMap() + if f.header.Flags &flagCustomPayload == flagCustomPayload { + f.header.CustomPayload = f.readBytesMap() } // assumes that the frame body has been read into rbuf - switch f.header.op { + switch f.header.Op { case opError: data = f.parseErrorFrame() case opReady: diff --git a/packetbeat/protos/cassandra/marshal.go b/packetbeat/protos/cassandra/internal/gocql/marshal.go similarity index 99% rename from packetbeat/protos/cassandra/marshal.go rename to packetbeat/protos/cassandra/internal/gocql/marshal.go index 809612ed04a..1342485a8f5 100644 --- a/packetbeat/protos/cassandra/marshal.go +++ b/packetbeat/protos/cassandra/internal/gocql/marshal.go @@ -1,5 +1,4 @@ package cassandra - // Copyright (c) 2012 The gocql Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -390,7 +389,7 @@ const ( type protoVersion byte -func (p protoVersion) request() bool { +func (p protoVersion) IsRequest() bool { v := p.version() if v < protoVersion1 || v > protoVersion4 { @@ -408,7 +407,7 @@ func (p protoVersion) request() bool { return p == 0x00 } -func (p protoVersion) response() bool { +func (p protoVersion) IsResponse() bool { v := p.version() if v < protoVersion1 || v > protoVersion4 { @@ -432,7 +431,7 @@ func (p protoVersion) version() byte { func (p protoVersion) String() string { dir := "REQ" - if p.response() { + if p.IsResponse() { dir = "RESP" } diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index f656ad026c2..e44cd196442 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -4,11 +4,11 @@ import ( "errors" "time" - "bytes" "fmt" "github.com/elastic/beats/libbeat/common/streambuf" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/packetbeat/protos/applayer" + . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" ) type parser struct { @@ -32,7 +32,7 @@ type message struct { failed bool data map[string]interface{} - header frameHeader + header map[string]interface{} // list element use by 'transactions' for correlation next *message @@ -55,6 +55,9 @@ func (p *parser) init( config: cfg, onMessage: onMessage, } + + isDebug = logp.IsDebug("cassandra") + } func (p *parser) append(data []byte) error { @@ -111,53 +114,50 @@ func (p *parser) newMessage(ts time.Time) *message { func (p *parser) parse() (*message, error) { - r := bytes.NewReader(p.buf.Bytes()) - head, err := readHeader(r, make([]byte, 9)) + if(!p.buf.Avail(9)){ + logp.Err("not enough bytes, ignore") + return nil, nil + } + + head, err := ReadHeader(&p.buf) if err != nil { - logp.Err(err.Error()) + logp.Err("%v", err) return nil, nil } - if logp.IsDebug("cassandra") { + if isDebug { logp.Debug("cassandra", fmt.Sprint(head)) } - framer := newFramer(r, p.config.compressor, byte(head.version)) - err = framer.readFrame(&head) + framer := NewFramer(&p.buf, p.config.compressor, byte(head.Version)) + err = framer.ReadFrame(head) if err != nil { - logp.Err(err.Error()) + logp.Err("%v", err) return nil, nil } msg := p.message - data, err := framer.parseFrame(msg) + data, err := framer.ParseFrame() if err != nil { - logp.Err(err.Error()) + logp.Err("%v", err) return nil, nil } dir := applayer.NetOriginalDirection isRequest := true - if head.version.response() { + if head.Version.IsResponse() { dir = applayer.NetReverseDirection isRequest = false } - //collect and wait for enough stream - _, err = p.buf.Collect(head.length + 9) - - if err == streambuf.ErrNoMoreBytes { - return nil, nil - } - msg.Size = uint64(p.buf.BufferConsumed()) msg.IsRequest = isRequest msg.Direction = dir msg.data = data - msg.header = head + msg.header = head.ToMap() if msg.IsRequest { p.message.results.requests.append(msg) @@ -165,8 +165,8 @@ func (p *parser) parse() (*message, error) { p.message.results.responses.append(msg) } - if logp.IsDebug("cassandra") { - logp.Debug("cassandra", fmt.Sprint(msg)) + if isDebug { + logp.Debug("cassandra", "%v",msg) } return msg, nil diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go index c763c318fd5..a0790b017e3 100644 --- a/packetbeat/protos/cassandra/pub.go +++ b/packetbeat/protos/cassandra/pub.go @@ -66,7 +66,7 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { if pub.sendRequest { if pub.sendRequestHeader { - requ.data["request_headers"] = requ.header.toMap() + requ.data["request_headers"] = requ.header } event["cassandra_request"] = requ.data @@ -74,13 +74,13 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { if pub.sendResponse { if pub.sendResponseHeader { - resp.data["response_headers"] = resp.header.toMap() + resp.data["response_headers"] = resp.header } event["cassandra_response"] = resp.data } - if logp.IsDebug("cassandra") { + if isDebug { logp.Debug("cassandra", fmt.Sprint(event)) } From 7616dfda2610ce53ec64ab872b68157ccac03342 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 20 Jul 2016 14:47:29 +0800 Subject: [PATCH 08/30] update config comments --- packetbeat/etc/beat.full.yml | 2 +- packetbeat/packetbeat.full.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 1d1c65b0d90..e55f156020e 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -438,5 +438,5 @@ packetbeat.protocols.cassandra: #send_response_header: true # If this option is enabled, and also the flag indicates that the frame body was compressed, and - # the stream will decode by the compressor, currently support: `snappy` , The default is `nil`. + # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index d9251e56dcb..3dd132c2f7d 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -438,7 +438,7 @@ packetbeat.protocols.cassandra: #send_response_header: true # If this option is enabled, and also the flag indicates that the frame body was compressed, and - # the stream will decode by the compressor, currently support: `snappy` , The default is `nil`. + # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" #================================ General ===================================== From 79cac0e7e45f517e6bd28fd4908f9cc3aa0806be Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 4 Aug 2016 07:52:51 +0800 Subject: [PATCH 09/30] refactor cassandra decoder --- packetbeat/etc/beat.full.yml | 5 + packetbeat/packetbeat.full.yml | 4 + packetbeat/protos/cassandra/README.md | 4 + packetbeat/protos/cassandra/cassandra.go | 25 +- packetbeat/protos/cassandra/config.go | 1 + .../cassandra/internal/gocql/array_decoder.go | 259 ++++++++ .../cassandra/internal/gocql/compressor.go | 4 + .../cassandra/internal/gocql/decoder.go | 39 ++ .../protos/cassandra/internal/gocql/frame.go | 553 ++++++------------ .../cassandra/internal/gocql/marshal.go | 39 +- .../internal/gocql/stream_decoder.go | 269 +++++++++ packetbeat/protos/cassandra/parser.go | 43 +- packetbeat/protos/cassandra/pub.go | 1 + packetbeat/protos/cassandra/trans.go | 14 - .../cassandra/v4/cassandra_create_index.pcap | Bin 8765 -> 3771 bytes .../v4/cassandra_create_keyspace.pcap | Bin 3346 -> 1314 bytes .../cassandra/v4/cassandra_create_table.pcap | Bin 8502 -> 3735 bytes .../pcaps/cassandra/v4/cassandra_insert.pcap | Bin 431 -> 287 bytes .../pcaps/cassandra/v4/cassandra_select.pcap | Bin 460 -> 316 bytes .../v4/cassandra_select_via_index.pcap | Bin 482 -> 338 bytes 20 files changed, 825 insertions(+), 435 deletions(-) create mode 100644 packetbeat/protos/cassandra/README.md create mode 100644 packetbeat/protos/cassandra/internal/gocql/array_decoder.go create mode 100644 packetbeat/protos/cassandra/internal/gocql/decoder.go create mode 100644 packetbeat/protos/cassandra/internal/gocql/stream_decoder.go diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index e55f156020e..2034d33bab2 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -440,3 +440,8 @@ packetbeat.protocols.cassandra: # If this option is enabled, and also the flag indicates that the frame body was compressed, and # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" + + # This option indicates which Operator/Operators will be ignored, + # multi value can be sperated by `,` + #ignored_ops: "SUPPORTED,OPTIONS" + diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 3dd132c2f7d..6ea8e17a90d 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -441,6 +441,10 @@ packetbeat.protocols.cassandra: # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" + # This option indicates which Operator/Operators will be ignored, + # multi value can be sperated by `,` + #ignored_ops: "SUPPORTED,OPTIONS" + #================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group diff --git a/packetbeat/protos/cassandra/README.md b/packetbeat/protos/cassandra/README.md new file mode 100644 index 00000000000..c5d41217342 --- /dev/null +++ b/packetbeat/protos/cassandra/README.md @@ -0,0 +1,4 @@ +#### Cassandra + +##Protocol spec +https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v4.spec diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go index 2e4c5b1fc46..20014250fea 100644 --- a/packetbeat/protos/cassandra/cassandra.go +++ b/packetbeat/protos/cassandra/cassandra.go @@ -6,10 +6,12 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + "fmt" "github.com/elastic/beats/packetbeat/protos" + . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" "github.com/elastic/beats/packetbeat/protos/tcp" "github.com/elastic/beats/packetbeat/publish" - . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" + "strings" ) // cassandra application level protocol analyzer plugin @@ -90,6 +92,27 @@ func (cassandra *cassandra) setFromConfig(config *cassandraConfig) error { parser.compressor = nil } + // parsed ignored ops + if len(config.IgnoredOPs) > 0 { + maps := make(map[string]interface{}) + + if strings.Contains(config.IgnoredOPs, ",") { + array := strings.Split(config.IgnoredOPs, ",") + for i := 0; i < len(array); i++ { + str := array[i] + if len(str) > 0 { + maps[strings.ToUpper(strings.TrimSpace(str))] = str + } + } + } else { + maps[strings.ToUpper(strings.TrimSpace(config.IgnoredOPs))] = config.IgnoredOPs + } + parser.ignoredOps = maps + if isDebug { + logp.Debug("cassandra", fmt.Sprintf("parsed config IgnoredOPs: %v ", parser.ignoredOps)) + } + } + // set transaction correlator configuration trans := &cassandra.transConfig trans.transactionTimeout = config.TransactionTimeout diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go index 1aa0741ea6f..a90ec05ab9c 100644 --- a/packetbeat/protos/cassandra/config.go +++ b/packetbeat/protos/cassandra/config.go @@ -12,6 +12,7 @@ type cassandraConfig struct { SendRequestHeader bool `config:"send_request_header"` SendResponseHeader bool `config:"send_response_header"` Compressor string `config:"compressor"` + IgnoredOPs string `config:"ignored_ops"` } var ( diff --git a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go new file mode 100644 index 00000000000..a0b14c16782 --- /dev/null +++ b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go @@ -0,0 +1,259 @@ +package cassandra + +import ( + "fmt" + "github.com/elastic/beats/libbeat/common/streambuf" + "io" + "net" +) + +type ByteArrayDecoder struct { + Data []byte +} + +func (f ByteArrayDecoder) ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { + p := make([]byte, 9) + header := &frameHeader{} + _, err = io.ReadFull(r, p[:1]) + if err != nil { + return header, err + } + + version := p[0] & protoVersionMask + + if version < protoVersion1 || version > protoVersion4 { + return header, fmt.Errorf("unsupported response version: %d", version) + } + + headSize := 9 + if version < protoVersion3 { + headSize = 8 + } + + _, err = io.ReadFull(r, p[1:headSize]) + if err != nil { + return header, err + } + + p = p[:headSize] + + v := p[0] + + head.Version = protoVersion(v) + + head.Flags = p[1] + + if version > protoVersion2 { + if len(p) != 9 { + return header, fmt.Errorf("not enough bytes to read header require 9 got: %d", len(p)) + } + + head.Stream = int(int16(p[2])<<8 | int16(p[3])) + head.Op = FrameOp(p[4]) + head.Length = int(readInt(p[5:])) + } else { + if len(p) != 8 { + return header, fmt.Errorf("not enough bytes to read header require 8 got: %d", len(p)) + } + + head.Stream = int(int8(p[2])) + head.Op = FrameOp(p[3]) + head.Length = int(readInt(p[4:])) + } + + return head, nil +} + +func (f ByteArrayDecoder) ReadByte() (b byte) { + if len(f.Data) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to Read byte require 1 got: %d", len(f.Data))) + } + + b = f.Data[0] + f.Data = f.Data[1:] + return +} + +func (f ByteArrayDecoder) ReadInt() (n int) { + if len(f.Data) < 4 { + panic(fmt.Errorf("not enough bytes in buffer to Read int require 4 got: %d", len(f.Data))) + } + + fmt.Println(f.Data) + + n = int(int32(f.Data[0])<<24 | int32(f.Data[1])<<16 | int32(f.Data[2])<<8 | int32(f.Data[3])) + f.Data = f.Data[4:] + fmt.Println(f.Data) + + return +} + +func (f ByteArrayDecoder) ReadShort() (n uint16) { + if len(f.Data) < 2 { + panic(fmt.Errorf("not enough bytes in buffer to Read short require 2 got: %d", len(f.Data))) + } + n = uint16(f.Data[0])<<8 | uint16(f.Data[1]) + f.Data = f.Data[2:] + return +} + +func (f ByteArrayDecoder) ReadLong() (n int64) { + if len(f.Data) < 8 { + panic(fmt.Errorf("not enough bytes in buffer to Read long require 8 got: %d", len(f.Data))) + } + n = int64(f.Data[0])<<56 | int64(f.Data[1])<<48 | int64(f.Data[2])<<40 | int64(f.Data[3])<<32 | + int64(f.Data[4])<<24 | int64(f.Data[5])<<16 | int64(f.Data[6])<<8 | int64(f.Data[7]) + f.Data = f.Data[8:] + return +} + +func (f ByteArrayDecoder) ReadString() (s string) { + size := f.ReadShort() + + if len(f.Data) < int(size) { + panic(fmt.Errorf("not enough bytes in buffer to Read string require %d got: %d", size, len(f.Data))) + } + + s = string(f.Data[:size]) + f.Data = f.Data[size:] + return +} + +func (f ByteArrayDecoder) ReadLongString() (s string) { + fmt.Println(f.Data) + size := f.ReadInt() + fmt.Println(f.Data) + + if len(f.Data) < size { + panic(fmt.Errorf("not enough bytes in buffer to Read long string require %d got: %d", size, len(f.Data))) + } + fmt.Println(f.Data) + + s = string(f.Data[:size]) + f.Data = f.Data[size:] + fmt.Println(f.Data) + + fmt.Println(size) + return +} + +func (f ByteArrayDecoder) ReadUUID() *UUID { + if len(f.Data) < 16 { + panic(fmt.Errorf("not enough bytes in buffer to Read uuid require %d got: %d", 16, len(f.Data))) + } + + u, _ := UUIDFromBytes(f.Data[:16]) + f.Data = f.Data[16:] + return &u +} + +func (f ByteArrayDecoder) ReadStringList() []string { + size := f.ReadShort() + + l := make([]string, size) + for i := 0; i < int(size); i++ { + l[i] = f.ReadString() + } + + return l +} + +func (f ByteArrayDecoder) ReadBytesInternal() []byte { + size := f.ReadInt() + if size < 0 { + return nil + } + + if len(f.Data) < size { + panic(fmt.Errorf("not enough bytes in buffer to Read bytes require %d got: %d", size, len(f.Data))) + } + + l := f.Data[:size] + f.Data = f.Data[size:] + + return l +} + +func (f ByteArrayDecoder) ReadBytes() []byte { + l := f.ReadBytesInternal() + + return l +} + +func (f ByteArrayDecoder) ReadShortBytes() []byte { + size := f.ReadShort() + if len(f.Data) < int(size) { + panic(fmt.Errorf("not enough bytes in buffer to Read short bytes: require %d got %d", size, len(f.Data))) + } + + l := f.Data[:size] + f.Data = f.Data[size:] + + return l +} + +func (f ByteArrayDecoder) ReadInet() (net.IP, int) { + if len(f.Data) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to Read inet size require %d got: %d", 1, len(f.Data))) + } + + size := f.Data[0] + f.Data = f.Data[1:] + + if !(size == 4 || size == 16) { + panic(fmt.Errorf("invalid IP size: %d", size)) + } + + if len(f.Data) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to Read inet require %d got: %d", size, len(f.Data))) + } + + ip := make([]byte, size) + copy(ip, f.Data[:size]) + f.Data = f.Data[size:] + + port := f.ReadInt() + return net.IP(ip), port +} + +func (f ByteArrayDecoder) ReadConsistency() Consistency { + return Consistency(f.ReadShort()) +} + +func (f ByteArrayDecoder) ReadStringMap() map[string]string { + size := f.ReadShort() + m := make(map[string]string) + + for i := 0; i < int(size); i++ { + k := f.ReadString() + v := f.ReadString() + m[k] = v + } + + return m +} + +func (f ByteArrayDecoder) ReadBytesMap() map[string][]byte { + size := f.ReadShort() + m := make(map[string][]byte) + + for i := 0; i < int(size); i++ { + k := f.ReadString() + v := f.ReadBytes() + m[k] = v + } + + return m +} + +func (f ByteArrayDecoder) ReadStringMultiMap() map[string][]string { + size := f.ReadShort() + m := make(map[string][]string) + + for i := 0; i < int(size); i++ { + k := f.ReadString() + v := f.ReadStringList() + m[k] = v + } + return m +} diff --git a/packetbeat/protos/cassandra/internal/gocql/compressor.go b/packetbeat/protos/cassandra/internal/gocql/compressor.go index 78c7e46a85b..4354a8d6c57 100644 --- a/packetbeat/protos/cassandra/internal/gocql/compressor.go +++ b/packetbeat/protos/cassandra/internal/gocql/compressor.go @@ -1,3 +1,7 @@ +// Copyright (c) 2012 The gocql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file.\ + package cassandra import ( diff --git a/packetbeat/protos/cassandra/internal/gocql/decoder.go b/packetbeat/protos/cassandra/internal/gocql/decoder.go new file mode 100644 index 00000000000..cf49453b019 --- /dev/null +++ b/packetbeat/protos/cassandra/internal/gocql/decoder.go @@ -0,0 +1,39 @@ +package cassandra + +import ( + "net" +) + +type Decoder interface { + ReadByte() byte + + ReadInt() (n int) + + ReadShort() (n uint16) + + ReadLong() (n int64) + + ReadString() (s string) + + ReadLongString() (s string) + + ReadUUID() *UUID + + ReadStringList() []string + + ReadBytesInternal() []byte + + ReadBytes() []byte + + ReadShortBytes() []byte + + ReadInet() (net.IP, int) + + ReadConsistency() Consistency + + ReadStringMap() map[string]string + + ReadBytesMap() map[string][]byte + + ReadStringMultiMap() map[string][]string +} diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index 13080dbd738..b4313c0a4b1 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -1,48 +1,28 @@ -/** -https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v4.spec +// Copyright (c) 2012 The gocql Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. - The CQL binary protocol is a frame based protocol. Frames are defined as: - - 0 8 16 24 32 40 - +---------+---------+---------+---------+---------+ - | version | flags | stream | opcode | - +---------+---------+---------+---------+---------+ - | length | - +---------+---------+---------+---------+ - | | - . ... body ... . - . . - . . - +---------------------------------------- - - The protocol is big-endian (network byte order). - - some code derived from https://github.com/gocql/gocql - -*/ package cassandra import ( "errors" "fmt" + "github.com/elastic/beats/libbeat/common/streambuf" "github.com/elastic/beats/libbeat/logp" - "io" - "io/ioutil" - "net" - "runtime" "sync" - "github.com/elastic/beats/libbeat/common/streambuf" ) var ( + ErrFrameLength = errors.New("frame body length can not be less than 0") ErrFrameTooBig = errors.New("frame length is bigger than the maximum allowed") + isDebug = false ) type frameHeader struct { Version protoVersion Flags byte Stream int - Op frameOp + Op FrameOp Length int CustomPayload map[string][]byte } @@ -77,60 +57,40 @@ var framerPool = sync.Pool{ // a framer is responsible for reading, writing and parsing frames on a single stream type framer struct { - r io.Reader - proto byte + // flags are for outgoing flags, enabling compression and tracing etc - flags byte - compres Compressor + flags byte + + compres Compressor + + isCompressed bool + headSize int // if this frame was read then the header will be here header *frameHeader - // if tracing flag is set this is not nil - traceID []byte - // holds a ref to the whole byte slice for rbuf so that it can be reset to // 0 after a read. readBuffer []byte - rbuf []byte -} + r *streambuf.Buffer -func NewFramer(r *streambuf.Buffer, compressor Compressor, version byte) *framer { - f := framerPool.Get().(*framer) - var flags byte - if compressor != nil { - flags |= flagCompress - } - - version &= protoVersionMask + decoder Decoder +} - headSize := 8 - if version > protoVersion2 { - headSize = 9 - } +func NewFramer(r *streambuf.Buffer, compressor Compressor) *framer { + f := framerPool.Get().(*framer) f.compres = compressor - f.proto = version - f.flags = flags - f.headSize = headSize - f.r = r - f.rbuf = f.readBuffer[:0] - - f.header = nil - f.traceID = nil return f } -type frame interface { - Header() frameHeader -} - -func ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { - v,err:=r.ReadByte() +// read header frame from stream +func (f *framer) ReadHeader() (head *frameHeader, err error) { + v, err := f.r.ReadByte() if err != nil { return nil, err } @@ -139,133 +99,139 @@ func ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { if version < protoVersion1 || version > protoVersion4 { return nil, fmt.Errorf("unsupported response version: %d", version) } + //fmt.Printf("Version Byte: %x \n",v) head = &frameHeader{} head.Version = protoVersion(v) - head.Flags,err = r.ReadByte() + head.Flags, err = f.r.ReadByte() if version > protoVersion2 { - stream,err := r.ReadNetUint16() + stream, err := f.r.ReadNetUint16() if err != nil { return nil, err } - head.Stream=int(stream) + head.Stream = int(stream) - b,err := r.ReadByte() + b, err := f.r.ReadByte() if err != nil { return nil, err } - head.Op = frameOp(b) - l,err := r.ReadNetUint32() + head.Op = FrameOp(b) + l, err := f.r.ReadNetUint32() if err != nil { return nil, err } - head.Length=int(l) + head.Length = int(l) } else { - stream,err := r.ReadNetUint8() + stream, err := f.r.ReadNetUint8() if err != nil { return nil, err } - head.Stream=int(stream) + head.Stream = int(stream) - - b,err := r.ReadByte() + b, err := f.r.ReadByte() if err != nil { return nil, err } - head.Op = frameOp(b) - l,err := r.ReadNetUint32() + head.Op = FrameOp(b) + l, err := f.r.ReadNetUint32() if err != nil { return nil, err } - head.Length=int(l) + head.Length = int(l) } - logp.Debug("cassandra","header: %v", head) + if head.Length < 0 { + return nil, fmt.Errorf("frame body length can not be less than 0: %d", head.Length) + } else if head.Length > maxFrameSize { + // need to free up the connection to be used again + logp.Err("head length is too large") + return nil, ErrFrameTooBig + } + if !f.r.Avail(head.Length) { + return nil, errors.New(fmt.Sprintf("frame length is not enough as expected length: %v", head.Length)) + } + + logp.Debug("cassandra", "header: %v", head) + f.header = head return head, nil } // reads a frame form the wire into the framers buffer -func (f *framer) ReadFrame(head *frameHeader) error { - if head.Length < 0 { - return fmt.Errorf("frame body length can not be less than 0: %d", head.Length) - } else if head.Length > maxFrameSize { - // need to free up the connection to be used again - _, err := io.CopyN(ioutil.Discard, f.r, int64(head.Length)) - if err != nil { - return fmt.Errorf("error whilst trying to discard frame with invalid length: %v", err) - } - return ErrFrameTooBig - } +func (f *framer) ReadFrame() (data map[string]interface{}, err error) { - if cap(f.readBuffer) >= head.Length { - f.rbuf = f.readBuffer[:head.Length] - } else { - f.readBuffer = make([]byte, head.Length) - f.rbuf = f.readBuffer + //defer func() { + // if r := recover(); r != nil { + // if _, ok := r.(runtime.Error); ok { + // panic(r) + // } + // err = r.(error) + // } + //}() + + if f.header.Length < 0 { + return nil, ErrFrameLength + } else if f.header.Length > maxFrameSize { + return nil, ErrFrameTooBig } - // assume the underlying reader takes care of timeouts and retries - n, err := io.ReadFull(f.r, f.rbuf) - if err != nil { - return fmt.Errorf("unable to read frame body: read %d/%d bytes: %v", n, head.Length, err) + var flags byte + version := byte(f.header.Version) + + if f.compres != nil { + flags |= flagCompress } - // dealing with compressed frame body - if head.Flags &flagCompress == flagCompress { - if f.compres == nil { - return errors.New("no compressor available with compressed frame body") - } + version &= protoVersionMask - f.rbuf, err = f.compres.Decode(f.rbuf) - if err != nil { - return err - } + headSize := 8 + if version > protoVersion2 { + headSize = 9 } - f.header = head - return nil -} + f.proto = version + f.flags = flags + f.headSize = headSize -func (f *framer) ParseFrame() (data map[string]interface{}, err error) { - defer func() { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } - err = r.(error) - } - }() + decoder := &StreamDecoder{} + decoder.r = f.r + f.decoder = decoder data = make(map[string]interface{}) - data["request_type"] = f.header.Op.String() + //Only QUERY, PREPARE and EXECUTE queries support tracing + //If a response frame has the tracing flag set, its body contains + //a tracing ID. The tracing ID is a [uuid] and is the first thing in + //the frame body. The rest of the body will then be the usual body + //corresponding to the response opcode. + if f.header.Flags&flagTracing == flagTracing && (f.header.Op == opQuery || f.header.Op == opExecute || f.header.Op == opPrepare) { - if f.header.Flags &flagTracing == flagTracing { - uuid, err := f.readUUID() - if err != nil { - f.traceID = uuid.Bytes() - // dealing with traceId - data["trace_id"] = uuid.String() - } + logp.Debug("cassandra", "tracing enabled") + + uid := f.decoder.ReadUUID() + + logp.Debug("cassandra", uid.String()) + data["trace_id"] = uid.String() } - if f.header.Flags &flagWarning == flagWarning { - warnings := f.readStringList() + if f.header.Flags&flagWarning == flagWarning { + warnings := f.decoder.ReadStringList() // dealing with warnings data["warnings"] = warnings } - if f.header.Flags &flagCustomPayload == flagCustomPayload { - f.header.CustomPayload = f.readBytesMap() + if f.header.Flags&flagCustomPayload == flagCustomPayload { + f.header.CustomPayload = f.decoder.ReadBytesMap() } + data["request_type"] = f.header.Op.String() + // assumes that the frame body has been read into rbuf switch f.header.Op { case opError: @@ -297,8 +263,8 @@ func (f *framer) ParseFrame() (data map[string]interface{}, err error) { func (f *framer) parseErrorFrame() (data map[string]interface{}) { - code := f.readInt() - msg := f.readString() + code := f.decoder.ReadInt() + msg := f.decoder.ReadString() errT := ErrType(code) @@ -309,18 +275,18 @@ func (f *framer) parseErrorFrame() (data map[string]interface{}) { switch errT { case errUnavailable: - cl := f.readConsistency() - required := f.readInt() - alive := f.readInt() + cl := f.decoder.ReadConsistency() + required := f.decoder.ReadInt() + alive := f.decoder.ReadInt() data["read_consistency"] = cl.String() data["required"] = required data["alive"] = alive case errWriteTimeout: - cl := f.readConsistency() - received := f.readInt() - blockfor := f.readInt() - writeType := f.readString() + cl := f.decoder.ReadConsistency() + received := f.decoder.ReadInt() + blockfor := f.decoder.ReadInt() + writeType := f.decoder.ReadString() data["read_consistency"] = cl.String() data["received"] = received @@ -328,10 +294,10 @@ func (f *framer) parseErrorFrame() (data map[string]interface{}) { data["write_type"] = writeType case errReadTimeout: - cl := f.readConsistency() - received := f.readInt() - blockfor := f.readInt() - dataPresent := f.readByte() + cl := f.decoder.ReadConsistency() + received := f.decoder.ReadInt() + blockfor := f.decoder.ReadInt() + dataPresent := f.decoder.ReadByte() data["read_consistency"] = cl.String() data["received"] = received @@ -339,34 +305,34 @@ func (f *framer) parseErrorFrame() (data map[string]interface{}) { data["data_present"] = dataPresent case errAlreadyExists: - ks := f.readString() - table := f.readString() + ks := f.decoder.ReadString() + table := f.decoder.ReadString() data["keyspace"] = ks data["table"] = table case errUnprepared: - stmtId := f.readShortBytes() + stmtId := f.decoder.ReadShortBytes() data["stmt_id"] = copyBytes(stmtId) case errReadFailure: - data["read_consistency"] = f.readConsistency().String() - data["received"] = f.readInt() - data["blockfor"] = f.readInt() - data["data_present"] = f.readByte() != 0 + data["read_consistency"] = f.decoder.ReadConsistency().String() + data["received"] = f.decoder.ReadInt() + data["blockfor"] = f.decoder.ReadInt() + data["data_present"] = f.decoder.ReadByte() != 0 case errWriteFailure: - data["read_consistency"] = f.readConsistency().String() - data["received"] = f.readInt() - data["blockfor"] = f.readInt() - data["num_failures"] = f.readInt() - data["write_type"] = f.readString() + data["read_consistency"] = f.decoder.ReadConsistency().String() + data["received"] = f.decoder.ReadInt() + data["blockfor"] = f.decoder.ReadInt() + data["num_failures"] = f.decoder.ReadInt() + data["write_type"] = f.decoder.ReadString() case errFunctionFailure: - data["keyspace"] = f.readString() - data["function"] = f.readString() - data["arg_types"] = f.readStringList() + data["keyspace"] = f.decoder.ReadString() + data["function"] = f.decoder.ReadString() + data["arg_types"] = f.decoder.ReadStringList() case errInvalid, errBootstrapping, errConfig, errCredentials, errOverloaded, errProtocol, errServer, errSyntax, errTruncate, errUnauthorized: @@ -379,33 +345,33 @@ func (f *framer) parseErrorFrame() (data map[string]interface{}) { func (f *framer) parseSupportedFrame() (data map[string]interface{}) { data = make(map[string]interface{}) - data["supported"] = f.readStringMultiMap() + data["supported"] = f.decoder.ReadStringMultiMap() return data } func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { meta := make(map[string]interface{}) - flags := f.readInt() + flags := f.decoder.ReadInt() meta["flags"] = flags - colCount := f.readInt() + colCount := f.decoder.ReadInt() meta["col_count"] = colCount if getPKinfo { //only for prepared result if f.proto >= protoVersion4 { - pkeyCount := f.readInt() + pkeyCount := f.decoder.ReadInt() pkeys := make([]int, pkeyCount) for i := 0; i < pkeyCount; i++ { - pkeys[i] = int(f.readShort()) + pkeys[i] = int(f.decoder.ReadShort()) } meta["pkey_columns"] = pkeys } } if flags&flagHasMorePages == flagHasMorePages { - meta["paging_state"] = fmt.Sprintf("%X", f.readBytes()) + meta["paging_state"] = fmt.Sprintf("%X", f.decoder.ReadBytes()) } if flags&flagNoMetaData == flagNoMetaData { @@ -415,8 +381,8 @@ func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { var keyspace, table string globalSpec := flags&flagGlobalTableSpec == flagGlobalTableSpec if globalSpec { - keyspace = f.readString() - table = f.readString() + keyspace = f.decoder.ReadString() + table = f.decoder.ReadString() meta["keyspace"] = keyspace meta["table"] = table } @@ -444,13 +410,13 @@ func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { func (f *framer) parseQueryFrame() (data map[string]interface{}) { data = make(map[string]interface{}) - data["query"] = string(f.readBytes()) + data["query"] = f.decoder.ReadLongString() return data } func (f *framer) parseResultFrame() (data map[string]interface{}) { - kind := f.readInt() + kind := f.decoder.ReadInt() data = make(map[string]interface{}) switch kind { @@ -461,7 +427,7 @@ func (f *framer) parseResultFrame() (data map[string]interface{}) { data["rows"] = f.parseResultRows() case resultKindSetKeyspace: data["result_type"] = "set_keyspace" - data["keyspace"] = f.readString() + data["keyspace"] = f.decoder.ReadString() case resultKindPrepared: data["result_type"] = "prepared" data["result"] = f.parseResultPrepared() @@ -477,7 +443,7 @@ func (f *framer) parseResultRows() map[string]interface{} { result := make(map[string]interface{}) result["meta"] = f.parseResultMetadata(false) - result["num_rows"] = f.readInt() + result["num_rows"] = f.decoder.ReadInt() return result } @@ -486,7 +452,7 @@ func (f *framer) parseResultPrepared() map[string]interface{} { result := make(map[string]interface{}) - result["prepared_id"] = string(f.readShortBytes()) + result["prepared_id"] = string(f.decoder.ReadShortBytes()) result["req_meta"] = f.parseResultMetadata(true) if f.proto < protoVersion2 { @@ -502,32 +468,32 @@ func (f *framer) parseResultSchemaChange() (data map[string]interface{}) { data = make(map[string]interface{}) if f.proto <= protoVersion2 { - change := f.readString() - keyspace := f.readString() - table := f.readString() + change := f.decoder.ReadString() + keyspace := f.decoder.ReadString() + table := f.decoder.ReadString() data["change"] = change data["keyspace"] = keyspace data["table"] = table } else { - change := f.readString() - target := f.readString() + change := f.decoder.ReadString() + target := f.decoder.ReadString() data["change"] = change data["type"] = target switch target { case "KEYSPACE": - data["keyspace"] = f.readString() + data["keyspace"] = f.decoder.ReadString() case "TABLE", "TYPE": - data["keyspace"] = f.readString() - data["object"] = f.readString() + data["keyspace"] = f.decoder.ReadString() + data["object"] = f.decoder.ReadString() case "FUNCTION", "AGGREGATE": - data["keyspace"] = f.readString() - data["name"] = f.readString() - data["args"] = f.readStringList() + data["keyspace"] = f.decoder.ReadString() + data["name"] = f.decoder.ReadString() + data["args"] = f.decoder.ReadStringList() default: logp.Warn("unknown SCHEMA_CHANGE target: %q change: %q", target, change) @@ -538,19 +504,19 @@ func (f *framer) parseResultSchemaChange() (data map[string]interface{}) { func (f *framer) parseAuthenticateFrame() (data map[string]interface{}) { data = make(map[string]interface{}) - data["class"] = f.readString() + data["class"] = f.decoder.ReadString() return data } func (f *framer) parseAuthSuccessFrame() (data map[string]interface{}) { data = make((map[string]interface{})) - data["data"] = fmt.Sprintf("%q", f.readBytes()) + data["data"] = fmt.Sprintf("%q", f.decoder.ReadBytes()) return data } func (f *framer) parseAuthChallengeFrame() (data map[string]interface{}) { data = make((map[string]interface{})) - data["data"] = fmt.Sprintf("%q", f.readBytes()) + data["data"] = fmt.Sprintf("%q", f.decoder.ReadBytes()) return data } @@ -558,19 +524,19 @@ func (f *framer) parseEventFrame() (data map[string]interface{}) { data = make((map[string]interface{})) - eventType := f.readString() + eventType := f.decoder.ReadString() data["event_type"] = eventType switch eventType { case "TOPOLOGY_CHANGE": - data["change"] = f.readString() - host, port := f.readInet() + data["change"] = f.decoder.ReadString() + host, port := f.decoder.ReadInet() data["host"] = host data["port"] = port case "STATUS_CHANGE": - data["change"] = f.readString() - host, port := f.readInet() + data["change"] = f.decoder.ReadString() + host, port := f.decoder.ReadInet() data["host"] = host data["port"] = port @@ -641,20 +607,20 @@ type ColumnInfo struct { func (f *framer) readCol(col *ColumnInfo, globalSpec bool, keyspace, table string) { if !globalSpec { - col.Keyspace = f.readString() - col.Table = f.readString() + col.Keyspace = f.decoder.ReadString() + col.Table = f.decoder.ReadString() } else { col.Keyspace = keyspace col.Table = table } - col.Name = f.readString() + col.Name = f.decoder.ReadString() col.TypeInfo = f.readTypeInfo() } func (f *framer) readTypeInfo() TypeInfo { - id := f.readShort() + id := f.decoder.ReadShort() simple := NativeType{ proto: f.proto, @@ -662,7 +628,7 @@ func (f *framer) readTypeInfo() TypeInfo { } if simple.typ == TypeCustom { - simple.custom = f.readString() + simple.custom = f.decoder.ReadString() if cassType := getApacheCassandraType(simple.custom); cassType != TypeCustom { simple.typ = cassType } @@ -670,7 +636,7 @@ func (f *framer) readTypeInfo() TypeInfo { switch simple.typ { case TypeTuple: - n := f.readShort() + n := f.decoder.ReadShort() tuple := TupleTypeInfo{ NativeType: simple, Elems: make([]TypeInfo, n), @@ -686,14 +652,14 @@ func (f *framer) readTypeInfo() TypeInfo { udt := UDTTypeInfo{ NativeType: simple, } - udt.KeySpace = f.readString() - udt.Name = f.readString() + udt.KeySpace = f.decoder.ReadString() + udt.Name = f.decoder.ReadString() - n := f.readShort() + n := f.decoder.ReadShort() udt.Elements = make([]UDTField, n) for i := 0; i < int(n); i++ { field := &udt.Elements[i] - field.Name = f.readString() + field.Name = f.decoder.ReadString() field.Type = f.readTypeInfo() } @@ -715,193 +681,6 @@ func (f *framer) readTypeInfo() TypeInfo { return simple } -func (f *framer) readByte() byte { - if len(f.rbuf) < 1 { - panic(fmt.Errorf("not enough bytes in buffer to read byte require 1 got: %d", len(f.rbuf))) - } - - b := f.rbuf[0] - f.rbuf = f.rbuf[1:] - return b -} - -func (f *framer) readInt() (n int) { - if len(f.rbuf) < 4 { - panic(fmt.Errorf("not enough bytes in buffer to read int require 4 got: %d", len(f.rbuf))) - } - - n = int(int32(f.rbuf[0])<<24 | int32(f.rbuf[1])<<16 | int32(f.rbuf[2])<<8 | int32(f.rbuf[3])) - f.rbuf = f.rbuf[4:] - return -} - -func (f *framer) readShort() (n uint16) { - if len(f.rbuf) < 2 { - panic(fmt.Errorf("not enough bytes in buffer to read short require 2 got: %d", len(f.rbuf))) - } - n = uint16(f.rbuf[0])<<8 | uint16(f.rbuf[1]) - f.rbuf = f.rbuf[2:] - return -} - -func (f *framer) readLong() (n int64) { - if len(f.rbuf) < 8 { - panic(fmt.Errorf("not enough bytes in buffer to read long require 8 got: %d", len(f.rbuf))) - } - n = int64(f.rbuf[0])<<56 | int64(f.rbuf[1])<<48 | int64(f.rbuf[2])<<40 | int64(f.rbuf[3])<<32 | - int64(f.rbuf[4])<<24 | int64(f.rbuf[5])<<16 | int64(f.rbuf[6])<<8 | int64(f.rbuf[7]) - f.rbuf = f.rbuf[8:] - return -} - -func (f *framer) readString() (s string) { - size := f.readShort() - - if len(f.rbuf) < int(size) { - panic(fmt.Errorf("not enough bytes in buffer to read string require %d got: %d", size, len(f.rbuf))) - } - - s = string(f.rbuf[:size]) - f.rbuf = f.rbuf[size:] - return -} - -func (f *framer) readLongString() (s string) { - size := f.readInt() - - if len(f.rbuf) < size { - panic(fmt.Errorf("not enough bytes in buffer to read long string require %d got: %d", size, len(f.rbuf))) - } - - s = string(f.rbuf[:size]) - f.rbuf = f.rbuf[size:] - return -} - -func (f *framer) readUUID() (*UUID, error) { - if len(f.rbuf) < 16 { - return nil, fmt.Errorf("not enough bytes in buffer to read uuid require %d got: %d", 16, len(f.rbuf)) - } - - u, _ := UUIDFromBytes(f.rbuf[:16]) - f.rbuf = f.rbuf[16:] - return &u, nil -} - -func (f *framer) readStringList() []string { - size := f.readShort() - - l := make([]string, size) - for i := 0; i < int(size); i++ { - l[i] = f.readString() - } - - return l -} - -func (f *framer) readBytesInternal() ([]byte, error) { - size := f.readInt() - if size < 0 { - return nil, nil - } - - if len(f.rbuf) < size { - return nil, fmt.Errorf("not enough bytes in buffer to read bytes require %d got: %d", size, len(f.rbuf)) - } - - l := f.rbuf[:size] - f.rbuf = f.rbuf[size:] - - return l, nil -} - -func (f *framer) readBytes() []byte { - l, err := f.readBytesInternal() - if err != nil { - panic(err) - } - - return l -} - -func (f *framer) readShortBytes() []byte { - size := f.readShort() - if len(f.rbuf) < int(size) { - panic(fmt.Errorf("not enough bytes in buffer to read short bytes: require %d got %d", size, len(f.rbuf))) - } - - l := f.rbuf[:size] - f.rbuf = f.rbuf[size:] - - return l -} - -func (f *framer) readInet() (net.IP, int) { - if len(f.rbuf) < 1 { - panic(fmt.Errorf("not enough bytes in buffer to read inet size require %d got: %d", 1, len(f.rbuf))) - } - - size := f.rbuf[0] - f.rbuf = f.rbuf[1:] - - if !(size == 4 || size == 16) { - panic(fmt.Errorf("invalid IP size: %d", size)) - } - - if len(f.rbuf) < 1 { - panic(fmt.Errorf("not enough bytes in buffer to read inet require %d got: %d", size, len(f.rbuf))) - } - - ip := make([]byte, size) - copy(ip, f.rbuf[:size]) - f.rbuf = f.rbuf[size:] - - port := f.readInt() - return net.IP(ip), port -} - -func (f *framer) readConsistency() Consistency { - return Consistency(f.readShort()) -} - -func (f *framer) readStringMap() map[string]string { - size := f.readShort() - m := make(map[string]string) - - for i := 0; i < int(size); i++ { - k := f.readString() - v := f.readString() - m[k] = v - } - - return m -} - -func (f *framer) readBytesMap() map[string][]byte { - size := f.readShort() - m := make(map[string][]byte) - - for i := 0; i < int(size); i++ { - k := f.readString() - v := f.readBytes() - m[k] = v - } - - return m -} - -func (f *framer) readStringMultiMap() map[string][]string { - size := f.readShort() - m := make(map[string][]string) - - for i := 0; i < int(size); i++ { - k := f.readString() - v := f.readStringList() - m[k] = v - } - return m -} - func readInt(p []byte) int32 { return int32(p[0])<<24 | int32(p[1])<<16 | int32(p[2])<<8 | int32(p[3]) } diff --git a/packetbeat/protos/cassandra/internal/gocql/marshal.go b/packetbeat/protos/cassandra/internal/gocql/marshal.go index 1342485a8f5..3fc5bbe3ad8 100644 --- a/packetbeat/protos/cassandra/internal/gocql/marshal.go +++ b/packetbeat/protos/cassandra/internal/gocql/marshal.go @@ -1,8 +1,9 @@ -package cassandra // Copyright (c) 2012 The gocql Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +package cassandra + import ( "bytes" "fmt" @@ -438,29 +439,29 @@ func (p protoVersion) String() string { return fmt.Sprintf("[version=%d direction=%s]", p.version(), dir) } -type frameOp byte +type FrameOp byte const ( // header ops - opError frameOp = 0x00 - opStartup frameOp = 0x01 - opReady frameOp = 0x02 - opAuthenticate frameOp = 0x03 - opOptions frameOp = 0x05 - opSupported frameOp = 0x06 - opQuery frameOp = 0x07 - opResult frameOp = 0x08 - opPrepare frameOp = 0x09 - opExecute frameOp = 0x0A - opRegister frameOp = 0x0B - opEvent frameOp = 0x0C - opBatch frameOp = 0x0D - opAuthChallenge frameOp = 0x0E - opAuthResponse frameOp = 0x0F - opAuthSuccess frameOp = 0x10 + opError FrameOp = 0x00 + opStartup FrameOp = 0x01 + opReady FrameOp = 0x02 + opAuthenticate FrameOp = 0x03 + opOptions FrameOp = 0x05 + opSupported FrameOp = 0x06 + opQuery FrameOp = 0x07 + opResult FrameOp = 0x08 + opPrepare FrameOp = 0x09 + opExecute FrameOp = 0x0A + opRegister FrameOp = 0x0B + opEvent FrameOp = 0x0C + opBatch FrameOp = 0x0D + opAuthChallenge FrameOp = 0x0E + opAuthResponse FrameOp = 0x0F + opAuthSuccess FrameOp = 0x10 ) -func (f frameOp) String() string { +func (f FrameOp) String() string { switch f { case opError: return "ERROR" diff --git a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go new file mode 100644 index 00000000000..221ad8f4189 --- /dev/null +++ b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go @@ -0,0 +1,269 @@ +package cassandra + +import ( + "errors" + "fmt" + "github.com/elastic/beats/libbeat/common/streambuf" + "github.com/elastic/beats/libbeat/logp" + "net" +) + +type StreamDecoder struct { + r *streambuf.Buffer +} + +func (f StreamDecoder) ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { + v, err := r.ReadByte() + if err != nil { + return nil, err + } + version := v & protoVersionMask + + if version < protoVersion1 || version > protoVersion4 { + return nil, fmt.Errorf("unsupported response version: %d", version) + } + + head = &frameHeader{} + + head.Version = protoVersion(v) + + head.Flags, err = r.ReadByte() + + if version > protoVersion2 { + stream, err := r.ReadNetUint16() + if err != nil { + return nil, err + } + head.Stream = int(stream) + + b, err := r.ReadByte() + if err != nil { + return nil, err + } + + head.Op = FrameOp(b) + l, err := r.ReadNetUint32() + if err != nil { + return nil, err + } + head.Length = int(l) + } else { + stream, err := r.ReadNetUint8() + if err != nil { + return nil, err + } + head.Stream = int(stream) + + b, err := r.ReadByte() + if err != nil { + return nil, err + } + + head.Op = FrameOp(b) + l, err := r.ReadNetUint32() + if err != nil { + return nil, err + } + head.Length = int(l) + } + + if head.Length < 0 { + return nil, fmt.Errorf("frame body length can not be less than 0: %d", head.Length) + } else if head.Length > maxFrameSize { + // need to free up the connection to be used again + logp.Err("head length is too large") + return nil, ErrFrameTooBig + } + + if !r.Avail(head.Length) { + return nil, errors.New(fmt.Sprintf("frame length is not enough as expected length: %v", head.Length)) + } + + logp.Debug("cassandra", "header: %v", head) + + return head, nil +} + +func (f StreamDecoder) ReadByte() byte { + + b, err := f.r.ReadByte() + if err != nil { + panic(err) + } + return b + +} + +func (f StreamDecoder) ReadInt() (n int) { + + data, err := f.r.ReadNetUint32() + if err != nil { + panic(err) + } + n = int(data) + + return +} + +func (f StreamDecoder) ReadShort() (n uint16) { + + data, err := f.r.ReadNetUint16() + if err != nil { + panic(err) + } + n = data + + return +} + +func (f StreamDecoder) ReadLong() (n int64) { + + data, err := f.r.ReadNetUint64() + if err != nil { + panic(err) + } + n = int64(data) + + return +} + +func (f StreamDecoder) ReadString() (s string) { + size := f.ReadShort() + + str := make([]byte, size) + _, err := f.r.Read(str) + if err != nil { + panic(err) + } + s = string(str) + + return +} + +func (f StreamDecoder) ReadLongString() (s string) { + + size := f.ReadInt() + + str := make([]byte, size) + _, err := f.r.Read(str) + if err != nil { + panic(err) + } + s = string(str) + + return +} + +func (f StreamDecoder) ReadUUID() *UUID { + + bytes := make([]byte, 16) + _, err := f.r.Read(bytes) + if err != nil { + panic(err) + } + u, _ := UUIDFromBytes(bytes) + return &u + +} + +func (f StreamDecoder) ReadStringList() []string { + size := f.ReadShort() + + l := make([]string, size) + for i := 0; i < int(size); i++ { + l[i] = f.ReadString() + } + + return l +} + +func (f StreamDecoder) ReadBytesInternal() []byte { + size := f.ReadInt() + if size < 0 { + return nil + } + + bytes := make([]byte, size) + _, err := f.r.Read(bytes) + if err != nil { + panic(err) + } + return bytes + +} + +func (f StreamDecoder) ReadBytes() []byte { + l := f.ReadBytesInternal() + return l +} + +func (f StreamDecoder) ReadShortBytes() []byte { + size := f.ReadShort() + + bytes := make([]byte, size) + _, err := f.r.Read(bytes) + if err != nil { + panic(err) + } + return bytes + +} + +func (f StreamDecoder) ReadInet() (net.IP, int) { + + size := f.ReadByte() + if !(size == 4 || size == 16) { + panic(fmt.Errorf("invalid IP size: %d", size)) + } + + ip := make([]byte, int(size)) + _, err := f.r.Read(ip) + if err != nil { + panic(err) + } + port := f.ReadInt() + return net.IP(ip), port + +} + +func (f StreamDecoder) ReadConsistency() Consistency { + return Consistency(f.ReadShort()) +} + +func (f StreamDecoder) ReadStringMap() map[string]string { + size := f.ReadShort() + m := make(map[string]string) + + for i := 0; i < int(size); i++ { + k := f.ReadString() + v := f.ReadString() + m[k] = v + } + + return m +} + +func (f StreamDecoder) ReadBytesMap() map[string][]byte { + size := f.ReadShort() + m := make(map[string][]byte) + + for i := 0; i < int(size); i++ { + k := f.ReadString() + v := f.ReadBytes() + m[k] = v + } + + return m +} + +func (f StreamDecoder) ReadStringMultiMap() map[string][]string { + size := f.ReadShort() + m := make(map[string][]string) + + for i := 0; i < int(size); i++ { + k := f.ReadString() + v := f.ReadStringList() + m[k] = v + } + return m +} diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index e44cd196442..01a602b816b 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -21,6 +21,7 @@ type parser struct { type parserConfig struct { maxBytes int compressor Compressor + ignoredOps map[string]interface{} } type message struct { @@ -114,36 +115,53 @@ func (p *parser) newMessage(ts time.Time) *message { func (p *parser) parse() (*message, error) { - if(!p.buf.Avail(9)){ + if !p.buf.Avail(9) { logp.Err("not enough bytes, ignore") + p.message = nil return nil, nil } - head, err := ReadHeader(&p.buf) + framer := NewFramer(&p.buf, p.config.compressor) + head, err := framer.ReadHeader() if err != nil { logp.Err("%v", err) + p.message = nil return nil, nil } - if isDebug { - logp.Debug("cassandra", fmt.Sprint(head)) + //check if the ops already ignored + if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { + + v := p.config.ignoredOps[head.Op.String()] + if v != nil { + logp.Debug("cassandra", fmt.Sprintf("Ops: %s was marked to be ignored, ignoring", head.Op.String())) + p.message = nil + return nil, nil + } } - framer := NewFramer(&p.buf, p.config.compressor, byte(head.Version)) - err = framer.ReadFrame(head) - if err != nil { - logp.Err("%v", err) + if !p.buf.Avail(head.Length) { + logp.Err("not enough bytes for frame body, ignore") + p.message = nil return nil, nil } - msg := p.message - data, err := framer.ParseFrame() + data, err := framer.ReadFrame() + frameLength := p.buf.BufferConsumed() if err != nil { + p.message = nil logp.Err("%v", err) return nil, nil } + // collect leftover + leftDataSize := head.Length + 9 - frameLength + if leftDataSize > 0 { + p.buf.Collect(leftDataSize) + + } + dir := applayer.NetOriginalDirection isRequest := true @@ -152,6 +170,7 @@ func (p *parser) parse() (*message, error) { isRequest = false } + msg := p.message msg.Size = uint64(p.buf.BufferConsumed()) msg.IsRequest = isRequest msg.Direction = dir @@ -165,9 +184,5 @@ func (p *parser) parse() (*message, error) { p.message.results.responses.append(msg) } - if isDebug { - logp.Debug("cassandra", "%v",msg) - } - return msg, nil } diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go index a0790b017e3..55b8bee4e6e 100644 --- a/packetbeat/protos/cassandra/pub.go +++ b/packetbeat/protos/cassandra/pub.go @@ -13,6 +13,7 @@ type transPub struct { sendResponse bool sendRequestHeader bool sendResponseHeader bool + ignoredOps string results publish.Transactions } diff --git a/packetbeat/protos/cassandra/trans.go b/packetbeat/protos/cassandra/trans.go index dde21a7e1ac..6b5b582371f 100644 --- a/packetbeat/protos/cassandra/trans.go +++ b/packetbeat/protos/cassandra/trans.go @@ -72,9 +72,6 @@ func (trans *transactions) onRequest( return err } if merged { - if isDebug { - debugf("request message got merged") - } msg = prev } else { trans.requests.append(msg) @@ -84,10 +81,6 @@ func (trans *transactions) onRequest( return nil } - if isDebug { - debugf("request message complete") - } - return trans.correlate() } @@ -104,9 +97,6 @@ func (trans *transactions) onResponse( return err } if merged { - if isDebug { - debugf("response message got merged") - } msg = prev } else { trans.responses.append(msg) @@ -116,10 +106,6 @@ func (trans *transactions) onResponse( return nil } - if isDebug { - debugf("response message complete") - } - return trans.correlate() } diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_index.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_index.pcap index 39906151574cf8c0d52c8ad5e9d08ed9fabe3447..5ea3accfc936c3ce1cb7a3ac0be65f288c7347ea 100644 GIT binary patch delta 43 zcmV+`0M!4zM7tf3ppoc!lc)iblY#<~lUoCslRN~LlNtrKvls@J0h9Izw6pREQ3$ZH B5`F*x delta 2042 zcmaKsZAepL6vv-ucT+Qqh1+%33{0ew%2`4|WQJN6OqVfCsMK`xrPS$qf7|T6C@LO` z=#A2_(kyEjQ4Pv;G6)UCBG3;h6h!#J(qQD)dG5Ws6&J_vj{h zi?y+KpXPnQsW#|Exi-kP5e>e^RyW9lamCP3#~R(x%`Qs0Amt)W_$qrnL@L(}sd>@r zlv-J2rQ$Yg9*{RW#t@gxcaq9DH<&DxTYon>5C zn326MXG0bWH=8oAj4yB$nN+gd+H*$Qq$esNk2bLNj~Sfmq#lthG^rS@0ogYejE+Mi zT7vAc!5PzVbik=YbA)5JJ)j^ZLZze#uXj))wqluCL!^|*k3VJ@E=ZdVuTVT`MuD~m zloJh{da$A;m>pfZT*~1mOq z;6u8zXZtZa4OX-Sv*YfqQjRcTJ?uUoP7loX1JP}xqRmpG$gCI4 zydpCSY^G-Mr!i^*D_VkCL-y`zIJ%_yUjMbASt|v>Yg)qrV1e2}YL$BOnc<(d0Ps_P zqE}Z+@xtJ}FnAX{?KY7nrMFVeTHuUg#1*fJNA&ll9AVrpc!*vn9Gy2XB++aI%`#H1 ztj6W=HfXIj0`Ho z74NTF;<+#l2M!>Ii4Xd>(?!4mSY4MQtPC!5S)(pu&hY1B`GyZ=NXh(W(jB&bUpE}W zJ&_>=x6=(DdV-OWL0s{M%*NvHQVu_67>Ybd$6Kl9P@Dk&|Zx=aZHNptF7k8v*AL4QK!W literal 3346 zcmds(Z%i9?7{`AFN?}lB1V)qb;sjicU>#)9F(uNJgD$S^Y$1fiq~6L2WQEqdiXn>Y z%z`m2DxewJrqLH_!XSz-G#E7tzQAZgVi;k;3<)H@(D?6-5u&Z%r+01dTGE(ozLDff z%W?PpJ)iIId(V#>Uw-sX9xGt!ALaXKtJHJk<3jd1{y>$7;u%|f;P*DxR>;^n#tc*| z<~nc9D!;T>dF@)%a&$8`kBf$)1Kr!M>)qRjtDshC!n4n#`W;^0-ogymbFJKSoHY;>WZBA{;HR<2MI<32_K#W{ zIjbawgM$GfG8h{6KP3brA&G8m;>I3khGST*j}}u zQ|DbV6G+W!M`yIW>4Vt4S{PSC=kYg8ECdZIRp+cPo~2V7Uc2?gPVey}QA^Wg%%9TP zg(({sSUg^e=WWdFb2xdI-S2SP-JLu$DGs%>qNGRcK-wh2cnQmAV&uK9Uw);u&Pz+9 zh3RN{!-NI{%e!_-=kTlv4v|YKj3fT z_kkV#no6EJI=2Q) zp;}b392mh+KnQX@PTtFNcpF=-i3?U0EN>wjtgKq!kOpi6V6*#yO)Qz%IvS!}wNy4GmGLc3qsB$)RI`PBdkgk#vxQYRZ8J@!DV&Cq zN2YSoxq?b>q*;lD%E*dSkj$L?n&4w*R@V27B%BF~{%57Zh$!PQ zEYX}T>~7p*hBgI6GmPm9UFt#6vc|+&g)a>usoy5xP#X|!0UTe#aGxaDD4jT>Z2EHy zw^wM^QqM75H8hbu+*-EH^S|lsQR39uyjMBzLY%(U*i81ANGG&o&Kj5##ta?pp<%(H zks6quX}o>Ah*K@a$&R}Hf%zUf%6w7opVfDS0MRKrjiW4#0KJkXqmDe&M79N;^XQ05 zeNo2#VtO6*I1DF_Ln2EJOhpG|bwt*Ny6u4(+Xrm5M&FPIY!P4=_XC^WHD?XX@7iVWDwqr+B-VpAXrC&cV+_9{(S|^%p5E)>oL{+TMVylVvsb9YwRM-8Pa*R zpe$=(@^9wQu|Ox9vgOTZg Bd)@#5 diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_table.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_create_table.pcap index f5caae7d4f3eea3ad27b8f6be38bdd9c6185dc38..adc90767d8edfd04ceebbf67cd7b41871ee0698b 100644 GIT binary patch delta 44 zcmV+{0Mq}rLYEzo&y(T-c$1<6l9PM`k&{&fnUgjJmXj0)wzCok0s^yK36cksHW9R< C;}F3B delta 1919 zcmaKsT}V@57{}kU&FR$TKsS_DkwY?9=)yiUKQ`xdCi7w@VU%f#LZt2^1o3O?k}e93 zR~;(Va?+g)Cz&LoL=X`fG)8z4W#*-lK^J*3d)~97p0l&Oc(!w~|L^&|&-=x%NDx0J&!+hDztV!r{Fri zNgYj;WlgdY(~y|PdTLa0xsxAxdjuwvBBx*|W2}%DLS7u8MikE@gf0$12>NJTyhx+q ziQ2m~_6k$v9S&KRn-!XYJ~Mb%Fs}IVfFJ5oL5Q5^DJ$ZGh!2klUn{o8_>sg) zP?M*;SNhB5;Y^qnoP*%pY`I7*P$Z1=lMSc#lI1o?-sdGv;?d}0mzSU1&JMdismiSM zm)mxd782pS^(@i?I4t0A()k{;ai4vBnCaPnVf_HB4Ts$Z&chf9>VvH##cXc3xG{{w z-l=4@+#Iy+t**xkRvr;Td3ca2=^?~rAck*g)i9UHybDX~)wvlku-*eR0`yo3CrTng zyQkA18t@UVz?Q$bBQc7JD7DxGP|mjpf6xW ziZSy~=?>u}YvkR*cl%~-Fda~TLK$3@QRyQ ziiF%ohPRk5K`%g$6r(u#ri!b`3@^li;hBP?`-<%Z1j8%9Yr;FU89L23D>=a>hK9FM z1J}9+!|Py~5nwwpO9jpVh3E>IQGJ76E{8E&qfbc8cH1~+?O;}jU7{9B3!iCN9aR>q zi-w~%LDgcnNQuXzN@?w!2-2-rv(9`}4g#42H1A=9fc9=Avr2$w Gl>h)6UMNQZ diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_select.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_select.pcap index 29ddb62f9974de2bdd231ccc94abbd050956e3d9..3181a4b938f213136b9f6535f89f92f5cbc6e2ac 100644 GIT binary patch delta 9 RcmX@ZyoYJRw2AkY0RR=c1jzsZ delta 128 zcmdnPbcT7tG%*h~UIq&W1~6s<(yj~)CiCqku8|fNFo^!A!N9=C$iWp{X#WL-Cr(=B z@##i>`1%M;O&uQ`7#!Fb80vvqK={l@SWHNG0qJF+B*0U~ph#V5nzcU<6_1pfevikEY5qWz;Qd z5HN83rvVgXnWPQO!E@Q8@@?4$!>D1_A9nB(qF_ HW|;s0Q|~9H From 864b1f54e9d730819150233ec12aa7682b2078e9 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 8 Aug 2016 20:02:54 +0800 Subject: [PATCH 10/30] bugfix and refactor --- .../cassandra/internal/gocql/array_decoder.go | 39 +- .../protos/cassandra/internal/gocql/frame.go | 353 +++++------------- .../cassandra/internal/gocql/marshal.go | 73 ++++ .../internal/gocql/stream_decoder.go | 74 ---- packetbeat/protos/cassandra/parser.go | 122 ++++-- packetbeat/protos/cassandra/pub.go | 84 +++-- packetbeat/protos/cassandra/trans.go | 25 +- .../tests/system/test_0062_cassandra.py | 24 +- 8 files changed, 350 insertions(+), 444 deletions(-) diff --git a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go index a0b14c16782..9014c7667c4 100644 --- a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go @@ -2,7 +2,6 @@ package cassandra import ( "fmt" - "github.com/elastic/beats/libbeat/common/streambuf" "io" "net" ) @@ -11,28 +10,33 @@ type ByteArrayDecoder struct { Data []byte } -func (f ByteArrayDecoder) ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { +func readInt(p []byte) int32 { + return int32(p[0])<<24 | int32(p[1])<<16 | int32(p[2])<<8 | int32(p[3]) +} + +func (f *Framer) ReadHeader1() (head *frameHeader, err error) { p := make([]byte, 9) - header := &frameHeader{} - _, err = io.ReadFull(r, p[:1]) + head = &frameHeader{} + _, err = io.ReadFull(f.r, p[:1]) if err != nil { - return header, err + return head, err } version := p[0] & protoVersionMask if version < protoVersion1 || version > protoVersion4 { - return header, fmt.Errorf("unsupported response version: %d", version) + return head, fmt.Errorf("unsupported response version: %d", version) } + f.proto = version headSize := 9 if version < protoVersion3 { headSize = 8 } - _, err = io.ReadFull(r, p[1:headSize]) + _, err = io.ReadFull(f.r, p[1:headSize]) if err != nil { - return header, err + return head, err } p = p[:headSize] @@ -45,22 +49,22 @@ func (f ByteArrayDecoder) ReadHeader(r *streambuf.Buffer) (head *frameHeader, er if version > protoVersion2 { if len(p) != 9 { - return header, fmt.Errorf("not enough bytes to read header require 9 got: %d", len(p)) + return head, fmt.Errorf("not enough bytes to read header require 9 got: %d", len(p)) } head.Stream = int(int16(p[2])<<8 | int16(p[3])) head.Op = FrameOp(p[4]) - head.Length = int(readInt(p[5:])) + head.BodyLength = int(readInt(p[5:])) } else { if len(p) != 8 { - return header, fmt.Errorf("not enough bytes to read header require 8 got: %d", len(p)) + return head, fmt.Errorf("not enough bytes to read header require 8 got: %d", len(p)) } head.Stream = int(int8(p[2])) head.Op = FrameOp(p[3]) - head.Length = int(readInt(p[4:])) + head.BodyLength = int(readInt(p[4:])) } - + f.Header = head return head, nil } @@ -79,11 +83,8 @@ func (f ByteArrayDecoder) ReadInt() (n int) { panic(fmt.Errorf("not enough bytes in buffer to Read int require 4 got: %d", len(f.Data))) } - fmt.Println(f.Data) - n = int(int32(f.Data[0])<<24 | int32(f.Data[1])<<16 | int32(f.Data[2])<<8 | int32(f.Data[3])) f.Data = f.Data[4:] - fmt.Println(f.Data) return } @@ -120,20 +121,14 @@ func (f ByteArrayDecoder) ReadString() (s string) { } func (f ByteArrayDecoder) ReadLongString() (s string) { - fmt.Println(f.Data) size := f.ReadInt() - fmt.Println(f.Data) if len(f.Data) < size { panic(fmt.Errorf("not enough bytes in buffer to Read long string require %d got: %d", size, len(f.Data))) } - fmt.Println(f.Data) s = string(f.Data[:size]) f.Data = f.Data[size:] - fmt.Println(f.Data) - - fmt.Println(size) return } diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index b4313c0a4b1..7600ba22d46 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -9,13 +9,13 @@ import ( "fmt" "github.com/elastic/beats/libbeat/common/streambuf" "github.com/elastic/beats/libbeat/logp" + "runtime" "sync" ) var ( - ErrFrameLength = errors.New("frame body length can not be less than 0") ErrFrameTooBig = errors.New("frame length is bigger than the maximum allowed") - isDebug = false + isDebug = logp.IsDebug("cassandra") ) type frameHeader struct { @@ -23,65 +23,50 @@ type frameHeader struct { Flags byte Stream int Op FrameOp - Length int + BodyLength int + HeadLength int CustomPayload map[string][]byte } func (f frameHeader) ToMap() map[string]interface{} { data := make(map[string]interface{}) data["version"] = fmt.Sprintf("%d", f.Version.version()) - data["flags"] = f.Flags + data["flags"] = getHeadFlagString(f.Flags) data["stream"] = f.Stream data["op"] = f.Op.String() - data["length"] = f.Length + data["length"] = f.BodyLength return data } func (f frameHeader) String() string { - return fmt.Sprintf("[header version=%s flags=0x%x stream=%d op=%s length=%d]", f.Version.String(), f.Flags, f.Stream, f.Op.String(), f.Length) + return fmt.Sprintf("version:%d, flags: %s, steam: %v, OP: %v, length: %v", f.Version.String(), getHeadFlagString(f.Flags), f.Stream, f.Op.String(), f.BodyLength) } -func (f frameHeader) Header() frameHeader { - return f -} - -const defaultBufSize = 128 - var framerPool = sync.Pool{ New: func() interface{} { - return &framer{ - readBuffer: make([]byte, defaultBufSize), - } + return &Framer{compres: nil, isCompressed: false, Header: nil, r: nil, decoder: nil} }, } // a framer is responsible for reading, writing and parsing frames on a single stream -type framer struct { +type Framer struct { proto byte - // flags are for outgoing flags, enabling compression and tracing etc - flags byte - compres Compressor isCompressed bool - headSize int // if this frame was read then the header will be here - header *frameHeader - - // holds a ref to the whole byte slice for rbuf so that it can be reset to - // 0 after a read. - readBuffer []byte + Header *frameHeader r *streambuf.Buffer decoder Decoder } -func NewFramer(r *streambuf.Buffer, compressor Compressor) *framer { +func NewFramer(r *streambuf.Buffer, compressor Compressor) *Framer { - f := framerPool.Get().(*framer) + f := framerPool.Get().(*Framer) f.compres = compressor f.r = r @@ -89,7 +74,7 @@ func NewFramer(r *streambuf.Buffer, compressor Compressor) *framer { } // read header frame from stream -func (f *framer) ReadHeader() (head *frameHeader, err error) { +func (f *Framer) ReadHeader() (head *frameHeader, err error) { v, err := f.r.ReadByte() if err != nil { return nil, err @@ -97,15 +82,20 @@ func (f *framer) ReadHeader() (head *frameHeader, err error) { version := v & protoVersionMask if version < protoVersion1 || version > protoVersion4 { - return nil, fmt.Errorf("unsupported response version: %d", version) + return nil, fmt.Errorf("unsupported version: %x ", v) } - //fmt.Printf("Version Byte: %x \n",v) + + f.proto = version head = &frameHeader{} head.Version = protoVersion(v) - head.Flags, err = f.r.ReadByte() + flag, err := f.r.ReadByte() + if err != nil { + return nil, err + } + head.Flags = flag if version > protoVersion2 { stream, err := f.r.ReadNetUint16() @@ -124,7 +114,7 @@ func (f *framer) ReadHeader() (head *frameHeader, err error) { if err != nil { return nil, err } - head.Length = int(l) + head.BodyLength = int(l) } else { stream, err := f.r.ReadNetUint8() if err != nil { @@ -142,61 +132,38 @@ func (f *framer) ReadHeader() (head *frameHeader, err error) { if err != nil { return nil, err } - head.Length = int(l) + head.BodyLength = int(l) } - if head.Length < 0 { - return nil, fmt.Errorf("frame body length can not be less than 0: %d", head.Length) - } else if head.Length > maxFrameSize { + if head.BodyLength < 0 { + return nil, fmt.Errorf("frame body length can not be less than 0: %d", head.BodyLength) + } else if head.BodyLength > maxFrameSize { // need to free up the connection to be used again logp.Err("head length is too large") return nil, ErrFrameTooBig } - if !f.r.Avail(head.Length) { - return nil, errors.New(fmt.Sprintf("frame length is not enough as expected length: %v", head.Length)) - } + headSize := f.r.BufferConsumed() + head.HeadLength = headSize - logp.Debug("cassandra", "header: %v", head) - f.header = head + if isDebug { + logp.Debug("cassandra", "header: %v", head) + } + f.Header = head return head, nil } // reads a frame form the wire into the framers buffer -func (f *framer) ReadFrame() (data map[string]interface{}, err error) { - - //defer func() { - // if r := recover(); r != nil { - // if _, ok := r.(runtime.Error); ok { - // panic(r) - // } - // err = r.(error) - // } - //}() - - if f.header.Length < 0 { - return nil, ErrFrameLength - } else if f.header.Length > maxFrameSize { - return nil, ErrFrameTooBig - } - - var flags byte - version := byte(f.header.Version) - - if f.compres != nil { - flags |= flagCompress - } +func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { - version &= protoVersionMask - - headSize := 8 - if version > protoVersion2 { - headSize = 9 - } - - f.proto = version - f.flags = flags - f.headSize = headSize + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() decoder := &StreamDecoder{} decoder.r = f.r @@ -209,35 +176,53 @@ func (f *framer) ReadFrame() (data map[string]interface{}, err error) { //a tracing ID. The tracing ID is a [uuid] and is the first thing in //the frame body. The rest of the body will then be the usual body //corresponding to the response opcode. - if f.header.Flags&flagTracing == flagTracing && (f.header.Op == opQuery || f.header.Op == opExecute || f.header.Op == opPrepare) { + if f.Header.Flags&flagTracing == flagTracing && (f.Header.Op&opQuery == opQuery || f.Header.Op&opExecute == opExecute || f.Header.Op&opPrepare == opPrepare) { - logp.Debug("cassandra", "tracing enabled") + if isDebug { + logp.Debug("cassandra", "tracing enabled") + } uid := f.decoder.ReadUUID() - - logp.Debug("cassandra", uid.String()) - + if isDebug { + logp.Debug("cassandra", uid.String()) + } data["trace_id"] = uid.String() } - if f.header.Flags&flagWarning == flagWarning { + if f.Header.Flags&flagWarning == flagWarning { + if isDebug { + logp.Debug("cassandra", "hit warning flags") + } warnings := f.decoder.ReadStringList() // dealing with warnings data["warnings"] = warnings } - if f.header.Flags&flagCustomPayload == flagCustomPayload { - f.header.CustomPayload = f.decoder.ReadBytesMap() + if f.Header.Flags&flagCustomPayload == flagCustomPayload { + if isDebug { + logp.Debug("cassandra", "hit custom payload flags") + } + f.Header.CustomPayload = f.decoder.ReadBytesMap() } - data["request_type"] = f.header.Op.String() + if f.Header.Flags&flagCompress == flagCompress { + //TODO decompress data and switch to use bytearray decoder + //decoder := &ByteArrayDecoder{} + //buf := make([]byte, f.header.BodyLength) + //f.r.Read(buf) + //decoder.Data = buf + //f.decoder = decoder + + if isDebug { + logp.Debug("cassandra", "hit compress flags") + } + return nil, errors.New("Compressed content not supported yet") + } // assumes that the frame body has been read into rbuf - switch f.header.Op { + switch f.Header.Op { case opError: data = f.parseErrorFrame() - case opReady: - // the body should be empty case opQuery: data = f.parseQueryFrame() case opResult: @@ -252,16 +237,23 @@ func (f *framer) ReadFrame() (data map[string]interface{}, err error) { data = f.parseAuthSuccessFrame() case opEvent: data = f.parseEventFrame() + case opReady: + // the body should be empty case opOptions: + //ignore case opStartup: + //ignore default: - return nil, errors.New(fmt.Sprintf("unknown op and not parsed,%s", f.header)) + //ignore + if isDebug { + logp.Debug("cassandra", "unknow ops, not processed, %v", f.Header) + } } return data, nil } -func (f *framer) parseErrorFrame() (data map[string]interface{}) { +func (f *Framer) parseErrorFrame() (data map[string]interface{}) { code := f.decoder.ReadInt() msg := f.decoder.ReadString() @@ -313,8 +305,7 @@ func (f *framer) parseErrorFrame() (data map[string]interface{}) { case errUnprepared: stmtId := f.decoder.ReadShortBytes() - - data["stmt_id"] = copyBytes(stmtId) + data["stmt_id"] = stmtId case errReadFailure: data["read_consistency"] = f.decoder.ReadConsistency().String() @@ -342,18 +333,18 @@ func (f *framer) parseErrorFrame() (data map[string]interface{}) { return data } -func (f *framer) parseSupportedFrame() (data map[string]interface{}) { +func (f *Framer) parseSupportedFrame() (data map[string]interface{}) { data = make(map[string]interface{}) data["supported"] = f.decoder.ReadStringMultiMap() return data } -func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { +func (f *Framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { meta := make(map[string]interface{}) flags := f.decoder.ReadInt() - meta["flags"] = flags + meta["flags"] = getRowFlagString(flags) colCount := f.decoder.ReadInt() meta["col_count"] = colCount @@ -372,6 +363,7 @@ func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { if flags&flagHasMorePages == flagHasMorePages { meta["paging_state"] = fmt.Sprintf("%X", f.decoder.ReadBytes()) + return meta } if flags&flagNoMetaData == flagNoMetaData { @@ -387,34 +379,16 @@ func (f *framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { meta["table"] = table } - var cols []ColumnInfo - if colCount < 1000 { - // preallocate columninfo to avoid excess copying - cols = make([]ColumnInfo, colCount) - for i := 0; i < colCount; i++ { - f.readCol(&cols[i], globalSpec, keyspace, table) - } - - } else { - // use append, huge number of columns usually indicates a corrupt frame or - // just a huge row. - for i := 0; i < colCount; i++ { - var col ColumnInfo - f.readCol(&col, globalSpec, keyspace, table) - cols = append(cols, col) - } - } - return meta } -func (f *framer) parseQueryFrame() (data map[string]interface{}) { +func (f *Framer) parseQueryFrame() (data map[string]interface{}) { data = make(map[string]interface{}) data["query"] = f.decoder.ReadLongString() return data } -func (f *framer) parseResultFrame() (data map[string]interface{}) { +func (f *Framer) parseResultFrame() (data map[string]interface{}) { kind := f.decoder.ReadInt() @@ -439,7 +413,7 @@ func (f *framer) parseResultFrame() (data map[string]interface{}) { return data } -func (f *framer) parseResultRows() map[string]interface{} { +func (f *Framer) parseResultRows() map[string]interface{} { result := make(map[string]interface{}) result["meta"] = f.parseResultMetadata(false) @@ -448,7 +422,7 @@ func (f *framer) parseResultRows() map[string]interface{} { return result } -func (f *framer) parseResultPrepared() map[string]interface{} { +func (f *Framer) parseResultPrepared() map[string]interface{} { result := make(map[string]interface{}) @@ -464,7 +438,7 @@ func (f *framer) parseResultPrepared() map[string]interface{} { return result } -func (f *framer) parseResultSchemaChange() (data map[string]interface{}) { +func (f *Framer) parseResultSchemaChange() (data map[string]interface{}) { data = make(map[string]interface{}) if f.proto <= protoVersion2 { @@ -502,25 +476,25 @@ func (f *framer) parseResultSchemaChange() (data map[string]interface{}) { return data } -func (f *framer) parseAuthenticateFrame() (data map[string]interface{}) { +func (f *Framer) parseAuthenticateFrame() (data map[string]interface{}) { data = make(map[string]interface{}) data["class"] = f.decoder.ReadString() return data } -func (f *framer) parseAuthSuccessFrame() (data map[string]interface{}) { +func (f *Framer) parseAuthSuccessFrame() (data map[string]interface{}) { data = make((map[string]interface{})) data["data"] = fmt.Sprintf("%q", f.decoder.ReadBytes()) return data } -func (f *framer) parseAuthChallengeFrame() (data map[string]interface{}) { +func (f *Framer) parseAuthChallengeFrame() (data map[string]interface{}) { data = make((map[string]interface{})) data["data"] = fmt.Sprintf("%q", f.decoder.ReadBytes()) return data } -func (f *framer) parseEventFrame() (data map[string]interface{}) { +func (f *Framer) parseEventFrame() (data map[string]interface{}) { data = make((map[string]interface{})) @@ -549,142 +523,3 @@ func (f *framer) parseEventFrame() (data map[string]interface{}) { return data } - -// explicitly enables tracing for the framers outgoing requests -func (f *framer) trace() { - f.flags |= flagTracing -} - -type UUID [16]byte - -// Bytes returns the raw byte slice for this UUID. A UUID is always 128 bits -// (16 bytes) long. -func (u UUID) Bytes() []byte { - return u[:] -} - -// String returns the UUID in it's canonical form, a 32 digit hexadecimal -// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. -func (u UUID) String() string { - var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} - const hexString = "0123456789abcdef" - r := make([]byte, 36) - for i, b := range u { - r[offsets[i]] = hexString[b>>4] - r[offsets[i]+1] = hexString[b&0xF] - } - r[8] = '-' - r[13] = '-' - r[18] = '-' - r[23] = '-' - return string(r) - -} - -// UUIDFromBytes converts a raw byte slice to an UUID. -func UUIDFromBytes(input []byte) (UUID, error) { - var u UUID - if len(input) != 16 { - return u, errors.New("UUIDs must be exactly 16 bytes long") - } - - copy(u[:], input) - return u, nil -} - -func copyBytes(p []byte) []byte { - b := make([]byte, len(p)) - copy(b, p) - return b -} - -type ColumnInfo struct { - Keyspace string - Table string - Name string - TypeInfo TypeInfo -} - -func (f *framer) readCol(col *ColumnInfo, globalSpec bool, keyspace, table string) { - if !globalSpec { - col.Keyspace = f.decoder.ReadString() - col.Table = f.decoder.ReadString() - } else { - col.Keyspace = keyspace - col.Table = table - } - - col.Name = f.decoder.ReadString() - col.TypeInfo = f.readTypeInfo() -} - -func (f *framer) readTypeInfo() TypeInfo { - - id := f.decoder.ReadShort() - - simple := NativeType{ - proto: f.proto, - typ: Type(id), - } - - if simple.typ == TypeCustom { - simple.custom = f.decoder.ReadString() - if cassType := getApacheCassandraType(simple.custom); cassType != TypeCustom { - simple.typ = cassType - } - } - - switch simple.typ { - case TypeTuple: - n := f.decoder.ReadShort() - tuple := TupleTypeInfo{ - NativeType: simple, - Elems: make([]TypeInfo, n), - } - - for i := 0; i < int(n); i++ { - tuple.Elems[i] = f.readTypeInfo() - } - - return tuple - - case TypeUDT: - udt := UDTTypeInfo{ - NativeType: simple, - } - udt.KeySpace = f.decoder.ReadString() - udt.Name = f.decoder.ReadString() - - n := f.decoder.ReadShort() - udt.Elements = make([]UDTField, n) - for i := 0; i < int(n); i++ { - field := &udt.Elements[i] - field.Name = f.decoder.ReadString() - field.Type = f.readTypeInfo() - } - - return udt - case TypeMap, TypeList, TypeSet: - collection := CollectionType{ - NativeType: simple, - } - - if simple.typ == TypeMap { - collection.Key = f.readTypeInfo() - } - - collection.Elem = f.readTypeInfo() - - return collection - } - - return simple -} - -func readInt(p []byte) int32 { - return int32(p[0])<<24 | int32(p[1])<<16 | int32(p[2])<<8 | int32(p[3]) -} - -func readShort(p []byte) uint16 { - return uint16(p[0])<<8 | uint16(p[1]) -} diff --git a/packetbeat/protos/cassandra/internal/gocql/marshal.go b/packetbeat/protos/cassandra/internal/gocql/marshal.go index 3fc5bbe3ad8..b0c99c39588 100644 --- a/packetbeat/protos/cassandra/internal/gocql/marshal.go +++ b/packetbeat/protos/cassandra/internal/gocql/marshal.go @@ -11,6 +11,7 @@ import ( "reflect" "time" + "errors" "github.com/elastic/beats/libbeat/logp" "gopkg.in/inf.v0" "strings" @@ -529,6 +530,34 @@ const ( flagWarning byte = 0x08 ) +func getHeadFlagString(f byte) string { + switch f { + case flagCompress: + return "Compress" + case flagTracing: + return "Tracing" + case flagCustomPayload: + return "CustomPayload" + case flagWarning: + return "Warning" + default: + return fmt.Sprintf("FLAG_%d", f) + } +} + +func getRowFlagString(f int) string { + switch f { + case flagGlobalTableSpec: + return "GlobalTableSpec" + case flagHasMorePages: + return "HasMorePages" + case flagNoMetaData: + return "NoMetaData" + default: + return fmt.Sprintf("FLAG_%d", f) + } +} + type Consistency uint16 const ( @@ -585,3 +614,47 @@ func (s SerialConsistency) String() string { return fmt.Sprintf("UNKNOWN_SERIAL_CONS_0x%x", uint16(s)) } } + +type UUID [16]byte + +// Bytes returns the raw byte slice for this UUID. A UUID is always 128 bits +// (16 bytes) long. +func (u UUID) Bytes() []byte { + return u[:] +} + +// String returns the UUID in it's canonical form, a 32 digit hexadecimal +// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. +func (u UUID) String() string { + var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} + const hexString = "0123456789abcdef" + r := make([]byte, 36) + for i, b := range u { + r[offsets[i]] = hexString[b>>4] + r[offsets[i]+1] = hexString[b&0xF] + } + r[8] = '-' + r[13] = '-' + r[18] = '-' + r[23] = '-' + return string(r) + +} + +// UUIDFromBytes converts a raw byte slice to an UUID. +func UUIDFromBytes(input []byte) (UUID, error) { + var u UUID + if len(input) != 16 { + return u, errors.New("UUIDs must be exactly 16 bytes long") + } + + copy(u[:], input) + return u, nil +} + +type ColumnInfo struct { + Keyspace string + Table string + Name string + TypeInfo TypeInfo +} diff --git a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go index 221ad8f4189..6e0212842de 100644 --- a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go @@ -1,10 +1,8 @@ package cassandra import ( - "errors" "fmt" "github.com/elastic/beats/libbeat/common/streambuf" - "github.com/elastic/beats/libbeat/logp" "net" ) @@ -12,78 +10,6 @@ type StreamDecoder struct { r *streambuf.Buffer } -func (f StreamDecoder) ReadHeader(r *streambuf.Buffer) (head *frameHeader, err error) { - v, err := r.ReadByte() - if err != nil { - return nil, err - } - version := v & protoVersionMask - - if version < protoVersion1 || version > protoVersion4 { - return nil, fmt.Errorf("unsupported response version: %d", version) - } - - head = &frameHeader{} - - head.Version = protoVersion(v) - - head.Flags, err = r.ReadByte() - - if version > protoVersion2 { - stream, err := r.ReadNetUint16() - if err != nil { - return nil, err - } - head.Stream = int(stream) - - b, err := r.ReadByte() - if err != nil { - return nil, err - } - - head.Op = FrameOp(b) - l, err := r.ReadNetUint32() - if err != nil { - return nil, err - } - head.Length = int(l) - } else { - stream, err := r.ReadNetUint8() - if err != nil { - return nil, err - } - head.Stream = int(stream) - - b, err := r.ReadByte() - if err != nil { - return nil, err - } - - head.Op = FrameOp(b) - l, err := r.ReadNetUint32() - if err != nil { - return nil, err - } - head.Length = int(l) - } - - if head.Length < 0 { - return nil, fmt.Errorf("frame body length can not be less than 0: %d", head.Length) - } else if head.Length > maxFrameSize { - // need to free up the connection to be used again - logp.Err("head length is too large") - return nil, ErrFrameTooBig - } - - if !r.Avail(head.Length) { - return nil, errors.New(fmt.Sprintf("frame length is not enough as expected length: %v", head.Length)) - } - - logp.Debug("cassandra", "header: %v", head) - - return head, nil -} - func (f StreamDecoder) ReadByte() byte { b, err := f.r.ReadByte() diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index 01a602b816b..ed121e67c15 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -14,6 +14,7 @@ import ( type parser struct { buf streambuf.Buffer config *parserConfig + framer *Framer message *message onMessage func(m *message) error } @@ -115,74 +116,115 @@ func (p *parser) newMessage(ts time.Time) *message { func (p *parser) parse() (*message, error) { - if !p.buf.Avail(9) { - logp.Err("not enough bytes, ignore") - p.message = nil - return nil, nil - } - - framer := NewFramer(&p.buf, p.config.compressor) - head, err := framer.ReadHeader() - if err != nil { - logp.Err("%v", err) - p.message = nil - return nil, nil + // if p.frame is nil then create a new framer, or continue to process the last message + if p.framer == nil { + p.framer = NewFramer(&p.buf, p.config.compressor) } - //check if the ops already ignored - if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { - - v := p.config.ignoredOps[head.Op.String()] - if v != nil { - logp.Debug("cassandra", fmt.Sprintf("Ops: %s was marked to be ignored, ignoring", head.Op.String())) - p.message = nil + // check if the frame header were parsed or not + if p.framer.Header == nil { + if isDebug { + logp.Debug("cassandra", "start to parse header") + } + if !p.buf.Avail(9) { + logp.Err("not enough head bytes, ignore") return nil, nil } - } - if !p.buf.Avail(head.Length) { - logp.Err("not enough bytes for frame body, ignore") - p.message = nil - return nil, nil + _, err := p.framer.ReadHeader() + if err != nil { + logp.Err("%v", err) + p.framer = nil + return nil, err + } } - data, err := framer.ReadFrame() + msg := p.message + msg.data = make(map[string]interface{}) - frameLength := p.buf.BufferConsumed() - if err != nil { - p.message = nil - logp.Err("%v", err) - return nil, nil - } + if p.framer.Header.BodyLength > 0 { - // collect leftover - leftDataSize := head.Length + 9 - frameLength - if leftDataSize > 0 { - p.buf.Collect(leftDataSize) + //let's wait for enough buf + if !p.buf.Avail(p.framer.Header.BodyLength) { + if isDebug { + logp.Debug("cassandra", " buf not enough for body, waiting for more, return") + } + return nil, nil + } else { + //check if the ops already ignored + if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { + v := p.config.ignoredOps[p.framer.Header.Op.String()] + if v != nil { + if isDebug { + logp.Debug("cassandra", fmt.Sprintf("Ops: %s was marked to be ignored, ignoring", p.framer.Header.Op.String())) + } + p.buf.Collect(p.framer.Header.BodyLength) + return nil, nil + } + } else { + + // start to prase body + data, err := p.framer.ReadFrame() + if err != nil { + // if the frame parsed failed, should ignore the whole message + p.framer = nil + return nil, err + } + + msg.data = data + + // dealing with un-parsed content + frameParsedLength := p.buf.BufferConsumed() + + // collect leftover + unParsedSize := p.framer.Header.BodyLength + p.framer.Header.HeadLength - frameParsedLength + if unParsedSize > 0 { + + // double check the buf size + if p.buf.Avail(unParsedSize) { + p.buf.Collect(unParsedSize) + consumedSize := p.buf.BufferConsumed() + if consumedSize-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + logp.Err("cassandra", "wrong, body_lenght:%d, body_read:%d, more collected:%d, only collect:%d", p.framer.Header.BodyLength, frameParsedLength, unParsedSize, consumedSize) + } + } else { + logp.Err("cassandra", " should be enough bytes for cleanup,but not enough") + return nil, errors.New("should be enough bytes,but not enough") + } + } + + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + logp.Err("cassandra", "wrong,body_length:%d, body_read:%d, all_consumed:%d", p.framer.Header.BodyLength, frameParsedLength, finalCollectedFrameLength) + return nil, errors.New("data messed while parse frame body") + } + + } + + } } dir := applayer.NetOriginalDirection isRequest := true - if head.Version.IsResponse() { + if p.framer.Header.Version.IsResponse() { dir = applayer.NetReverseDirection isRequest = false } - msg := p.message msg.Size = uint64(p.buf.BufferConsumed()) msg.IsRequest = isRequest msg.Direction = dir - msg.data = data - msg.header = head.ToMap() + msg.header = p.framer.Header.ToMap() + //? how to make sure the two pair an added to the same result ? if msg.IsRequest { p.message.results.requests.append(msg) } else { p.message.results.responses.append(msg) } - + p.framer = nil return msg, nil } diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go index 55b8bee4e6e..04bcbae447c 100644 --- a/packetbeat/protos/cassandra/pub.go +++ b/packetbeat/protos/cassandra/pub.go @@ -1,9 +1,7 @@ package cassandra import ( - "fmt" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/packetbeat/publish" ) @@ -35,44 +33,62 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { status = common.ERROR_STATUS } - // resp_time in milliseconds - responseTime := int32(resp.Ts.Sub(requ.Ts).Nanoseconds() / 1e6) - - src := &common.Endpoint{ - Ip: requ.Tuple.Src_ip.String(), - Port: requ.Tuple.Src_port, - Proc: string(requ.CmdlineTuple.Src), - } - dst := &common.Endpoint{ - Ip: requ.Tuple.Dst_ip.String(), - Port: requ.Tuple.Dst_port, - Proc: string(requ.CmdlineTuple.Dst), - } - event := common.MapStr{ - "@timestamp": common.Time(requ.Ts), - "type": "cassandra", - "status": status, - "responsetime": responseTime, - "bytes_in": requ.Size, - "bytes_out": resp.Size, - "src": src, - "dst": dst, + "type": "cassandra", + "status": status, } - // add processing notes/errors to event - if len(requ.Notes)+len(resp.Notes) > 0 { - event["notes"] = append(requ.Notes, resp.Notes...) - } + //requ can be null, if the message is a PUSHed message + if requ != nil { + // resp_time in milliseconds + responseTime := int32(resp.Ts.Sub(requ.Ts).Nanoseconds() / 1e6) + + src := &common.Endpoint{ + Ip: requ.Tuple.Src_ip.String(), + Port: requ.Tuple.Src_port, + Proc: string(requ.CmdlineTuple.Src), + } + + event["@timestamp"] = common.Time(requ.Ts) + event["responsetime"] = responseTime + event["bytes_in"] = requ.Size + event["src"] = src + + // add processing notes/errors to event + if len(requ.Notes)+len(resp.Notes) > 0 { + event["notes"] = append(requ.Notes, resp.Notes...) + } + + if pub.sendRequest { + if pub.sendRequestHeader { + requ.data["request_headers"] = requ.header + } - if pub.sendRequest { - if pub.sendRequestHeader { - requ.data["request_headers"] = requ.header + event["cassandra_request"] = requ.data } - event["cassandra_request"] = requ.data + dst := &common.Endpoint{ + Ip: requ.Tuple.Dst_ip.String(), + Port: requ.Tuple.Dst_port, + Proc: string(requ.CmdlineTuple.Dst), + } + event["dst"] = dst + + } else { + //dealing with PUSH message + event["no_request"] = true + event["@timestamp"] = common.Time(resp.Ts) + + dst := &common.Endpoint{ + Ip: resp.Tuple.Dst_ip.String(), + Port: resp.Tuple.Dst_port, + Proc: string(resp.CmdlineTuple.Dst), + } + event["dst"] = dst } + event["bytes_out"] = resp.Size + if pub.sendResponse { if pub.sendResponseHeader { resp.data["response_headers"] = resp.header @@ -81,9 +97,5 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { event["cassandra_response"] = resp.data } - if isDebug { - logp.Debug("cassandra", fmt.Sprint(event)) - } - return event } diff --git a/packetbeat/protos/cassandra/trans.go b/packetbeat/protos/cassandra/trans.go index 6b5b582371f..d7909bee9dc 100644 --- a/packetbeat/protos/cassandra/trans.go +++ b/packetbeat/protos/cassandra/trans.go @@ -128,10 +128,33 @@ func (trans *transactions) correlate() error { requests := &trans.requests responses := &trans.responses + + // drop responses with missing requests if requests.empty() { for !responses.empty() { - logp.Warn("Response from unknown transaction. Ignoring.") + + //if the response is EVENT, which pushed from server, we can accept that + resp := responses.first() + if !resp.isComplete { + break + } + + if(resp.header["op"]=="EVENT"){ + if(isDebug){ + logp.Debug("cassandra","server pushed message,%v",resp.header) + } + + responses.pop() + + if err := trans.onTransaction(nil, resp); err != nil { + return err + } + + return nil + } + + logp.Warn("Response from unknown transaction. Ignoring. %v", resp.header) responses.pop() } return nil diff --git a/packetbeat/tests/system/test_0062_cassandra.py b/packetbeat/tests/system/test_0062_cassandra.py index 51448fc8f11..6eaf31dad48 100644 --- a/packetbeat/tests/system/test_0062_cassandra.py +++ b/packetbeat/tests/system/test_0062_cassandra.py @@ -32,7 +32,7 @@ def test_create_keyspace(self): assert h["version"] == "4" assert h["op"] == "QUERY" assert h["length"] == 124 - assert h["flags"] == 0 + assert h["flags"] == "FLAG_0" assert h["stream"] == 20 @@ -43,7 +43,7 @@ def test_create_keyspace(self): assert h2["version"] == "4" assert h2["length"] == 35 assert h2["op"] == "RESULT" - assert h2["flags"] == 0 + assert h2["flags"] == "FLAG_0" assert h2["stream"] == 20 def test_create_table(self): @@ -70,7 +70,7 @@ def test_create_table(self): assert h["version"] == "4" assert h["op"] == "QUERY" assert h["length"] == 98 - assert h["flags"] == 0 + assert h["flags"] == "FLAG_0" assert h["stream"] == 49 @@ -80,7 +80,7 @@ def test_create_table(self): assert h2["version"] == "4" assert h2["length"] == 39 assert h2["op"] == "RESULT" - assert h2["flags"] == 0 + assert h2["flags"] == "FLAG_0" assert h2["stream"] == 49 @@ -108,7 +108,7 @@ def test_insert_data(self): assert h["version"] == "4" assert h["op"] == "QUERY" assert h["length"] == 97 - assert h["flags"] == 0 + assert h["flags"] == "FLAG_0" assert h["stream"] == 252 r=o["cassandra_response"] @@ -117,7 +117,7 @@ def test_insert_data(self): assert h2["version"] == "4" assert h2["length"] == 4 assert h2["op"] == "RESULT" - assert h2["flags"] == 0 + assert h2["flags"] == "FLAG_0" assert h2["stream"] == 252 @@ -144,7 +144,7 @@ def test_select_data(self): assert h["version"] == "4" assert h["op"] == "QUERY" assert h["length"] == 41 - assert h["flags"] == 0 + assert h["flags"] == "FLAG_0" assert h["stream"] == 253 @@ -154,7 +154,7 @@ def test_select_data(self): assert h2["version"] == "4" assert h2["length"] == 89 assert h2["op"] == "RESULT" - assert h2["flags"] == 0 + assert h2["flags"] == "FLAG_0" assert h2["stream"] == 253 def test_create_index(self): @@ -180,7 +180,7 @@ def test_create_index(self): assert h["version"] == "4" assert h["op"] == "QUERY" assert h["length"] == 51 - assert h["flags"] == 0 + assert h["flags"] == "FLAG_0" assert h["stream"] == 92 r=o["cassandra_response"] @@ -189,7 +189,7 @@ def test_create_index(self): assert h2["version"] == "4" assert h2["length"] == 39 assert h2["op"] == "RESULT" - assert h2["flags"] == 0 + assert h2["flags"] == "FLAG_0" assert h2["stream"] == 92 def test_select_use_index(self): @@ -216,7 +216,7 @@ def test_select_use_index(self): assert h["version"] == "4" assert h["op"] == "QUERY" assert h["length"] == 63 - assert h["flags"] == 0 + assert h["flags"] == "FLAG_0" assert h["stream"] == 262 @@ -225,7 +225,7 @@ def test_select_use_index(self): assert h2["version"] == "4" assert h2["length"] == 89 assert h2["op"] == "RESULT" - assert h2["flags"] == 0 + assert h2["flags"] == "FLAG_0" assert h2["stream"] == 262 assert r["result_type"]=="rows" From 34ca244a3f3d1c9005299d182db9cc2e35ea2e27 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 8 Aug 2016 20:14:04 +0800 Subject: [PATCH 11/30] cleanup comments --- packetbeat/protos/cassandra/parser.go | 1 - packetbeat/protos/cassandra/trans.go | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index ed121e67c15..b9b8d7ce668 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -219,7 +219,6 @@ func (p *parser) parse() (*message, error) { msg.header = p.framer.Header.ToMap() - //? how to make sure the two pair an added to the same result ? if msg.IsRequest { p.message.results.requests.append(msg) } else { diff --git a/packetbeat/protos/cassandra/trans.go b/packetbeat/protos/cassandra/trans.go index d7909bee9dc..63541173d15 100644 --- a/packetbeat/protos/cassandra/trans.go +++ b/packetbeat/protos/cassandra/trans.go @@ -128,8 +128,6 @@ func (trans *transactions) correlate() error { requests := &trans.requests responses := &trans.responses - - // drop responses with missing requests if requests.empty() { for !responses.empty() { @@ -140,9 +138,9 @@ func (trans *transactions) correlate() error { break } - if(resp.header["op"]=="EVENT"){ - if(isDebug){ - logp.Debug("cassandra","server pushed message,%v",resp.header) + if resp.header["op"] == "EVENT" { + if isDebug { + logp.Debug("cassandra", "server pushed message,%v", resp.header) } responses.pop() From beef417aa61114c018da5fec7b0e193325ce0e4c Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 9 Aug 2016 14:13:55 +0800 Subject: [PATCH 12/30] bugfix --- packetbeat/protos/cassandra/cassandra.go | 10 +---- .../protos/cassandra/internal/gocql/frame.go | 35 +++++++----------- packetbeat/protos/cassandra/parser.go | 37 ++++++++++++++----- packetbeat/protos/cassandra/pub.go | 9 ++++- 4 files changed, 49 insertions(+), 42 deletions(-) diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go index 20014250fea..6c7f41ebfdd 100644 --- a/packetbeat/protos/cassandra/cassandra.go +++ b/packetbeat/protos/cassandra/cassandra.go @@ -6,7 +6,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "fmt" "github.com/elastic/beats/packetbeat/protos" . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" "github.com/elastic/beats/packetbeat/protos/tcp" @@ -35,15 +34,10 @@ type stream struct { var ( debugf = logp.MakeDebug("cassandra") - - // use isDebug/isDetailed to guard debugf/detailedf to minimize allocations - // (garbage collection) when debug log is disabled. - isDebug = false ) func init() { protos.Register("cassandra", New) - isDebug = logp.IsDebug("cassandra") } // New create and initializes a new cassandra protocol analyzer instance. @@ -108,9 +102,7 @@ func (cassandra *cassandra) setFromConfig(config *cassandraConfig) error { maps[strings.ToUpper(strings.TrimSpace(config.IgnoredOPs))] = config.IgnoredOPs } parser.ignoredOps = maps - if isDebug { - logp.Debug("cassandra", fmt.Sprintf("parsed config IgnoredOPs: %v ", parser.ignoredOps)) - } + debugf("parsed config IgnoredOPs: %v ", parser.ignoredOps) } // set transaction correlator configuration diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index 7600ba22d46..013e0947f3b 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -15,7 +15,6 @@ import ( var ( ErrFrameTooBig = errors.New("frame length is bigger than the maximum allowed") - isDebug = logp.IsDebug("cassandra") ) type frameHeader struct { @@ -146,9 +145,8 @@ func (f *Framer) ReadHeader() (head *frameHeader, err error) { headSize := f.r.BufferConsumed() head.HeadLength = headSize - if isDebug { - logp.Debug("cassandra", "header: %v", head) - } + logp.Debug("cassandra", "header: %v", head) + f.Header = head return head, nil } @@ -178,30 +176,25 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { //corresponding to the response opcode. if f.Header.Flags&flagTracing == flagTracing && (f.Header.Op&opQuery == opQuery || f.Header.Op&opExecute == opExecute || f.Header.Op&opPrepare == opPrepare) { - if isDebug { - logp.Debug("cassandra", "tracing enabled") - } + logp.Debug("cassandra", "tracing enabled") uid := f.decoder.ReadUUID() - if isDebug { - logp.Debug("cassandra", uid.String()) - } + logp.Debug("cassandra", uid.String()) + data["trace_id"] = uid.String() } if f.Header.Flags&flagWarning == flagWarning { - if isDebug { - logp.Debug("cassandra", "hit warning flags") - } + logp.Debug("cassandra", "hit warning flags") + warnings := f.decoder.ReadStringList() // dealing with warnings data["warnings"] = warnings } if f.Header.Flags&flagCustomPayload == flagCustomPayload { - if isDebug { - logp.Debug("cassandra", "hit custom payload flags") - } + logp.Debug("cassandra", "hit custom payload flags") + f.Header.CustomPayload = f.decoder.ReadBytesMap() } @@ -213,9 +206,8 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { //decoder.Data = buf //f.decoder = decoder - if isDebug { - logp.Debug("cassandra", "hit compress flags") - } + logp.Debug("cassandra", "hit compress flags") + return nil, errors.New("Compressed content not supported yet") } @@ -245,9 +237,8 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { //ignore default: //ignore - if isDebug { - logp.Debug("cassandra", "unknow ops, not processed, %v", f.Header) - } + logp.Debug("cassandra", "unknow ops, not processed, %v", f.Header) + } return data, nil diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index b9b8d7ce668..3aeda941870 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -32,7 +32,9 @@ type message struct { // (if false) to be merged to generate full message. isComplete bool - failed bool + failed bool + ignored bool + data map[string]interface{} header map[string]interface{} // list element use by 'transactions' for correlation @@ -46,6 +48,7 @@ type message struct { // Error code if stream exceeds max allowed size on append. var ( ErrStreamTooLarge = errors.New("Stream data too large") + isDebug = false ) func (p *parser) init( @@ -118,6 +121,9 @@ func (p *parser) parse() (*message, error) { // if p.frame is nil then create a new framer, or continue to process the last message if p.framer == nil { + if isDebug { + logp.Debug("cassandra", "start new framer") + } p.framer = NewFramer(&p.buf, p.config.compressor) } @@ -153,17 +159,30 @@ func (p *parser) parse() (*message, error) { } else { //check if the ops already ignored if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { + if isDebug { + logp.Debug("cassandra", "ignoreOPS configed, let's check") + } v := p.config.ignoredOps[p.framer.Header.Op.String()] if v != nil { if isDebug { - logp.Debug("cassandra", fmt.Sprintf("Ops: %s was marked to be ignored, ignoring", p.framer.Header.Op.String())) + logp.Debug("cassandra", fmt.Sprintf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest())) } p.buf.Collect(p.framer.Header.BodyLength) - return nil, nil + + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + return nil, errors.New("data messed while parse frame body") + } + // as we already igore the content, we now mark the result is completed + p.message.ignored = true } - } else { + } - // start to prase body + // start to parse body + if !p.message.ignored { + if isDebug { + logp.Debug("cassandra", "start read frame") + } data, err := p.framer.ReadFrame() if err != nil { // if the frame parsed failed, should ignore the whole message @@ -185,22 +204,20 @@ func (p *parser) parse() (*message, error) { p.buf.Collect(unParsedSize) consumedSize := p.buf.BufferConsumed() if consumedSize-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("cassandra", "wrong, body_lenght:%d, body_read:%d, more collected:%d, only collect:%d", p.framer.Header.BodyLength, frameParsedLength, unParsedSize, consumedSize) + logp.Err("body_lenght:%d, body_read:%d, more collected:%d, only collect:%d", p.framer.Header.BodyLength, frameParsedLength, unParsedSize, consumedSize) } } else { - logp.Err("cassandra", " should be enough bytes for cleanup,but not enough") + logp.Err("should be enough bytes for cleanup,but not enough") return nil, errors.New("should be enough bytes,but not enough") } } finalCollectedFrameLength := p.buf.BufferConsumed() if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("cassandra", "wrong,body_length:%d, body_read:%d, all_consumed:%d", p.framer.Header.BodyLength, frameParsedLength, finalCollectedFrameLength) + logp.Err("body_length:%d, body_read:%d, all_consumed:%d", p.framer.Header.BodyLength, frameParsedLength, finalCollectedFrameLength) return nil, errors.New("data messed while parse frame body") } - } - } } diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go index 04bcbae447c..360286ec4a7 100644 --- a/packetbeat/protos/cassandra/pub.go +++ b/packetbeat/protos/cassandra/pub.go @@ -22,7 +22,9 @@ func (pub *transPub) onTransaction(requ, resp *message) error { } event := pub.createEvent(requ, resp) - pub.results.PublishTransaction(event) + if event != nil { + pub.results.PublishTransaction(event) + } return nil } @@ -33,6 +35,11 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { status = common.ERROR_STATUS } + //ignore + if (resp != nil && resp.ignored) || (requ != nil && requ.ignored) { + return nil + } + event := common.MapStr{ "type": "cassandra", "status": status, From 6c4003cd5d913626241f10174c069719208b0acc Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 10 Aug 2016 18:27:53 +0800 Subject: [PATCH 13/30] prefer to use debugf, instead of logp.Debug() --- .../protos/cassandra/internal/gocql/frame.go | 15 ++++++++------- packetbeat/protos/cassandra/parser.go | 14 +++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index 013e0947f3b..fa4965143b2 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -15,6 +15,7 @@ import ( var ( ErrFrameTooBig = errors.New("frame length is bigger than the maximum allowed") + debugf = logp.MakeDebug("cassandra") ) type frameHeader struct { @@ -145,7 +146,7 @@ func (f *Framer) ReadHeader() (head *frameHeader, err error) { headSize := f.r.BufferConsumed() head.HeadLength = headSize - logp.Debug("cassandra", "header: %v", head) + debugf("header: %v", head) f.Header = head return head, nil @@ -176,16 +177,16 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { //corresponding to the response opcode. if f.Header.Flags&flagTracing == flagTracing && (f.Header.Op&opQuery == opQuery || f.Header.Op&opExecute == opExecute || f.Header.Op&opPrepare == opPrepare) { - logp.Debug("cassandra", "tracing enabled") + debugf("tracing enabled") uid := f.decoder.ReadUUID() - logp.Debug("cassandra", uid.String()) + debugf(uid.String()) data["trace_id"] = uid.String() } if f.Header.Flags&flagWarning == flagWarning { - logp.Debug("cassandra", "hit warning flags") + debugf("hit warning flags") warnings := f.decoder.ReadStringList() // dealing with warnings @@ -193,7 +194,7 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { } if f.Header.Flags&flagCustomPayload == flagCustomPayload { - logp.Debug("cassandra", "hit custom payload flags") + debugf("hit custom payload flags") f.Header.CustomPayload = f.decoder.ReadBytesMap() } @@ -206,7 +207,7 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { //decoder.Data = buf //f.decoder = decoder - logp.Debug("cassandra", "hit compress flags") + debugf("hit compress flags") return nil, errors.New("Compressed content not supported yet") } @@ -237,7 +238,7 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { //ignore default: //ignore - logp.Debug("cassandra", "unknow ops, not processed, %v", f.Header) + debugf("unknow ops, not processed, %v", f.Header) } diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index 3aeda941870..9fb3bea0b3f 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -122,7 +122,7 @@ func (p *parser) parse() (*message, error) { // if p.frame is nil then create a new framer, or continue to process the last message if p.framer == nil { if isDebug { - logp.Debug("cassandra", "start new framer") + debugf("start new framer") } p.framer = NewFramer(&p.buf, p.config.compressor) } @@ -130,10 +130,10 @@ func (p *parser) parse() (*message, error) { // check if the frame header were parsed or not if p.framer.Header == nil { if isDebug { - logp.Debug("cassandra", "start to parse header") + debugf("start to parse header") } if !p.buf.Avail(9) { - logp.Err("not enough head bytes, ignore") + debugf("not enough head bytes, ignore") return nil, nil } @@ -153,19 +153,19 @@ func (p *parser) parse() (*message, error) { //let's wait for enough buf if !p.buf.Avail(p.framer.Header.BodyLength) { if isDebug { - logp.Debug("cassandra", " buf not enough for body, waiting for more, return") + debugf("buf not enough for body, waiting for more, return") } return nil, nil } else { //check if the ops already ignored if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { if isDebug { - logp.Debug("cassandra", "ignoreOPS configed, let's check") + debugf("ignoreOPS configed, let's check") } v := p.config.ignoredOps[p.framer.Header.Op.String()] if v != nil { if isDebug { - logp.Debug("cassandra", fmt.Sprintf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest())) + debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) } p.buf.Collect(p.framer.Header.BodyLength) @@ -181,7 +181,7 @@ func (p *parser) parse() (*message, error) { // start to parse body if !p.message.ignored { if isDebug { - logp.Debug("cassandra", "start read frame") + debugf("start read frame") } data, err := p.framer.ReadFrame() if err != nil { From 448fd4892ffa01b22039f0f58098f4608258462f Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 10 Aug 2016 20:26:59 +0800 Subject: [PATCH 14/30] remove unused import --- packetbeat/protos/cassandra/parser.go | 1 - 1 file changed, 1 deletion(-) diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index 9fb3bea0b3f..182997fc525 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -4,7 +4,6 @@ import ( "errors" "time" - "fmt" "github.com/elastic/beats/libbeat/common/streambuf" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/packetbeat/protos/applayer" From 81ad366d9cbf6d08078c6904b8b908341a8edba4 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 10 Aug 2016 21:20:26 +0800 Subject: [PATCH 15/30] minor improve to parser --- packetbeat/protos/cassandra/parser.go | 104 +++++++++++++------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index 182997fc525..c1bee71a49d 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -85,6 +85,7 @@ func (p *parser) feed(ts time.Time, data []byte) error { if p.message == nil { // allocate new message object to be used by parser with current timestamp p.message = p.newMessage(ts) + p.message.data = map[string]interface{}{} } msg, err := p.parse() @@ -145,7 +146,6 @@ func (p *parser) parse() (*message, error) { } msg := p.message - msg.data = make(map[string]interface{}) if p.framer.Header.BodyLength > 0 { @@ -155,67 +155,67 @@ func (p *parser) parse() (*message, error) { debugf("buf not enough for body, waiting for more, return") } return nil, nil - } else { - //check if the ops already ignored - if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { + } + + //check if the ops already ignored + if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { + if isDebug { + debugf("ignoreOPS configed, let's check") + } + v := p.config.ignoredOps[p.framer.Header.Op.String()] + if v != nil { if isDebug { - debugf("ignoreOPS configed, let's check") + debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) } - v := p.config.ignoredOps[p.framer.Header.Op.String()] - if v != nil { - if isDebug { - debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) - } - p.buf.Collect(p.framer.Header.BodyLength) + p.buf.Collect(p.framer.Header.BodyLength) - finalCollectedFrameLength := p.buf.BufferConsumed() - if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - return nil, errors.New("data messed while parse frame body") - } - // as we already igore the content, we now mark the result is completed - p.message.ignored = true + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + return nil, errors.New("data messed while parse frame body") } + // as we already igore the content, we now mark the result is completed + p.message.ignored = true } + } - // start to parse body - if !p.message.ignored { - if isDebug { - debugf("start read frame") - } - data, err := p.framer.ReadFrame() - if err != nil { - // if the frame parsed failed, should ignore the whole message - p.framer = nil - return nil, err - } + // start to parse body + if !p.message.ignored { + if isDebug { + debugf("start read frame") + } + data, err := p.framer.ReadFrame() + if err != nil { + // if the frame parsed failed, should ignore the whole message + p.framer = nil + return nil, err + } + + msg.data = data + + // dealing with un-parsed content + frameParsedLength := p.buf.BufferConsumed() - msg.data = data - - // dealing with un-parsed content - frameParsedLength := p.buf.BufferConsumed() - - // collect leftover - unParsedSize := p.framer.Header.BodyLength + p.framer.Header.HeadLength - frameParsedLength - if unParsedSize > 0 { - - // double check the buf size - if p.buf.Avail(unParsedSize) { - p.buf.Collect(unParsedSize) - consumedSize := p.buf.BufferConsumed() - if consumedSize-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("body_lenght:%d, body_read:%d, more collected:%d, only collect:%d", p.framer.Header.BodyLength, frameParsedLength, unParsedSize, consumedSize) - } - } else { - logp.Err("should be enough bytes for cleanup,but not enough") - return nil, errors.New("should be enough bytes,but not enough") + // collect leftover + unParsedSize := p.framer.Header.BodyLength + p.framer.Header.HeadLength - frameParsedLength + if unParsedSize > 0 { + + // double check the buf size + if p.buf.Avail(unParsedSize) { + p.buf.Collect(unParsedSize) + consumedSize := p.buf.BufferConsumed() + if consumedSize-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + logp.Err("body_lenght:%d, body_read:%d, more collected:%d, only collect:%d", p.framer.Header.BodyLength, frameParsedLength, unParsedSize, consumedSize) } + } else { + logp.Err("should be enough bytes for cleanup,but not enough") + return nil, errors.New("should be enough bytes,but not enough") } + } - finalCollectedFrameLength := p.buf.BufferConsumed() - if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("body_length:%d, body_read:%d, all_consumed:%d", p.framer.Header.BodyLength, frameParsedLength, finalCollectedFrameLength) - return nil, errors.New("data messed while parse frame body") - } + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + logp.Err("body_length:%d, body_read:%d, all_consumed:%d", p.framer.Header.BodyLength, frameParsedLength, finalCollectedFrameLength) + return nil, errors.New("data messed while parse frame body") } } From 34ad18fa8fdaf0125d074cfe42a00b7ecc7d5a5e Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 11 Aug 2016 00:17:08 +0800 Subject: [PATCH 16/30] refactor framops ignoring, add test methond --- packetbeat/protos/cassandra/cassandra.go | 15 +- packetbeat/protos/cassandra/config.go | 2 +- .../cassandra/internal/gocql/marshal.go | 41 +++++ packetbeat/protos/cassandra/parser.go | 63 +++---- .../tests/system/config/packetbeat.yml.j2 | 1 + .../cassandra/v4/cassandra_mixed_frame.pcap | Bin 0 -> 114224 bytes .../tests/system/test_0062_cassandra.py | 154 ++++++++++++++++++ 7 files changed, 240 insertions(+), 36 deletions(-) create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_mixed_frame.pcap diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go index 6c7f41ebfdd..4483c3ce82f 100644 --- a/packetbeat/protos/cassandra/cassandra.go +++ b/packetbeat/protos/cassandra/cassandra.go @@ -87,19 +87,22 @@ func (cassandra *cassandra) setFromConfig(config *cassandraConfig) error { } // parsed ignored ops - if len(config.IgnoredOPs) > 0 { - maps := make(map[string]interface{}) + if len(config.OPsIgnored) > 0 { + maps := map[FrameOp]bool{} - if strings.Contains(config.IgnoredOPs, ",") { - array := strings.Split(config.IgnoredOPs, ",") + if strings.Contains(config.OPsIgnored, ",") { + array := strings.Split(config.OPsIgnored, ",") for i := 0; i < len(array); i++ { str := array[i] if len(str) > 0 { - maps[strings.ToUpper(strings.TrimSpace(str))] = str + op := FrameOpFromString(strings.ToUpper(strings.TrimSpace(str))) + maps[op] = true } } } else { - maps[strings.ToUpper(strings.TrimSpace(config.IgnoredOPs))] = config.IgnoredOPs + op := FrameOpFromString(strings.ToUpper(strings.TrimSpace(config.OPsIgnored))) + maps[op] = true + } parser.ignoredOps = maps debugf("parsed config IgnoredOPs: %v ", parser.ignoredOps) diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go index a90ec05ab9c..1224378887c 100644 --- a/packetbeat/protos/cassandra/config.go +++ b/packetbeat/protos/cassandra/config.go @@ -12,7 +12,7 @@ type cassandraConfig struct { SendRequestHeader bool `config:"send_request_header"` SendResponseHeader bool `config:"send_response_header"` Compressor string `config:"compressor"` - IgnoredOPs string `config:"ignored_ops"` + OPsIgnored string `config:"ignored_ops"` } var ( diff --git a/packetbeat/protos/cassandra/internal/gocql/marshal.go b/packetbeat/protos/cassandra/internal/gocql/marshal.go index b0c99c39588..3beb3ac3889 100644 --- a/packetbeat/protos/cassandra/internal/gocql/marshal.go +++ b/packetbeat/protos/cassandra/internal/gocql/marshal.go @@ -460,6 +460,7 @@ const ( opAuthChallenge FrameOp = 0x0E opAuthResponse FrameOp = 0x0F opAuthSuccess FrameOp = 0x10 + opUnknown FrameOp = 0xFF ) func (f FrameOp) String() string { @@ -501,6 +502,46 @@ func (f FrameOp) String() string { } } +func FrameOpFromString(str string) FrameOp { + switch str { + case "ERROR": + return opError + case "STARTUP": + return opStartup + case "READY": + return opReady + case "AUTHENTICATE": + return opAuthenticate + case "OPTIONS": + return opOptions + case "SUPPORTED": + return opSupported + case "QUERY": + return opQuery + case "RESULT": + return opResult + case "PREPARE": + return opPrepare + case "EXECUTE": + return opExecute + case "REGISTER": + return opRegister + case "EVENT": + return opEvent + case "BATCH": + return opBatch + case "AUTH_CHALLENGE": + return opAuthChallenge + case "AUTH_RESPONSE": + return opAuthResponse + case "AUTH_SUCCESS": + return opAuthSuccess + default: + debugf("unknown Op while convert: %s", str) + return opUnknown + } +} + const ( // result kind resultKindVoid = 1 diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index c1bee71a49d..26ba9f7ee9b 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -21,7 +21,20 @@ type parser struct { type parserConfig struct { maxBytes int compressor Compressor - ignoredOps map[string]interface{} + ignoredOps map[FrameOp]bool +} + +// check whether this ops is enabled or not +func (p *parser) FrameOpsIgnored() bool { + + if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { + //default map value is false + v := p.config.ignoredOps[p.framer.Header.Op] + if v { + return true + } + } + return false } type message struct { @@ -145,6 +158,15 @@ func (p *parser) parse() (*message, error) { } } + //check if the ops need to be ignored + if p.FrameOpsIgnored() { + // as we already ignore the content, we now mark the result is ignored + p.message.ignored = true + if isDebug { + debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) + } + } + msg := p.message if p.framer.Header.BodyLength > 0 { @@ -158,31 +180,17 @@ func (p *parser) parse() (*message, error) { } //check if the ops already ignored - if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { - if isDebug { - debugf("ignoreOPS configed, let's check") + if p.message.ignored { + p.buf.Collect(p.framer.Header.BodyLength) + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + return nil, errors.New("data messed while parse frame body") } - v := p.config.ignoredOps[p.framer.Header.Op.String()] - if v != nil { - if isDebug { - debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) - } - p.buf.Collect(p.framer.Header.BodyLength) - finalCollectedFrameLength := p.buf.BufferConsumed() - if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - return nil, errors.New("data messed while parse frame body") - } - // as we already igore the content, we now mark the result is completed - p.message.ignored = true - } } // start to parse body if !p.message.ignored { - if isDebug { - debugf("start read frame") - } data, err := p.framer.ReadFrame() if err != nil { // if the frame parsed failed, should ignore the whole message @@ -202,21 +210,18 @@ func (p *parser) parse() (*message, error) { // double check the buf size if p.buf.Avail(unParsedSize) { p.buf.Collect(unParsedSize) - consumedSize := p.buf.BufferConsumed() - if consumedSize-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("body_lenght:%d, body_read:%d, more collected:%d, only collect:%d", p.framer.Header.BodyLength, frameParsedLength, unParsedSize, consumedSize) - } } else { logp.Err("should be enough bytes for cleanup,but not enough") return nil, errors.New("should be enough bytes,but not enough") } } - finalCollectedFrameLength := p.buf.BufferConsumed() - if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("body_length:%d, body_read:%d, all_consumed:%d", p.framer.Header.BodyLength, frameParsedLength, finalCollectedFrameLength) - return nil, errors.New("data messed while parse frame body") - } + } + + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { + logp.Err("body_length:%d, head_length:%d, all_consumed:%d", p.framer.Header.BodyLength, p.framer.Header.HeadLength, finalCollectedFrameLength) + return nil, errors.New("data messed while parse frame body") } } diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 9542792dfb2..29c738ec29a 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -124,6 +124,7 @@ packetbeat.protocols.cassandra: {% if cassandra_send_response %} send_response: true{% endif %} {% if cassandra_send_request_header %} send_request_header: true{% endif %} {% if cassandra_send_response_header %} send_response_header: true{% endif %} +{% if cassandra_ignored_ops %} ignored_ops: {{cassandra_ignored_ops}}{% endif %} {% if procs_enabled %} diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_mixed_frame.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_mixed_frame.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a0417059a942ffb1d94b138be12eaaf4df06ed40 GIT binary patch literal 114224 zcmeF437A|}nfEJXUrEwY;$C*Z1=HQNFHx2lB5Oz_VG+c#x~jUnNOe_HOF99=ridGe zxS)*dprei`j(`dZ;yNya8=#Eh_=z)aaWv|L|4dAWPQkt6lI79wKk^)icjyx= zp{-)UIs0QKjzilOLR&YSkAQO%w8+Py=f?PnCXPYRYUsJ%r00Z<>yHZ$g-_VFZOc$} zFuHWn`c`wSRxe+WZSE08|6ayC1;-2%%7P^ZQ8bRr+k(!I-%BRt_xRcg?wd9 zzGh>Q$kIg{YSm_`+8j8gR2^-Otqbd>YN?>XwwFp11NlmMM``JzrHe!qw_8Nv~gBdgYvoSOcTdL&yRd^ZsRPxAQ+Fn|1%%XMQirn@xU3M$KJZdqrb{A zH+EcV!MyAd6B8aUdiSeN`EPK)&?O%I6`B3j#k>wozg_)m(!|`O>HGq?zky$GJW^O9 z|BqO3-uZqL$D!>vLR&YSITwaPpN41oI8J!;FJ5Ef81zUy`cT^m6+Opq+!h|nRcpmk z<8cd0N5Y3+u76!%^3jpUnJ@JDM_aCUUla;`c$NNWd93=tk|+1TAtT7^QX(HLpzpCkElrBv)+7umhGzffs4 znx%TKnjbIquj?;W@{MM>&?x2Wg|Ys%{Yqm3c53F#a;}s-VQZsYEWPOD6Km(!k1wy6 zBhHmuDy>m2Xx8)9#zd{&%;k&4I-LROMUawV35OYrdI2@Mhv=sBr zJb%j7(GujGig|b)pwK&N(>HW=VG~ftX|3&Cu%he@527ce6uiy==pkntJEsx8s!Uw zjA%q@$oUl$Z8=~g<((MnZjg3Pl-@Mx)8s^^=fVz@b0ZiKy}9OTMEtx`if{cGZ>d@3;_SV9yn zXANvY%HdPJvWLrjr82CfZ#`kR^KWf`P074o8X+Kz<;oP3^&R;NUvlH+N~O$EF#$YE z2cb>*LU{Am@L3`w2}1}nlggzM{O&}BDDfmZn38jL5YO_%NN=KElQ>tagP7nzMG9Hi zH4k7-&WU0j`=+W)s7z+knYI1n;B5!Be;uqhN&ua=13h2ZKhZhIL*Za%Fp^8hgzRxQ zz;2!>RZlwZftjs>61Da$C88T*uZc!=^m&Sm2TEw(5i&D?jnSZ zH`azvtQJIZm@fghSspJLGEJl-@kk;Y&t%g{h%YHp$4dDL&MG(M)ZtxC35u~)CJ|32 zk|{!g93Ic_^d5~yW2tB+k>zl;R?S%vlF4K?g3Q||m>4LxSgtc*=!%UnM+-Sju)b$=6RUO~N1p5XaUx0d3u(d@{+fl2x8#ip)8s1zfhL0~*rQ246$O5S@ zX0BE*maBOvsSTeeJWkY<87k@^Z{^Ho%yO*?V;({bey}aY4}ZO#lH6g)abnN4U6_bXi3t@a z%Z2W6E-}%VUn=p4gmy25O5&6GI@;!rgr9SS~GpSbzj zuI;H2Vj5Dd?zP4)8t@CuZbswzDk;028+EzT1(Db`1;ZRIc`F}7V6HZOrsW81xK&Yw zhs~P1T$vP`Y*h0Tjj@`y*Def9Mb-csrPwgFPO($(p3{HMNGTIpw=R0FTV$W({oyOY z&m{%6hVj+04n(j_CrgenQXq_7C?%L%CHCrC*V<0k`iG}$Y1miS`Xb35lxQ;Rl&h#& zCr?6(MiQB1I;A{+f?-FwD$&0^5@}amoownYs&P+7p@n*RLT0ZBWE;I>p72I0P0Fca zSXDey)Dw_poJ87i>XTZ1)GZUWCl0FWbnq;>#JOTOe2&+OEj{RH(?(onurV-NuWbi( ziUgjhE(*7+8>+W_Gu&!0mKLBeBMvI6Y9rjNg&WLfHN&a~+!xWBmO+6S)Lk6USC{^^ zZJI0j>gYger%IqAu_@DiN~=IB2zM@Rge5PIQi6gLCW(a&+3!=#T5E$Hz(KSnc^$Ye zbi-Y3$F8Z_7?Vt;~x6O)>=gV127Jkn!zm&SwX5NN_7g%Jmuxo zN#4@FCanEA%)~|`T*y!4hs*fdF7Y~5xrE8K^)gN=e93nXjg=aucKa}PHBl;*M|R>m6M>WDjlrX+SoQ$3ZJGgGI(3RI6sDG z`8YQ$e)z5LnK%YLa>L?V?Fo8CkG^5y%#IH+&ha4{CBt0wOPQ3zQu#}U; zA>*!mBd7ai`ZF@f1YhT0O68MBF&Mw!zuKP%wBO(Vnk()Y!|=Z9F}z15hKS#9y*7yR z$v2vY=lH9}Sz*Vy*n%_spoya$;f^ulhukRkI)c#isL=Cz3(n8Bnm7(UKkCAl!ueMV z&Wo-yaSVD^*y%as?Lqmx`EnD-Lr>WvXGPCh&t4W-kaPBvd^ne^+51mp{e+Zm`DC&o z$K2R|rv-Dv1tumuUh?xhhil+oW6vj(jl}q5!^ONGOut=y)mInn(R5w|?gadL@`=V0 zJ?^pKoO8UDwh5uF8&2Xm%K{5>R=vc;G3YTC|8- z+Hwu8TNYT5bK7=P9tMuFAZKz?bUodL>&$|jnFTrdxVuh7-zG5D_Apt)wBO8voS6kV zGYfKN7Uaw<$eCG?GqWJ4>#Cxe1v%@cvPi_gvTJ5Rj8U4{KlRpI zesd2Kpq>ReYT|NICYIlRR1l}{escoUnV{A<#)6y<&c9o5;wckHPo)0l9wuZz?VFI5 ziBv_;&n!6aI@ZK-==rpTubnQ=vyKk(bO;uNv-m*sC^Em` zq-B9sIG6pt2ad4{XYxX^gXzARF|!KCx6&@S%(j+}*__jQx=iqnGu|K^waa9&9gx$U;^ zN}}j;*PC5VI!WXL3+9vGWn#kPE6;jx&C|fWyGs(;jqG>3N#reH`t9oepPHC^G@W;Y z`*isACJ}`tcJ)3B&hyurI1X)37uvev{1u$lQkLAn9LVhxv$nS9*w~KmOkrKrogmieWr5OgNM&Y@ZavNW1JL$T_l#i`J5n5 z_UIlsE9^K|S#Un|Iul15!r}9ex$QIKK0|2J^G*xS>t18xIDC!w;_HVNoDZ)taSVFQ z2->D6{L-L&Uh*ds$3xG8ZgT!sF&IG~@vZaR^Qtv_Z#?w}mrIoF5arghTZE`>M)VdF>OmTLSX6Bzb81WNiNN#w@^H3# zE|f6Ss*cN}Ma6bt;^7*ehu^*XToo>!G!j(nD<3e$*ipw1FGn5U@`Q`FS})qZEc6ek zLOzZ6no4a|x1 z2pAp77CS7G9~(GQR^&M5!L!hu+kY+^S1SJ^s`7hs;rXV@wdefs@|7RB{{~e4ZyuG4 zsxB2zUSAF3+~BYMROE7SR@iZ_wBXzn93&F0?!V#2Q-3HC!E1tTdfsip89mm@7ZIvE zU!S+&ME_#qXqRy4Sz)K=VGGW!{s|T>AED=?HaXvdoIi2eg1}1mA^%GE%h&Av^4RAN zmn>k<5W8=aV-n2;Gqg2`dA@HII6PjS{M*JHxJTGC#O~V?zE!Xv%Gm zrt=7JNH?6mAh6Q?6-#>H7%SZ!Q9Kqa-G#|2#QE*! zwoN8+j-K9&&ToN9&r0_Wli@RiIQ>WVz%f?3@BWO~ve5H#3(gMT6Dp3-Z$aoXHuef# za6ThA6${R?v#j*I&%#$5=ff79B`ZxFgC1kzSn;LkdBlQq=t(AyBWI!KlRi1O>G^*z z3&xJu`k!!m^_sn3e_QoGalLNw=(BRnjUCU;1ux>?CS>nmB304TwBY2wXr*Vw!k6IOA$;9p!TI@Y6Gu-FJM@@0 zEZR7Kw&2`&o{8hgS?KwxPtI+erRN84E-e0?59f+Cd*g5Z%fB)=-|dFQopQ{*xlplS zp7j$G6CSVFdF_fXg8S7jH!SW%W_P-nmw@TFtN(nZiMdD9`BiYg1i#)J779z`{{{=r z=U;8&IJA99XzPaa6L4hPN%=T8EPlA4&%`n4ksB86ttWRYdh9FR|L(S)6tOL+2gUz& z-+joA*dDAQHqWAW&930w*<27<^!}86RfN7H#9xPO3OPd?FDLfaQCIIEv$;yASZL1^oS^BZvd z3vz}(V&WL|7z=VVJ&)sp91+E=p9V!bc&EuiM-+E%MHHU3Asvxk{h#v!3v$9Q=z(J_ z$ZKNp%f4qw|We6?}jZ^3!tZ%rJ79;3ch zd?|WHwp{dud+54wcmy24v#iL{8m>WCpvS1$a zX_Gd1eC>Vj-zd1*F7fDCWPhxSc|VwbyGr{P4rw~G;QkJNz41t4i5`D$!P)K`7dW*2 zPH5|fv-~&n0t<59>mSE?=`j}MT&n1Kycgt%JfiU7m+No-Y}J*><5r)4wB?);RMn*s*+^+S7c=?#zOmnFTqf zMa(S7nOTt2oLP|bIJ^s1CP3fK#HD`%^d8?6ntSv_?ydK||LkEVKs^g`)Wqd_!8!L& zL7Y=gGAB|E9AiOF2WPhh=d-?d3F-;zvxk|Gz0EfvD-)@To+~Uk?|Pw?p4)oyb(00> zO}@Qz40_BP7HxWdWx@Hqf3F%XXQ4;l9c7jC^3eRi4b<0M@8j!rQ1siko+URW>^mA< zdYl|{Z!Y94m;-4O6U5gIeygtt?n}Ddus9CcALn9@gXy=c2mWAU?$LC<1YCL2)qBH2 zVToPswBS7NSQE#gtt7N{!}%aM{;el}?O!0|rAKa9e93)+_)FO$JUb~tIt_v zb?#@n%Ic5E%Cq=D^C&p)m_0wR3g_Z$df*tVa5T-<$O4pz(xL2^xR$ zIfBr$QRsQ21?SE)OdN-=KXu_t;oM}wd4X>`3xghG6^`Od;VhgNl+ROrcW*uPY_`Z* z;kj6hZkmFTS)b@YgRVgUm9wp4^!;1!dqE zTTfnkf*8ScXOO?!ttUnBA4Xt)!GG}^robG_`Vf}B7Yp@lJ*fnCg5V7I1#xz4=z(Kw zJ=wvz*n;!RohFW>K#3B*&3ijIue0E+^KX``J@~`D`1*nc=gN2VpvR1$Vo*vx3swc? zbIUy@jw5HGN7m6;<@~m%2P5eBed{XsykX7W2X5an&?ADL?2VwmwqV{K%vx_4`T3_d zgFDzIf}RYoC%X~!Phk4(>T5M0znacLaJTp(pTZKmdc=ZrZ*ZNA(6&Wr>xNT&#{9t6 zlk@yrqk8Eff)0^<)Rq?K561;)A{sG58OT}5y5MMVmgYRuUl}A z{IQAS&_jgk&ezW^IA0$%aSVElttUHtt$0>YK5skT#PQH`b2mAE{=S8QiJG7KYsD+q zoVNIDl|?-=#KYbU@fQo`)w@mF)||HI$}e8?FW&}tahDA7FtUHx%@FhcePO_^D*l(a zYdRN$`yKf8*4PS5?CMYp&L!Ws()JyptsBm}ey}jK9;W5P0{_s)Q0S=dztF_l}WefXx_MPE+6%Hz=r|!JI(`(O+ZIzLfEQqgx&v>g`pOh@^J_|KfA35ju2LA6ZWVG z{Z(8$k52-G#Tzzn+H%^)ty@prd@4WZHL7ejyel-jazTP)FFqxA#>Ugsv00(H@xjDk zOr*Tn7Q|n=NLphDmeH4=fcpL5F#IknXRY#p20J4R)fQM zPwzQ=sW9{C9Qn!hPYlg63GR^lOS054NGPu;UHR0mT&jvT8;`s+z; ztr|zU)a7j+^lUcRU7=8DCF|RVx854 zYr~DgSZO@(zIY(~qSH2S3O9B&nx*kU0czj_BE>r))~}7L?=_irv=K%d--YS#bPWD7 zA@&f#`35*&g;@DGOn>QUUt#75@4HHP!t|#NrVoMXIl}Z6d@gaAE>v33T+hi{&3ZCF zk*_z)O}0!>aw%0x`9{g*zNa)QwL-oUKJ$c)r)>;xFYP+&*{b2$j(omjkZt_$y9r+yhr-h*#;7dM^d_Hu^Tf4~_oYmmGvn`+5B4@01_5!Zo?qEnPn@1d0 zS`E$BYMmp0Ld!#oJ%$om8k!@f!ryt?SVH}LU+gg$u2^Q8OlS%4%0|YZ-ESjrKgXn9 z>vnt?-M+Nh1phTO{ToF89 z4tu#KHneIW8cE05r9Bpjr!uK%GLcLsIZ@iPkyJ91$tDxgSTY?+Mg)NNXgV6p#4}uz zNF}n#Y&5IeV{#yyi6j!4OfnNoMjf1ZB$|#z6Pai<9!W;yv1nYMACG51PsO6VE+)&D zjA*rQAPRJf9%7MbG#TYRBME(aBAQJkQqT#d*-R`OPdn|&cq9RS0&=41csv==?a5?1 zk&2{KnMf=diD&q#+o3rV%_h^ScsiYpXJcuHoK!LvjYc_|%BF?0cub$4&Lm@5s7uCE z(J0bLIqYWOD3!^^Q;~QgnMtOiLO$ZjB-61p{}50tmF5x$CkuQ!o=V$oPU9f?6uBra0sxat=a7(n`wbR?6G@gt?% zV~H%>rZbUL0`W)EO3bSUVxsO;Jef+OuQ)%H+WQ8eE*?koSvpN+vr$K!u|z7J;)+-- zn@%Q>TiWS1kw(Ah1Wq9y`8#ByX|x`VAs+sSf;2hFL?#gt8E{=Bo1%N#=@#`z0HyC( zJer{+eL5yd4`9b)anv7AXd{eAQEVb1%dxSEG}71P#1mL;CXW8(3@j$8wa&F!EEq#W ze#irXD*SjNBW4@HRHO~DIqg^;a!7C-6T;jasbK@zcodt-#uHhLGnUlW6Gt|YC?=SQ zMG}x3i90yyXqLt#eWT!1B$;*2PeVF#i{o5E5Z5a)#BnSvDwR%SYMDelu8kR&Q0FGG zOghc*e@%WSn@+{DNz{SQW#f^UZb!n{I5fqvQA`4BSMbULa5(HLh4>R`=X9Jk6H8}d zE1Qa9qcJVD1npT2BLP0Xl|nPBTYMvqeJ4>0CK6A^)7sUAUVx={3>QDmXl@Yzb`wcq z%1JCg9(O{3#F|Jfh6Tpa0G&J1L^hzKy99oVN-#a$FD4j?adtGGLSz}#rQ0*~8OKcG z_$Ts@Cmi~*9^wW3MhO{&Atz=?fPzsplgwgiC|X+^`it{D2~rjt&!BdVpG`qGmKBdl z*hmrnR68Lto*~X51UO8#eJhD!$A~+4Q9Oo|WfIyt@d?gFDnbt|qj`-J%@7&DNl1`L zrm*UEJCQeql_BW_CgijuO&lnhB3`Ex5#qBp6G9ZyrGpd^0LGD&#=$@$a1@Q;7g-`< z#5o`5N+C)toFIzZrA2OiyuS;nXS1;+K1WD#og$gRTzUC9XdES?WVJaYGg-Jq*6ApU z<5<#Ri{Pq+rRb8>pvB3>2~3Q>3Gi4u-K%cp8$l0o5h1fMrKwMW32_)}JWUcvgQK{@ zQqzQKp3_5x=!&qU+r^M*gw|M$*VS;PZWr?;s7Zt%U}sZl9g@I6(HL0-tHYQx5p7i| z&P~NqaWO0cC?4qIkn#xDII4J!WHgPRl(>LCh|iH|GNajAHIPao+-x>WTU4&VRyesU{8ATtjlu)dX&e#(A)fdk=?F2?v%u0=X{Bpk`{?k^i240y2UgJ zY>FW43*~BCh5$?tYl7p1(CWB~C>on5o0BuCDCkFVi*bm0k#hti!ls?bMra|=6J#=^ zZgNl3@fB<}hK*BaA=6kqi-0r^MB}pLR6;pHUot}y&h`1!M+8$5HpYmcQd%!?g570tBO)-2xlTvfMeriH zQz&4j^snj9MlnvJQ6!y4f`kIcDQG8<5@H~o!U9lbZD?F9jX**kkz}H)nFkQCq~srIuZw=%n+}FBvnE7QQ?w6F)*pP$vU{bIy*@bN>!?I z1h$UKRI}7-giQ(w(l9bpg71ssKAb{!CC8vb$l~l3xF5li`U@iw!D$H8OWL4;bWfHc zTygG7(x%j2@FcktEs1-tB+&9kKoTQiRH_~AeNvF37*-#V={Xa=gg7TK2o(@c&7r)e z4*^giAwYs0A`-o-v*Y9?sfMu_EQ8RZo5^mRE75^kFAJkiGf7SAFzi{16Zldf#5;@u zg-};X@um!-k9db`0!pfrDv44-oh|xGOQ3-aEEI3k&6I5v7*a+OsmNkZb4H@0gj7N> zrMkloewE>LiGWn9Sc9a$KI$xy6DQ(Cl&hp4)vTNhK|?z^nfR$Gks%7Tl$Z(eE{$QR zv(Y%pCGy~Dl(0HF_L1MPI&_Tn2$2+oayID)CK-%~ousMNmG|^P1N=xqLnT5oNBx|= zf>bL-9Z;AX3Q{T~^f3&Iq8BA)6nLG3}cfEE+5=$bI$ znV2gzE3uBW45U7rKp>`s2#^zHvXY2Y*y5pdhjkNZRf6f`G?E&*mxv~Vsfe}`8ZdT= zVpQT8I?kw)S8_2{N020~##6*B!Cr}sBpFG`C4)mk+j7lVGmoyc^K9Sv{iQtunrgAK*-sd{R(p zLxVxG76Kq9P+~|as<5$h2HjL~q*1~v^r&XW31ksM9WjuI2jRMz(Fw#zz(a&_)S#N_ zL<-OpJ#s2$sX;JG@qW}3!GduU=WH1zOMIr9m5NN2rF{(3)KE`EO0FfvD8Hc@;-q%O z#i+FCNxEe4A-|<=pzt6Ys)4CA6MZoTNWel+us$0>QBBYrlyF?tqF9h| zk&dlNsW~Z)Wgta%bgBW)ma?8mj~ZZ-h^)Aj&Zzz<8Sw?Q>0&1#NewO|42~dNrI3{j zS;=OS5D*i!i&i}LNb*3PV(?1V|`nSWfM!nyEL5!vp|CNU}g_su>51 z;3a5Fjj=H>X(kS%14M;O;*m@gDDOkB$O(z_o75%gPo2%I1R))n%9v0_bjn^a8M=~~ z&HsreYJl1YTbKf(lMZCGr3sPV7>SS%@IEPkRh7~wDGjq{?ua3))Fe7e(M(-Rn3T~J zH6;Rqkd+dM$u1NRRG-*RR#^ky$N`eJ2vv$XRVef^g2L4?dI}jLE48&!jf@4TNtiYv z!SkKKr+9=#l#Q!O)g^YM?2_RoCM_>9CR{KArutH6ll+;BAQV&Y;>jFS%_s%KMj})o zX*@@jN_~`%$eoBtmVyw-)gZ(XSc`y8{U-LW!W4;(Du6ly@4(5$yaij5v;=*+#jHqp zYM@OsN}+~>ZgeW(`fMpY5|n6+qp))3GZdG|Na|>!D3M9#HIk}JaRQl=qr#Ao=@il{ z@l7HU#*Q^pOA-SWAtV%1fTTW&Gr|zfoGb%GNJn}M(m0|{#y1&lAUzC-i3zMqH)CRq zuS7%?##pP6x00e=QX4A43e|v1pPZHoR#l7yegc6KAA*J(NtK9z(2kR70>K3i>23#B^Z94v79tg+UDlNX@3jhv{NEM0-gHadf1) zVbYFBM+Jx?$mA593R}kG6tH+1Lukp-T7EK`k;01zi`_HSR-TGQiI+-#CrSx#nr=i! z?IHyUG$koJHQfXTxWi0{{%F$a5*wyA6jLMYkR0$uC|Rje7Q{*T!LyyQJ%bt`&k+ug(fFtKn9_mvMEh1z|0-;Rv=^zy261u9Ubd= zgFY!l87&hp2quV@;zFIx_=mcVqE@mhSxc4NoK3Qk0h9E~IKxpT&0-F)fn#B;2u*=x z2s*?rNpa+SLXwh{bVhlLOk^a=G=wT$`>@ainUC;9-aZ61Ci9Gkd1`GK{ zBVR4n^Ml2)=HRAQeY{nVZ|U460%Uf4Pz-MQ)!%r3@xE`zPrB{LJ2yXb-O$_QX2svQ zSs{v&k8{H&@qy*$&5F)VkFP(-O^;=s8)NO89{q4m#;s_9hYli z;u{_&r1QlhjN`yLkIv=eFfsCqtcl|YF>ovGe2A@ zxg4Bq;ouEdm_2(q_!=C1&BnoL;K+S5_ud(+G)7;$-o$Y@_*$ETa{?U97Y>f(v!)jZ zg<7RGUUgYG%fiB~CzxG(Soj+({Hu+Hmx1$cx|WZlgzK)zm^cm#e`~YQxqYUTi4qnF z3(w*6%w8-wLE?77v(;I133v&C_<$BD*_4;?2)ONV8%8k=6*|@HPQ-xajI9#81 zhi^rO=DNzwN5}MD7Es(GVPb&KvwCrzA04fiM)S?K`86zDeEH6rrOk9!>lk zJbd29!&|`dyU_E7O&o`Ze{1s)bRiyXn1TzH93njLp^P>aC89X1A%&s-Gx zK9tKxGjPb=L+AIvSslGY)?_$#v{VB6tB3*ILJy5NLXex-b3#k?G6E-)gy!l1u++^c zY5@j!!52!q`(yD)xl}2tUyV>$ZnWzLeXTU`kM0&qcQ^78lBK%^{_Ylb!bhOXrS37e z+hzIsXltA$FzP+`y5~-ttQ660bB1fhT@bN~1s;`BF;}bRgyjX5e08*y zA1w*x^-{A{uPVeg-HU|o5B8#aktbpV2w&Dd*o*Q-Bjswok`pHBctN>ZZt}+XVvwdq zyzaeOa!+)AiKyH3E)jY^=b?A5U(WMgzX}k$NTpqu&_(Jz_t%{5{G^xpwFw>KpUDqV zuIO^n_%7E-_%*l1BSHGACg0|& zlcnA`2Q&BAE#KZ_nz;@&wa6E*xr?wHdC=o7V&+Q)XG#B}P-`|zAo*x_nSK5#|7GH6 zx4Y}@i@%23;V!fA3y-@j#eU}^*V$5xy7oKUFV}^x2pXY)WS13Nbt!3bll8J>mN{Z< z6P0oyFTxDDvXq#z*m1&qt2xF`wOa8^r{B4S3KcCs>_u*cWlYUleQ;}ee4H0xjcLpU*oTK3@*=X?J*}wW@YIJVn+|dHQRMLV9?tIauiktXOW4 z^@dx`Qt>hFH4eVJw!|meSv>dp!L;(eHP2b)^vKY@xKf*h`0HL8A`R2Lakx^ejps(n6yIZ+2+;XdyQ$#+YlAB=I#k>k!3ZU%ARczG5jy`p&sysyU&RY=TD) zr#|HWFuEgO;gY9}m&UbpMp%kBmYd*hl(iyfX=%+Mn1+w#>k{B^Z zayx%#uRrA!{;aImCL3~XuDxVbPC9k-smv*aGRiQ+wUKafI4r$1!jt9Z7>l=6Ut;%R ztR1^8hHKSuKJ1djwY&LHsB{wVG1kOe93JfWEcyO|(zY_DS=`WeWKT9;D$ddzqnEK- zMRMaIYUR~ZaR+bnEGcgotSOFH7RFlD?YT;+I@%l)7ur57N7M{-yDAv85?8J%;v{?w zo^p0VXp--$=2&pdUSRY;3BvBbInEdN?{a>tQG&3^cF02ZoY0OB%4y4UL$`nQO~1>& z==LM-{lU7ICV#DtF`!U?Q@^=H{|?DtZy!st*)1DS+pzJ}Z6}^0dh%Ju5@isCkYQcD z9<&`iO#s6V;8=O05hgTmZ%u?%IDmi|Wg`Us^;GtAIN3Oo_*E|9- z9V(xzLNQne?bynA!c;dhPZ^Xe=i^s%{(<z5SWp23;XxF}dOrvABq?a_PTx=T1~`8B7Ue#PLr#5w=6 zWaS@T@stm}m9}ZoIHMPRe`Olfy;5V%sMk>Il+=6p@~0m0j~`q2=PN#c^Rl1ZaQ!jU zrQSn}_1eS)^R?5c=EEG!u%mA7^2m&l%#Y0kZ>9P!g_z^YFKfN>mOuQ@y5C)w{nSss z_s8EoW4d(hnfIRtXMUy;yWQIcGrp}9?E19(w`b^n+X?@$>ykT;UiHCQBk#TS32)gl zUHWd{f0zbsKSj?MBU{kng9>iN(;XWkHfMrP=?@v9HNd+x;0bm_T$ z6F??T4!Y)E<=$PHab?l){;BvQH=J?VfBaMQ%eN$N`rfx5KJ=6N`u!e52kLHjVsjew zyFyKh%qVyNm3!ll9(vbjpB@>zfi=>CcNt zw$F{d_^t1Y-E!>QhfkO;1v~3Ira`+;Gm0)*k1(U>{nzu^uYc-?Mh-b<^1U0cY23Zw zO&3g?p7p)OY0&d>L7GwR{;T%dSKsi|ADtDO^})M7dU|pB*DgOm)wWA2xi7V#Ag_rm z)z$Zdy=NsXJ?^96)es>!edd>{a>KO9*SRD43in#YcK7ms{cpiBx4uLEUB^$i#C883 zN}IxPq@zw*Dn)H*dHVL@_ntkp;%A>rJ!Q+e*BmbEjkwGygX9bicL{%0qcrf(f8WT^ z!a`PLRg3w0aYhXyD_0Gjzpm`({9^eu$lku#=Ly_~m9>D*3WPD{b!uv2=b(oj%T7~^ z8p0jO$K6Z*uJ6!_4d7}2@saN*ZhhDD-+9e{J$%icvp=_rwvMMw{SK{N+skIu{CMAp zS6U7&Z@uTyj}D(4z4Vgb{PnuizVoMNPpg_&sHIQCtuo7tW!YV(# znu%ig45?>S7vXjnGY;-(mp{QJ7|yZNFT3SESDl*t$*q0szW#$3?~!nSAP(MHMY-@q zS^03%BPMS%@d)m7A}Zzvrce<~^6ko_pP* zEpNH;y6^n-pcT73zaw9+sKw8$c^Xw}IB5Nz+6I;V8rAb|LmEBIN^2H5hKFmlCUF9W!>b6BYJ*lDHGn7bt~PYO>J!pgsO+M;^D4ou@af-?s6%Bf~Con5w^Ce*8oG z{}vq|a&_nh*Itx;#SKS4{P3Mm8Ii{YSY~ziE>54=z>DoyhO3O^c&HrCc4DsMpvhq0tB{)G$9MY7OSOymXD z$3)C$SWYSJV7qnu11$d=s^?FAYVQ{wUX)n=hPg-oY1L<5F>ui8S}dFR=a9-YMd!mb z6oVq3YGdk)@vnYcz>GgmDe=DK$G`g3m7j>Of75T5zoU8Is#hJff}NK&Ohe3F8;rTz ztF_6hRxlA+g<$7}2+%zfe-Q%zddA=MpP&5o-CxK&|AX`Dr@e0Prp=sxaPfDsnxM+< z9GSMq%hYrl4t;)$*H7TmB;GJnUeyL&K<t6h~Xg{7;ox2ykaDwhpD-p`%@hc05b z5k68nZwxJ3{K?lI8#;DgD0DHEh{3BXVF&&>qx-<}a7|Hex% z4CPWgKmGo#e|Yoat6mMl!I{qT1~;K(mk@ate@D3~Qse{a(0ETxpZ$#AkKXYAj#+tm;aeZe&;ANqiyfqW~m&~7ap{GZZek2PO*-*pxqU9e#Gv1exwdeN7=_PxDl&b;Xxnk^Ghxnb3Y{6D2x z?TqUGZr*kO+BK^$SpJ%4{oj9n7uA@QzbiJ$b#3(cH@NlGbIvqzbeZ|&otCYq4t2JknosHZ($B(-l&Xq(|87C; zN1OMtT@ZUT$s^G2=1|L>@jY=QSF0Bp0WJ-#@SKtx-X#^^Z1%EYJq!mIFkz>*FJNn= zIqf|J{{&gSjZvnsWezieYEvCiQkC~_@^^u!ZE1U(oE*zFidc=yunz?-cHjjr-)0NrWbAM|!rz2jevmz`){$H zZgg4IVZr@vquQx@0uJDT>r`hQNN<}3b~Xz+_8|-JhLdfTnvFq?g4r&idXcP)$PT&g zZwski4aBnXi_hYm;=|V(q*@~L{&cgX=xjM7yOqkdPATPYw{WviE@A3+e1hu&60)!q zuspZ3z=8cYNlSO8f*k>xjY0Xr%6`eu)wauu``aR~u4*y1g7I(X-cd>Lh^3P$X5}tj zN`#j2zay+pkM4n>!1kjQFgn#+)OWaZe_ME(n&9kv-FG_ZcHGjTrydU2wZVjMgZ{SQ z0_{P$^)xiI6L+n@9`klMng+KqI_kU?pwox9%_6C(sm52lxW3|R-XD0+ZaUr#``ZdI z8-vpC874j80TH{j2M+9i$ntdKW=$8}{e6HUY466}(W9ry0(zKJ%a7+LR4KnUtj5UJ zg5E>68BC`m!nPkWmMG+ZeL#ns;1LT`J-yiVDd{JmOIHnn1N*OJgCPvQ^Y()dH@;>I zEBm9L&h9kbcGm9n-2Ds-5%Y?*7Po0dVCEZB)SE2SR^L3t?ryi<>BQ4yK|L_mC>ixO z?%r(?{|9bcHGimw=spsSUkJ=ce?QI@sBSA=O2T{B15^Ot%0`FnX}t`r=NC% zrHd6#EN{e}{Vh)$38qX*WJ9mL>B@T0($iFu zgQI*rC))rkvqA6$Aj9(yc}+}n?*Ds?J1*5qrpNNb9NaeqhdVB1xR@oqV5TG`)@9!vXbbK`koURxzb zTTmZ89mIKA5ap-m{Xa%t2^_HKq-%<4!v|h3)KR(Rkgbbu6P@iYy4T&V&byys>99v+ zYOS{R^OV?1_hBOOs7u9UF03Q(fzVh4c%-1|!z1a$H zw@ceGOTSameD7-8?XdS*2iSGDlH2XH+i^>G-AbvBojmpxdAl!LUF`B?J9Z%a9Bj$l zCf52gu})0ua@az0uL+fQZ;xqeaUYPk7vQc2>_;sO?bD|AQFgl>?&Gur>DgX>yB*q( z9#B8l>bu>K^{Azvsl*{`_3id#J!t7@-}%Dx?!ZpjA%^2lpC4%1VZH~i<3RrQ9#pqy z1N}Q$9rea_F#_`zFL0Oj}^yx4!1F&sQ zu9LSLEcWFxZQq@;>t3YJYlcSh~(qj z+2@?z{m-LT+c6942jfn%RmkaGD$157G0`Kf@jI?;^vb&w4t7Pjz z8v*S*B3%_7++){SlIICu9${M#S~c#SXteVl?3!5zo@gjn<>mICPJPW5M(32PC2{qs zc4Q5KZRvIbSSNVeI&~(!rrK$@z~}bc?f5iw?F)TwkG^J0M^lK1K{e`eyi*0}ANKNc z-w`1QKJ_?jHA{@7eM4fZiZjBSD?6jnBDn+MZHan(rcj>XozdQ!!QXD_PK7t|0=f4& zRmPm1Du#JMp7-GiU$dpR-uWu1w1p~!XK~UMYPiFjc8r4+k+ezw%uT{)w!kd z3AH0d2h6s$^E6sc?~djL%AZb)g?tqV)e;i~(?RxZKx~_4?kZVt7`LJ=9B;{_Rjuj` z*hZs;$VKWAqMlUFXA{K+u#&Hwfaz(nU^-K~9U{Hk^3Eydn|TkE``d!!j(Y7t`z1U# zAlacj>`ViPize&kX0ufF)#yg6g~-QHe}SjXLiN1iG8Kpv@ExM<;0rYnyG?q~ zjmCq5w+?SyMbmu1feetry*&NqDuJpHCy69rHc9|Wd%i2Mw?hzL9rz)Yv_1a{E z8LaxK;S9cJO9#tko&$?o8`hR3MR7+R?VWJv%iguV8mryndV%aM5N1z+=+i>#qqm-V z+Q0*^u3-{IlnkHf4u8BhOqX3pk!?IpR{<=|Sq4 zKt5=LFK|;9XSv4hZLcxfg zD;`5|L4D2pi75p^uE2&-#>4H4&F1~XTrpZ}G|CgX(NeXm@cjYXTQclW_fIS6|*l6r-OouTK##Y+0WmvPu1|Jp^=y8gzukKtd zZg1RSnuFQAf0%RF>`1+Mz$std-}{GhDEru!@{N*(d&{By!(LpUC|DHdYqnu_*V;0@ z%YlX5;1L_LGeK?%F3Xx@_3}v5h8#FzL+-BqdCl5(-Vfx7g`E})k};KTA(U09t*WYE z&0^f&Zh=-eC`1HgZh1@qk~|%_8(YsUj2KzF{X3Wm0SH;*fQz)!vn5|{IA5vMyoIIx zsDB#@*Gdu=*4=}(u*YtyFEN1h+)dU>j{>A_QB1QEq!&$6gM>Y zKFiW;Fz~y~Eg%A_O~*Y3>TR=t^$^veE?Q9EmO~coeTZjy8$T!%+c8TI-9<5Y3xQ6Fzw$K|Kezdpf_7oH@o~Ox%6^z#|eJ*b?{p~giyGzi`$S#`u+lCU%9xk7jH0%Oe4(%VdFBQAEzGfR{cdmnZ*o6%qu_1TY zcqePRh@BP-QkOjH+dq6s=iaw*``O=a>%e;h+T;8BZj8G@`C^s*oo(YfrsZqyZnc5f z65JVk>azviy9pRzu$#Vdw)D!iYu21>o9%IyE{83GvrR|^yIbDwsXBaW8O%b}+_Sq{ss>oGaVLfQ+G@z1RJw7I?>DA$)ip*50@GLIj z=*VofFctLsUY!_I|88)Z`s(p6`(7_1D~#rv<)+#`p-sBc+Jp2is0(CNIjUdz?qYh| zEQpSVy8yCkl^HN2a(kLAaIS2-;4KELzAM1DuX+D*=Z-kf-MU=28x8=g;>U6u_8)7G z56b<$|40ia@{PtM%fCF9?rqzD#D$C*1B>>IMhm8KXQhjh=gE$s#sbJri#7Pek{kTp z=lpHki+kP%;i`For_DmGYmL27)n4PqbjbowlRf)-hqyhk7dd-fYjl@pv$d;0+*z3( z`11mr4R=~>(r{}29*D~{VvcuXO*D)f3&A6n9-Ls(ql0-&#qru{j}doTEP!1V=-r!q zUZGMpV^pWbhS^OFUQD9g_;}t}kL78yp*r<6Sd5x!;{`c!CMz$?G~w-vl? z7Tj(M^Af#?GCn_E;zo#f2Gne|KwH*Q_mm9~>ita~)9^G|I+&7en9k-3+-%){7c|6k zu#2DN2I?`~K=p6V{*M1KZ=iNIXP-RklI734!LvEL+(124a9;4iMWN%6pnUX=*wC?e zJmkZY)) zq3tT6tsBlwA6^t%56|*(`x!0zx)<+hF z&V&&8ILvIl^c_8LR)ceVo0&mJ%ClrV`b1tDU=Z9P_tkXcCpY$QW;c%PzkyLa1i9by zkSpB`3fq4G$8Qv4hxWi(4Nk`>6b`bVC9Eo=n0G^vu<>h6Ku3-bA;)`sa%>ZJ_N|LT zmw_oCM~)Xg`1+P~;c+Xxya85&1nwT=mh7aCWyx z}KF7mC43ak$57R zNv5JgKK*8r=~$Y72q>0HbBTkK1wI{5BBMk!6-RguU$mz(h$o$nfs&27{6a-6mWV`B z*+g1|m(Vz|Xe^$N#2_dV7b$aG^$Q9NApJ->l1az-k<#t4L>6w-nMf*u_#Si~1vg(swK#&CroP9h0O7uw$_}>W?S1 z5yqn^HjzlA5k90MeN9e0fz@W>=ughTVv<_xT${y$F*M|dJP@eDk0&x>wh>H4+7O%5 zj^!bT1jjKU%*~M+Hjs@+v6*Z&?@KaqA$$5}J6bQZR~WiNs=9U>pt5 zxg$+v13J1(;J2s*)6@N8f{_?!N8>3(mO)**Jwu;y%p{I~BL8^8p&#oZUchgZkUnqRp{*02;9R64^uRKj*ErD(kpY~91c_t{t8TXwc~e*!l1^YkPCL@X zfs!fWbvh9tK5H`}L?K-|ND%>G97$;$3?u?a(FlH#B@#xQ^Kq^eqQt@pqPSgJ()ie?uJB$$hNV2Aj-LrzNUCK|zk zF+-xFBTZ!c9>CfisHPl2aX=H1rU_B>D0Mi*(zC7E(o*UeUO(B7SUJg2lBQO544kf$Oz;v;+|VhzzG(ZM9M@Ok{nS>!_y9w z#44f|5g;mB=^H>?l1P&zVr-4Cnya)hiu04qh|UCgC;f>l;7=F<;YEU$gPumANzq4? zszGWKZJQ)*@64(?$*cZyxwhRH79@YfM38B?- z6;U)cPc|oKQc=*4;uhl&^&;m8Mubf}k&Vzoo+rp;NZsU~q~j~tY785v&O)ZKcoqR^ z9EijhkrsiB5XN!cjz)lzNKGggFzpj4l@OSYaSR{{&TczNi>#f*)FhcggM*WfVDu?s zuh@&!Rk}S*Oh`!yKwU>ZBr4GoztQJoNLd)b3?nhIB*%C#x~PP5g1%&i zB%JH>sgDSzB5aHiL8Y`_-~_wN;zmSZ7;~MDvWwtFaHmkfOzB_KpN(RiM59PLjRXk= zj#JQ1ASJ{=I)w$G%G%JlSQ>$ZJR-?NS2s~~f`8%=350|RMO46(;wU2_4b7rBj82TI zk8~ssLYX061xc!c?4!aZfns1%ag%j$dv$h_B9y9B`IP7g^YDL%8DHm84Cnz2Hf5Ct4EsUP+*h zQ>0;MGEyKhDhMe^QA~mooPVn972=%0AXGp&HHY$=J_JC8ga8S0h)DFR&W@9pq#DLz zunag4lld-OP zBScaV%GsnJm}D>_c9N!6SKiYH4e%oc4V4JV9QAYd3R0~UbwFWmC`j2Vp^sru6!kFB zNW-rRTzxWzL)PguN=s575b;!p32G0j1+>PWqV+5FVUz@+2A`M_1)w^c z^nd~&nT#f8tBS}zKv5)mK(Z1%h=xK4(!p|ZungkB{nRM3kEAUg347Qo&cm=ypk;7D zM5I`kV2dH^vL0I}WG0vJP6m#j7}g%0v;laqXyMXCsKf>=#f(~OAUfiiua?Q2o{W+ zIA_Z!S>iL*tW;#GEbU{MriOYVQgSURM)?iR5GS=GE=HwAPtqlW5BV*11BD0KPz_9_ zndpl#Kmrzmg7w)5ifV%1poHT(FZ2-yaU*KC6vcv!i*#&FO3g`WECVUBqf-rVwv_cm zdei`uL}bOKbVl_@$%rqYO&2=}NosHzVQ>WDDut|M$VxVogn*c+U9{q{N0JBXB)`f~ zl?bGmL}b+DBtSwr!*Xg*)l9uf93}uDLXrhaQ_VP71TR5bYK)D6Ni%U69Uv-P5|3n} zKzSc}MNUYZ-=r={f9h;zB?#%rRK|odqEq&Y$KO`lEYjEqvy zCwU>J10!}o^k*szYA`@*HZ49(7takjAazX}V5ZWo5($oUX+#zoTLe`)w9GKJ5v7R^RlR}iyGVy|7f@mo&)Y*)GsOuXeP$5Xu6mSex-T!@_WTUTIpO4?#1xyxv#CUgtpZdoa!wm zjzinULR&YSv%t9tp5^1n?Z#E_HgWc7xoLV7wzCf)tSr>j`zELiOUvlPY&HR~()TV74cgkn!q7ztNwl1u08OqK)>!gxj zx@d!5P&jZ(sXE#mTNl<%Rc=3WOJ^N78cP!c`3lchr^ev3#9P=8igj+lvM=ozp$cYbIX~aehD9^bwOdM^CTEZsgAJJsm%}LWpaL zu-^CEAkM&^9ylxPIA5{gEPcI+;|Tp8?C%Xep||O|$AWYE^(KzP*Bg5A^*0O7C!S~G z81$_0-e2y>C%!i*pIzTJaXj?g+fB|d{&O%MU77XaT)k%R8$NQuFR{07@n}+xxv}E| z7R<`qOiXyZ`tlRM`2%o&)g>NHVhfWl=51j5?do%1HZk{TI)4T358>Avj}(^J)mJPy z&)8w&IJEsxXzPY^=wFDv@GKw432*NAIiW$1#G?+)`9T$<+ca#fC3=U@J zV!2u^9Gp{iV-o2kSE*t6_f7{*hqONhPoT1ofE7X zkt?gUW_d*Rk|JTt??%NrkT0m`)ue5_RxPvLD!Np^^ls6)VreAbsx*~)%RD1NU&44} z-O!M1Q&lbuOioS?I@%a4)P`8RUurgnhAT~mzLC<%a3)eH<%f%jw5ziL6jVDR7I{wW zxr2=zqsN?o6udybEXe|X=YXObc53|cQr_UCe3PxD$HrZi$YUkBO1TP`ZMu8}jk1YA zX{b~k(!fKgsm|p?yH^R=-MWtf6IaY3j zy#x(%CHryI&`$rFcq*Stj0lzxMax+OTaa@2RIlvelD!3nmGrGA>~{XG?XM}B^RzF! z(~Pl;7XHVc_P5DB2^VaZLA|nYy2r`q(r4sz^M1?5vBs!Rq zb9NBV^1QiJb`_ERg==*X6C7ac1HEN~aR6&_ev5VNo2oLQk^y~YZT~oU+d=JL2kVUz zKftjs>61Da z$C88T*uZc!=^m&Sm2TEw(5i&D?jnSZH`azvtQJIZYBe(t=!};PnaWxdW*B6Mn}qn1 zB6X~kpWv)An;TX2^RfNw7!b>ShGdFRAcx2EJH1C`O$@{TEQfhqD`!PuLXNo>&j}_5 z$}N`JETY*J8)1$Xc;vJqs~VLC&*^8`*r}yS7-RtA*!re?ef!o)1{@NwVTtFaauQ$A z_&Jv3yK%#& zt>Mj;V)*z{Rl038h%B<_1rM+jB8oY^!84T9hR+ioCu+*Zp^5m}DycZCBB3Hx48{`* zjZgkT|3`FJf5`j3^V4PFTIV<5k zFpUCZIYOsHiwlJP%Rn!u26}Lo9OsMxwB9*L2YDM=F_$sRwJMBx2+`5sAp_e|{P5S? zDT6u~a-7(6Z5Jl!ehN&eI9V=qhjWQxve*Y0k4R|uQm7<8nXj|uTX#TL$Wx=wDbKoS zKJ7?~zg{|A(_z`Cs!ts49F(9;L#ow{Lf~M`h{t$%!9zPY>T;zEsl>J^>C{{KOvSVu zfep7Rs_?K`vug)FqfClT$}0WFSk2pO7Y3%HbO4P~Y?x>B*}GqF&?T|&oRLx{vTj}U zT(`(R$NR%qf}cwYFXaOx9T?Q9h7gQ_|mJWDQduGkHqMh?4^S~w*9YA4598^@*Mz~oEH<-<8hE)x?FQPRqg90(A zyEvY&F8yoUG*@_4!$4`LN_J9oPnqshTC626g*%rv!jcz9^#gBVl33V~{XWI4wKmuR z97J1^gCVO6-Eddiv1{ryBLdD3JN8i>ogRuc6@wbQ3#H>M$9GDXDcZ#*=Zfj3)}@5i z`(*G~+`hB7ZqOe$b#wgbq8bb_~3VWVJ zv>`THkU)%ofVrH?!E$#%;ErEQWkPJUptaIinix$aM`D8%#%N(e1IEJQI%%-8Ac61A z%+9?xJG(X}l3X!A_PaN4-jU+DLm81n3}3SP*f!L^xwdNt`Pq88~@A&NC0r;R}hQ3F`Lf)7MeeA!MyQ z&pkMWj}pg>tC}^g4<4Ma4<(L}MH6{x| zhgcjS_ITjhEh1fTbNlf1|CiU;^Qi4qQ4ZIBbF=M5Bv>rY$3>! z6{o&AjYuM#n1*x=;}HUiWmI`ui5Rt_(~3uc;I^ueN3?;^tJK1Y$wo=V<*JqXDb8)N z3~}%vI%!rQrd|OdhE$@3gW|68yt#u4^yk$81YhTGNUbMr7>w)py?>;^>}$Vo<+g$V zhWD8n9vRFk`Tei%Bu>w422S3Ob2*>Hd3#>sXh#SFIKrmm2v(jO9-K3GC5{=_X4beK cdvLa$qw_+Zyl_k_&vy?_?@+2d1)rXe0h`dyi2wiq literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/test_0062_cassandra.py b/packetbeat/tests/system/test_0062_cassandra.py index 6eaf31dad48..4006f63b7d1 100644 --- a/packetbeat/tests/system/test_0062_cassandra.py +++ b/packetbeat/tests/system/test_0062_cassandra.py @@ -229,3 +229,157 @@ def test_select_use_index(self): assert h2["stream"] == 262 assert r["result_type"]=="rows" + + def test_ops_mixed(self): + """ + Some mixed operation happened in Cassandra + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + + self.run_packetbeat(pcap="cassandra/v4/cassandra_mixed_frame.pcap",debug_selectors=["*"]) + objs = self.read_output() + + o = objs[0] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + assert o["bytes_in"] == 9 + assert o["bytes_out"] == 61 + + q=o["cassandra_request"] + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "OPTIONS" + assert h["length"] == 0 + assert h["flags"] == "FLAG_0" + assert h["stream"] == 0 + + + r=o["cassandra_response"] + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 52 + assert h2["op"] == "SUPPORTED" + assert h2["flags"] == "FLAG_0" + assert h2["stream"] == 0 + + o = objs[1] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + assert o["bytes_in"] == 31 + assert o["bytes_out"] == 9 + + q=o["cassandra_request"] + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "STARTUP" + assert h["length"] == 22 + assert h["flags"] == "FLAG_0" + assert h["stream"] == 1 + + + r=o["cassandra_response"] + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 0 + assert h2["op"] == "READY" + assert h2["flags"] == "FLAG_0" + assert h2["stream"] == 1 + + o = objs[2] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + assert o["bytes_in"] == 58 + assert o["bytes_out"] == 9 + + q=o["cassandra_request"] + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "REGISTER" + assert h["length"] == 49 + assert h["flags"] == "FLAG_0" + assert h["stream"] == 2 + + + r=o["cassandra_response"] + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 0 + assert h2["op"] == "READY" + assert h2["flags"] == "FLAG_0" + assert h2["stream"] == 2 + + + + def test_ops_ignored(self): + """ + Should correctly ignore OPTIONS and REGISTER operation + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + cassandra_ignored_ops= "OPTIONS,REGISTER", + ) + + self.run_packetbeat(pcap="cassandra/v4/cassandra_mixed_frame.pcap",debug_selectors=["*"]) + objs = self.read_output() + + o = objs[0] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + assert o["bytes_in"] == 31 + assert o["bytes_out"] == 9 + + q=o["cassandra_request"] + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "STARTUP" + assert h["length"] == 22 + assert h["flags"] == "FLAG_0" + assert h["stream"] == 1 + + + r=o["cassandra_response"] + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 0 + assert h2["op"] == "READY" + assert h2["flags"] == "FLAG_0" + assert h2["stream"] == 1 + + o = objs[1] + print o + assert o["type"] == "cassandra" + assert o["port"] == 9042 + assert o["bytes_in"] == 101 + assert o["bytes_out"] == 116 + + q=o["cassandra_request"] + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 92 + assert h["flags"] == "FLAG_0" + assert h["stream"] == 3 + + + r=o["cassandra_response"] + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 107 + assert h2["op"] == "RESULT" + assert h2["flags"] == "FLAG_0" + assert h2["stream"] == 3 + From 0297c362d1ff352943676a4cc0aa85fcd227fc5a Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 12 Aug 2016 10:45:11 +0800 Subject: [PATCH 17/30] fix build --- .../cassandra/internal/gocql/array_decoder.go | 6 +++--- .../protos/cassandra/internal/gocql/decoder.go | 2 +- .../protos/cassandra/internal/gocql/frame.go | 13 ++++++++++--- .../cassandra/internal/gocql/stream_decoder.go | 18 ++++++++---------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go index 9014c7667c4..76a6604c583 100644 --- a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go @@ -68,14 +68,14 @@ func (f *Framer) ReadHeader1() (head *frameHeader, err error) { return head, nil } -func (f ByteArrayDecoder) ReadByte() (b byte) { +func (f ByteArrayDecoder) ReadByte() (byte, error) { if len(f.Data) < 1 { panic(fmt.Errorf("not enough bytes in buffer to Read byte require 1 got: %d", len(f.Data))) } - b = f.Data[0] + b := f.Data[0] f.Data = f.Data[1:] - return + return b, nil } func (f ByteArrayDecoder) ReadInt() (n int) { diff --git a/packetbeat/protos/cassandra/internal/gocql/decoder.go b/packetbeat/protos/cassandra/internal/gocql/decoder.go index cf49453b019..8c0ed5e429c 100644 --- a/packetbeat/protos/cassandra/internal/gocql/decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/decoder.go @@ -5,7 +5,7 @@ import ( ) type Decoder interface { - ReadByte() byte + ReadByte() (byte, error) ReadInt() (n int) diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index fa4965143b2..d672d42df19 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -39,7 +39,7 @@ func (f frameHeader) ToMap() map[string]interface{} { } func (f frameHeader) String() string { - return fmt.Sprintf("version:%d, flags: %s, steam: %v, OP: %v, length: %v", f.Version.String(), getHeadFlagString(f.Flags), f.Stream, f.Op.String(), f.BodyLength) + return fmt.Sprintf("version:%s, flags: %s, steam: %v, OP: %v, length: %v", f.Version.String(), getHeadFlagString(f.Flags), f.Stream, f.Op.String(), f.BodyLength) } var framerPool = sync.Pool{ @@ -281,7 +281,10 @@ func (f *Framer) parseErrorFrame() (data map[string]interface{}) { cl := f.decoder.ReadConsistency() received := f.decoder.ReadInt() blockfor := f.decoder.ReadInt() - dataPresent := f.decoder.ReadByte() + dataPresent, err := f.decoder.ReadByte() + if err != nil { + panic(err) + } data["read_consistency"] = cl.String() data["received"] = received @@ -303,7 +306,11 @@ func (f *Framer) parseErrorFrame() (data map[string]interface{}) { data["read_consistency"] = f.decoder.ReadConsistency().String() data["received"] = f.decoder.ReadInt() data["blockfor"] = f.decoder.ReadInt() - data["data_present"] = f.decoder.ReadByte() != 0 + b, err := f.decoder.ReadByte() + if err != nil { + panic(err) + } + data["data_present"] = b != 0 case errWriteFailure: data["read_consistency"] = f.decoder.ReadConsistency().String() diff --git a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go index 6e0212842de..d272bf02f81 100644 --- a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go @@ -10,14 +10,8 @@ type StreamDecoder struct { r *streambuf.Buffer } -func (f StreamDecoder) ReadByte() byte { - - b, err := f.r.ReadByte() - if err != nil { - panic(err) - } - return b - +func (f StreamDecoder) ReadByte() (byte, error) { + return f.r.ReadByte() } func (f StreamDecoder) ReadInt() (n int) { @@ -137,13 +131,17 @@ func (f StreamDecoder) ReadShortBytes() []byte { func (f StreamDecoder) ReadInet() (net.IP, int) { - size := f.ReadByte() + size, err := f.ReadByte() + if err != nil { + panic(err) + } + if !(size == 4 || size == 16) { panic(fmt.Errorf("invalid IP size: %d", size)) } ip := make([]byte, int(size)) - _, err := f.r.Read(ip) + _, err = f.r.Read(ip) if err != nil { panic(err) } From d0b915cb3f63bc8f593018799fddad8f462070d9 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 12 Aug 2016 12:25:06 +0800 Subject: [PATCH 18/30] fix array_decoder --- .../cassandra/internal/gocql/array_decoder.go | 162 +++++++----------- .../protos/cassandra/internal/gocql/frame.go | 142 +++++++-------- 2 files changed, 134 insertions(+), 170 deletions(-) diff --git a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go index 76a6604c583..3d612fe22d6 100644 --- a/packetbeat/protos/cassandra/internal/gocql/array_decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/array_decoder.go @@ -2,143 +2,94 @@ package cassandra import ( "fmt" - "io" "net" ) type ByteArrayDecoder struct { - Data []byte + Data *[]byte } func readInt(p []byte) int32 { return int32(p[0])<<24 | int32(p[1])<<16 | int32(p[2])<<8 | int32(p[3]) } -func (f *Framer) ReadHeader1() (head *frameHeader, err error) { - p := make([]byte, 9) - head = &frameHeader{} - _, err = io.ReadFull(f.r, p[:1]) - if err != nil { - return head, err - } - - version := p[0] & protoVersionMask - - if version < protoVersion1 || version > protoVersion4 { - return head, fmt.Errorf("unsupported response version: %d", version) - } - f.proto = version - - headSize := 9 - if version < protoVersion3 { - headSize = 8 - } - - _, err = io.ReadFull(f.r, p[1:headSize]) - if err != nil { - return head, err - } - - p = p[:headSize] - - v := p[0] - - head.Version = protoVersion(v) - - head.Flags = p[1] - - if version > protoVersion2 { - if len(p) != 9 { - return head, fmt.Errorf("not enough bytes to read header require 9 got: %d", len(p)) - } - - head.Stream = int(int16(p[2])<<8 | int16(p[3])) - head.Op = FrameOp(p[4]) - head.BodyLength = int(readInt(p[5:])) - } else { - if len(p) != 8 { - return head, fmt.Errorf("not enough bytes to read header require 8 got: %d", len(p)) - } - - head.Stream = int(int8(p[2])) - head.Op = FrameOp(p[3]) - head.BodyLength = int(readInt(p[4:])) - } - f.Header = head - return head, nil -} - func (f ByteArrayDecoder) ReadByte() (byte, error) { - if len(f.Data) < 1 { - panic(fmt.Errorf("not enough bytes in buffer to Read byte require 1 got: %d", len(f.Data))) + data := *f.Data + if len(data) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to Read byte require 1 got: %d", len(data))) } - b := f.Data[0] - f.Data = f.Data[1:] + b := data[0] + *f.Data = data[1:] return b, nil } func (f ByteArrayDecoder) ReadInt() (n int) { - if len(f.Data) < 4 { - panic(fmt.Errorf("not enough bytes in buffer to Read int require 4 got: %d", len(f.Data))) + data := *f.Data + if len(data) < 4 { + panic(fmt.Errorf("not enough bytes in buffer to Read int require 4 got: %d", len(data))) } - n = int(int32(f.Data[0])<<24 | int32(f.Data[1])<<16 | int32(f.Data[2])<<8 | int32(f.Data[3])) - f.Data = f.Data[4:] + n = int(int32(data[0])<<24 | int32(data[1])<<16 | int32(data[2])<<8 | int32(data[3])) + *f.Data = data[4:] return } func (f ByteArrayDecoder) ReadShort() (n uint16) { - if len(f.Data) < 2 { - panic(fmt.Errorf("not enough bytes in buffer to Read short require 2 got: %d", len(f.Data))) + data := *f.Data + if len(data) < 2 { + panic(fmt.Errorf("not enough bytes in buffer to Read short require 2 got: %d", len(data))) } - n = uint16(f.Data[0])<<8 | uint16(f.Data[1]) - f.Data = f.Data[2:] + n = uint16(data[0])<<8 | uint16(data[1]) + *f.Data = data[2:] return } func (f ByteArrayDecoder) ReadLong() (n int64) { - if len(f.Data) < 8 { - panic(fmt.Errorf("not enough bytes in buffer to Read long require 8 got: %d", len(f.Data))) + data := *f.Data + if len(data) < 8 { + panic(fmt.Errorf("not enough bytes in buffer to Read long require 8 got: %d", len(data))) } - n = int64(f.Data[0])<<56 | int64(f.Data[1])<<48 | int64(f.Data[2])<<40 | int64(f.Data[3])<<32 | - int64(f.Data[4])<<24 | int64(f.Data[5])<<16 | int64(f.Data[6])<<8 | int64(f.Data[7]) - f.Data = f.Data[8:] + n = int64(data[0])<<56 | int64(data[1])<<48 | int64(data[2])<<40 | int64(data[3])<<32 | + int64(data[4])<<24 | int64(data[5])<<16 | int64(data[6])<<8 | int64(data[7]) + *f.Data = data[8:] return } func (f ByteArrayDecoder) ReadString() (s string) { size := f.ReadShort() - - if len(f.Data) < int(size) { - panic(fmt.Errorf("not enough bytes in buffer to Read string require %d got: %d", size, len(f.Data))) + data := *f.Data + if len(data) < int(size) { + panic(fmt.Errorf("not enough bytes in buffer to Read string require %d got: %d", size, len(data))) } - s = string(f.Data[:size]) - f.Data = f.Data[size:] + s = string(data[:size]) + *f.Data = data[size:] return } func (f ByteArrayDecoder) ReadLongString() (s string) { size := f.ReadInt() - - if len(f.Data) < size { - panic(fmt.Errorf("not enough bytes in buffer to Read long string require %d got: %d", size, len(f.Data))) + data := *f.Data + if len(data) < size { + panic(fmt.Errorf("not enough bytes in buffer to Read long string require %d got: %d", size, len(data))) } - s = string(f.Data[:size]) - f.Data = f.Data[size:] + s = string(data[:size]) + *f.Data = data[size:] return } func (f ByteArrayDecoder) ReadUUID() *UUID { - if len(f.Data) < 16 { - panic(fmt.Errorf("not enough bytes in buffer to Read uuid require %d got: %d", 16, len(f.Data))) + data := *f.Data + + if len(data) < 16 { + panic(fmt.Errorf("not enough bytes in buffer to Read uuid require %d got: %d", 16, len(data))) } - u, _ := UUIDFromBytes(f.Data[:16]) - f.Data = f.Data[16:] + u, _ := UUIDFromBytes(data[:16]) + *f.Data = data[16:] return &u } @@ -158,13 +109,14 @@ func (f ByteArrayDecoder) ReadBytesInternal() []byte { if size < 0 { return nil } + data := *f.Data - if len(f.Data) < size { - panic(fmt.Errorf("not enough bytes in buffer to Read bytes require %d got: %d", size, len(f.Data))) + if len(data) < size { + panic(fmt.Errorf("not enough bytes in buffer to Read bytes require %d got: %d", size, len(data))) } - l := f.Data[:size] - f.Data = f.Data[size:] + l := data[:size] + *f.Data = data[size:] return l } @@ -177,35 +129,39 @@ func (f ByteArrayDecoder) ReadBytes() []byte { func (f ByteArrayDecoder) ReadShortBytes() []byte { size := f.ReadShort() - if len(f.Data) < int(size) { - panic(fmt.Errorf("not enough bytes in buffer to Read short bytes: require %d got %d", size, len(f.Data))) + data := *f.Data + if len(data) < int(size) { + panic(fmt.Errorf("not enough bytes in buffer to Read short bytes: require %d got %d", size, len(data))) } - l := f.Data[:size] - f.Data = f.Data[size:] + l := data[:size] + *f.Data = data[size:] return l } func (f ByteArrayDecoder) ReadInet() (net.IP, int) { - if len(f.Data) < 1 { - panic(fmt.Errorf("not enough bytes in buffer to Read inet size require %d got: %d", 1, len(f.Data))) + data := *f.Data + + if len(data) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to Read inet size require %d got: %d", 1, len(data))) } - size := f.Data[0] - f.Data = f.Data[1:] + size := data[0] + *f.Data = data[1:] if !(size == 4 || size == 16) { panic(fmt.Errorf("invalid IP size: %d", size)) } - if len(f.Data) < 1 { - panic(fmt.Errorf("not enough bytes in buffer to Read inet require %d got: %d", size, len(f.Data))) + data = *f.Data + if len(data) < 1 { + panic(fmt.Errorf("not enough bytes in buffer to Read inet require %d got: %d", size, len(data))) } ip := make([]byte, size) - copy(ip, f.Data[:size]) - f.Data = f.Data[size:] + copy(ip, data[:size]) + *f.Data = data[size:] port := f.ReadInt() return net.IP(ip), port diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index d672d42df19..3b14dbb294f 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -164,8 +164,14 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { } }() - decoder := &StreamDecoder{} - decoder.r = f.r + //decoder := &StreamDecoder{} + //decoder.r = f.r + //f.decoder = decoder + + decoder := &ByteArrayDecoder{} + buf := make([]byte, f.Header.BodyLength) + f.r.Read(buf) + decoder.Data = &buf f.decoder = decoder data = make(map[string]interface{}) @@ -179,7 +185,7 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { debugf("tracing enabled") - uid := f.decoder.ReadUUID() + uid := decoder.ReadUUID() debugf(uid.String()) data["trace_id"] = uid.String() @@ -188,7 +194,7 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { if f.Header.Flags&flagWarning == flagWarning { debugf("hit warning flags") - warnings := f.decoder.ReadStringList() + warnings := decoder.ReadStringList() // dealing with warnings data["warnings"] = warnings } @@ -196,7 +202,7 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { if f.Header.Flags&flagCustomPayload == flagCustomPayload { debugf("hit custom payload flags") - f.Header.CustomPayload = f.decoder.ReadBytesMap() + f.Header.CustomPayload = decoder.ReadBytesMap() } if f.Header.Flags&flagCompress == flagCompress { @@ -247,8 +253,9 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { func (f *Framer) parseErrorFrame() (data map[string]interface{}) { - code := f.decoder.ReadInt() - msg := f.decoder.ReadString() + decoder := f.decoder + code := decoder.ReadInt() + msg := decoder.ReadString() errT := ErrType(code) @@ -259,18 +266,18 @@ func (f *Framer) parseErrorFrame() (data map[string]interface{}) { switch errT { case errUnavailable: - cl := f.decoder.ReadConsistency() - required := f.decoder.ReadInt() - alive := f.decoder.ReadInt() + cl := decoder.ReadConsistency() + required := decoder.ReadInt() + alive := decoder.ReadInt() data["read_consistency"] = cl.String() data["required"] = required data["alive"] = alive case errWriteTimeout: - cl := f.decoder.ReadConsistency() - received := f.decoder.ReadInt() - blockfor := f.decoder.ReadInt() - writeType := f.decoder.ReadString() + cl := decoder.ReadConsistency() + received := decoder.ReadInt() + blockfor := decoder.ReadInt() + writeType := decoder.ReadString() data["read_consistency"] = cl.String() data["received"] = received @@ -278,10 +285,10 @@ func (f *Framer) parseErrorFrame() (data map[string]interface{}) { data["write_type"] = writeType case errReadTimeout: - cl := f.decoder.ReadConsistency() - received := f.decoder.ReadInt() - blockfor := f.decoder.ReadInt() - dataPresent, err := f.decoder.ReadByte() + cl := decoder.ReadConsistency() + received := decoder.ReadInt() + blockfor := decoder.ReadInt() + dataPresent, err := decoder.ReadByte() if err != nil { panic(err) } @@ -292,37 +299,37 @@ func (f *Framer) parseErrorFrame() (data map[string]interface{}) { data["data_present"] = dataPresent case errAlreadyExists: - ks := f.decoder.ReadString() - table := f.decoder.ReadString() + ks := decoder.ReadString() + table := decoder.ReadString() data["keyspace"] = ks data["table"] = table case errUnprepared: - stmtId := f.decoder.ReadShortBytes() + stmtId := decoder.ReadShortBytes() data["stmt_id"] = stmtId case errReadFailure: - data["read_consistency"] = f.decoder.ReadConsistency().String() - data["received"] = f.decoder.ReadInt() - data["blockfor"] = f.decoder.ReadInt() - b, err := f.decoder.ReadByte() + data["read_consistency"] = decoder.ReadConsistency().String() + data["received"] = decoder.ReadInt() + data["blockfor"] = decoder.ReadInt() + b, err := decoder.ReadByte() if err != nil { panic(err) } data["data_present"] = b != 0 case errWriteFailure: - data["read_consistency"] = f.decoder.ReadConsistency().String() - data["received"] = f.decoder.ReadInt() - data["blockfor"] = f.decoder.ReadInt() - data["num_failures"] = f.decoder.ReadInt() - data["write_type"] = f.decoder.ReadString() + data["read_consistency"] = decoder.ReadConsistency().String() + data["received"] = decoder.ReadInt() + data["blockfor"] = decoder.ReadInt() + data["num_failures"] = decoder.ReadInt() + data["write_type"] = decoder.ReadString() case errFunctionFailure: - data["keyspace"] = f.decoder.ReadString() - data["function"] = f.decoder.ReadString() - data["arg_types"] = f.decoder.ReadStringList() + data["keyspace"] = decoder.ReadString() + data["function"] = decoder.ReadString() + data["arg_types"] = decoder.ReadStringList() case errInvalid, errBootstrapping, errConfig, errCredentials, errOverloaded, errProtocol, errServer, errSyntax, errTruncate, errUnauthorized: @@ -335,33 +342,34 @@ func (f *Framer) parseErrorFrame() (data map[string]interface{}) { func (f *Framer) parseSupportedFrame() (data map[string]interface{}) { data = make(map[string]interface{}) - data["supported"] = f.decoder.ReadStringMultiMap() + data["supported"] = (f.decoder).ReadStringMultiMap() return data } func (f *Framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { + decoder := f.decoder meta := make(map[string]interface{}) - flags := f.decoder.ReadInt() + flags := decoder.ReadInt() meta["flags"] = getRowFlagString(flags) - colCount := f.decoder.ReadInt() + colCount := decoder.ReadInt() meta["col_count"] = colCount if getPKinfo { //only for prepared result if f.proto >= protoVersion4 { - pkeyCount := f.decoder.ReadInt() + pkeyCount := decoder.ReadInt() pkeys := make([]int, pkeyCount) for i := 0; i < pkeyCount; i++ { - pkeys[i] = int(f.decoder.ReadShort()) + pkeys[i] = int(decoder.ReadShort()) } meta["pkey_columns"] = pkeys } } if flags&flagHasMorePages == flagHasMorePages { - meta["paging_state"] = fmt.Sprintf("%X", f.decoder.ReadBytes()) + meta["paging_state"] = fmt.Sprintf("%X", decoder.ReadBytes()) return meta } @@ -372,8 +380,8 @@ func (f *Framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { var keyspace, table string globalSpec := flags&flagGlobalTableSpec == flagGlobalTableSpec if globalSpec { - keyspace = f.decoder.ReadString() - table = f.decoder.ReadString() + keyspace = decoder.ReadString() + table = decoder.ReadString() meta["keyspace"] = keyspace meta["table"] = table } @@ -383,13 +391,13 @@ func (f *Framer) parseResultMetadata(getPKinfo bool) map[string]interface{} { func (f *Framer) parseQueryFrame() (data map[string]interface{}) { data = make(map[string]interface{}) - data["query"] = f.decoder.ReadLongString() + data["query"] = (f.decoder).ReadLongString() return data } func (f *Framer) parseResultFrame() (data map[string]interface{}) { - kind := f.decoder.ReadInt() + kind := (f.decoder).ReadInt() data = make(map[string]interface{}) switch kind { @@ -400,7 +408,7 @@ func (f *Framer) parseResultFrame() (data map[string]interface{}) { data["rows"] = f.parseResultRows() case resultKindSetKeyspace: data["result_type"] = "set_keyspace" - data["keyspace"] = f.decoder.ReadString() + data["keyspace"] = (f.decoder).ReadString() case resultKindPrepared: data["result_type"] = "prepared" data["result"] = f.parseResultPrepared() @@ -416,7 +424,7 @@ func (f *Framer) parseResultRows() map[string]interface{} { result := make(map[string]interface{}) result["meta"] = f.parseResultMetadata(false) - result["num_rows"] = f.decoder.ReadInt() + result["num_rows"] = (f.decoder).ReadInt() return result } @@ -425,7 +433,7 @@ func (f *Framer) parseResultPrepared() map[string]interface{} { result := make(map[string]interface{}) - result["prepared_id"] = string(f.decoder.ReadShortBytes()) + result["prepared_id"] = string((f.decoder).ReadShortBytes()) result["req_meta"] = f.parseResultMetadata(true) if f.proto < protoVersion2 { @@ -439,34 +447,34 @@ func (f *Framer) parseResultPrepared() map[string]interface{} { func (f *Framer) parseResultSchemaChange() (data map[string]interface{}) { data = make(map[string]interface{}) - + decoder := f.decoder if f.proto <= protoVersion2 { - change := f.decoder.ReadString() - keyspace := f.decoder.ReadString() - table := f.decoder.ReadString() + change := decoder.ReadString() + keyspace := decoder.ReadString() + table := decoder.ReadString() data["change"] = change data["keyspace"] = keyspace data["table"] = table } else { - change := f.decoder.ReadString() - target := f.decoder.ReadString() + change := decoder.ReadString() + target := decoder.ReadString() data["change"] = change data["type"] = target switch target { case "KEYSPACE": - data["keyspace"] = f.decoder.ReadString() + data["keyspace"] = decoder.ReadString() case "TABLE", "TYPE": - data["keyspace"] = f.decoder.ReadString() - data["object"] = f.decoder.ReadString() + data["keyspace"] = decoder.ReadString() + data["object"] = decoder.ReadString() case "FUNCTION", "AGGREGATE": - data["keyspace"] = f.decoder.ReadString() - data["name"] = f.decoder.ReadString() - data["args"] = f.decoder.ReadStringList() + data["keyspace"] = decoder.ReadString() + data["name"] = decoder.ReadString() + data["args"] = decoder.ReadStringList() default: logp.Warn("unknown SCHEMA_CHANGE target: %q change: %q", target, change) @@ -477,39 +485,39 @@ func (f *Framer) parseResultSchemaChange() (data map[string]interface{}) { func (f *Framer) parseAuthenticateFrame() (data map[string]interface{}) { data = make(map[string]interface{}) - data["class"] = f.decoder.ReadString() + data["class"] = (f.decoder).ReadString() return data } func (f *Framer) parseAuthSuccessFrame() (data map[string]interface{}) { data = make((map[string]interface{})) - data["data"] = fmt.Sprintf("%q", f.decoder.ReadBytes()) + data["data"] = fmt.Sprintf("%q", (f.decoder).ReadBytes()) return data } func (f *Framer) parseAuthChallengeFrame() (data map[string]interface{}) { data = make((map[string]interface{})) - data["data"] = fmt.Sprintf("%q", f.decoder.ReadBytes()) + data["data"] = fmt.Sprintf("%q", (f.decoder).ReadBytes()) return data } func (f *Framer) parseEventFrame() (data map[string]interface{}) { data = make((map[string]interface{})) - - eventType := f.decoder.ReadString() + decoder := f.decoder + eventType := decoder.ReadString() data["event_type"] = eventType switch eventType { case "TOPOLOGY_CHANGE": - data["change"] = f.decoder.ReadString() - host, port := f.decoder.ReadInet() + data["change"] = decoder.ReadString() + host, port := decoder.ReadInet() data["host"] = host data["port"] = port case "STATUS_CHANGE": - data["change"] = f.decoder.ReadString() - host, port := f.decoder.ReadInet() + data["change"] = decoder.ReadString() + host, port := decoder.ReadInet() data["host"] = host data["port"] = port From ff7ed3df42803ef6b7b997bfd083f8de57a0a676 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 12 Aug 2016 16:18:24 +0800 Subject: [PATCH 19/30] refactor and fix tracing protocol bug --- packetbeat/protos/cassandra/config.go | 6 +- .../protos/cassandra/internal/gocql/frame.go | 34 +++--- .../internal/gocql/stream_decoder.go | 5 + packetbeat/protos/cassandra/parser.go | 102 ++++++++++-------- 4 files changed, 84 insertions(+), 63 deletions(-) diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go index 1224378887c..033338db1b2 100644 --- a/packetbeat/protos/cassandra/config.go +++ b/packetbeat/protos/cassandra/config.go @@ -19,9 +19,11 @@ var ( defaultConfig = cassandraConfig{ ProtocolCommon: config.ProtocolCommon{ TransactionTimeout: protos.DefaultTransactionExpiration, + SendRequest: true, + SendResponse: true, }, - SendRequestHeader: false, - SendResponseHeader: false, + SendRequestHeader: true, + SendResponseHeader: true, } ) diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index 3b14dbb294f..c6204655a57 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -164,14 +164,8 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { } }() - //decoder := &StreamDecoder{} - //decoder.r = f.r - //f.decoder = decoder - - decoder := &ByteArrayDecoder{} - buf := make([]byte, f.Header.BodyLength) - f.r.Read(buf) - decoder.Data = &buf + decoder := &StreamDecoder{} + decoder.r = f.r f.decoder = decoder data = make(map[string]interface{}) @@ -185,10 +179,9 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { debugf("tracing enabled") - uid := decoder.ReadUUID() - debugf(uid.String()) - - data["trace_id"] = uid.String() + //seems no UUID to read, protocol incorrect? + //uid := decoder.ReadUUID() + //data["trace_id"] = uid.String() } if f.Header.Flags&flagWarning == flagWarning { @@ -206,12 +199,17 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { } if f.Header.Flags&flagCompress == flagCompress { - //TODO decompress data and switch to use bytearray decoder - //decoder := &ByteArrayDecoder{} - //buf := make([]byte, f.header.BodyLength) - //f.r.Read(buf) - //decoder.Data = buf - //f.decoder = decoder + //decompress data and switch to use bytearray decoder + decoder := &ByteArrayDecoder{} + buf := make([]byte, f.Header.BodyLength) + f.r.Read(buf) + dec, err := f.compres.Decode(buf) + if err != nil { + return nil, err + } + + decoder.Data = &dec + f.decoder = decoder debugf("hit compress flags") diff --git a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go index d272bf02f81..711539c3f66 100644 --- a/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go +++ b/packetbeat/protos/cassandra/internal/gocql/stream_decoder.go @@ -3,6 +3,7 @@ package cassandra import ( "fmt" "github.com/elastic/beats/libbeat/common/streambuf" + "github.com/pkg/errors" "net" ) @@ -64,6 +65,9 @@ func (f StreamDecoder) ReadLongString() (s string) { size := f.ReadInt() + if !f.r.Avail(size) { + panic(errors.New(fmt.Sprintf("not enough buf to readLongString,need:%d,actual:%d", size, f.r.Len()))) + } str := make([]byte, size) _, err := f.r.Read(str) if err != nil { @@ -81,6 +85,7 @@ func (f StreamDecoder) ReadUUID() *UUID { if err != nil { panic(err) } + u, _ := UUIDFromBytes(bytes) return &u diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index 26ba9f7ee9b..d3d51ca43f1 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -2,12 +2,11 @@ package cassandra import ( "errors" - "time" - "github.com/elastic/beats/libbeat/common/streambuf" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/packetbeat/protos/applayer" . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" + "time" ) type parser struct { @@ -130,46 +129,10 @@ func (p *parser) newMessage(ts time.Time) *message { } } -func (p *parser) parse() (*message, error) { - - // if p.frame is nil then create a new framer, or continue to process the last message - if p.framer == nil { - if isDebug { - debugf("start new framer") - } - p.framer = NewFramer(&p.buf, p.config.compressor) - } - - // check if the frame header were parsed or not - if p.framer.Header == nil { - if isDebug { - debugf("start to parse header") - } - if !p.buf.Avail(9) { - debugf("not enough head bytes, ignore") - return nil, nil - } - - _, err := p.framer.ReadHeader() - if err != nil { - logp.Err("%v", err) - p.framer = nil - return nil, err - } - } - - //check if the ops need to be ignored - if p.FrameOpsIgnored() { - // as we already ignore the content, we now mark the result is ignored - p.message.ignored = true - if isDebug { - debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) - } - } - - msg := p.message +func (p *parser) parserBody() (map[string]interface{}, error) { if p.framer.Header.BodyLength > 0 { + debugf("bodyLength: %d", p.framer.Header.BodyLength) //let's wait for enough buf if !p.buf.Avail(p.framer.Header.BodyLength) { @@ -181,6 +144,7 @@ func (p *parser) parse() (*message, error) { //check if the ops already ignored if p.message.ignored { + debugf("message marked to be ignored, let's do this") p.buf.Collect(p.framer.Header.BodyLength) finalCollectedFrameLength := p.buf.BufferConsumed() if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { @@ -198,8 +162,6 @@ func (p *parser) parse() (*message, error) { return nil, err } - msg.data = data - // dealing with un-parsed content frameParsedLength := p.buf.BufferConsumed() @@ -215,7 +177,7 @@ func (p *parser) parse() (*message, error) { return nil, errors.New("should be enough bytes,but not enough") } } - + return data, nil } finalCollectedFrameLength := p.buf.BufferConsumed() @@ -226,6 +188,60 @@ func (p *parser) parse() (*message, error) { } + return map[string]interface{}{}, nil +} + +func (p *parser) parse() (*message, error) { + + // if p.frame is nil then create a new framer, or continue to process the last message + if p.framer == nil { + if isDebug { + debugf("start new framer") + } + p.framer = NewFramer(&p.buf, p.config.compressor) + } + + // check if the frame header were parsed or not + if p.framer.Header == nil { + if isDebug { + debugf("start to parse header") + } + if !p.buf.Avail(9) { + debugf("not enough head bytes, ignore") + return nil, nil + } + + _, err := p.framer.ReadHeader() + if err != nil { + logp.Err("%v", err) + p.framer = nil + return nil, err + } + } + + //check if the ops need to be ignored + if p.FrameOpsIgnored() { + // as we already ignore the content, we now mark the result is ignored + p.message.ignored = true + if isDebug { + debugf("Ops: %s was marked to be ignored, ignoring, request:%v", p.framer.Header.Op.String(), p.framer.Header.Version.IsRequest()) + } + } + + msg := p.message + + data, err := p.parserBody() + if err != nil { + return nil, err + } + + //ignore and wait for more data + if data == nil { + return nil, nil + } + + msg.data = data + dir := applayer.NetOriginalDirection isRequest := true From 665de42a64a4dd455bbf7485ea7bb24bbc7cac1e Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 15 Aug 2016 13:02:32 +0800 Subject: [PATCH 20/30] minor improvments, add trace test --- packetbeat/etc/beat.full.yml | 5 +- packetbeat/packetbeat.full.yml | 8 +- packetbeat/protos/cassandra/cassandra.go | 15 +-- packetbeat/protos/cassandra/config.go | 9 +- .../protos/cassandra/internal/gocql/frame.go | 2 - .../cassandra/internal/gocql/marshal.go | 77 +++++++------- packetbeat/protos/cassandra/parser.go | 99 +++++++++--------- packetbeat/protos/cassandra/pub.go | 17 ++- .../cassandra/v4/cassandra_trace_err.pcap | Bin 0 -> 285 bytes .../tests/system/test_0062_cassandra.py | 46 ++++++-- 10 files changed, 154 insertions(+), 124 deletions(-) create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_trace_err.pcap diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 2034d33bab2..00d2526ee94 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -441,7 +441,6 @@ packetbeat.protocols.cassandra: # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" - # This option indicates which Operator/Operators will be ignored, - # multi value can be sperated by `,` - #ignored_ops: "SUPPORTED,OPTIONS" + # This option indicates which Operator/Operators will be ignored + #ignored_ops: ["SUPPORTED","OPTIONS"] diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 6ea8e17a90d..95a1f0613be 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -438,12 +438,12 @@ packetbeat.protocols.cassandra: #send_response_header: true # If this option is enabled, and also the flag indicates that the frame body was compressed, and - # the stream will decode by the compressor, currently support: `snappy` , The default is empty. + # the stream will decode by the compressor, currently support: `snappy` , The default compressor is nil. #compressor: "snappy" - # This option indicates which Operator/Operators will be ignored, - # multi value can be sperated by `,` - #ignored_ops: "SUPPORTED,OPTIONS" + # This option indicates which Operator/Operators will be ignored + #ignored_ops: ["SUPPORTED","OPTIONS"] + #================================ General ===================================== diff --git a/packetbeat/protos/cassandra/cassandra.go b/packetbeat/protos/cassandra/cassandra.go index 4483c3ce82f..fb84a619e48 100644 --- a/packetbeat/protos/cassandra/cassandra.go +++ b/packetbeat/protos/cassandra/cassandra.go @@ -10,7 +10,6 @@ import ( . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" "github.com/elastic/beats/packetbeat/protos/tcp" "github.com/elastic/beats/packetbeat/publish" - "strings" ) // cassandra application level protocol analyzer plugin @@ -89,20 +88,8 @@ func (cassandra *cassandra) setFromConfig(config *cassandraConfig) error { // parsed ignored ops if len(config.OPsIgnored) > 0 { maps := map[FrameOp]bool{} - - if strings.Contains(config.OPsIgnored, ",") { - array := strings.Split(config.OPsIgnored, ",") - for i := 0; i < len(array); i++ { - str := array[i] - if len(str) > 0 { - op := FrameOpFromString(strings.ToUpper(strings.TrimSpace(str))) - maps[op] = true - } - } - } else { - op := FrameOpFromString(strings.ToUpper(strings.TrimSpace(config.OPsIgnored))) + for _, op := range config.OPsIgnored { maps[op] = true - } parser.ignoredOps = maps debugf("parsed config IgnoredOPs: %v ", parser.ignoredOps) diff --git a/packetbeat/protos/cassandra/config.go b/packetbeat/protos/cassandra/config.go index 033338db1b2..43e1f693e36 100644 --- a/packetbeat/protos/cassandra/config.go +++ b/packetbeat/protos/cassandra/config.go @@ -4,15 +4,16 @@ import ( "fmt" "github.com/elastic/beats/packetbeat/config" "github.com/elastic/beats/packetbeat/protos" + . "github.com/elastic/beats/packetbeat/protos/cassandra/internal/gocql" "github.com/pkg/errors" ) type cassandraConfig struct { config.ProtocolCommon `config:",inline"` - SendRequestHeader bool `config:"send_request_header"` - SendResponseHeader bool `config:"send_response_header"` - Compressor string `config:"compressor"` - OPsIgnored string `config:"ignored_ops"` + SendRequestHeader bool `config:"send_request_header"` + SendResponseHeader bool `config:"send_response_header"` + Compressor string `config:"compressor"` + OPsIgnored []FrameOp `config:"ignored_ops"` } var ( diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index c6204655a57..1db7da02b92 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -212,8 +212,6 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { f.decoder = decoder debugf("hit compress flags") - - return nil, errors.New("Compressed content not supported yet") } // assumes that the frame body has been read into rbuf diff --git a/packetbeat/protos/cassandra/internal/gocql/marshal.go b/packetbeat/protos/cassandra/internal/gocql/marshal.go index 3beb3ac3889..09b68c66d1c 100644 --- a/packetbeat/protos/cassandra/internal/gocql/marshal.go +++ b/packetbeat/protos/cassandra/internal/gocql/marshal.go @@ -502,44 +502,47 @@ func (f FrameOp) String() string { } } -func FrameOpFromString(str string) FrameOp { - switch str { - case "ERROR": - return opError - case "STARTUP": - return opStartup - case "READY": - return opReady - case "AUTHENTICATE": - return opAuthenticate - case "OPTIONS": - return opOptions - case "SUPPORTED": - return opSupported - case "QUERY": - return opQuery - case "RESULT": - return opResult - case "PREPARE": - return opPrepare - case "EXECUTE": - return opExecute - case "REGISTER": - return opRegister - case "EVENT": - return opEvent - case "BATCH": - return opBatch - case "AUTH_CHALLENGE": - return opAuthChallenge - case "AUTH_RESPONSE": - return opAuthResponse - case "AUTH_SUCCESS": - return opAuthSuccess - default: - debugf("unknown Op while convert: %s", str) - return opUnknown +var frameOps = map[string]FrameOp{ + "ERROR": opError, + "STARTUP": opStartup, + "READY": opReady, + "AUTHENTICATE": opAuthenticate, + "OPTIONS": opOptions, + "SUPPORTED": opSupported, + "QUERY": opQuery, + "RESULT": opResult, + "PREPARE": opPrepare, + "EXECUTE": opExecute, + "REGISTER": opRegister, + "EVENT": opEvent, + "BATCH": opBatch, + "AUTH_CHALLENGE": opAuthChallenge, + "AUTH_RESPONSE": opAuthResponse, + "AUTH_SUCCESS": opAuthSuccess, +} + +func FrameOpFromString(s string) (FrameOp, error) { + s = strings.ToUpper(strings.TrimSpace(s)) + op, found := frameOps[s] + if !found { + return opUnknown, fmt.Errorf("unknown frame op: %v", s) + } + return op, nil +} + +func (f *FrameOp) Unpack(in interface{}) error { + s, ok := in.(string) + if !ok { + return errors.New("expected string") } + + op, err := FrameOpFromString(s) + if err != nil { + return err + } + + *f = op + return nil } const ( diff --git a/packetbeat/protos/cassandra/parser.go b/packetbeat/protos/cassandra/parser.go index d3d51ca43f1..eed70874092 100644 --- a/packetbeat/protos/cassandra/parser.go +++ b/packetbeat/protos/cassandra/parser.go @@ -24,7 +24,7 @@ type parserConfig struct { } // check whether this ops is enabled or not -func (p *parser) FrameOpsIgnored() bool { +func (p *parser) CheckFrameOpsIgnored() bool { if p.config.ignoredOps != nil && len(p.config.ignoredOps) > 0 { //default map value is false @@ -97,7 +97,6 @@ func (p *parser) feed(ts time.Time, data []byte) error { if p.message == nil { // allocate new message object to be used by parser with current timestamp p.message = p.newMessage(ts) - p.message.data = map[string]interface{}{} } msg, err := p.parse() @@ -129,66 +128,64 @@ func (p *parser) newMessage(ts time.Time) *message { } } -func (p *parser) parserBody() (map[string]interface{}, error) { +func (p *parser) parserBody() (bool, error) { - if p.framer.Header.BodyLength > 0 { - debugf("bodyLength: %d", p.framer.Header.BodyLength) + headLen := p.framer.Header.HeadLength + bdyLen := p.framer.Header.BodyLength + if bdyLen <= 0 { + return true, nil + } - //let's wait for enough buf - if !p.buf.Avail(p.framer.Header.BodyLength) { - if isDebug { - debugf("buf not enough for body, waiting for more, return") - } - return nil, nil + //let's wait for enough buf + debugf("bodyLength: %d", bdyLen) + if !p.buf.Avail(bdyLen) { + if isDebug { + debugf("buf not enough for body, waiting for more, return") } + return false, nil + } - //check if the ops already ignored - if p.message.ignored { + //check if the ops already ignored + if p.message.ignored { + if isDebug { debugf("message marked to be ignored, let's do this") - p.buf.Collect(p.framer.Header.BodyLength) - finalCollectedFrameLength := p.buf.BufferConsumed() - if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - return nil, errors.New("data messed while parse frame body") - } - } - + p.buf.Collect(bdyLen) + } else { // start to parse body - if !p.message.ignored { - data, err := p.framer.ReadFrame() - if err != nil { - // if the frame parsed failed, should ignore the whole message - p.framer = nil - return nil, err - } - - // dealing with un-parsed content - frameParsedLength := p.buf.BufferConsumed() + data, err := p.framer.ReadFrame() + if err != nil { + // if the frame parsed failed, should ignore the whole message + p.framer = nil + return false, err + } - // collect leftover - unParsedSize := p.framer.Header.BodyLength + p.framer.Header.HeadLength - frameParsedLength - if unParsedSize > 0 { + // dealing with un-parsed content + frameParsedLength := p.buf.BufferConsumed() - // double check the buf size - if p.buf.Avail(unParsedSize) { - p.buf.Collect(unParsedSize) - } else { - logp.Err("should be enough bytes for cleanup,but not enough") - return nil, errors.New("should be enough bytes,but not enough") - } + // collect leftover + unParsedSize := bdyLen + headLen - frameParsedLength + if unParsedSize > 0 { + if !p.buf.Avail(unParsedSize) { + err := errors.New("should be enough bytes for cleanup,but not enough") + logp.Err("Finishing frame failed with: %v", err) + return false, err } - return data, nil - } - finalCollectedFrameLength := p.buf.BufferConsumed() - if finalCollectedFrameLength-p.framer.Header.HeadLength != p.framer.Header.BodyLength { - logp.Err("body_length:%d, head_length:%d, all_consumed:%d", p.framer.Header.BodyLength, p.framer.Header.HeadLength, finalCollectedFrameLength) - return nil, errors.New("data messed while parse frame body") + p.buf.Collect(unParsedSize) } + p.message.data = data + } + + finalCollectedFrameLength := p.buf.BufferConsumed() + if finalCollectedFrameLength-headLen != bdyLen { + logp.Err("body_length:%d, head_length:%d, all_consumed:%d", + bdyLen, headLen, finalCollectedFrameLength) + return false, errors.New("data messed while parse frame body") } - return map[string]interface{}{}, nil + return true, nil } func (p *parser) parse() (*message, error) { @@ -220,7 +217,7 @@ func (p *parser) parse() (*message, error) { } //check if the ops need to be ignored - if p.FrameOpsIgnored() { + if p.CheckFrameOpsIgnored() { // as we already ignore the content, we now mark the result is ignored p.message.ignored = true if isDebug { @@ -230,18 +227,16 @@ func (p *parser) parse() (*message, error) { msg := p.message - data, err := p.parserBody() + finished, err := p.parserBody() if err != nil { return nil, err } //ignore and wait for more data - if data == nil { + if !finished { return nil, nil } - msg.data = data - dir := applayer.NetOriginalDirection isRequest := true diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go index 360286ec4a7..c55723323df 100644 --- a/packetbeat/protos/cassandra/pub.go +++ b/packetbeat/protos/cassandra/pub.go @@ -67,11 +67,17 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { } if pub.sendRequest { + if requ.data == nil { + requ.data = map[string]interface{}{} + } + if pub.sendRequestHeader { requ.data["request_headers"] = requ.header } - event["cassandra_request"] = requ.data + if len(requ.data) > 0 { + event["cassandra_request"] = requ.data + } } dst := &common.Endpoint{ @@ -97,11 +103,18 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { event["bytes_out"] = resp.Size if pub.sendResponse { + if resp.data == nil { + resp.data = map[string]interface{}{} + } + if pub.sendResponseHeader { resp.data["response_headers"] = resp.header } - event["cassandra_response"] = resp.data + if len(resp.data) > 0 { + event["cassandra_response"] = resp.data + } + } return event diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_trace_err.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_trace_err.pcap new file mode 100644 index 0000000000000000000000000000000000000000..8d006781352516aa49eeec2ebffc10020c52e795 GIT binary patch literal 285 zcmca|c+)~A1{MYcfUsRYEDFEL#Ke#flmKBSAa-S7$bKs3z~I2fz);V?zzD+Ef|Y}k zQ|Gka+b>buAfeCvFCHk!$ibz@vVcjCr6`MqiBXsxs8_^oO@Q(#3>*xsmX`lkPdoy$UOkqHp$sSt!eHx}7>WX~qgt;VbS)TU zHP8ur8YB!_{-wdK7Y8|^g@I8RsGY%-fkBypLEkwsFE77DA*Cq4Kp`(bPa(A;v$!NP PFI@rV5(RaHtJL)XYQ03Z literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/test_0062_cassandra.py b/packetbeat/tests/system/test_0062_cassandra.py index 4006f63b7d1..1ebb644e546 100644 --- a/packetbeat/tests/system/test_0062_cassandra.py +++ b/packetbeat/tests/system/test_0062_cassandra.py @@ -83,7 +83,6 @@ def test_create_table(self): assert h2["flags"] == "FLAG_0" assert h2["stream"] == 49 - def test_insert_data(self): """ Should correctly insert record into table in Cassandra @@ -120,7 +119,6 @@ def test_insert_data(self): assert h2["flags"] == "FLAG_0" assert h2["stream"] == 252 - def test_select_data(self): """ Should correctly select record from table in Cassandra @@ -192,6 +190,45 @@ def test_create_index(self): assert h2["flags"] == "FLAG_0" assert h2["stream"] == 92 + def test_trace_error(self): + """ + Should correctly catch a error message and trace flag was enabled + """ + self.render_config_template( + cassandra_ports=[9042], + cassandra_send_request=True, + cassandra_send_response=True, + cassandra_send_request_header=True, + cassandra_send_response_header=True, + ) + self.run_packetbeat(pcap="cassandra/v4/cassandra_trace_err.pcap",debug_selectors=["*"]) + objs = self.read_output() + o = objs[0] + assert o["type"] == "cassandra" + assert o["port"] == 9042 + + q=o["cassandra_request"] + assert o["bytes_in"] == 55 + assert o["bytes_out"] == 62 + assert q["query"] == "DROP KEYSPACE mykeyspace;" + h= q["request_headers"] + assert h["version"] == "4" + assert h["op"] == "QUERY" + assert h["length"] == 46 + assert h["flags"] == "Tracing" + assert h["stream"] == 275 + + r=o["cassandra_response"] + assert r["err_code"]==8960 + assert r["err_msg"]=="Cannot drop non existing keyspace 'mykeyspace'." + assert r["err_type"]=="errConfig" + h2= r["response_headers"] + assert h2["version"] == "4" + assert h2["length"] == 53 + assert h2["op"] == "ERROR" + assert h2["flags"] == "FLAG_0" + assert h2["stream"] == 275 + def test_select_use_index(self): """ Should correctly select record from table (use index) in Cassandra @@ -229,7 +266,6 @@ def test_select_use_index(self): assert h2["stream"] == 262 assert r["result_type"]=="rows" - def test_ops_mixed(self): """ Some mixed operation happened in Cassandra @@ -317,8 +353,6 @@ def test_ops_mixed(self): assert h2["flags"] == "FLAG_0" assert h2["stream"] == 2 - - def test_ops_ignored(self): """ Should correctly ignore OPTIONS and REGISTER operation @@ -329,7 +363,7 @@ def test_ops_ignored(self): cassandra_send_response=True, cassandra_send_request_header=True, cassandra_send_response_header=True, - cassandra_ignored_ops= "OPTIONS,REGISTER", + cassandra_ignored_ops= ["OPTIONS","REGISTER"] ) self.run_packetbeat(pcap="cassandra/v4/cassandra_mixed_frame.pcap",debug_selectors=["*"]) From 9c6547f07870e9122c43e485d6acb8f0fd5d8057 Mon Sep 17 00:00:00 2001 From: medcl Date: Mon, 15 Aug 2016 21:25:20 +0800 Subject: [PATCH 21/30] minor change to pub.go --- packetbeat/protos/cassandra/pub.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packetbeat/protos/cassandra/pub.go b/packetbeat/protos/cassandra/pub.go index c55723323df..be6a2cec336 100644 --- a/packetbeat/protos/cassandra/pub.go +++ b/packetbeat/protos/cassandra/pub.go @@ -67,11 +67,10 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { } if pub.sendRequest { - if requ.data == nil { - requ.data = map[string]interface{}{} - } - if pub.sendRequestHeader { + if requ.data == nil { + requ.data = map[string]interface{}{} + } requ.data["request_headers"] = requ.header } @@ -103,11 +102,12 @@ func (pub *transPub) createEvent(requ, resp *message) common.MapStr { event["bytes_out"] = resp.Size if pub.sendResponse { - if resp.data == nil { - resp.data = map[string]interface{}{} - } if pub.sendResponseHeader { + if resp.data == nil { + resp.data = map[string]interface{}{} + } + resp.data["response_headers"] = resp.header } From 0739c75c9debff5b49a01b5cee14f97b3890ca0c Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 16 Aug 2016 09:52:48 +0800 Subject: [PATCH 22/30] throw error while compressor is not set but hit compress flag --- packetbeat/protos/cassandra/internal/gocql/frame.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index 1db7da02b92..13e2e6e3f70 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -200,6 +200,11 @@ func (f *Framer) ReadFrame() (data map[string]interface{}, err error) { if f.Header.Flags&flagCompress == flagCompress { //decompress data and switch to use bytearray decoder + if f.compres == nil { + logp.Err("hit compress flag, but comprossor was not set") + panic(errors.New("hit compress flag, but comprossor was not set")) + } + decoder := &ByteArrayDecoder{} buf := make([]byte, f.Header.BodyLength) f.r.Read(buf) From e54663f9b3afd44a4068a36e0cf80c6817240d6a Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 16 Aug 2016 11:37:12 +0800 Subject: [PATCH 23/30] add test for compressed frame --- .../tests/system/config/packetbeat.yml.j2 | 1 + .../cassandra/v4/cassandra_compressed.pcap | Bin 0 -> 23389 bytes .../tests/system/test_0062_cassandra.py | 99 +++++++++++++++++- 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 packetbeat/tests/system/pcaps/cassandra/v4/cassandra_compressed.pcap diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 29c738ec29a..45da0466035 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -125,6 +125,7 @@ packetbeat.protocols.cassandra: {% if cassandra_send_request_header %} send_request_header: true{% endif %} {% if cassandra_send_response_header %} send_response_header: true{% endif %} {% if cassandra_ignored_ops %} ignored_ops: {{cassandra_ignored_ops}}{% endif %} +{% if cassandra_compressor %} compressor: {{cassandra_compressor}}{% endif %} {% if procs_enabled %} diff --git a/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_compressed.pcap b/packetbeat/tests/system/pcaps/cassandra/v4/cassandra_compressed.pcap new file mode 100644 index 0000000000000000000000000000000000000000..fc9c1ebc3e4cf188f8be37b3d17e577b24ee9cd2 GIT binary patch literal 23389 zcmd74dt6l2`Ukw$Two6~vzgfp<3NBj1_}z}>^*lx1r-t$R8%x?;{XGW!hkd2tupCi zS?5^VRnv-2+Qlq0wYz2=D=Rfi&CE19r%pFHp6;D$e7_420sZ+o?;r2y^ZMBXxM%NY zJ(uruSwX>X-y$QXRVJ~Dg?3CX|WntYNUL&)`nQ1*Fa z*_GvwC?p)M?YZW7pQ@LD8DHmrceQq)PP5}apW>)IOGk(li;}^E5r6>A@K6)5WJ_@`&CvyZF2pbcQGjW_9zkt3x*2OYs>33hL35#``J? z%O@94C?iU{)n>Kf+=SAJ<%N?w&c#GR{>H|oOolfwc3)j=xAY#nhYasPhNC@n!N%Uj z=sY}2OGaUQ_UV#gL`(K%=$Jr9>XzXxM&~97!0x3TmVW~j`*PN>sW-(gt-J#(P41&z zM)57i*YWUToP!WYdZ?ETevHvM zzW!qMK4`iO9TVtWP;Unvs5k$DdVc}MzMKvFcW;Va>OBYb&IV`4F2&hXyR_sHsD^#I z)cfFHB|ZQ z%+tmemKT~A1eOjT)Nx`Ez`~5f?!MBp0zZ>4d3R$V5RP;K>nJ$xD|uj7Z;rccGTU?2 zjvyp|zlY?E<4>7Qe%Y-hpJGJz>5}~8zg^lUopel~bD&%D(~J(Y35WYPtfP%xWSAGL zNKB{LMA^x?g2tX*OmswD?Qin?ssf>=K$vJr?7W6ZlP_3JG(^)F4%7s~;Xt)7*oZ3? zVSm*EToNA+)CK&JfNv2r5NrtHlsep4}wBB?& zmH)G>L-|lz!i8A(8KVQG5hyKj@bA*usvcSsE@-U-97GPFKm<)?Lqk)fDU6sGsK)>G zmEi_|b(J5gyPyS4Rb4ZH7WRew^#P`&s)cpuRYKz%d+S|a+YpHa8-2BbP$2AYqUqyR zJ#E3)1wks5I@i-L)n$#(#E|Vl$ z%o@Ts%?k%>ni7Rf?FGYFRAZXL&7mrP6YTA8A{HP(o--^*%m;W_s2^WArI6)e#r0WN zr)Z`GB2DJebs3gyoI*x$SP>gYhA}IVh}1S~+Vq%_xb3G)s-8cJ^SX; zKWK_}{QVRenha?O*INCJejvpP#YOy~>agEhJ+H}H+8nNL4%;UV7hn>lMWB9-+-`NJ zj$fjiJNnkQ7u4kaasQ9sT)$_-j?JJGBiXDX(h=!5OZDn>V#pC~4#{P6I9!t5?s0g% zHm8%QOnF|f+aWncugB|^><+hwZk&i~MZ3o%*(968ZgYq>m(62W+44nvjZ;pS8|UqI z$*vtcMsj+*xY+4-x$Iui=@DJHQ1XbP=#XqKr^oHa6{5xI0a4uK24$B+^olfl8gAh)N#S*nGFmD~UFn+itgsqT3~TE!y!3Ik>>zt5j3kL7bYYV)lG6)4dQ^K9lAXSsEqU!C zxb{j=n%0(Y$8Bz}+v&7>+@c2xw4^bbc8}<=xjc3!WOQIC2QGEE?QR$c3Ui7!i0snZ z3=X%`tK}@mg|Cq6^BlxpLfN?2--taX`^-j~j*&ot6n>a3|CW z%ex#lkJsgeO4^CtA8*6ZPKRi>c_hi}$SO12VO^KqWpi16)40Gv&hmWd)#jEw;0w1o z;VU^YW9&{S*p6Xv#^rIrCH&dh7{ulB+F>NGjXF5HC~1A1Taqj%Fx0_B0y3hd=foH=q#&9mr2`oUZcDMzLW8wkb7^hQ$t3z9o%Z?ZXsj^b@op2ch zE?^kG4(X(=Eu7Qow7iiC40>Gvsn_G7HV;@q5JTjIfCzZ4Ep&6H0q2~!$PTSd(0N>< z*Q>oF9}b3~0>z1v2&LGadK1-WdJtIQU@pWTZ65I**f z$SPB~ME@5VaRvk&=v{e-$&llc5FA>c;oMHCrG-e^xy0pvKO3rWc^qzF!tT~S&bb_( zmNhLuAkrd$AJ+M=(*VJe|4+alp6U%I;HG#3KkIxG*j})=ghlIK-Ax z%W@3{^A>E)hSS**SP(fRmp=jS4PC>*JxDb$A)Nor8RAlYY4EsRFk;Ixh25*dBmt;# zwdNX-(f|&66%_5ZA-O=q&Vv0KqzS2BNJka{)De7bt?M|LBylX~@z^YPrrVr|i4G{s zTe!mvzjiqBx4mV*2IjRvdpP59AzlNy5)9})5aockeQoIoVHm^#i>{&yx6_}hbGQ%z z_ia$PyxOsuPCJ4DVvg4>wQW&&oqE6zvD*XIZ4ShS*5?CZ9Y%IJTZi)&l>vzYzN|7C zkSrw0Wx2zIK#zdpeIl3-2Xu>GrwukmD1moveH+eZLmq|$BPt`dwcey0o9+>ju~{y( zx2_Wgss})Xf@q;Odcu1sfPiw0|bRc|-swHL*!YOi|Xlr|exM*J1 zGdd64&&EH>>74Ktw}rRV5(71HZUnJk4UhvtG~;?|VxbsW(<9?)4p8EEsT4M+|Dig0|H1@z@09W1s{H5-}aQ7Miepq7!hJ z#BbqJfCJG5OACKhYXp=P1^4S9v&+ew)kG|)VDsCnsuy=N6($Ia;^fnTB4pjflFOwgd7a2z<3A_5vnM6TBT zbR2Oj3q67eF#$6PWuD9KYzt+S=OeTrJ20#RM<@YZs^1IN=(|F@*CP8K6RTUCD{~ZS~0o%&&j2@LP z0gCeYhjH*@%Rmlki$09|9JT_7!G>HQy4$W*-ewXI0ieS`t3q<{8#D!G4%gf~F$Ez4 z$wib}Gb~v|vOGpSt-DRA(@@(;!dlgs1Xg9)Pyiz5iO2(PsZdS(nZ2miP~SL2ItdC4 zV?dXqEjFE1QwU6`@b;ag{W6h@BxoMF(#5~6ZJCNO-GW9<956bY7WFqZ%zSCZ#|;$*!FEEj>7vu z+RKOH6n1;yEh6pxF{>a2#zJjv_wZR-4Ty`{Lpm5sbn>@rO{jH|?L5MRDn)|b65>!y z@|(39Fk>_RHO`TMV2x4^(FnDr7YZky!XFI?gQ(5C(YH-f!|0okhjt#{Rfag*k zqB#ni*rqn2o`=$$SvxtXlds|waMLpy&~U7pL+rx6yu#(#`wjxR+r3KU4H^!0eam_s z(ydbz4)FO25ZZ~14{t=Bf($PT1xw-d7L+!2iO=E#$2q&pOaCyT7DMQ^^O>r>Khb`~ z#a{t0vhhUAK{>)QEqv!XPGNWWPqw7tmwp$fDSJ)nDAX`bK|*upOcVNPGf{y%Mc@~n zt^Fs|?nF*N$U90re49FOmO~j^@NGK$iPf_xW}X;ra^QNe-OE3!VHG8i^hMd&1k`t! zzo1}YRg@D!$kDn`BcO8OUyoC`CHfJE%3;$V;7YFMP`~hBY7EF@DAa;y3)UJ?yCBOK ztkWPYI5Hj}j*P_`xRjTls8e{orO%*5!kEY^n4kfn4!fAOpF^T+Ij2E5cD3G4oXNMx z@n7+r%_Z>*)u)S$w&^WVoK(ju~Yd3#*OyE;^@PD!{%i$W0Tj9;l~aq%~C9O4ZBdb|O# z4Dmr29+#4U{0pdyoBo3NRpBqxN}Rl!BO-qrXF|%y+{Kf%QsET&a*YmCGOx&MRm5p* zRVxylmX8qgorw8>v*-vePeCc*Moe zBvxR-w(q~V(b>|-;XYn1XfU7k0?khR?{k5np$!1@*tD~VoBvth4*=nIK0zR2;nb7_ zC(3Qq0Q_W~24Had|KRHqHJA>%`M+yWC}Vohx2v)WFd?)!b@-p?q;g^t{>=jxdVLB| zkD0nn;(yfM$Ke&G?;0 zK#=DDoq#yRf6Q@c2?zm|Edgx>3E2v6?h)HwBwk?wOzc8dMODVnQyEahVJaa|4VU0@ z3#IxL7z~K>xC*W+m<(2teVqIQ@rV!nAuZTYJ2^Yk8*eM2<+S9_KfzTK9862>%-gQ44=0!rogl zk=Ox6HYK%}mSYt3DI9KLw?P5T-lb|8$0fL>y%v;=PRndKSHbg^pEZ!q&VQ>jKq!RG z=vSY<oj+>up)S%>W1}qvq;F0x7Habm+>JIT7{;}5U zX0%@W_PwVT^lrU&w{UM#JJG_;?9swyt=G>Oopbtl@;O?t?9hI8ck$GP{piJWGhb z2suX|?rrgorae&OxoTGekn_d`ixa43p+4~PZ12!SnG5`l2p@mkuVC(_Gtu6vlR~KY8Lp575Gr*R0SXl!6L(O${zF?@axe0dX>inVFX5byy0D(=-;ZO&W4)if5M=_MI zIwJ4sn#R6NDmz2jMGSVi<```yTd!oA33RS)#;5AF>1T_Km30jb^}d>59b2cXY4lai z^M|SefPHKgR;GfX+AdFst!k)e0)nkL{E6(*!%fsw(6N=zU}8eLt}0vwDuJp6Nt4GX zS66i`8~VZle>HwI`h(%5v?fD!pvK=^2WpRM{tQONI$Hv+gL7YYYI^1wY zeV{%oEx!g!k@I|wfpD+^tGOXxJuTM=^}$e)6V6or3fZ8KGzO}g>pIsKVTW+XGUwuX zfpEYFXGe(YvS-VJWbKa?^D8?)opI&L*Vfmbo0;a$+ELsPen}M&=ZB8_>mi)pvQYCDxt3{ z#dLVdC7eC3i6-g!|3CZ8*?c)s*3|iHJNFq}+JD&2hHg(`{?f};GLpymF7}mSJ$z*w zsk@}Fptgfr)j5Be$WkT#OhZU$Xb!`D!wq!-+GMH^Mv(m@ss(xYmK%A~vbL+9He7?h z#hj0b!txS>io9U_c>EH=4T~c*&*1A=+Rs=H(u%SPWpEJU3DR#&=7t(`b)}ism?P%J z!KQiU#(=VJ46Dn`h!_y5+T0K_`_1~hG`QtD8c4yfj-*Y)b4Ol!<-O8s`B&C&HYDw3 zfhj@wef8*uN&MU%*;~s34E7ELBJ&#Rs_6#`yQE~*iPR5Ga<-;FlbY#2va9AbhZgwi z0-@Tbd6v5s3o2?FRr0EM4#6v$fCz+dcMv#^wq6v{^KorHRI@s+*!`=R37C7or1X9SEzJA zJEu(4(rk^_JF&2QbYa<);_;TpbCYF6W7%TzAq-Xq!j@(2oUD)|sc1@?wUe^v1sfw~ zq<{s@jpo9iqGxFL@!%Qi;SW4{%dfwWn0xEtrr%xBS^kd{3X*mFn=_Q?EF1i<`=kD~ zg&Y3+o1FA7`&_PQYPcsh`UCAg9{hpk<^KM!hb(uCk8SGp-q3#snHje-N)J&m3cGwWy(%vIL|!2#kBgnZ@1lf;$xcY zQ)Lp;9nGSj(X7vO16T!_^%)1VC(cp{(eG*ZvHp9iQ02B=pRZdxe(EjOT*u63HNp?q zUh()nxJ3NkfW(IBMqpX{A$i~s5me)&r)l?b@HEX!*QI1U@^tPmx9r`)w>|Z@k@!+P zZH})FH?X}gzN`mY{q6sv8UG^vv{Q&#HN{{?r5}v&gEi4b>h5FtB6a<{d2^a?-1)&?={hP5ezZ3FCC&JfSfp2!`kB#_^6ul{NqOGAV;@+$V)qEc zW3=Ys7ZdNE7(GkHvm{pNA0K@(_Uk9f6~_5GjX%89tgMKBMZ1rKU(vkyZ>K#a_V)~Z zss7IN50#BW@um0`QSlBv-!4Rt%LB)WL0=aA2kkx%{)6Uid8*=uQ=7$oJDtxRJM?G# zE_^BeBkf~DyE7pf9TL5esE>Y1yN`pP(!6Ir{e9yrX|{PQD%y{|A$v)MFU3y{C_#g@ zwNXt%vL*o~erdFgb{_}ZXkO_r>Y4>gY0^Ed($0M4pJVZ**rs7~jp#<@6B|jY)U2Y@ zx+TINOsE;Rr#xOG8@c%5!nKhPxi!mh+WD1+RYB34W52kW=x9mw6WV)JDR_pfPeBu*o=$xDI<5Elr52%GIqsM5*F-oI*44udJJqB*%i|Z&Q>$6rW zY1)e9iY3WJ%O|SG5Z&{K%ipgm)_-|_{E$al#GF`Fe)9cB6{TbO-vs81BbFpv-=ygY zJtjTI^}6T;6+Z!2MUPxK)*hjnUzhwj7w2-1P?9Jep=mejz)a6$`!Q-TZ&j13lIW9i z=}8nw(lN>*QbrHcp2zmXR5N7PtbClyJxqy^b+~iZ6kVq5c^q7()Kp!U@P70|n)@Lo z2J443x>VWo_+Y70Ge7IA*P=&h?omqOtw(8eiL&Rhdx?@;xac5GjaZ^|=odBz!b>Zg zgLT!YCM};vOJaK-+e>0Ki)Mc`vHcNw{Ubz3o|Z+b`7x=wjyVkfw?U>vM0?bjX%hz< z=3#Qv5aw^nY~P(K>`qni<-*?v6LZq!=xI&ovHi4$e_yMmd=*cro~w9{S7GDNsVyHl zUfeir!`d(YT>I^eJqGDHe)Ev5igrr)WOugr?&!a0&*Rj;XkPO}=eAak6I)ia|95@) z;hzWLd+A@4==q-$h;P%g1|KVLnxl&ZVUaL4trVqBIOwlKlBhOU0B>ex6)`7PMcd@v z$J#czXX*_*A6i}JYw_ni}yzHLc7*nbF#dEyWPHBlAwyRDkyP)$ShzSy?=h{1A% zG^ItZ6dvs_L`>0>wEGZG@`ti{honLsl=St+Mf(n&AHW#SucuubNBYWp=?5m z65HG~&);N*`px{iB3a5?a!i#0GotDuVKf&_mUSPKlEWXxdiHt$*m7)!c(}#>L4~q< zM0={NIMona>2Io<7d6O=4G4zg4QBshf3T@Qk?~(XHV(ny`%fwVlN3G8j__$oMe}rP z){on_%dY82#gPoJ|c{V(A%)^K$Vwb2ihtsgj22ahSlR_oCP*nv0L zte;^p118~MVSe zvC=#3XK2P5of4C(rsym3z$+wO`kPWY?YF-usssFc!-RYEWQ}Ua!1ulF^|{IQxm;`@ z#8<`fT0x_%@z)7Q;-W3NI1aYtk|e%8x6S4skld16v`L*eu;z1R?uEnM&CXAI3W`zsron_%vMSvgl@4mDKs)41p{W%sf6 zn9{TC=;*bteCphMQ@=9BlDW_Ezo!0N8Lv||`j<3Bzm^BSPPrl&%E1QATHZIDAmy0N z_)lX)#9v3}lG4AvHelOul2^WMxT|!8ZhT--pl(LVi0BA4j@2X7dCxz8_VqvG9J;?K zNBnGf`P!WJyX1kp%xbnR*N2Ov4=T4lNcyS9>5xh5{JcD$YicmlO3Sm+JLTQS+B@ZW zSD*goo4;|G<5Dr z<@o~S&+cRGS(@kmCHbA}UiS=pEGArj+u70y?PqDmS)vwN1Y4gcwWnH9=NG#&{vc+$fS9cdNc4of`&fHIp69=#f5C_!zN?}; zV!Mw=cEsk*=>L7~Q=g3#Zmv4?v|s+_oc3>M#y7+ytP_m)r0_2n6JJ`aIyy|t57R>I z*~U9kbP;n^bC^G3=4YsdpgL+B(S6)t8^N6U`S15~OAh3%*l_fiYWu5(Q*Rcgof%D< z_!;9Z*O3@wzB>9fwR}x@I+brVB~FgMpzJ=@zM#yzb>LTDx6OBF-ZFXE-ai)RWt=!D zA9aE%xQOxjG6ckolSC(;Or07%MZ1r+r>JM&*AE$2F7_IpU4Cy&!lWzt!KDXRsW^1G z_$>~;d_sx}@ogz~4@XZKjHe7pyZgJ2?fdDeXB?Ws1r}TTCsOh!((wH{B_=4f(Ff&$ z2XW!P3&*|tC|5FM8%|*!S%7(D;;;>SEG74IhYaYMkyo>cbx+Y#7 zy^$rjnhR?Opx_~jP(z4vRs+##**r^Gr}3e~MKh40s6oM+ujCyxW)<{ycD39esiG6L z@g&Zxq()U>P7NU%>dfw%TCi?tj7ptUk);2AI?-M@W0H2} z2opIrhvna!IA24Jbr-SXw&^7F9!-2foTT5T*M07%nQFu_wh4TmKhiQ#KOO9rJVO@# zl>4h9{ld(n<#EINzqN&6h4oKrA4^;PX0t?}tOWPlM&mV6WIU0D=^zfRQ> zX)f1%U9OQ-2=dC$<*9F$Y~YgKDo-V4L#ct)zl0Qxlcfent&=UR$Ea^^m!;Mkr>TIG7bUK`kWl5Dmlo2XlL!E z`o^-`WhEbyQOjh}yJWNPDrq3DHkwnwhRVlKPv=vyUH>jdveC+ExUDUoze{8J^|$N! z&oxxg&n!wNB>}Sj-q;xt()tLE^mxs+(dmom1ln>ETriag-t-j7MtEf2Zn(m|4PzPU^Hi!P?D z!SvqON$#thein@Pv=7aA;imY)r%0Sqo!dg0Fw1GduF6+P>g?#XG;}RZt^JRSIns#) z%Atk5W`e#vHKAlGHv1>Pr{?PYjY3*Hmu+t1)8oqzt5ctMl`4sR|KAv;Yrst9HJJP^ zNd8mbvQpQv;KBc{=dj>mOx5ZSGL;-9;Wy>hc_jJ;4ShkSAW1pD3S!wfSmGG;-z85! zOi7GzhO^{PxpcqSdP-h1fRx-!3^COW&6Ra@u9M#Ns0QBDnj&I-y86aq5Mr{(riO#hmCLSr-%gjJUy;q zJVnULp;RFYTJP4Y_>yA6DVm#=S@Zph=vHNDtJ0f@Zkm6I@vDC3?(W-ln=8^t$@^qw z64kJdHJf&HwBh)#j49_^*49;Y*&y=4HKd~(7y2hDOC!gEB&=1in}cL+iz5G`)||Ck z1NSbOPgdu|WbNc1)eihc4S2ltJJ&TcH{P8T{X!P{LI&;!8{^iz0lOsAqSq9psWL0h z8YHe{1mgC=amj<@tO%Z(oME|l&=LJe(Ha_#)evRcPyG3%zuDHilvCtL?0?!s@6xC!IqHK|D}sUYW+eW=EOB* zPDHJ5X7FerbH=Hz&Wx^AuUxB6tSd9A^+#~S!S{(VL#?lXb;pvG?<=x_Lrk32vM6Qa52})XIWfMv9m0||Q7iDEr7=eRo5ZFx0tTOcMe(CCOoRD38 zqN0LK+AmVQZv7U5p2ijuJuaJle9m-o^*;Tx;O059{3F?%S~BTPeE=kYCW~}TmbF7v zl^>O%#bKxIM`XoEN+yuvefn20;)k-5$z;;&`l+~DPs=`#^=)&|_CFMV{iA#;(XWd> zL99=plf-|U&{m+IihyyBeEK{};OzqILkr0**7OU@w)~^Ul&+SCFZmUduA}E66dH)I z|A3*!UmvVnD$M1q*Aj-BRYck~_@x_(P9IoO6%am2)uHZ+M4qVDM;TUH$RyQE*Fxf? zYYoVkRSTMUyXnts0OIHkBrskOOL5gQ&m2R6-qvKVDbw|FU_VVtn#th4?iKNqs1Uik~83fo?k2F_dt4XA$xvuC3!b*)6GT@UdD&JI~Jldm3 zo;7&mp6g!4`SS0nRq9vq2T~WwR97{i6vpCHQ-gJwjq&?6vUD&M@PGSYkd(`nQM1xz zRu-KibAD6kmq5<4vt;iBPojVJyFwpkHw4LTZ^(+iCdTa?+^0NVrL7njzfw>X%_f`Y zDa-GZQM!8sl};4Z-T*SP{}d0^j;$5K@SGQz_@G47g$( zjC_c^K8mV>Bp@vmaeq73y#lT~mEz+pfz*-@{vjhVe5^>Ct&dXRu9_^ep{XRS5`;Nw z-mbz{hK@%~zs4LfRj8OCPy0NvB1TzWL!xE!8_MM6i-m%_uI}a8OrgV}5NH;!~lDkX?Hp+JpN|)gQNhQ`oo2_rf z0+1o9QwHiAn|Q4y>520#qspwn{(@hlxw6^0i-IWQ6C>i=;CT<(q)J<8NSe{r+EKo= zf33b$#aGUt*0+@9WkjQ@Z*CGE&$1-yr!pOGLJ25n+~Lhg^LRLxcMTkToeCj&3Q4vl zByA#`@&`p-gHV;FVGn<%j)O~$C20l5iei#ARR0wy52tI^RE#63hx8M1-+B`M93=Ta zhHzLI&V5vk8W+T*s50qsqHbNSX!5nLvmB3}k(Hm3@bF~>MW z1~0p!@&J)yl?P@ag3g!|gDQ5bIcYljIu8ssazh56(-BT69!Y%C^gLFj?IN0(%3!T= zqjNbPT34xNlLqNoX5;UDr2MqTdDE!l`bEP%%im+kaO8-`bqppSkQuvV6>Dg4lm1N= z|KxM>im7C?Acqbg94AXxv-K=EY`9j^c4SCD5j;j6T0z<7;u+{Bd+*@T88XgQlUSWxwYTiV-C`dYfkVC2;-8r^~Jqm2i zH0yP{iiob`k!}9^tQX_NT__@U%>wUuF1SR+I7LRO*@f|pzOk6-vB74LDAjd5$}KI`z(GogaD|qZ<;KS@e8QyZe0;cQ4VMc%%p}6qCX1Y84>)5P z9;acgv)1jLvZ_W3#U|>C;$kT+nL_m?oPZ!Vm(=c68+?`6&(P3%Vv?T){O8GxgRzM< zq@KS?uLuWP8_?)^b}$J_HHZSFqz3hNtiF;Y7L6gUy_!l3DVax>VaAkPQb&q5%2COb zEFwj}DfOS4xX$jJB-j~FO3C~oXy)t7iq2B#qGT3X)*q=iac)r}W#PP%)TpT@7Klwd zIqtwKsR=c&#qd|^r02K`Jv!s-aaaV|7N>~#k~b9m`8=!Dp22*Wccuq-0=v11irjgu zPFS4k)9Y$pB1Q9+baYEh?o*togUU;`k&-y2e#+oU9CP%t@r3>=&7g_n^(XbkdfmbA zNOsSs@A>E`{S*efM@Y>m`k%G>GE7H{NbA{(uSm%)vHD4!HqE^~wqv@^4Ei%Ex;0im zt<#z{-_ze*`tlBo%@`Y->`qcFsWCp_B$|Oc(j<2pBkdnYd{rfA&|5Co%PD?c!WbLs3w01U|4*Ll2*RA#u z*AkO<4x7x)Mj*!z*6}lbV&VrI6pMnwOjG%|1o1|eI_JtN4wABkh?V6hc^3Jrh~>^v zAH|kclIRe5`4Bl8DhgGYzpoU}Zrb<_x(Q_na~#5)(>jX)A?s%n zbOOEoj31`(YU6;X)U0;VdvE8kbBUil<+^W~6E~AlchOLoL?58#4^V7ZQ^eKrPYRnJ zN|k zd=7J|>158c)VhrWnti2y(JN0V0-o`lgx@9M{M%CxYSn>`21vT zn7ZoaiLaK92(EkcM!EF`;(F)Gq)J|{N*i-U$!;Q*syjNF)13T0Gj$!EOkQa$IH6Pe z>nfY~{sbItRw0_)r?572b0$C^{QKBFWvSFoOwt!kB6u`LteP_k+k&LVAu9fcGCHS} z=wexuGi$BLX5gdgvjcDe)?0&3!r5YNJE)xj*h+E`1@&<-gO!PZWwK*B4hIx!*~oEqvKM#JNR( zKWdLMqU-1?vOPWi{+(|N@&H>he6Y2{am#_;dwMSH4YL2dLN~On$KD{er>BU~Ip|kh z^oFFi#eL{pb7lGIxGO%wF#pFJl6FpK-?6tDF`824#Wd$l>P-{-jjl;5PK$!>@k_rU zY3FoH>CWlrX%>JcleMS3AN`{@%{MyvJPx|-wKYB7khJq^MvL(_@e*|Ii0MtIi??%( zw@c8un$c+tDlU3M(tx$S>Ga~K*Bg>{&Hz7GD+MzM}@cIFiYLHO_ZbfJ4ep>!EKOuIgC(eXa4ogqd?(X}(bYgc*%FP*7s zGWUF^iWyDi`d)AI=@KL9)r-ZLJF@rioeJ)+IvtlX8e;T#M^eY|AtuJ&cNAnf#9*H; zF_vw6_cC3%}|{mltAj+B~lC>d6xeMi-iki_B#crkD$-7f+rt*&ONE2foBSt$4~< zENXUdGdB-6Uo)sewtkcwX0L@jSuB$x# zIumCx#ObS+cmLg6oGxQefU(*8#tvPGq&t~7EsV~!f|8&i)bZ(3%V_7y%g`}_j=Nhe zH!wOd_6jUCkQfE487L#$3bK9K`}oqX@D8jtr;pVb*&7(0+AQTon`NJw-vz&& zR(q|Ry;~R^SnV}twLAVw&!Uw{mUWqJO2Me{|A)=2ELld(nZasvb2ENSEH5rCD8HIn zIvY8&4m&5%JkDnMxW4O8yo?C%22yGsZo*DX?4jsd*2-pcUNc){$?9dsD$9pVzEzN~ zubv*+*IT|WGtN03ms$xuz1zc#Oukzfon=EWHskI&m!V?A zSIaV26pWu*IN6+u*FWHi?;3ZG6mU9n@EQkajwEe)7EKG+wvpfVV)#<`M)$Qw^IV(SjR@-GO42W4L~Xuf~h zG8U!rh@gGYb$wvqUB>kD3iQ)g&zm0Tt)DI!TniYmVr@?k7%=_(h0&Qi{$dzxf9NuF zOrUcC2CEqzz@Q@!_Xy1`AlsL{`)hlX?ScVn(*C#ifdM1Cnz3h`q$Fr5bbLB}nhr8d zxeOf!gDbjWa0jCU7<44mZ@#2wzW@wAyc7(2IdQhR(?OX(UIG&r&>&S+d8?`_-FUH$ zB<48qXAa)<;?Aj&q)L}xti*%v)!F~wL#7Wxihp9N8wS<&H9^Jn-m2?D$nB@&`o9Yk zJl_K$Om%lKIy0`m7$L*^JZjO4jtO)wAY=`r1BCQV-Yp>8m%TIL-t2WEr2o4>$a=8% zK@avA*)@#4Pv*Nu?1j1Caek-TfF&Q^lV#6Hhkc9vMy z3lN#s@nTD)?BVAB1p&Qt^7QeH?LWYFU-2LQs5jePT3vrS&YTPJck~dSvAvej*}6bU zqTrc*y2StQ^rFkqF@a8dxA=E6IvBwOTkn>xenj&~k ziaqDSX7zRKft-$hK|TJ6=?s|otD_6h`7js;0wG=Y{`a1*w2XiE##_E0GXDx^Db5)# z6(ez8e^HQR1o}ovIyJAk*7%Ec%2kOEku@iGZ$31UY2zTY(O0W?(%#zW1eG~Asn|Rh zT79pFR+%>LW^}$?aWS1K>n=k_-;2(=^K=qgE~ayXw>O Date: Tue, 16 Aug 2016 21:02:41 +0800 Subject: [PATCH 24/30] add docs --- packetbeat/docs/gettingstarted.asciidoc | 4 ++ .../configuration/packetbeat-options.asciidoc | 54 +++++++++++++++++++ packetbeat/docs/shared-protocol-list.asciidoc | 2 +- packetbeat/etc/beat.full.yml | 18 +++---- packetbeat/etc/beat.yml | 4 ++ packetbeat/packetbeat.full.yml | 20 +++---- 6 files changed, 82 insertions(+), 20 deletions(-) diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index fd929685214..7398790ab26 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -165,6 +165,10 @@ packetbeat.protocols.thrift: packetbeat.protocols.mongodb: ports: [27017] + +packetbeat.protocols.cassandra: + ports: [9042] + ---------------------------------------------------------------------- + . Set the IP address and port where Packetbeat can find the Elasticsearch diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index 9bc9f66fbf4..4dc8df43a02 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -248,6 +248,10 @@ packetbeat.protocols.pgsql: packetbeat.protocols.thrift: ports: [9090] + +packetbeat.protocols.cassandra: + ports: [9042] + ------------------------------------------------------------------------------ ==== Common Protocol Options @@ -678,6 +682,56 @@ at the end of the document. Note that limiting documents in this way means that they are no longer correctly formatted JSON objects. + +[[configuration-cassandra]] +==== Cassandra Configuration Options + +The following settings are specific to the Cassandra protocol. Here is a sample +configuration for the `cassandra` section of the +{beatname_lc}.yml+ config file: + +[source,yaml] +------------------------------------------------------------------------------ +packetbeat.protocols.cassandra: + ports: [9042] + send_request: true + send_request_header: true + send_response: true + send_response_header: true + compressor: "snappy" + ignored_ops: ["SUPPORTED","OPTIONS"] +------------------------------------------------------------------------------ + + +===== send_request + +If this option is enabled, the raw message of the request (`cassandra_request` field) +is included in published events. The default is true. + +===== send_request_header + +If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) +is sent to Elasticsearch. The default is true. enable `send_request` first before enable this option. + +===== send_response + +If this option is enabled, the raw message of the response (`cassandra_response` field) +is included in published events. The default is true. + +===== send_response_header + +If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) +is included in published events. The default is true. enable `send_response` first before enable this option. + +===== ignored_ops + +This option indicates which Operator/Operators will be ignored. + +===== compressor + +If this option is enabled, and also the flag indicates that the frame body was compressed, and +the stream will decode by the compressor, currently only support `snappy` , The default compressor is nil. + + [[configuration-processes]] === Monitored Processes Configuration diff --git a/packetbeat/docs/shared-protocol-list.asciidoc b/packetbeat/docs/shared-protocol-list.asciidoc index acf541d2315..f8b7f333b02 100644 --- a/packetbeat/docs/shared-protocol-list.asciidoc +++ b/packetbeat/docs/shared-protocol-list.asciidoc @@ -14,4 +14,4 @@ - Thrift-RPC - MongoDB - Memcache - \ No newline at end of file + - Cassandra diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 00d2526ee94..5ddca23c3bc 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -422,25 +422,25 @@ packetbeat.protocols.cassandra: ports: [9042] # If this option is enabled, the raw message of the request (`cassandra_request` field) - # is included in published events. The default is false. + # is included in published events. The default is true. #send_request: true + # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) + # is included in published events. The default is true. enable `send_request` first before enable this option. + #send_request_header: true + # If this option is enabled, the raw message of the response (`cassandra_response` field) - # is included in published events. The default is false. + # is included in published events. The default is true. #send_response: true - # If this option is enabled, the raw message of the response (`cassandra_request_header` field) - # is included in published events. The default is false. - #send_request_header: true - - # If this option is enabled, the raw message of the response (`cassandra_response_header` field) - # is included in published events. The default is false. + # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) + # is included in published events. The default is true. enable `send_response` first before enable this option. #send_response_header: true # If this option is enabled, and also the flag indicates that the frame body was compressed, and # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" - # This option indicates which Operator/Operators will be ignored + # This option indicates which Operator/Operators will be ignored. #ignored_ops: ["SUPPORTED","OPTIONS"] diff --git a/packetbeat/etc/beat.yml b/packetbeat/etc/beat.yml index c2848648969..65017004cbc 100644 --- a/packetbeat/etc/beat.yml +++ b/packetbeat/etc/beat.yml @@ -87,3 +87,7 @@ packetbeat.protocols.nfs: # Configure the ports where to listen for NFS traffic. You can disable # the NFS protocol by commenting out the list of ports. ports: [2049] + +packetbeat.protocols.cassandra: + #Cassandra port for traffic monitoring. + ports: [9042] diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 95a1f0613be..29a03ea8312 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -422,26 +422,26 @@ packetbeat.protocols.cassandra: ports: [9042] # If this option is enabled, the raw message of the request (`cassandra_request` field) - # is included in published events. The default is false. + # is included in published events. The default is true. #send_request: true + # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) + # is included in published events. The default is true. enable `send_request` first before enable this option. + #send_request_header: true + # If this option is enabled, the raw message of the response (`cassandra_response` field) - # is included in published events. The default is false. + # is included in published events. The default is true. #send_response: true - # If this option is enabled, the raw message of the response (`cassandra_request_header` field) - # is sent to Elasticsearch. The default is false. - #send_request_header: true - - # If this option is enabled, the raw message of the response (`cassandra_response_header` field) - # is included in published events. The default is false. + # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) + # is included in published events. The default is true. enable `send_response` first before enable this option. #send_response_header: true # If this option is enabled, and also the flag indicates that the frame body was compressed, and - # the stream will decode by the compressor, currently support: `snappy` , The default compressor is nil. + # the stream will decode by the compressor, currently support: `snappy` , The default is empty. #compressor: "snappy" - # This option indicates which Operator/Operators will be ignored + # This option indicates which Operator/Operators will be ignored. #ignored_ops: ["SUPPORTED","OPTIONS"] From d1c467ffc16ec90a2bb6dd947ff8a6fda22a13cb Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 16 Aug 2016 21:10:57 +0800 Subject: [PATCH 25/30] add ignore ops list --- .../docs/reference/configuration/packetbeat-options.asciidoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index 4dc8df43a02..cf9cd7a2aa7 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -724,7 +724,10 @@ is included in published events. The default is true. enable `send_response` fir ===== ignored_ops -This option indicates which Operator/Operators will be ignored. +This option indicates which Operator/Operators captured will be ignored. currently support: +`ERROR` ,`STARTUP` ,`READY` ,`AUTHENTICATE` ,`OPTIONS` ,`SUPPORTED` , +`QUERY` ,`RESULT` ,`PREPARE` ,`EXECUTE` ,`REGISTER` ,`EVENT` , +`BATCH` ,`AUTH_CHALLENGE`,`AUTH_RESPONSE` ,`AUTH_SUCCESS` . ===== compressor From 2117cb694a92b14d680f33dba217e7cfe9e1a790 Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 16 Aug 2016 21:29:36 +0800 Subject: [PATCH 26/30] update section in alphabetical order --- .../configuration/packetbeat-options.asciidoc | 96 ++++++++----------- packetbeat/docs/shared-protocol-list.asciidoc | 2 +- packetbeat/etc/beat.full.yml | 54 +++++------ packetbeat/etc/beat.yml | 8 +- packetbeat/etc/fields.yml | 52 +++++----- packetbeat/packetbeat.full.yml | 54 +++++------ .../tests/system/config/packetbeat.yml.j2 | 18 ++-- 7 files changed, 137 insertions(+), 147 deletions(-) diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index cf9cd7a2aa7..ab900c9c16f 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -234,6 +234,9 @@ packetbeat.protocols.http: packetbeat.protocols.amqp: ports: [5672] +packetbeat.protocols.cassandra: + ports: [9042] + packetbeat.protocols.memcache: ports: [11211] @@ -249,8 +252,6 @@ packetbeat.protocols.pgsql: packetbeat.protocols.thrift: ports: [9090] -packetbeat.protocols.cassandra: - ports: [9042] ------------------------------------------------------------------------------ @@ -509,6 +510,45 @@ If set to false, the connection layer methods of the protocol are also displayed, such as the opening and closing of connections and channels by clients, or the quality of service negotiation. The default is true. + +[[configuration-cassandra]] +==== Cassandra Configuration Options + +The following settings are specific to the Cassandra protocol. Here is a sample +configuration for the `cassandra` section of the +{beatname_lc}.yml+ config file: + +[source,yaml] +------------------------------------------------------------------------------ +packetbeat.protocols.cassandra: + send_request_header: true + send_response_header: true + compressor: "snappy" + ignored_ops: ["SUPPORTED","OPTIONS"] +------------------------------------------------------------------------------ + +===== send_request_header + +If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) +is sent to Elasticsearch. The default is true. enable `send_request` first before enable this option. + +===== send_response_header + +If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) +is included in published events. The default is true. enable `send_response` first before enable this option. + +===== ignored_ops + +This option indicates which Operator/Operators captured will be ignored. currently support: +`ERROR` ,`STARTUP` ,`READY` ,`AUTHENTICATE` ,`OPTIONS` ,`SUPPORTED` , +`QUERY` ,`RESULT` ,`PREPARE` ,`EXECUTE` ,`REGISTER` ,`EVENT` , +`BATCH` ,`AUTH_CHALLENGE`,`AUTH_RESPONSE` ,`AUTH_SUCCESS` . + +===== compressor + +Configures the default compression algorithm being used to uncompress compressed frames by name. Currently only `snappy` is can be configured. +By default no compressor is configured. + + ==== Memcache Configuration Options The `memcache` section of the +{beatname_lc}.yml+ config file specifies configuration options for the memcache @@ -683,58 +723,6 @@ Note that limiting documents in this way means that they are no longer correctly formatted JSON objects. -[[configuration-cassandra]] -==== Cassandra Configuration Options - -The following settings are specific to the Cassandra protocol. Here is a sample -configuration for the `cassandra` section of the +{beatname_lc}.yml+ config file: - -[source,yaml] ------------------------------------------------------------------------------- -packetbeat.protocols.cassandra: - ports: [9042] - send_request: true - send_request_header: true - send_response: true - send_response_header: true - compressor: "snappy" - ignored_ops: ["SUPPORTED","OPTIONS"] ------------------------------------------------------------------------------- - - -===== send_request - -If this option is enabled, the raw message of the request (`cassandra_request` field) -is included in published events. The default is true. - -===== send_request_header - -If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) -is sent to Elasticsearch. The default is true. enable `send_request` first before enable this option. - -===== send_response - -If this option is enabled, the raw message of the response (`cassandra_response` field) -is included in published events. The default is true. - -===== send_response_header - -If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) -is included in published events. The default is true. enable `send_response` first before enable this option. - -===== ignored_ops - -This option indicates which Operator/Operators captured will be ignored. currently support: -`ERROR` ,`STARTUP` ,`READY` ,`AUTHENTICATE` ,`OPTIONS` ,`SUPPORTED` , -`QUERY` ,`RESULT` ,`PREPARE` ,`EXECUTE` ,`REGISTER` ,`EVENT` , -`BATCH` ,`AUTH_CHALLENGE`,`AUTH_RESPONSE` ,`AUTH_SUCCESS` . - -===== compressor - -If this option is enabled, and also the flag indicates that the frame body was compressed, and -the stream will decode by the compressor, currently only support `snappy` , The default compressor is nil. - - [[configuration-processes]] === Monitored Processes Configuration diff --git a/packetbeat/docs/shared-protocol-list.asciidoc b/packetbeat/docs/shared-protocol-list.asciidoc index f8b7f333b02..16da553167c 100644 --- a/packetbeat/docs/shared-protocol-list.asciidoc +++ b/packetbeat/docs/shared-protocol-list.asciidoc @@ -8,10 +8,10 @@ - DNS - HTTP - AMQP 0.9.1 + - Cassandra - Mysql - PostgreSQL - Redis - Thrift-RPC - MongoDB - Memcache - - Cassandra diff --git a/packetbeat/etc/beat.full.yml b/packetbeat/etc/beat.full.yml index 5ddca23c3bc..4c4f334cf9c 100644 --- a/packetbeat/etc/beat.full.yml +++ b/packetbeat/etc/beat.full.yml @@ -99,6 +99,33 @@ packetbeat.protocols.amqp: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s +packetbeat.protocols.cassandra: + #Cassandra port for traffic monitoring. + ports: [9042] + + # If this option is enabled, the raw message of the request (`cassandra_request` field) + # is included in published events. The default is true. + #send_request: true + + # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) + # is included in published events. The default is true. enable `send_request` first before enable this option. + #send_request_header: true + + # If this option is enabled, the raw message of the response (`cassandra_response` field) + # is included in published events. The default is true. + #send_response: true + + # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) + # is included in published events. The default is true. enable `send_response` first before enable this option. + #send_response_header: true + + # Configures the default compression algorithm being used to uncompress compressed frames by name. Currently only `snappy` is can be configured. + # By default no compressor is configured. + #compressor: "snappy" + + # This option indicates which Operator/Operators will be ignored. + #ignored_ops: ["SUPPORTED","OPTIONS"] + packetbeat.protocols.dns: # Enable DNS monitoring. Default: true #enabled: true @@ -417,30 +444,3 @@ packetbeat.protocols.nfs: # - process: app # cmdline_grep: gunicorn -packetbeat.protocols.cassandra: - #Cassandra port for traffic monitoring. - ports: [9042] - - # If this option is enabled, the raw message of the request (`cassandra_request` field) - # is included in published events. The default is true. - #send_request: true - - # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) - # is included in published events. The default is true. enable `send_request` first before enable this option. - #send_request_header: true - - # If this option is enabled, the raw message of the response (`cassandra_response` field) - # is included in published events. The default is true. - #send_response: true - - # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) - # is included in published events. The default is true. enable `send_response` first before enable this option. - #send_response_header: true - - # If this option is enabled, and also the flag indicates that the frame body was compressed, and - # the stream will decode by the compressor, currently support: `snappy` , The default is empty. - #compressor: "snappy" - - # This option indicates which Operator/Operators will be ignored. - #ignored_ops: ["SUPPORTED","OPTIONS"] - diff --git a/packetbeat/etc/beat.yml b/packetbeat/etc/beat.yml index 65017004cbc..a67aa82e85c 100644 --- a/packetbeat/etc/beat.yml +++ b/packetbeat/etc/beat.yml @@ -35,6 +35,10 @@ packetbeat.protocols.amqp: # the AMQP protocol by commenting out the list of ports. ports: [5672] +packetbeat.protocols.cassandra: + #Cassandra port for traffic monitoring. + ports: [9042] + packetbeat.protocols.dns: # Configure the ports where to listen for DNS traffic. You can disable # the DNS protocol by commenting out the list of ports. @@ -87,7 +91,3 @@ packetbeat.protocols.nfs: # Configure the ports where to listen for NFS traffic. You can disable # the NFS protocol by commenting out the list of ports. ports: [2049] - -packetbeat.protocols.cassandra: - #Cassandra port for traffic monitoring. - ports: [9042] diff --git a/packetbeat/etc/fields.yml b/packetbeat/etc/fields.yml index 32b805cafa7..60c0fba1e7a 100644 --- a/packetbeat/etc/fields.yml +++ b/packetbeat/etc/fields.yml @@ -831,6 +831,33 @@ description: > Creating application id. + +- key: cassandra + title: "Cassandra" + description: Cassandra v4/3 specific event fields. + fields: + - name: cassandra_request + type: dict + dict-type: keyword + description: > + Cassandra request field table. + - name: cassandra_request.request_headers + type: dict + dict-type: keyword + description: > + Cassandra request header field table. + - name: cassandra_response + type: dict + dict-type: keyword + description: > + Cassandra response field table. + - name: cassandra_response.response_headers + type: dict + dict-type: keyword + description: > + Cassandra response header field table. + + - key: http title: "HTTP" description: HTTP-specific event fields. @@ -1390,31 +1417,6 @@ description: NFS operation reply status. -- key: cassandra - title: "Cassandra" - description: Cassandra v4/3 specific event fields. - fields: - - name: cassandra_request - type: dict - dict-type: keyword - description: > - Cassandra request field table. - - name: cassandra_request.request_headers - type: dict - dict-type: keyword - description: > - Cassandra request header field table. - - name: cassandra_response - type: dict - dict-type: keyword - description: > - Cassandra response field table. - - name: cassandra_response.response_headers - type: dict - dict-type: keyword - description: > - Cassandra response header field table. - - key: raw title: Raw diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index 29a03ea8312..0f25204eb10 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -99,6 +99,33 @@ packetbeat.protocols.amqp: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s +packetbeat.protocols.cassandra: + #Cassandra port for traffic monitoring. + ports: [9042] + + # If this option is enabled, the raw message of the request (`cassandra_request` field) + # is included in published events. The default is true. + #send_request: true + + # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) + # is included in published events. The default is true. enable `send_request` first before enable this option. + #send_request_header: true + + # If this option is enabled, the raw message of the response (`cassandra_response` field) + # is included in published events. The default is true. + #send_response: true + + # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) + # is included in published events. The default is true. enable `send_response` first before enable this option. + #send_response_header: true + + # Configures the default compression algorithm being used to uncompress compressed frames by name. Currently only `snappy` is can be configured. + # By default no compressor is configured. + #compressor: "snappy" + + # This option indicates which Operator/Operators will be ignored. + #ignored_ops: ["SUPPORTED","OPTIONS"] + packetbeat.protocols.dns: # Enable DNS monitoring. Default: true #enabled: true @@ -417,33 +444,6 @@ packetbeat.protocols.nfs: # - process: app # cmdline_grep: gunicorn -packetbeat.protocols.cassandra: - #Cassandra port for traffic monitoring. - ports: [9042] - - # If this option is enabled, the raw message of the request (`cassandra_request` field) - # is included in published events. The default is true. - #send_request: true - - # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) - # is included in published events. The default is true. enable `send_request` first before enable this option. - #send_request_header: true - - # If this option is enabled, the raw message of the response (`cassandra_response` field) - # is included in published events. The default is true. - #send_response: true - - # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) - # is included in published events. The default is true. enable `send_response` first before enable this option. - #send_response_header: true - - # If this option is enabled, and also the flag indicates that the frame body was compressed, and - # the stream will decode by the compressor, currently support: `snappy` , The default is empty. - #compressor: "snappy" - - # This option indicates which Operator/Operators will be ignored. - #ignored_ops: ["SUPPORTED","OPTIONS"] - #================================ General ===================================== diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 45da0466035..3a8ec4959db 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -34,6 +34,15 @@ packetbeat.protocols.amqp: {% if amqp_send_request %} send_request: true{%- endif %} {% if amqp_send_response %} send_response: true{%- endif %} +packetbeat.protocols.cassandra: + ports: [{{ cassandra_ports|default([9042])|join(", ") }}] +{% if cassandra_send_request %} send_request: true{%endif %} +{% if cassandra_send_response %} send_response: true{% endif %} +{% if cassandra_send_request_header %} send_request_header: true{% endif %} +{% if cassandra_send_response_header %} send_response_header: true{% endif %} +{% if cassandra_ignored_ops %} ignored_ops: {{cassandra_ignored_ops}}{% endif %} +{% if cassandra_compressor %} compressor: {{cassandra_compressor}}{% endif %} + packetbeat.protocols.http: ports: [{{ http_ports|default([80])|join(", ") }}] {% if http_send_request %} send_request: true{%- endif %} @@ -118,15 +127,6 @@ packetbeat.protocols.mongodb: {% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} {% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} -packetbeat.protocols.cassandra: - ports: [{{ cassandra_ports|default([9042])|join(", ") }}] -{% if cassandra_send_request %} send_request: true{%endif %} -{% if cassandra_send_response %} send_response: true{% endif %} -{% if cassandra_send_request_header %} send_request_header: true{% endif %} -{% if cassandra_send_response_header %} send_response_header: true{% endif %} -{% if cassandra_ignored_ops %} ignored_ops: {{cassandra_ignored_ops}}{% endif %} -{% if cassandra_compressor %} compressor: {{cassandra_compressor}}{% endif %} - {% if procs_enabled %} #=========================== Monitored processes ============================== From 615ce3a489f76da45d523ac6fad60c70e8f9e46d Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 16 Aug 2016 21:41:43 +0800 Subject: [PATCH 27/30] add sample output and kibana dashboard --- .../dashboard/Packetbeat-Cassandra.json | 19 +++ packetbeat/etc/kibana/search/Cassandra.json | 70 +++++++++ .../etc/kibana/visualization/Cassandra.json | 142 ++++++++++++++++++ packetbeat/etc/sample_outputs/cassandra.json | 49 ++++++ 4 files changed, 280 insertions(+) create mode 100644 packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json create mode 100644 packetbeat/etc/kibana/search/Cassandra.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra.json create mode 100644 packetbeat/etc/sample_outputs/cassandra.json diff --git a/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json b/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json new file mode 100644 index 00000000000..77e3843f7a7 --- /dev/null +++ b/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json @@ -0,0 +1,19 @@ +[ + { + "_id": "Cassandra", + "_type": "dashboard", + "_source": { + "title": "Cassandra", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":7,\"id\":\"Cassandra:-ResponseKeyspace\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-ResponseType\",\"panelIndex\":4,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Cassandra:-ErrType\",\"panelIndex\":6,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.err_type\",\"cassandra_response.err_msg\"],\"id\":\"Cassandra:-ErrorView\",\"panelIndex\":7,\"row\":8,\"size_x\":12,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.rows.meta.keyspace\",\"cassandra_response.rows.meta.table\",\"cassandra_response.rows.num_rows\"],\"id\":\"Cassandra:-QueryView\",\"panelIndex\":8,\"row\":12,\"size_x\":12,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"Cassandra:-ResponseTime\",\"panelIndex\":9,\"row\":3,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCount\",\"panelIndex\":10,\"row\":3,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-Ops\",\"panelIndex\":11,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"cassandra_response.type\",\"cassandra_response.change\",\"cassandra_response.keyspace\",\"cassandra_response.object\"],\"id\":\"Cassandra:-EventView\",\"panelIndex\":12,\"row\":10,\"size_x\":12,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"Cassandra:-ResponseCountByType\",\"panelIndex\":13,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountByType\",\"panelIndex\":14,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-13\":{\"vis\":{\"legendOpen\":false}},\"P-14\":{\"vis\":{\"legendOpen\":false}}}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } + } + } +] diff --git a/packetbeat/etc/kibana/search/Cassandra.json b/packetbeat/etc/kibana/search/Cassandra.json new file mode 100644 index 00000000000..7dbb6f5826b --- /dev/null +++ b/packetbeat/etc/kibana/search/Cassandra.json @@ -0,0 +1,70 @@ +[ + { + "_id": "Cassandra:-QueryView", + "_type": "search", + "_source": { + "title": "Cassandra: QueryView", + "description": "", + "hits": 0, + "columns": [ + "cassandra_request.query", + "cassandra_response.rows.meta.keyspace", + "cassandra_response.rows.meta.table", + "cassandra_response.rows.num_rows" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"cassandra_request.request_headers.op\",\"value\":\"QUERY\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_request.request_headers.op\":{\"query\":\"QUERY\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"negate\":true,\"index\":\"packetbeat-*\",\"key\":\"cassandra_response.response_headers.op\",\"value\":\"ERROR\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_response.response_headers.op\":{\"query\":\"ERROR\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + } + } + }, + { + "_id": "Cassandra:-ErrorView", + "_type": "search", + "_source": { + "title": "Cassandra: ErrorView", + "description": "", + "hits": 0, + "columns": [ + "cassandra_request.query", + "cassandra_response.err_type", + "cassandra_response.err_msg" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":true,\"index\":\"packetbeat-*\",\"key\":\"cassandra_request.request_headers.op\",\"negate\":false,\"value\":\"QUERY\"},\"query\":{\"match\":{\"cassandra_request.request_headers.op\":{\"query\":\"QUERY\",\"type\":\"phrase\"}}}},{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"type\",\"value\":\"cassandra\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"type\":{\"query\":\"cassandra\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"cassandra_response.response_headers.op\",\"value\":\"ERROR\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_response.response_headers.op\":{\"query\":\"ERROR\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + } + } + }, + { + "_id": "Cassandra:-EventView", + "_type": "search", + "_source": { + "title": "Cassandra: EventView", + "description": "", + "hits": 0, + "columns": [ + "cassandra_response.type", + "cassandra_response.change", + "cassandra_response.keyspace", + "cassandra_response.object" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"cassandra_response.response_headers.op\",\"value\":\"EVENT\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_response.response_headers.op\":{\"query\":\"EVENT\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + } + } + } +] diff --git a/packetbeat/etc/kibana/visualization/Cassandra.json b/packetbeat/etc/kibana/visualization/Cassandra.json new file mode 100644 index 00000000000..d781f9ea64f --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra.json @@ -0,0 +1,142 @@ +[ + { + "_id": "Cassandra:-ResponseKeyspace", + "_type": "visualization", + "_source": { + "title": "Cassandra: ResponseKeyspace", + "visState": "{\"title\":\"Cassandra: ResponseKeyspace\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.rows.meta.keyspace\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.rows.meta.table\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-ResponseOPs", + "_type": "visualization", + "_source": { + "title": "Cassandra: ResponseOPs", + "visState": "{\"title\":\"Cassandra: ResponseOPs\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-ResponseType", + "_type": "visualization", + "_source": { + "title": "Cassandra: ResponseType", + "visState": "{\"title\":\"Cassandra: ResponseType\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.result_type\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-RequestOps", + "_type": "visualization", + "_source": { + "title": "Cassandra: RequestOps", + "visState": "{\"title\":\"Cassandra: RequestOps\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-ErrType", + "_type": "visualization", + "_source": { + "title": "Cassandra: ErrType", + "visState": "{\"title\":\"Cassandra: ErrType\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.err_type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-ResponseTime", + "_type": "visualization", + "_source": { + "title": "Cassandra: ResponseTime", + "visState": "{\"title\":\"Cassandra: ResponseTime\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"square root\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"percentiles\",\"schema\":\"metric\",\"params\":{\"field\":\"responsetime\",\"percents\":[25,50,75,95,100]}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-RequestCount", + "_type": "visualization", + "_source": { + "title": "Cassandra: RequestCount", + "visState": "{\"title\":\"Cassandra: RequestCount\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"square root\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-Ops", + "_type": "visualization", + "_source": { + "title": "Cassandra: Ops", + "visState": "{\"title\":\"Cassandra: Ops\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-ResponseCountByType", + "_type": "visualization", + "_source": { + "title": "Cassandra: ResponseCountByType", + "visState": "{\"title\":\"Cassandra: ResponseCountByType\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"log\",\"drawLinesBetweenPoints\":false,\"radiusRatio\":\"15\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"count\",\"schema\":\"radius\",\"params\":{}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-RequestCountByType", + "_type": "visualization", + "_source": { + "title": "Cassandra: RequestCountByType", + "visState": "{\"title\":\"Cassandra: RequestCountByType\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"log\",\"drawLinesBetweenPoints\":false,\"radiusRatio\":\"13\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"count\",\"schema\":\"radius\",\"params\":{}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } + } + } +] diff --git a/packetbeat/etc/sample_outputs/cassandra.json b/packetbeat/etc/sample_outputs/cassandra.json new file mode 100644 index 00000000000..14537ef3852 --- /dev/null +++ b/packetbeat/etc/sample_outputs/cassandra.json @@ -0,0 +1,49 @@ +{ + "@timestamp": "2016-08-16T03:01:07.105Z", + "beat": { + "hostname": "Medcl.local", + "name": "Medcl.local" + }, + "bytes_in": 111, + "bytes_out": 198, + "cassandra_request": { + "query": "SELECT * FROM simplex.playlists WHERE id = 2cc9ccb7-6221-4ccb-8387-f22b6a1b354d;", + "request_headers": { + "flags": "Compress", + "length": 102, + "op": "QUERY", + "stream": 448, + "version": "4" + } + }, + "cassandra_response": { + "response_headers": { + "flags": "Compress", + "length": 189, + "op": "RESULT", + "stream": 448, + "version": "4" + }, + "result_type": "rows", + "rows": { + "meta": { + "col_count": 5, + "flags": "GlobalTableSpec", + "keyspace": "simplex", + "table": "playlists" + }, + "num_rows": 158052 + } + }, + "client_ip": "127.0.0.1", + "client_port": 50043, + "client_proc": "", + "client_server": "Medcl.local", + "ip": "127.0.0.1", + "port": 9042, + "proc": "", + "responsetime": 3, + "server": "Medcl.local", + "status": "OK", + "type": "cassandra" +} From 2675b388a3784cb83794c31e8a327cfc04b06efa Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 17 Aug 2016 14:01:46 +0800 Subject: [PATCH 28/30] update kibana dashboard --- .../dashboard/Packetbeat-Cassandra.json | 8 +-- .../etc/kibana/visualization/Cassandra.json | 56 +++++++++++++++++++ .../etc/kibana/visualization/Navigation.json | 26 +++++---- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json b/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json index 77e3843f7a7..678dd059e67 100644 --- a/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json +++ b/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json @@ -1,14 +1,14 @@ [ { - "_id": "Cassandra", + "_id": "Packetbeat-Cassandra", "_type": "dashboard", "_source": { - "title": "Cassandra", + "title": "Packetbeat Cassandra", "hits": 0, "description": "", - "panelsJSON": "[{\"col\":7,\"id\":\"Cassandra:-ResponseKeyspace\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-ResponseType\",\"panelIndex\":4,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Cassandra:-ErrType\",\"panelIndex\":6,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.err_type\",\"cassandra_response.err_msg\"],\"id\":\"Cassandra:-ErrorView\",\"panelIndex\":7,\"row\":8,\"size_x\":12,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.rows.meta.keyspace\",\"cassandra_response.rows.meta.table\",\"cassandra_response.rows.num_rows\"],\"id\":\"Cassandra:-QueryView\",\"panelIndex\":8,\"row\":12,\"size_x\":12,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"Cassandra:-ResponseTime\",\"panelIndex\":9,\"row\":3,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCount\",\"panelIndex\":10,\"row\":3,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-Ops\",\"panelIndex\":11,\"row\":1,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"cassandra_response.type\",\"cassandra_response.change\",\"cassandra_response.keyspace\",\"cassandra_response.object\"],\"id\":\"Cassandra:-EventView\",\"panelIndex\":12,\"row\":10,\"size_x\":12,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"Cassandra:-ResponseCountByType\",\"panelIndex\":13,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountByType\",\"panelIndex\":14,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":10,\"id\":\"Cassandra:-ResponseKeyspace\",\"panelIndex\":3,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cassandra:-ResponseType\",\"panelIndex\":4,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-ErrType\",\"panelIndex\":6,\"row\":7,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"cassandra_request.query\",\"cassandra_response.err_type\",\"cassandra_response.err_msg\"],\"id\":\"Cassandra:-ErrorView\",\"panelIndex\":7,\"row\":5,\"size_x\":9,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.rows.meta.keyspace\",\"cassandra_response.rows.meta.table\",\"cassandra_response.rows.num_rows\"],\"id\":\"Cassandra:-QueryView\",\"panelIndex\":8,\"row\":18,\"size_x\":12,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Cassandra:-ResponseTime\",\"panelIndex\":9,\"row\":22,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-RequestCount\",\"panelIndex\":10,\"row\":1,\"size_x\":9,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-Ops\",\"panelIndex\":11,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"cassandra_response.type\",\"cassandra_response.change\",\"cassandra_response.keyspace\",\"cassandra_response.object\"],\"id\":\"Cassandra:-EventView\",\"panelIndex\":12,\"row\":24,\"size_x\":9,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountStackByType\",\"panelIndex\":15,\"row\":9,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-ResponseCountStackByType\",\"panelIndex\":16,\"row\":11,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountByType\",\"panelIndex\":17,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cassandra:-ResponseCountByType\",\"panelIndex\":18,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Navigation\",\"panelIndex\":19,\"row\":1,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-EventType\",\"panelIndex\":20,\"row\":16,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-Flags\",\"panelIndex\":21,\"row\":5,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", - "uiStateJSON": "{\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-13\":{\"vis\":{\"legendOpen\":false}},\"P-14\":{\"vis\":{\"legendOpen\":false}}}", + "uiStateJSON": "{\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-17\":{\"vis\":{\"legendOpen\":false}},\"P-18\":{\"vis\":{\"legendOpen\":false}}}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { diff --git a/packetbeat/etc/kibana/visualization/Cassandra.json b/packetbeat/etc/kibana/visualization/Cassandra.json index d781f9ea64f..25a0b20f8fe 100644 --- a/packetbeat/etc/kibana/visualization/Cassandra.json +++ b/packetbeat/etc/kibana/visualization/Cassandra.json @@ -138,5 +138,61 @@ "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" } } + }, + { + "_id": "Cassandra:-ResponseCountStackByType", + "_type": "visualization", + "_source": { + "title": "Cassandra: ResponseCountStackByType", + "visState": "{\"title\":\"Cassandra: ResponseCountStackByType\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-RequestCountStackByType", + "_type": "visualization", + "_source": { + "title": "Cassandra: RequestCountStackByType", + "visState": "{\"title\":\"Cassandra: RequestCountStackByType\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-Flags", + "_type": "visualization", + "_source": { + "title": "Cassandra: Flags", + "visState": "{\"title\":\"Cassandra: Flags\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_request.request_headers.flags\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.response_headers.flags\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } + }, + { + "_id": "Cassandra:-EventType", + "_type": "visualization", + "_source": { + "title": "Cassandra: EventType", + "visState": "{\"title\":\"Cassandra: EventType\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.change\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } + } } ] diff --git a/packetbeat/etc/kibana/visualization/Navigation.json b/packetbeat/etc/kibana/visualization/Navigation.json index 309872d51c2..6ceaac298f7 100644 --- a/packetbeat/etc/kibana/visualization/Navigation.json +++ b/packetbeat/etc/kibana/visualization/Navigation.json @@ -1,10 +1,16 @@ -{ - "visState": "{\"title\":\"Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"###Packetbeat:\\n\\n[Overview](/#/dashboard/Packetbeat-Dashboard)\\n\\n[Web transactions](/#/dashboard/Packetbeat-HTTP)\\n\\n[MySQL performance](/#/dashboard/Packetbeat-MySQL-performance)\\n\\n[PostgreSQL performance](/#/dashboard/Packetbeat-PgSQL-performance)\\n\\n[MongoDB performance](/#/dashboard/Packetbeat-MongoDB-performance)\\n\\n[Thrift-RPC performance](/#/dashboard/Packetbeat-Thrift-performance)\\n\\n[NFS transactions](/#/dashboard/Packetbeat-NFS)\"},\"aggs\":[],\"listeners\":{}}", - "description": "", - "title": "Navigation", - "uiStateJSON": "{}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" - } -} \ No newline at end of file +[ + { + "_id": "Navigation", + "_type": "visualization", + "_source": { + "title": "Navigation", + "visState": "{\"title\":\"Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"###Packetbeat:\\n\\n[Overview](/#/dashboard/Packetbeat-Dashboard)\\n\\n[Web transactions](/#/dashboard/Packetbeat-HTTP)\\n\\n[MySQL performance](/#/dashboard/Packetbeat-MySQL-performance)\\n\\n[PostgreSQL performance](/#/dashboard/Packetbeat-PgSQL-performance)\\n\\n[MongoDB performance](/#/dashboard/Packetbeat-MongoDB-performance)\\n\\n[Thrift-RPC performance](/#/dashboard/Packetbeat-Thrift-performance)\\n\\n[NFS transactions](/#/dashboard/Packetbeat-NFS)\\n\\n[Cassandra performance](/#/dashboard/Packetbeat-Cassandra)\"},\"aggs\":[],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } + } + } +] From ccce922154dd0b1cd7ecb31535f6c566b86d81d3 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 17 Aug 2016 18:24:01 +0800 Subject: [PATCH 29/30] update changelog --- CHANGELOG.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7dbfa468759..490ed17fe75 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -79,6 +79,8 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha5...master[Check the HEAD d *Packetbeat* + - Add cassandra protocol analyzer to packetbeat. {pull}1959[1959] + *Topbeat* *Filebeat* From d5f9fbd4f834e3a41f1eb6a2a212939c239f8ac5 Mon Sep 17 00:00:00 2001 From: medcl Date: Wed, 17 Aug 2016 21:18:41 +0800 Subject: [PATCH 30/30] update dashboard --- .../dashboard/Packetbeat-Cassandra.json | 32 ++++++++----------- .../etc/kibana/index-pattern/packetbeat.json | 3 +- .../kibana/search/Cassandra:-ErrorView.json | 18 +++++++++++ .../kibana/search/Cassandra:-EventView.json | 19 +++++++++++ .../kibana/search/Cassandra:-QueryView.json | 19 +++++++++++ .../visualization/Cassandra:-ErrType.json | 10 ++++++ .../visualization/Cassandra:-EventType.json | 10 ++++++ .../visualization/Cassandra:-Flags.json | 10 ++++++ .../kibana/visualization/Cassandra:-Ops.json | 10 ++++++ .../Cassandra:-RequestCount.json | 10 ++++++ .../Cassandra:-RequestCountByType.json | 10 ++++++ .../Cassandra:-RequestCountStackByType.json | 10 ++++++ .../Cassandra:-ResponseCountByType.json | 10 ++++++ .../Cassandra:-ResponseCountStackByType.json | 10 ++++++ .../Cassandra:-ResponseKeyspace.json | 10 ++++++ .../Cassandra:-ResponseTime.json | 10 ++++++ .../Cassandra:-ResponseType.json | 10 ++++++ 17 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 packetbeat/etc/kibana/search/Cassandra:-ErrorView.json create mode 100644 packetbeat/etc/kibana/search/Cassandra:-EventView.json create mode 100644 packetbeat/etc/kibana/search/Cassandra:-QueryView.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-ErrType.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-EventType.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-Flags.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-Ops.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-RequestCount.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-RequestCountByType.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-RequestCountStackByType.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountByType.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountStackByType.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-ResponseKeyspace.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-ResponseTime.json create mode 100644 packetbeat/etc/kibana/visualization/Cassandra:-ResponseType.json diff --git a/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json b/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json index 678dd059e67..b9007ebb354 100644 --- a/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json +++ b/packetbeat/etc/kibana/dashboard/Packetbeat-Cassandra.json @@ -1,19 +1,13 @@ -[ - { - "_id": "Packetbeat-Cassandra", - "_type": "dashboard", - "_source": { - "title": "Packetbeat Cassandra", - "hits": 0, - "description": "", - "panelsJSON": "[{\"col\":10,\"id\":\"Cassandra:-ResponseKeyspace\",\"panelIndex\":3,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cassandra:-ResponseType\",\"panelIndex\":4,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-ErrType\",\"panelIndex\":6,\"row\":7,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"cassandra_request.query\",\"cassandra_response.err_type\",\"cassandra_response.err_msg\"],\"id\":\"Cassandra:-ErrorView\",\"panelIndex\":7,\"row\":5,\"size_x\":9,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.rows.meta.keyspace\",\"cassandra_response.rows.meta.table\",\"cassandra_response.rows.num_rows\"],\"id\":\"Cassandra:-QueryView\",\"panelIndex\":8,\"row\":18,\"size_x\":12,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Cassandra:-ResponseTime\",\"panelIndex\":9,\"row\":22,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-RequestCount\",\"panelIndex\":10,\"row\":1,\"size_x\":9,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-Ops\",\"panelIndex\":11,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"cassandra_response.type\",\"cassandra_response.change\",\"cassandra_response.keyspace\",\"cassandra_response.object\"],\"id\":\"Cassandra:-EventView\",\"panelIndex\":12,\"row\":24,\"size_x\":9,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountStackByType\",\"panelIndex\":15,\"row\":9,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-ResponseCountStackByType\",\"panelIndex\":16,\"row\":11,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountByType\",\"panelIndex\":17,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cassandra:-ResponseCountByType\",\"panelIndex\":18,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Navigation\",\"panelIndex\":19,\"row\":1,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-EventType\",\"panelIndex\":20,\"row\":16,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-Flags\",\"panelIndex\":21,\"row\":5,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"}]", - "optionsJSON": "{\"darkTheme\":false}", - "uiStateJSON": "{\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-17\":{\"vis\":{\"legendOpen\":false}},\"P-18\":{\"vis\":{\"legendOpen\":false}}}", - "version": 1, - "timeRestore": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" - } - } - } -] +{ + "hits": 0, + "timeRestore": false, + "description": "", + "title": "Packetbeat Cassandra", + "uiStateJSON": "{\"P-10\":{\"vis\":{\"legendOpen\":false}},\"P-17\":{\"vis\":{\"legendOpen\":false}},\"P-18\":{\"vis\":{\"legendOpen\":false}}}", + "panelsJSON": "[{\"col\":10,\"id\":\"Cassandra:-ResponseKeyspace\",\"panelIndex\":3,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cassandra:-ResponseType\",\"panelIndex\":4,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-ErrType\",\"panelIndex\":6,\"row\":7,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"cassandra_request.query\",\"cassandra_response.err_type\",\"cassandra_response.err_msg\"],\"id\":\"Cassandra:-ErrorView\",\"panelIndex\":7,\"row\":5,\"size_x\":9,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"columns\":[\"cassandra_request.query\",\"cassandra_response.rows.meta.keyspace\",\"cassandra_response.rows.meta.table\",\"cassandra_response.rows.num_rows\"],\"id\":\"Cassandra:-QueryView\",\"panelIndex\":8,\"row\":18,\"size_x\":12,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Cassandra:-ResponseTime\",\"panelIndex\":9,\"row\":22,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-RequestCount\",\"panelIndex\":10,\"row\":1,\"size_x\":9,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Cassandra:-Ops\",\"panelIndex\":11,\"row\":3,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":4,\"columns\":[\"cassandra_response.type\",\"cassandra_response.change\",\"cassandra_response.keyspace\",\"cassandra_response.object\"],\"id\":\"Cassandra:-EventView\",\"panelIndex\":12,\"row\":24,\"size_x\":9,\"size_y\":2,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountStackByType\",\"panelIndex\":15,\"row\":9,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-ResponseCountStackByType\",\"panelIndex\":16,\"row\":11,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-RequestCountByType\",\"panelIndex\":17,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Cassandra:-ResponseCountByType\",\"panelIndex\":18,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Navigation\",\"panelIndex\":19,\"row\":1,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-EventType\",\"panelIndex\":20,\"row\":16,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Cassandra:-Flags\",\"panelIndex\":21,\"row\":5,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"}]", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/index-pattern/packetbeat.json b/packetbeat/etc/kibana/index-pattern/packetbeat.json index c78835d0999..63ab4c4ccc6 100644 --- a/packetbeat/etc/kibana/index-pattern/packetbeat.json +++ b/packetbeat/etc/kibana/index-pattern/packetbeat.json @@ -1,6 +1,5 @@ { - "fields": "[{\"count\": 0, \"name\": \"beat.name\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"beat.hostname\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"@timestamp\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"tags\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"fields\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"server\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"client_server\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"service\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"client_service\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"client_ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"real_ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"client_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"client_port\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"transport\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"port\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"proc\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"client_proc\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"release\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"@timestamp\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"start_time\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"last_time\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"final\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"flow_id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"vlan\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"outer_vlan\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.mac\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.ip_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"source.outer_ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.outer_ip_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"source.ipv6\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.ipv6_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"source.outer_ipv6\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.outer_ipv6_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"source.port\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.stats.net_packets_total\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"source.stats.net_bytes_total\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.mac\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.ip_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dest.outer_ip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.outer_ip_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dest.ipv6\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.ipv6_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dest.outer_ipv6\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.outer_ipv6_location\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dest.port\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.stats.net_packets_total\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dest.stats.net_bytes_total\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp_id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"transport\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"connection_id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"@timestamp\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"direction\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"status\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"method\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"resource\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"path\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"query\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"params\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"notes\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.version\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.request.message\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.request.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.request.code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.response.message\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.response.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"icmp.response.code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.op_code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.flags.authoritative\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.flags.recursion_available\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.flags.recursion_desired\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.flags.authentic_data\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.flags.checking_disabled\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.flags.truncated_response\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.response_code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.question.name\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.question.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.question.class\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.question.etld_plus_one\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers_count\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.name\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.class\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.ttl\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.data\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.authorities\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.authorities_count\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.authorities.name\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.authorities.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.authorities.class\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.ttl\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.answers.data\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals_count\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals.name\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals.class\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals.ttl\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.additionals.data\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.opt.version\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.opt.do\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"dns.opt.ext_rcode\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"dns.opt.udp_size\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.reply-code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.reply-text\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.class-id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.method-id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.exchange\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.exchange-type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.passive\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.durable\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.exclusive\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.auto-delete\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.no-wait\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.consumer-tag\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.delivery-tag\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.message-count\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.consumer-count\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.routing-key\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.no-ack\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.no-local\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.if-unused\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.if-empty\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.queue\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.redelivered\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.multiple\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.arguments\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.mandatory\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.immediate\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.content-type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.content-encoding\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.headers\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.delivery-mode\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.priority\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.correlation-id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.reply-to\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.expiration\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.message-id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.timestamp\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.user-id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"amqp.app-id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"http.code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"http.phrase\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"http.request_headers\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"http.response_headers\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"http.content_length\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.protocol_type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.line\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.command\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.command\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.type\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.error_msg\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.opcode\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.opcode\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.opcode_value\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.opcode_value\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.opaque\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.opaque\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.vbucket\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.status\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.status_code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.keys\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.keys\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.count_values\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.count_values\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.values\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.values\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.bytes\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.bytes\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.delta\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.initial\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.verbosity\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.raw_args\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.source_class\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.dest_class\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.automove\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.flags\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.flags\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.exptime\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.sleep_us\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.value\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.noreply\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.quiet\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.request.cas_unique\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.cas_unique\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.stats\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"memcache.response.version\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.iserror\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.affected_rows\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.insert_id\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.num_fields\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.num_rows\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.query\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.error_code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"mysql.error_message\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.query\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.iserror\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.error_code\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.error_message\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.error_severity\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.num_fields\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"pgsql.num_rows\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"thrift.params\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"thrift.service\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"thrift.return_value\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"thrift.exceptions\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"redis.return_value\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"redis.error\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.error\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.fullCollectionName\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.numberToSkip\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.numberToReturn\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.numberReturned\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.startingFrom\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.query\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.returnFieldsSelector\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.selector\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.update\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"mongodb.cursorId\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.xid\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.call_size\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.reply_size\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.status\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.time\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.time_str\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.auth_flavor\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.cred.uid\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.cred.gid\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.cred.gids\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.cred.stamp\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"rpc.cred.machinename\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"nfs.version\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"nfs.minor_version\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"nfs.tag\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"nfs.opcode\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"nfs.status\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"request\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"response\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"name\": \"responsetime\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"cpu_time\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"bytes_in\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"bytes_out\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"dnstime\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"connecttime\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"loadtime\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"name\": \"domloadtime\", \"analyzed\": false, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}]", - "fieldFormatMap": "{\"client_ip\": {\"id\": \"dotted notation.\"}, \"memcache.request.bytes\": {\"id\": \"bytes\"}, \"start_time\": {\"id\": \"YYYY-MM-DDTHH:MM:SS.milliZ\"}, \"real_ip\": {\"id\": \"Dotted notation.\"}, \"ip\": {\"id\": \"dotted notation.\"}, \"@timestamp\": {\"id\": \"YYYY-MM-DDTHH:MM:SS.milliZ\"}, \"bytes_out\": {\"id\": \"bytes\"}, \"last_time\": {\"id\": \"YYYY-MM-DDTHH:MM:SS.milliZ\"}, \"client_port\": {\"id\": \"dotted notation.\"}, \"memcache.response.bytes\": {\"id\": \"bytes\"}, \"bytes_in\": {\"id\": \"bytes\"}, \"port\": {\"id\": \"dotted notation.\"}}", + "fields": "[{\"name\":\"thrift.exceptions\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.exptime\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.if-unused\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.answers.data\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.cas_unique\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.reply-text\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.dest_class\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"nfs.version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.opcode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.iserror\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.ipv6_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"thrift.service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.error\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.method-id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.correlation-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.additionals.class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"beat.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.additionals.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.port\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.stats.net_packets_total\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.response.code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.class-id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.question.etld_plus_one\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.opt.udp_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.immediate\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.flags.recursion_desired\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.error_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.request.type\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.message-count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.authorities_count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.cred.stamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.count_values\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"connecttime\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.outer_ip_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.question.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.cursorId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.cred.gids\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"http.code\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.flags\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.opaque\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.sleep_us\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.delivery-tag\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.iserror\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.value\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.error_severity\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"notes\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.error_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.no-local\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.vbucket\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.verbosity\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.flags.authoritative\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.additionals_count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"client_ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.stats.net_bytes_total\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.answers.class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.quiet\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"loadtime\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"nfs.status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.line\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.if-empty\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.numberReturned\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.response.message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"start_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.ipv6_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.authorities.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.consumer-tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.flags\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"port\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"final\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.redelivered\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.num_rows\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.delivery-mode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.opcode_value\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.protocol_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.cred.machinename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.passive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.initial\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.additionals.ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.priority\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"last_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.automove\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.opt.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.outer_ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.port\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cpu_time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.mandatory\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.startingFrom\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"client_proc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"beat.hostname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.affected_rows\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.timestamp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.request.code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.auto-delete\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"nfs.opcode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"http.phrase\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.additionals.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.user-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"real_ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.answers_count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"release\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.routing-key\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.outer_ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"nfs.minor_version\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.flags.authentic_data\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.ipv6\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.content-encoding\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.num_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.call_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.exchange\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"method\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.xid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.command\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.auth_flavor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"params\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"amqp.exchange-type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.answers.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.returnFieldsSelector\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"redis.return_value\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.cas_unique\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.request.message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.outer_ipv6_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.error_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.question.class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.opcode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.opcode_value\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"redis.error\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"outer_vlan\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"vlan\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"responsetime\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dnstime\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.outer_ipv6_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.insert_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"proc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.cred.gid\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.outer_ip_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"http.content_length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.content-type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.no-ack\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.opt.do\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.outer_ipv6\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.delta\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp.response.type\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.stats.net_bytes_total\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.num_rows\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"client_server\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.opaque\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.consumer-count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.status_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"connection_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.update\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.stats.net_packets_total\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.response_code\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.error_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.opt.ext_rcode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.expiration\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"nfs.tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.mac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.numberToReturn\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.noreply\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mysql.error_message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"thrift.params\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.reply-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.answers.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"domloadtime\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"flow_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.flags.checking_disabled\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dest.ip_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.additionals.data\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.answers.ttl\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.raw_args\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.response.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.app-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"resource\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.fullCollectionName\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.time_str\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.source_class\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pgsql.num_fields\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.queue\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.op_code\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.durable\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.flags.recursion_available\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes_out\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.outer_ipv6\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.exclusive\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.numberToSkip\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"bytes_in\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"client_service\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.reply-code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"icmp_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"memcache.request.count_values\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"client_port\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.no-wait\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"mongodb.selector\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"client_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.question.name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"direction\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.authorities.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.ipv6\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"transport\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"source.ip_location\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"dns.flags.truncated_response\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.cred.uid\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"amqp.multiple\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"dns.authorities.class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"rpc.reply_size\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"thrift.return_value\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.err_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.data_present\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.keyspace\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.response_headers.flags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.rows.meta.keyspace\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_request.request_headers.flags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.supported.CQL_VERSION\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.supported.COMPRESSION\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.keyspace\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.rows.num_rows\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_request.request_headers.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.err_code\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_request.request_headers.op\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.read_consistency\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.response_headers.version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"no_request\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_request.request_headers.length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.blockfor\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.response_headers.op\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_request.request_headers.stream\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.rows.meta.table\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.rows.meta.flags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.object\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.object\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.response_headers.length\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.received\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.response_headers.stream\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.err_msg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.change\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.change\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_request.query\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.rows.meta.col_count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.resp_meta.col_count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.req_meta.flags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.req_meta.table\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.req_meta.keyspace\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.req_meta.col_count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.prepared_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.resp_meta.flags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.req_meta.pkey_columns\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"cassandra_response.result.resp_meta.paging_state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]", "timeFieldName": "@timestamp", "title": "packetbeat-*" } \ No newline at end of file diff --git a/packetbeat/etc/kibana/search/Cassandra:-ErrorView.json b/packetbeat/etc/kibana/search/Cassandra:-ErrorView.json new file mode 100644 index 00000000000..0a9941bfe7e --- /dev/null +++ b/packetbeat/etc/kibana/search/Cassandra:-ErrorView.json @@ -0,0 +1,18 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Cassandra: ErrorView", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":true,\"index\":\"packetbeat-*\",\"key\":\"cassandra_request.request_headers.op\",\"negate\":false,\"value\":\"QUERY\"},\"query\":{\"match\":{\"cassandra_request.request_headers.op\":{\"query\":\"QUERY\",\"type\":\"phrase\"}}}},{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"type\",\"value\":\"cassandra\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"type\":{\"query\":\"cassandra\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"cassandra_response.response_headers.op\",\"value\":\"ERROR\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_response.response_headers.op\":{\"query\":\"ERROR\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "cassandra_request.query", + "cassandra_response.err_type", + "cassandra_response.err_msg" + ] +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/search/Cassandra:-EventView.json b/packetbeat/etc/kibana/search/Cassandra:-EventView.json new file mode 100644 index 00000000000..dfcc66b65b7 --- /dev/null +++ b/packetbeat/etc/kibana/search/Cassandra:-EventView.json @@ -0,0 +1,19 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Cassandra: EventView", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"cassandra_response.response_headers.op\",\"value\":\"EVENT\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_response.response_headers.op\":{\"query\":\"EVENT\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "cassandra_response.type", + "cassandra_response.change", + "cassandra_response.keyspace", + "cassandra_response.object" + ] +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/search/Cassandra:-QueryView.json b/packetbeat/etc/kibana/search/Cassandra:-QueryView.json new file mode 100644 index 00000000000..fa24fcd11ef --- /dev/null +++ b/packetbeat/etc/kibana/search/Cassandra:-QueryView.json @@ -0,0 +1,19 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Cassandra: QueryView", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"packetbeat-*\",\"key\":\"cassandra_request.request_headers.op\",\"value\":\"QUERY\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_request.request_headers.op\":{\"query\":\"QUERY\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}},{\"meta\":{\"negate\":true,\"index\":\"packetbeat-*\",\"key\":\"cassandra_response.response_headers.op\",\"value\":\"ERROR\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"cassandra_response.response_headers.op\":{\"query\":\"ERROR\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" + }, + "columns": [ + "cassandra_request.query", + "cassandra_response.rows.meta.keyspace", + "cassandra_response.rows.meta.table", + "cassandra_response.rows.num_rows" + ] +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-ErrType.json b/packetbeat/etc/kibana/visualization/Cassandra:-ErrType.json new file mode 100644 index 00000000000..4de4034bef8 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-ErrType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: ErrType\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.err_type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: ErrType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-EventType.json b/packetbeat/etc/kibana/visualization/Cassandra:-EventType.json new file mode 100644 index 00000000000..6088cd79069 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-EventType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: EventType\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.change\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: EventType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-Flags.json b/packetbeat/etc/kibana/visualization/Cassandra:-Flags.json new file mode 100644 index 00000000000..98c96124ab9 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-Flags.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: Flags\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_request.request_headers.flags\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.response_headers.flags\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: Flags", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-Ops.json b/packetbeat/etc/kibana/visualization/Cassandra:-Ops.json new file mode 100644 index 00000000000..4f1faa9bcfa --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-Ops.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: Ops\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: Ops", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-RequestCount.json b/packetbeat/etc/kibana/visualization/Cassandra:-RequestCount.json new file mode 100644 index 00000000000..651a3d170d8 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-RequestCount.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: RequestCount\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"square root\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: RequestCount", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-RequestCountByType.json b/packetbeat/etc/kibana/visualization/Cassandra:-RequestCountByType.json new file mode 100644 index 00000000000..4b9d48a688a --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-RequestCountByType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: RequestCountByType\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"log\",\"drawLinesBetweenPoints\":false,\"radiusRatio\":\"13\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"count\",\"schema\":\"radius\",\"params\":{}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: RequestCountByType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-RequestCountStackByType.json b/packetbeat/etc/kibana/visualization/Cassandra:-RequestCountStackByType.json new file mode 100644 index 00000000000..5eb31ef0076 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-RequestCountStackByType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: RequestCountStackByType\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_request.request_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: RequestCountStackByType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountByType.json b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountByType.json new file mode 100644 index 00000000000..83cad64bbfe --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountByType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: ResponseCountByType\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"log\",\"drawLinesBetweenPoints\":false,\"radiusRatio\":\"15\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"count\",\"schema\":\"radius\",\"params\":{}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: ResponseCountByType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountStackByType.json b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountStackByType.json new file mode 100644 index 00000000000..bd0465d31a1 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseCountStackByType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: ResponseCountStackByType\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cassandra_response.response_headers.op\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: ResponseCountStackByType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-ResponseKeyspace.json b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseKeyspace.json new file mode 100644 index 00000000000..6cf91b85bb5 --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseKeyspace.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: ResponseKeyspace\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.rows.meta.keyspace\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.rows.meta.table\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: ResponseKeyspace", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-ResponseTime.json b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseTime.json new file mode 100644 index 00000000000..a87b17f867f --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseTime.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: ResponseTime\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"square root\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"percentiles\",\"schema\":\"metric\",\"params\":{\"field\":\"responsetime\",\"percents\":[25,50,75,95,100]}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: ResponseTime", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/packetbeat/etc/kibana/visualization/Cassandra:-ResponseType.json b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseType.json new file mode 100644 index 00000000000..257ad52b50c --- /dev/null +++ b/packetbeat/etc/kibana/visualization/Cassandra:-ResponseType.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Cassandra: ResponseType\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"cassandra_response.result_type\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "description": "", + "title": "Cassandra: ResponseType", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"packetbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file