diff --git a/x-pack/filebeat/input/azureblobstorage/config.go b/x-pack/filebeat/input/azureblobstorage/config.go index a780e60216d2..7aff4111941c 100644 --- a/x-pack/filebeat/input/azureblobstorage/config.go +++ b/x-pack/filebeat/input/azureblobstorage/config.go @@ -8,6 +8,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/reader/parser" ) // MaxWorkers, Poll, PollInterval, FileSelectors, TimeStampEpoch & ExpandEventListFromField can @@ -22,6 +23,7 @@ type config struct { PollInterval *time.Duration `config:"poll_interval"` Containers []container `config:"containers" validate:"required"` FileSelectors []fileSelectorConfig `config:"file_selectors"` + ReaderConfig readerConfig `config:",inline"` TimeStampEpoch *int64 `config:"timestamp_epoch"` ExpandEventListFromField string `config:"expand_event_list_from_field"` } @@ -33,6 +35,7 @@ type container struct { Poll *bool `config:"poll"` PollInterval *time.Duration `config:"poll_interval"` FileSelectors []fileSelectorConfig `config:"file_selectors"` + ReaderConfig readerConfig `config:",inline"` TimeStampEpoch *int64 `config:"timestamp_epoch"` ExpandEventListFromField string `config:"expand_event_list_from_field"` } @@ -43,6 +46,12 @@ type fileSelectorConfig struct { // TODO: Add support for reader config in future } +// readerConfig defines the options for reading the content of an azure container. +type readerConfig struct { + Parsers parser.Config `config:",inline"` + Decoding decoderConfig `config:"decoding"` +} + type authConfig struct { SharedCredentials *sharedKeyConfig `config:"shared_credentials,omitempty"` ConnectionString *connectionStringConfig `config:"connection_string,omitempty"` diff --git a/x-pack/filebeat/input/azureblobstorage/decoding.go b/x-pack/filebeat/input/azureblobstorage/decoding.go new file mode 100644 index 000000000000..77f0a1e8c2c7 --- /dev/null +++ b/x-pack/filebeat/input/azureblobstorage/decoding.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azureblobstorage + +import ( + "fmt" + "io" +) + +// decoder is an interface for decoding data from an io.Reader. +type decoder interface { + // decode reads and decodes data from an io reader based on the codec type. + // It returns the decoded data and an error if the data cannot be decoded. + decode() ([]byte, error) + // next advances the decoder to the next data item and returns true if there is more data to be decoded. + next() bool + // close closes the decoder and releases any resources associated with it. + // It returns an error if the decoder cannot be closed. + + // more returns whether there are more records to read. + more() bool + + close() error +} + +// valueDecoder is a decoder that can decode directly to a JSON serialisable value. +type valueDecoder interface { + decoder + + decodeValue() ([]byte, map[string]any, error) +} + +// newDecoder creates a new decoder based on the codec type. +// It returns a decoder type and an error if the codec type is not supported. +// If the reader config codec option is not set, it returns a nil decoder and nil error. +func newDecoder(cfg decoderConfig, r io.Reader) (decoder, error) { + switch { + case cfg.Codec == nil: + return nil, nil + case cfg.Codec.CSV != nil: + return newCSVDecoder(cfg, r) + default: + return nil, fmt.Errorf("unsupported config value: %v", cfg) + } +} diff --git a/x-pack/filebeat/input/azureblobstorage/decoding_config.go b/x-pack/filebeat/input/azureblobstorage/decoding_config.go new file mode 100644 index 000000000000..3f680873bf78 --- /dev/null +++ b/x-pack/filebeat/input/azureblobstorage/decoding_config.go @@ -0,0 +1,54 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azureblobstorage + +import ( + "fmt" + "unicode/utf8" +) + +// decoderConfig contains the configuration options for instantiating a decoder. +type decoderConfig struct { + Codec *codecConfig `config:"codec"` +} + +// codecConfig contains the configuration options for different codecs used by a decoder. +type codecConfig struct { + CSV *csvCodecConfig `config:"csv"` +} + +// csvCodecConfig contains the configuration options for the CSV codec. +type csvCodecConfig struct { + Enabled bool `config:"enabled"` + + // Fields is the set of field names. If it is present + // it is used to specify the object names of returned + // values and the FieldsPerRecord field in the csv.Reader. + // Otherwise, names are obtained from the first + // line of the CSV data. + Fields []string `config:"fields_names"` + + // The fields below have the same meaning as the + // fields of the same name in csv.Reader. + Comma *configRune `config:"comma"` + Comment configRune `config:"comment"` + LazyQuotes bool `config:"lazy_quotes"` + TrimLeadingSpace bool `config:"trim_leading_space"` +} + +type configRune rune + +func (r *configRune) Unpack(s string) error { + if s == "" { + return nil + } + n := utf8.RuneCountInString(s) + if n != 1 { + return fmt.Errorf("single character option given more than one character: %q", s) + } + _r, _ := utf8.DecodeRuneInString(s) + *r = configRune(_r) + return nil +} diff --git a/x-pack/filebeat/input/azureblobstorage/decoding_csv.go b/x-pack/filebeat/input/azureblobstorage/decoding_csv.go new file mode 100644 index 000000000000..7409e96f1d55 --- /dev/null +++ b/x-pack/filebeat/input/azureblobstorage/decoding_csv.go @@ -0,0 +1,139 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azureblobstorage + +import ( + "bytes" + "encoding/csv" + "fmt" + "io" + "slices" +) + +// csvDecoder is a decoder for CSV data. +type csvDecoder struct { + r *csv.Reader + + header []string + current []string + coming []string + + err error +} + +// newParquetDecoder creates a new CSV decoder. +func newCSVDecoder(config decoderConfig, r io.Reader) (decoder, error) { + d := csvDecoder{r: csv.NewReader(r)} + d.r.ReuseRecord = true + if config.Codec.CSV.Comma != nil { + d.r.Comma = rune(*config.Codec.CSV.Comma) + } + d.r.Comment = rune(config.Codec.CSV.Comment) + d.r.LazyQuotes = config.Codec.CSV.LazyQuotes + d.r.TrimLeadingSpace = config.Codec.CSV.TrimLeadingSpace + if len(config.Codec.CSV.Fields) != 0 { + d.r.FieldsPerRecord = len(config.Codec.CSV.Fields) + d.header = config.Codec.CSV.Fields + } else { + h, err := d.r.Read() + if err != nil { + return nil, err + } + d.header = slices.Clone(h) + } + var err error + d.coming, err = d.r.Read() + if err != nil { + return nil, err + } + d.current = make([]string, 0, len(d.header)) + return &d, nil +} + +func (d *csvDecoder) more() bool { return len(d.coming) == len(d.header) } + +// next advances the decoder to the next data item and returns true if +// there is more data to be decoded. +func (d *csvDecoder) next() bool { + if !d.more() && d.err != nil { + return false + } + d.current = d.current[:len(d.header)] + copy(d.current, d.coming) + d.coming, d.err = d.r.Read() + if d.err == io.EOF { + d.coming = nil + } + return true +} + +// decode returns the JSON encoded value of the current CSV line. next must +// have been called before any calls to decode. +func (d *csvDecoder) decode() ([]byte, error) { + err := d.check() + if err != nil { + return nil, err + } + var buf bytes.Buffer + buf.WriteByte('{') + for i, n := range d.header { + if i != 0 { + buf.WriteByte(',') + } + buf.WriteByte('"') + buf.WriteString(n) + buf.WriteString(`":"`) + buf.WriteString(d.current[i]) + buf.WriteByte('"') + } + buf.WriteByte('}') + d.current = d.current[:0] + return buf.Bytes(), nil +} + +// decodeValue returns the value of the current CSV line interpreted as +// an object with fields based on the header held by the receiver. next must +// have been called before any calls to decode. +func (d *csvDecoder) decodeValue() ([]byte, map[string]any, error) { + err := d.check() + if err != nil { + return nil, nil, err + } + m := make(map[string]any, len(d.header)) + for i, n := range d.header { + m[n] = d.current[i] + } + d.current = d.current[:0] + b, err := d.decode() + if err != nil { + return nil, nil, err + } + return b, m, nil +} + +func (d *csvDecoder) check() error { + if d.err != nil { + if d.err == io.EOF && d.coming == nil { + return nil + } + return d.err + } + if len(d.current) == 0 { + return fmt.Errorf("decode called before next") + } + // By the time we are here, current must be the same + // length as header; if it was not read, it would be + // zero, but if it was, it must match by the contract + // of the csv.Reader. + return nil +} + +// close closes the parquet decoder and releases the resources. +func (d *csvDecoder) close() error { + if d.err == io.EOF { + return nil + } + return d.err +} diff --git a/x-pack/filebeat/input/azureblobstorage/decoding_test.go b/x-pack/filebeat/input/azureblobstorage/decoding_test.go new file mode 100644 index 000000000000..e541fc742796 --- /dev/null +++ b/x-pack/filebeat/input/azureblobstorage/decoding_test.go @@ -0,0 +1,237 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azureblobstorage + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + azcontainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +// all test files are read from the "testdata" directory +const testDataPath = "testdata" + +func TestDecoding(t *testing.T) { + logp.TestingSetup() + log := logp.L() + + testCases := []struct { + name string + file string + content string + contentType string + numEvents int + assertAgainst string + config decoderConfig + }{ + { + name: "gzip_csv", + file: "txn.csv.gz", + content: "text/csv", + numEvents: 4, + assertAgainst: "txn.json", + config: decoderConfig{ + Codec: &codecConfig{ + CSV: &csvCodecConfig{ + Enabled: true, + Comma: ptr[configRune](' '), + }, + }, + }, + }, + { + name: "csv", + file: "txn.csv", + content: "text/csv", + numEvents: 4, + assertAgainst: "txn.json", + config: decoderConfig{ + Codec: &codecConfig{ + CSV: &csvCodecConfig{ + Enabled: true, + Comma: ptr[configRune](' '), + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + file := filepath.Join(testDataPath, tc.file) + if tc.contentType == "" { + tc.contentType = "application/octet-stream" + } + f, err := os.Open(file) + if err != nil { + t.Fatalf("failed to open test data: %v", err) + } + defer f.Close() + p := &pub{t: t} + item := &azcontainer.BlobItem{ + Name: ptr("test_blob"), + Properties: &azcontainer.BlobProperties{ + ContentType: ptr(tc.content), + LastModified: &time.Time{}, + }, + } + j := newJob(&blob.Client{}, item, "https://foo.blob.core.windows.net/", newState(), &Source{}, p, log) + j.src.ReaderConfig.Decoding = tc.config + err = j.decode(context.Background(), f, "test") + if err != nil { + t.Errorf("unexpected error calling decode: %v", err) + } + + events := p.events + if tc.assertAgainst != "" { + targetData := readJSONFromFile(t, filepath.Join(testDataPath, tc.assertAgainst)) + assert.Equal(t, len(targetData), len(events)) + + for i, event := range events { + msg, err := event.Fields.GetValue("message") + assert.NoError(t, err) + assert.JSONEq(t, targetData[i], msg.(string)) + } + } + }) + } +} + +type pub struct { + t *testing.T + events []beat.Event +} + +func (p *pub) Publish(e beat.Event, _cursor interface{}) error { + p.t.Logf("%v\n", e.Fields) + p.events = append(p.events, e) + return nil +} + +// readJSONFromFile reads the json file and returns the data as a slice of strings +func readJSONFromFile(t *testing.T, filepath string) []string { + fileBytes, err := os.ReadFile(filepath) + assert.NoError(t, err) + var rawMessages []json.RawMessage + err = json.Unmarshal(fileBytes, &rawMessages) + assert.NoError(t, err) + var data []string + + for _, rawMsg := range rawMessages { + data = append(data, string(rawMsg)) + } + return data +} + +var codecConfigTests = []struct { + name string + yaml string + want decoderConfig + wantErr error +}{ + { + name: "handle_rune", + yaml: ` +codec: + csv: + enabled: true + comma: ' ' + comment: '#' +`, + want: decoderConfig{&codecConfig{ + CSV: &csvCodecConfig{ + Enabled: true, + Comma: ptr[configRune](' '), + Comment: '#', + }, + }}, + }, + { + name: "no_comma", + yaml: ` +codec: + csv: + enabled: true +`, + want: decoderConfig{&codecConfig{ + CSV: &csvCodecConfig{ + Enabled: true, + }, + }}, + }, + { + name: "null_comma", + yaml: ` +codec: + csv: + enabled: true + comma: "\u0000" +`, + want: decoderConfig{&codecConfig{ + CSV: &csvCodecConfig{ + Enabled: true, + Comma: ptr[configRune]('\x00'), + }, + }}, + }, + { + name: "bad_rune", + yaml: ` +codec: + csv: + enabled: true + comma: 'this is too long' +`, + wantErr: errors.New(`single character option given more than one character: "this is too long" accessing 'codec.csv.comma'`), + }, +} + +func TestCodecConfig(t *testing.T) { + for _, test := range codecConfigTests { + t.Run(test.name, func(t *testing.T) { + c, err := conf.NewConfigWithYAML([]byte(test.yaml), "") + if err != nil { + t.Fatalf("unexpected error unmarshaling config: %v", err) + } + + var got decoderConfig + err = c.Unpack(&got) + if !sameError(err, test.wantErr) { + t.Errorf("unexpected error unpacking config: got:%v want:%v", err, test.wantErr) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("unexpected result\n--- want\n+++ got\n%s", cmp.Diff(test.want, got)) + } + }) + } +} + +func sameError(a, b error) bool { + switch { + case a == nil && b == nil: + return true + case a == nil, b == nil: + return false + default: + return a.Error() == b.Error() + } +} + +func ptr[T any](v T) *T { return &v } diff --git a/x-pack/filebeat/input/azureblobstorage/input.go b/x-pack/filebeat/input/azureblobstorage/input.go index 245daaef6e5b..7c7989ccbce3 100644 --- a/x-pack/filebeat/input/azureblobstorage/input.go +++ b/x-pack/filebeat/input/azureblobstorage/input.go @@ -71,6 +71,7 @@ func configure(cfg *conf.C) ([]cursor.Source, cursor.Input, error) { TimeStampEpoch: container.TimeStampEpoch, ExpandEventListFromField: container.ExpandEventListFromField, FileSelectors: container.FileSelectors, + ReaderConfig: container.ReaderConfig, }) } @@ -120,6 +121,7 @@ func tryOverrideOrDefault(cfg config, c container) container { if len(c.FileSelectors) == 0 && len(cfg.FileSelectors) != 0 { c.FileSelectors = cfg.FileSelectors } + c.ReaderConfig = cfg.ReaderConfig return c } diff --git a/x-pack/filebeat/input/azureblobstorage/input_stateless.go b/x-pack/filebeat/input/azureblobstorage/input_stateless.go index 73ae14a62e58..8bff050c1123 100644 --- a/x-pack/filebeat/input/azureblobstorage/input_stateless.go +++ b/x-pack/filebeat/input/azureblobstorage/input_stateless.go @@ -57,6 +57,7 @@ func (in *statelessInput) Run(inputCtx v2.Context, publisher stateless.Publisher TimeStampEpoch: container.TimeStampEpoch, ExpandEventListFromField: container.ExpandEventListFromField, FileSelectors: container.FileSelectors, + ReaderConfig: container.ReaderConfig, } st := newState() diff --git a/x-pack/filebeat/input/azureblobstorage/job.go b/x-pack/filebeat/input/azureblobstorage/job.go index ae1c2c1e6173..fe5be18f357a 100644 --- a/x-pack/filebeat/input/azureblobstorage/job.go +++ b/x-pack/filebeat/input/azureblobstorage/job.go @@ -134,20 +134,47 @@ func (j *job) processAndPublishData(ctx context.Context, id string) error { } }() - err = j.readJsonAndPublish(ctx, reader, id) - if err != nil { - return fmt.Errorf("failed to read data from blob with error: %w", err) - } - - return err + return j.decode(ctx, reader, id) } -func (j *job) readJsonAndPublish(ctx context.Context, r io.Reader, id string) error { +func (j *job) decode(ctx context.Context, r io.Reader, id string) error { r, err := j.addGzipDecoderIfNeeded(bufio.NewReader(r)) if err != nil { return fmt.Errorf("failed to add gzip decoder to blob: %s, with error: %w", *j.blob.Name, err) } + dec, err := newDecoder(j.src.ReaderConfig.Decoding, r) + if err != nil { + return err + } + var evtOffset int64 + switch dec := dec.(type) { + case decoder: + defer dec.close() + + for dec.next() { + msg, err := dec.decode() + if err != nil { + if err == io.EOF { + return nil + } + break + } + evt := j.createEvent(string(msg), evtOffset) + j.publish(evt, !dec.more(), id) + } + default: + err = j.readJsonAndPublish(ctx, r, id) + if err != nil { + return fmt.Errorf("failed to read data from blob with error: %w", err) + } + } + + return err +} + +func (j *job) readJsonAndPublish(ctx context.Context, r io.Reader, id string) error { + var err error // checks if the root element is an array or not r, j.isRootArray, err = evaluateJSON(bufio.NewReader(r)) if err != nil { @@ -183,22 +210,25 @@ func (j *job) readJsonAndPublish(ctx context.Context, r io.Reader, id string) er return err } evt := j.createEvent(string(data), offset) + j.publish(evt, !dec.More(), id) + } + return nil +} - if !dec.More() { - // if this is the last object, then perform a complete state save - cp, done := j.state.saveForTx(*j.blob.Name, *j.blob.Properties.LastModified) - if err := j.publisher.Publish(evt, cp); err != nil { - j.log.Errorf(jobErrString, id, err) - } - done() - } else { - // since we don't update the cursor checkpoint, lack of a lock here should be fine - if err := j.publisher.Publish(evt, nil); err != nil { - j.log.Errorf(jobErrString, id, err) - } +func (j *job) publish(evt beat.Event, last bool, id string) { + if last { + // if this is the last object, then perform a complete state save + cp, done := j.state.saveForTx(*j.blob.Name, *j.blob.Properties.LastModified) + if err := j.publisher.Publish(evt, cp); err != nil { + j.log.Errorf(jobErrString, id, err) } + done() + return + } + // since we don't update the cursor checkpoint, lack of a lock here should be fine + if err := j.publisher.Publish(evt, nil); err != nil { + j.log.Errorf(jobErrString, id, err) } - return nil } // splitEventList splits the event list into individual events and publishes them diff --git a/x-pack/filebeat/input/azureblobstorage/testdata/txn.csv b/x-pack/filebeat/input/azureblobstorage/testdata/txn.csv new file mode 100644 index 000000000000..80ca65df21ef --- /dev/null +++ b/x-pack/filebeat/input/azureblobstorage/testdata/txn.csv @@ -0,0 +1,5 @@ +date time time-taken cs-bytes sc-bytes bytes c-ip s-ip cs-username cs-method cs-uri-scheme cs-uri-query cs-user-agent cs-content-type sc-status sc-content-type cs-dns cs-host cs-uri cs-uri-port cs-referer x-cs-session-id x-cs-access-method x-cs-app x-s-country x-s-latitude x-s-longitude x-s-location x-s-region x-s-zipcode x-c-country x-c-latitude x-c-longitude x-c-location x-c-region x-c-zipcode x-c-os x-c-browser x-c-browser-version x-c-device x-cs-site x-cs-timestamp x-cs-page-id x-cs-userip x-cs-traffic-type x-cs-tunnel-id x-category x-other-category x-type x-server-ssl-err x-client-ssl-err x-transaction-id x-request-id x-cs-sni x-cs-domain-fronted-sni x-category-id x-other-category-id x-sr-headers-name x-sr-headers-value x-cs-ssl-ja3 x-sr-ssl-ja3s x-ssl-bypass x-ssl-bypass-reason x-r-cert-subject-cn x-r-cert-issuer-cn x-r-cert-startdate x-r-cert-enddate x-r-cert-valid x-r-cert-expired x-r-cert-untrusted-root x-r-cert-incomplete-chain x-r-cert-self-signed x-r-cert-revoked x-r-cert-revocation-check x-r-cert-mismatch x-cs-ssl-fronting-error x-cs-ssl-handshake-error x-sr-ssl-handshake-error x-sr-ssl-client-certificate-error x-sr-ssl-malformed-ssl x-s-custom-signing-ca-error x-cs-ssl-engine-action x-cs-ssl-engine-action-reason x-sr-ssl-engine-action x-sr-ssl-engine-action-reason x-ssl-policy-src-ip x-ssl-policy-dst-ip x-ssl-policy-dst-host x-ssl-policy-dst-host-source x-ssl-policy-categories x-ssl-policy-action x-ssl-policy-name x-cs-ssl-version x-cs-ssl-cipher x-sr-ssl-version x-sr-ssl-cipher x-cs-src-ip-egress x-s-dp-name x-cs-src-ip x-cs-src-port x-cs-dst-ip x-cs-dst-port x-sr-src-ip x-sr-src-port x-sr-dst-ip x-sr-dst-port x-cs-ip-connect-xff x-cs-ip-xff x-cs-connect-host x-cs-connect-port x-cs-connect-user-agent x-cs-url x-cs-uri-path x-cs-http-version rs-status x-cs-app-category x-cs-app-cci x-cs-app-ccl x-cs-app-tags x-cs-app-suite x-cs-app-instance-id x-cs-app-instance-name x-cs-app-instance-tag x-cs-app-activity x-cs-app-from-user x-cs-app-to-user x-cs-app-object-type x-cs-app-object-name x-cs-app-object-id x-rs-file-type x-rs-file-category x-rs-file-language x-rs-file-size x-rs-file-md5 x-rs-file-sha256 x-error x-c-local-time x-policy-action x-policy-name x-policy-src-ip x-policy-dst-ip x-policy-dst-host x-policy-dst-host-source x-policy-justification-type x-policy-justification-reason x-sc-notification-name +2024-08-05 16:24:20 64 2971 2050 5021 10.5.78.159 204.79.197.237 "vikash.ranjan@riverbed.com" GET https cc=US&setlang=en-US "Mozilla/5.0 (Windows NT 10.0; Win64; x64; Cortana 1.14.7.19041; 10.0.0.0.19045.2006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19045" - 200 "application/json; charset=utf-8" www.bing.com www.bing.com /client/config?cc=US&setlang=en-US 443 - 3683772769278232507 "Client" "Microsoft Bing" "US" 47.682899 -122.120903 "Redmond" "Washington" "N/A" "US" 29.775400 -95.598000 "Houston" "Texas" "77079" "Windows 10" "Edge" "18.19045" "Windows Device" "bing" 1722875060 5762388460300455936 10.5.78.159 CloudApp - "Search Engines" - http_transaction - - 2696581500064586450 2901306739654139904 www.bing.com - 551 - - - 28a2c9bd18a11de089ef85a160da29e4 NotAvailable No - "NotChecked" "NotChecked" "NotChecked" "NotChecked" NotChecked NotChecked NotChecked NotChecked NotChecked NotChecked "NotChecked" NotChecked No No NotChecked NotChecked NotChecked No Allow "Established" None "NotEstablished" 10.5.78.159 69.192.139.97 www.bing.com Sni "Search Engines" Decrypt - TLSv1.2 ECDHE-RSA-AES256-GCM-SHA384 NotChecked NotChecked 208.185.23.18 "US-ATL2" 10.5.78.159 25941 69.192.139.97 443 - - 10.144.54.201 842 - - - - - https://www.bing.com/client/config?cc=US&setlang=en-US /client/config HTTP1.1 200 "Search Engines" 58 low "Consumer,Unsanctioned" - - - - "Browse" - - - - - - - - - - - - "2024-08-05 11:24:00" "allow" "NetskopeAllow" 10.5.78.159 204.79.197.237 www.bing.com HttpHostHeader - - - +2024-08-05 16:24:19 - 18 0 18 10.70.0.19 - "nadav@skyformation.onmicrosoft.com" PRI - - - - - - - us-west1-b-osconfig.googleapis.com * 443 - 0 "Client" - - - - - - - "US" 45.605600 -121.180700 "The Dalles" "Oregon" "97058" - - - - - 1722875059 0 10.70.0.19 - - "Technology" "Cloud Storage" http_transaction - - 2035489204758272484 0 us-west1-b-osconfig.googleapis.com - 564 "7" - - 7a15285d4efc355608b304698cd7f9ab NotAvailable No - "NotChecked" "NotChecked" "NotChecked" "NotChecked" NotChecked NotChecked NotChecked NotChecked NotChecked NotChecked "NotChecked" NotChecked No No NotChecked NotChecked NotChecked No Allow "Established" None "NotEstablished" 10.70.0.19 142.250.99.95 us-west1-b-osconfig.googleapis.com Sni "Technology, Cloud Storage" Decrypt - TLSv1.3 TLS_AES_256_GCM_SHA384 NotChecked NotChecked 34.82.190.203 "US-SEA2" 10.70.0.19 32951 142.250.99.95 443 - - - - - - - - - - - HTTP1.1 - - - - - - - - - - - - - - - - - - - - - http-malformed "NotChecked" NotChecked - - - - - - - - +2024-08-05 16:24:20 - 0 0 0 10.0.20.111 - "levente.fangli@cososys.com" - - - - - - - - achecker-alliances.eu.goskope.com - 443 - 0 "Client" - - - - - - - "RO" 46.765700 23.594300 "Cluj-Napoca" "Cluj County" "400027" - - - - - 1722875060 0 10.0.20.111 - - - - http_transaction - "HsFailure (error:14094418:SSL routines:ssl3_read_bytes:tlsv1 alert unknown ca)" 1350739992944030464 0 achecker-alliances.eu.goskope.com - - - - - bc29aa426fc99c0be1b9be941869f88a NotAvailable No - "NotChecked" "NotChecked" "NotChecked" "NotChecked" NotChecked NotChecked NotChecked NotChecked NotChecked NotChecked "NotChecked" NotChecked No Yes NotChecked NotChecked NotChecked No Block "SSL Error - SSL Handshake Error" None "NotEstablished" - - - Unknown - Decrypt - - - NotChecked NotChecked 81.196.156.53 "AT-VIE1" 10.0.20.111 57897 31.186.239.94 443 - - - - - - - - - - - UNKNOWN - - - - - - - - - - - - - - - - - - - - - client-ssl "NotChecked" NotChecked - - - - - - - - +2024-08-05 16:24:23 - 0 0 0 10.0.20.111 - "levente.fangli@cososys.com" - - - - - - - - achecker-alliances.eu.goskope.com - 443 - 0 "Client" - - - - - - - "RO" 46.765700 23.594300 "Cluj-Napoca" "Cluj County" "400027" - - - - - 1722875063 0 10.0.20.111 - - - - http_transaction - "HsFailure (error:14094418:SSL routines:ssl3_read_bytes:tlsv1 alert unknown ca)" 1615432978285898071 0 achecker-alliances.eu.goskope.com - - - - - bc29aa426fc99c0be1b9be941869f88a NotAvailable No - "NotChecked" "NotChecked" "NotChecked" "NotChecked" NotChecked NotChecked NotChecked NotChecked NotChecked NotChecked "NotChecked" NotChecked No Yes NotChecked NotChecked NotChecked No Block "SSL Error - SSL Handshake Error" None "NotEstablished" - - - Unknown - Decrypt - - - NotChecked NotChecked 81.196.156.53 "AT-VIE1" 10.0.20.111 57897 31.186.239.94 443 - - - - - - - - - - - UNKNOWN - - - - - - - - - - - - - - - - - - - - - client-ssl "NotChecked" NotChecked - - - - - - - - diff --git a/x-pack/filebeat/input/azureblobstorage/testdata/txn.csv.gz b/x-pack/filebeat/input/azureblobstorage/testdata/txn.csv.gz new file mode 100644 index 000000000000..52e8fb20539a Binary files /dev/null and b/x-pack/filebeat/input/azureblobstorage/testdata/txn.csv.gz differ diff --git a/x-pack/filebeat/input/azureblobstorage/testdata/txn.json b/x-pack/filebeat/input/azureblobstorage/testdata/txn.json new file mode 100644 index 000000000000..57e92b1c7601 --- /dev/null +++ b/x-pack/filebeat/input/azureblobstorage/testdata/txn.json @@ -0,0 +1,594 @@ +[ + { + "bytes": "5021", + "c-ip": "10.5.78.159", + "cs-bytes": "2971", + "cs-content-type": "-", + "cs-dns": "www.bing.com", + "cs-host": "www.bing.com", + "cs-method": "GET", + "cs-referer": "-", + "cs-uri": "/client/config?cc=US&setlang=en-US", + "cs-uri-port": "443", + "cs-uri-query": "cc=US&setlang=en-US", + "cs-uri-scheme": "https", + "cs-user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; Cortana 1.14.7.19041; 10.0.0.0.19045.2006) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19045", + "cs-username": "vikash.ranjan@riverbed.com", + "date": "2024-08-05", + "rs-status": "200", + "s-ip": "204.79.197.237", + "sc-bytes": "2050", + "sc-content-type": "application/json; charset=utf-8", + "sc-status": "200", + "time": "16:24:20", + "time-taken": "64", + "x-c-browser": "Edge", + "x-c-browser-version": "18.19045", + "x-c-country": "US", + "x-c-device": "Windows Device", + "x-c-latitude": "29.775400", + "x-c-local-time": "2024-08-05 11:24:00", + "x-c-location": "Houston", + "x-c-longitude": "-95.598000", + "x-c-os": "Windows 10", + "x-c-region": "Texas", + "x-c-zipcode": "77079", + "x-category": "Search Engines", + "x-category-id": "551", + "x-client-ssl-err": "-", + "x-cs-access-method": "Client", + "x-cs-app": "Microsoft Bing", + "x-cs-app-activity": "Browse", + "x-cs-app-category": "Search Engines", + "x-cs-app-cci": "58", + "x-cs-app-ccl": "low", + "x-cs-app-from-user": "-", + "x-cs-app-instance-id": "-", + "x-cs-app-instance-name": "-", + "x-cs-app-instance-tag": "-", + "x-cs-app-object-id": "-", + "x-cs-app-object-name": "-", + "x-cs-app-object-type": "-", + "x-cs-app-suite": "-", + "x-cs-app-tags": "Consumer,Unsanctioned", + "x-cs-app-to-user": "-", + "x-cs-connect-host": "-", + "x-cs-connect-port": "-", + "x-cs-connect-user-agent": "-", + "x-cs-domain-fronted-sni": "-", + "x-cs-dst-ip": "69.192.139.97", + "x-cs-dst-port": "443", + "x-cs-http-version": "HTTP1.1", + "x-cs-ip-connect-xff": "-", + "x-cs-ip-xff": "-", + "x-cs-page-id": "5762388460300455936", + "x-cs-session-id": "3683772769278232507", + "x-cs-site": "bing", + "x-cs-sni": "www.bing.com", + "x-cs-src-ip": "10.5.78.159", + "x-cs-src-ip-egress": "208.185.23.18", + "x-cs-src-port": "25941", + "x-cs-ssl-cipher": "ECDHE-RSA-AES256-GCM-SHA384", + "x-cs-ssl-engine-action": "Allow", + "x-cs-ssl-engine-action-reason": "Established", + "x-cs-ssl-fronting-error": "No", + "x-cs-ssl-handshake-error": "No", + "x-cs-ssl-ja3": "28a2c9bd18a11de089ef85a160da29e4", + "x-cs-ssl-version": "TLSv1.2", + "x-cs-timestamp": "1722875060", + "x-cs-traffic-type": "CloudApp", + "x-cs-tunnel-id": "-", + "x-cs-uri-path": "/client/config", + "x-cs-url": "https://www.bing.com/client/config?cc=US&setlang=en-US", + "x-cs-userip": "10.5.78.159", + "x-error": "-", + "x-other-category": "-", + "x-other-category-id": "-", + "x-policy-action": "allow", + "x-policy-dst-host": "www.bing.com", + "x-policy-dst-host-source": "HttpHostHeader", + "x-policy-dst-ip": "204.79.197.237", + "x-policy-justification-reason": "-", + "x-policy-justification-type": "-", + "x-policy-name": "NetskopeAllow", + "x-policy-src-ip": "10.5.78.159", + "x-r-cert-enddate": "NotChecked", + "x-r-cert-expired": "NotChecked", + "x-r-cert-incomplete-chain": "NotChecked", + "x-r-cert-issuer-cn": "NotChecked", + "x-r-cert-mismatch": "NotChecked", + "x-r-cert-revocation-check": "NotChecked", + "x-r-cert-revoked": "NotChecked", + "x-r-cert-self-signed": "NotChecked", + "x-r-cert-startdate": "NotChecked", + "x-r-cert-subject-cn": "NotChecked", + "x-r-cert-untrusted-root": "NotChecked", + "x-r-cert-valid": "NotChecked", + "x-request-id": "2901306739654139904", + "x-rs-file-category": "-", + "x-rs-file-language": "-", + "x-rs-file-md5": "-", + "x-rs-file-sha256": "-", + "x-rs-file-size": "-", + "x-rs-file-type": "-", + "x-s-country": "US", + "x-s-custom-signing-ca-error": "No", + "x-s-dp-name": "US-ATL2", + "x-s-latitude": "47.682899", + "x-s-location": "Redmond", + "x-s-longitude": "-122.120903", + "x-s-region": "Washington", + "x-s-zipcode": "N/A", + "x-sc-notification-name": "-", + "x-server-ssl-err": "-", + "x-sr-dst-ip": "10.144.54.201", + "x-sr-dst-port": "842", + "x-sr-headers-name": "-", + "x-sr-headers-value": "-", + "x-sr-src-ip": "-", + "x-sr-src-port": "-", + "x-sr-ssl-cipher": "NotChecked", + "x-sr-ssl-client-certificate-error": "NotChecked", + "x-sr-ssl-engine-action": "None", + "x-sr-ssl-engine-action-reason": "NotEstablished", + "x-sr-ssl-handshake-error": "NotChecked", + "x-sr-ssl-ja3s": "NotAvailable", + "x-sr-ssl-malformed-ssl": "NotChecked", + "x-sr-ssl-version": "NotChecked", + "x-ssl-bypass": "No", + "x-ssl-bypass-reason": "-", + "x-ssl-policy-action": "Decrypt", + "x-ssl-policy-categories": "Search Engines", + "x-ssl-policy-dst-host": "www.bing.com", + "x-ssl-policy-dst-host-source": "Sni", + "x-ssl-policy-dst-ip": "69.192.139.97", + "x-ssl-policy-name": "-", + "x-ssl-policy-src-ip": "10.5.78.159", + "x-transaction-id": "2696581500064586450", + "x-type": "http_transaction" + }, + { + "bytes": "18", + "c-ip": "10.70.0.19", + "cs-bytes": "18", + "cs-content-type": "-", + "cs-dns": "-", + "cs-host": "us-west1-b-osconfig.googleapis.com", + "cs-method": "PRI", + "cs-referer": "-", + "cs-uri": "*", + "cs-uri-port": "443", + "cs-uri-query": "-", + "cs-uri-scheme": "-", + "cs-user-agent": "-", + "cs-username": "nadav@skyformation.onmicrosoft.com", + "date": "2024-08-05", + "rs-status": "-", + "s-ip": "-", + "sc-bytes": "0", + "sc-content-type": "-", + "sc-status": "-", + "time": "16:24:19", + "time-taken": "-", + "x-c-browser": "-", + "x-c-browser-version": "-", + "x-c-country": "US", + "x-c-device": "-", + "x-c-latitude": "45.605600", + "x-c-local-time": "NotChecked", + "x-c-location": "The Dalles", + "x-c-longitude": "-121.180700", + "x-c-os": "-", + "x-c-region": "Oregon", + "x-c-zipcode": "97058", + "x-category": "Technology", + "x-category-id": "564", + "x-client-ssl-err": "-", + "x-cs-access-method": "Client", + "x-cs-app": "-", + "x-cs-app-activity": "-", + "x-cs-app-category": "-", + "x-cs-app-cci": "-", + "x-cs-app-ccl": "-", + "x-cs-app-from-user": "-", + "x-cs-app-instance-id": "-", + "x-cs-app-instance-name": "-", + "x-cs-app-instance-tag": "-", + "x-cs-app-object-id": "-", + "x-cs-app-object-name": "-", + "x-cs-app-object-type": "-", + "x-cs-app-suite": "-", + "x-cs-app-tags": "-", + "x-cs-app-to-user": "-", + "x-cs-connect-host": "-", + "x-cs-connect-port": "-", + "x-cs-connect-user-agent": "-", + "x-cs-domain-fronted-sni": "-", + "x-cs-dst-ip": "142.250.99.95", + "x-cs-dst-port": "443", + "x-cs-http-version": "HTTP1.1", + "x-cs-ip-connect-xff": "-", + "x-cs-ip-xff": "-", + "x-cs-page-id": "0", + "x-cs-session-id": "0", + "x-cs-site": "-", + "x-cs-sni": "us-west1-b-osconfig.googleapis.com", + "x-cs-src-ip": "10.70.0.19", + "x-cs-src-ip-egress": "34.82.190.203", + "x-cs-src-port": "32951", + "x-cs-ssl-cipher": "TLS_AES_256_GCM_SHA384", + "x-cs-ssl-engine-action": "Allow", + "x-cs-ssl-engine-action-reason": "Established", + "x-cs-ssl-fronting-error": "No", + "x-cs-ssl-handshake-error": "No", + "x-cs-ssl-ja3": "7a15285d4efc355608b304698cd7f9ab", + "x-cs-ssl-version": "TLSv1.3", + "x-cs-timestamp": "1722875059", + "x-cs-traffic-type": "-", + "x-cs-tunnel-id": "-", + "x-cs-uri-path": "-", + "x-cs-url": "-", + "x-cs-userip": "10.70.0.19", + "x-error": "http-malformed", + "x-other-category": "Cloud Storage", + "x-other-category-id": "7", + "x-policy-action": "NotChecked", + "x-policy-dst-host": "-", + "x-policy-dst-host-source": "-", + "x-policy-dst-ip": "-", + "x-policy-justification-reason": "-", + "x-policy-justification-type": "-", + "x-policy-name": "-", + "x-policy-src-ip": "-", + "x-r-cert-enddate": "NotChecked", + "x-r-cert-expired": "NotChecked", + "x-r-cert-incomplete-chain": "NotChecked", + "x-r-cert-issuer-cn": "NotChecked", + "x-r-cert-mismatch": "NotChecked", + "x-r-cert-revocation-check": "NotChecked", + "x-r-cert-revoked": "NotChecked", + "x-r-cert-self-signed": "NotChecked", + "x-r-cert-startdate": "NotChecked", + "x-r-cert-subject-cn": "NotChecked", + "x-r-cert-untrusted-root": "NotChecked", + "x-r-cert-valid": "NotChecked", + "x-request-id": "0", + "x-rs-file-category": "-", + "x-rs-file-language": "-", + "x-rs-file-md5": "-", + "x-rs-file-sha256": "-", + "x-rs-file-size": "-", + "x-rs-file-type": "-", + "x-s-country": "-", + "x-s-custom-signing-ca-error": "No", + "x-s-dp-name": "US-SEA2", + "x-s-latitude": "-", + "x-s-location": "-", + "x-s-longitude": "-", + "x-s-region": "-", + "x-s-zipcode": "-", + "x-sc-notification-name": "-", + "x-server-ssl-err": "-", + "x-sr-dst-ip": "-", + "x-sr-dst-port": "-", + "x-sr-headers-name": "-", + "x-sr-headers-value": "-", + "x-sr-src-ip": "-", + "x-sr-src-port": "-", + "x-sr-ssl-cipher": "NotChecked", + "x-sr-ssl-client-certificate-error": "NotChecked", + "x-sr-ssl-engine-action": "None", + "x-sr-ssl-engine-action-reason": "NotEstablished", + "x-sr-ssl-handshake-error": "NotChecked", + "x-sr-ssl-ja3s": "NotAvailable", + "x-sr-ssl-malformed-ssl": "NotChecked", + "x-sr-ssl-version": "NotChecked", + "x-ssl-bypass": "No", + "x-ssl-bypass-reason": "-", + "x-ssl-policy-action": "Decrypt", + "x-ssl-policy-categories": "Technology, Cloud Storage", + "x-ssl-policy-dst-host": "us-west1-b-osconfig.googleapis.com", + "x-ssl-policy-dst-host-source": "Sni", + "x-ssl-policy-dst-ip": "142.250.99.95", + "x-ssl-policy-name": "-", + "x-ssl-policy-src-ip": "10.70.0.19", + "x-transaction-id": "2035489204758272484", + "x-type": "http_transaction" + }, + { + "bytes": "0", + "c-ip": "10.0.20.111", + "cs-bytes": "0", + "cs-content-type": "-", + "cs-dns": "-", + "cs-host": "achecker-alliances.eu.goskope.com", + "cs-method": "-", + "cs-referer": "-", + "cs-uri": "-", + "cs-uri-port": "443", + "cs-uri-query": "-", + "cs-uri-scheme": "-", + "cs-user-agent": "-", + "cs-username": "levente.fangli@cososys.com", + "date": "2024-08-05", + "rs-status": "-", + "s-ip": "-", + "sc-bytes": "0", + "sc-content-type": "-", + "sc-status": "-", + "time": "16:24:20", + "time-taken": "-", + "x-c-browser": "-", + "x-c-browser-version": "-", + "x-c-country": "RO", + "x-c-device": "-", + "x-c-latitude": "46.765700", + "x-c-local-time": "NotChecked", + "x-c-location": "Cluj-Napoca", + "x-c-longitude": "23.594300", + "x-c-os": "-", + "x-c-region": "Cluj County", + "x-c-zipcode": "400027", + "x-category": "-", + "x-category-id": "-", + "x-client-ssl-err": "HsFailure (error:14094418:SSL routines:ssl3_read_bytes:tlsv1 alert unknown ca)", + "x-cs-access-method": "Client", + "x-cs-app": "-", + "x-cs-app-activity": "-", + "x-cs-app-category": "-", + "x-cs-app-cci": "-", + "x-cs-app-ccl": "-", + "x-cs-app-from-user": "-", + "x-cs-app-instance-id": "-", + "x-cs-app-instance-name": "-", + "x-cs-app-instance-tag": "-", + "x-cs-app-object-id": "-", + "x-cs-app-object-name": "-", + "x-cs-app-object-type": "-", + "x-cs-app-suite": "-", + "x-cs-app-tags": "-", + "x-cs-app-to-user": "-", + "x-cs-connect-host": "-", + "x-cs-connect-port": "-", + "x-cs-connect-user-agent": "-", + "x-cs-domain-fronted-sni": "-", + "x-cs-dst-ip": "31.186.239.94", + "x-cs-dst-port": "443", + "x-cs-http-version": "UNKNOWN", + "x-cs-ip-connect-xff": "-", + "x-cs-ip-xff": "-", + "x-cs-page-id": "0", + "x-cs-session-id": "0", + "x-cs-site": "-", + "x-cs-sni": "achecker-alliances.eu.goskope.com", + "x-cs-src-ip": "10.0.20.111", + "x-cs-src-ip-egress": "81.196.156.53", + "x-cs-src-port": "57897", + "x-cs-ssl-cipher": "-", + "x-cs-ssl-engine-action": "Block", + "x-cs-ssl-engine-action-reason": "SSL Error - SSL Handshake Error", + "x-cs-ssl-fronting-error": "No", + "x-cs-ssl-handshake-error": "Yes", + "x-cs-ssl-ja3": "bc29aa426fc99c0be1b9be941869f88a", + "x-cs-ssl-version": "-", + "x-cs-timestamp": "1722875060", + "x-cs-traffic-type": "-", + "x-cs-tunnel-id": "-", + "x-cs-uri-path": "-", + "x-cs-url": "-", + "x-cs-userip": "10.0.20.111", + "x-error": "client-ssl", + "x-other-category": "-", + "x-other-category-id": "-", + "x-policy-action": "NotChecked", + "x-policy-dst-host": "-", + "x-policy-dst-host-source": "-", + "x-policy-dst-ip": "-", + "x-policy-justification-reason": "-", + "x-policy-justification-type": "-", + "x-policy-name": "-", + "x-policy-src-ip": "-", + "x-r-cert-enddate": "NotChecked", + "x-r-cert-expired": "NotChecked", + "x-r-cert-incomplete-chain": "NotChecked", + "x-r-cert-issuer-cn": "NotChecked", + "x-r-cert-mismatch": "NotChecked", + "x-r-cert-revocation-check": "NotChecked", + "x-r-cert-revoked": "NotChecked", + "x-r-cert-self-signed": "NotChecked", + "x-r-cert-startdate": "NotChecked", + "x-r-cert-subject-cn": "NotChecked", + "x-r-cert-untrusted-root": "NotChecked", + "x-r-cert-valid": "NotChecked", + "x-request-id": "0", + "x-rs-file-category": "-", + "x-rs-file-language": "-", + "x-rs-file-md5": "-", + "x-rs-file-sha256": "-", + "x-rs-file-size": "-", + "x-rs-file-type": "-", + "x-s-country": "-", + "x-s-custom-signing-ca-error": "No", + "x-s-dp-name": "AT-VIE1", + "x-s-latitude": "-", + "x-s-location": "-", + "x-s-longitude": "-", + "x-s-region": "-", + "x-s-zipcode": "-", + "x-sc-notification-name": "-", + "x-server-ssl-err": "-", + "x-sr-dst-ip": "-", + "x-sr-dst-port": "-", + "x-sr-headers-name": "-", + "x-sr-headers-value": "-", + "x-sr-src-ip": "-", + "x-sr-src-port": "-", + "x-sr-ssl-cipher": "NotChecked", + "x-sr-ssl-client-certificate-error": "NotChecked", + "x-sr-ssl-engine-action": "None", + "x-sr-ssl-engine-action-reason": "NotEstablished", + "x-sr-ssl-handshake-error": "NotChecked", + "x-sr-ssl-ja3s": "NotAvailable", + "x-sr-ssl-malformed-ssl": "NotChecked", + "x-sr-ssl-version": "NotChecked", + "x-ssl-bypass": "No", + "x-ssl-bypass-reason": "-", + "x-ssl-policy-action": "Decrypt", + "x-ssl-policy-categories": "-", + "x-ssl-policy-dst-host": "-", + "x-ssl-policy-dst-host-source": "Unknown", + "x-ssl-policy-dst-ip": "-", + "x-ssl-policy-name": "-", + "x-ssl-policy-src-ip": "-", + "x-transaction-id": "1350739992944030464", + "x-type": "http_transaction" + }, + { + "bytes": "0", + "c-ip": "10.0.20.111", + "cs-bytes": "0", + "cs-content-type": "-", + "cs-dns": "-", + "cs-host": "achecker-alliances.eu.goskope.com", + "cs-method": "-", + "cs-referer": "-", + "cs-uri": "-", + "cs-uri-port": "443", + "cs-uri-query": "-", + "cs-uri-scheme": "-", + "cs-user-agent": "-", + "cs-username": "levente.fangli@cososys.com", + "date": "2024-08-05", + "rs-status": "-", + "s-ip": "-", + "sc-bytes": "0", + "sc-content-type": "-", + "sc-status": "-", + "time": "16:24:23", + "time-taken": "-", + "x-c-browser": "-", + "x-c-browser-version": "-", + "x-c-country": "RO", + "x-c-device": "-", + "x-c-latitude": "46.765700", + "x-c-local-time": "NotChecked", + "x-c-location": "Cluj-Napoca", + "x-c-longitude": "23.594300", + "x-c-os": "-", + "x-c-region": "Cluj County", + "x-c-zipcode": "400027", + "x-category": "-", + "x-category-id": "-", + "x-client-ssl-err": "HsFailure (error:14094418:SSL routines:ssl3_read_bytes:tlsv1 alert unknown ca)", + "x-cs-access-method": "Client", + "x-cs-app": "-", + "x-cs-app-activity": "-", + "x-cs-app-category": "-", + "x-cs-app-cci": "-", + "x-cs-app-ccl": "-", + "x-cs-app-from-user": "-", + "x-cs-app-instance-id": "-", + "x-cs-app-instance-name": "-", + "x-cs-app-instance-tag": "-", + "x-cs-app-object-id": "-", + "x-cs-app-object-name": "-", + "x-cs-app-object-type": "-", + "x-cs-app-suite": "-", + "x-cs-app-tags": "-", + "x-cs-app-to-user": "-", + "x-cs-connect-host": "-", + "x-cs-connect-port": "-", + "x-cs-connect-user-agent": "-", + "x-cs-domain-fronted-sni": "-", + "x-cs-dst-ip": "31.186.239.94", + "x-cs-dst-port": "443", + "x-cs-http-version": "UNKNOWN", + "x-cs-ip-connect-xff": "-", + "x-cs-ip-xff": "-", + "x-cs-page-id": "0", + "x-cs-session-id": "0", + "x-cs-site": "-", + "x-cs-sni": "achecker-alliances.eu.goskope.com", + "x-cs-src-ip": "10.0.20.111", + "x-cs-src-ip-egress": "81.196.156.53", + "x-cs-src-port": "57897", + "x-cs-ssl-cipher": "-", + "x-cs-ssl-engine-action": "Block", + "x-cs-ssl-engine-action-reason": "SSL Error - SSL Handshake Error", + "x-cs-ssl-fronting-error": "No", + "x-cs-ssl-handshake-error": "Yes", + "x-cs-ssl-ja3": "bc29aa426fc99c0be1b9be941869f88a", + "x-cs-ssl-version": "-", + "x-cs-timestamp": "1722875063", + "x-cs-traffic-type": "-", + "x-cs-tunnel-id": "-", + "x-cs-uri-path": "-", + "x-cs-url": "-", + "x-cs-userip": "10.0.20.111", + "x-error": "client-ssl", + "x-other-category": "-", + "x-other-category-id": "-", + "x-policy-action": "NotChecked", + "x-policy-dst-host": "-", + "x-policy-dst-host-source": "-", + "x-policy-dst-ip": "-", + "x-policy-justification-reason": "-", + "x-policy-justification-type": "-", + "x-policy-name": "-", + "x-policy-src-ip": "-", + "x-r-cert-enddate": "NotChecked", + "x-r-cert-expired": "NotChecked", + "x-r-cert-incomplete-chain": "NotChecked", + "x-r-cert-issuer-cn": "NotChecked", + "x-r-cert-mismatch": "NotChecked", + "x-r-cert-revocation-check": "NotChecked", + "x-r-cert-revoked": "NotChecked", + "x-r-cert-self-signed": "NotChecked", + "x-r-cert-startdate": "NotChecked", + "x-r-cert-subject-cn": "NotChecked", + "x-r-cert-untrusted-root": "NotChecked", + "x-r-cert-valid": "NotChecked", + "x-request-id": "0", + "x-rs-file-category": "-", + "x-rs-file-language": "-", + "x-rs-file-md5": "-", + "x-rs-file-sha256": "-", + "x-rs-file-size": "-", + "x-rs-file-type": "-", + "x-s-country": "-", + "x-s-custom-signing-ca-error": "No", + "x-s-dp-name": "AT-VIE1", + "x-s-latitude": "-", + "x-s-location": "-", + "x-s-longitude": "-", + "x-s-region": "-", + "x-s-zipcode": "-", + "x-sc-notification-name": "-", + "x-server-ssl-err": "-", + "x-sr-dst-ip": "-", + "x-sr-dst-port": "-", + "x-sr-headers-name": "-", + "x-sr-headers-value": "-", + "x-sr-src-ip": "-", + "x-sr-src-port": "-", + "x-sr-ssl-cipher": "NotChecked", + "x-sr-ssl-client-certificate-error": "NotChecked", + "x-sr-ssl-engine-action": "None", + "x-sr-ssl-engine-action-reason": "NotEstablished", + "x-sr-ssl-handshake-error": "NotChecked", + "x-sr-ssl-ja3s": "NotAvailable", + "x-sr-ssl-malformed-ssl": "NotChecked", + "x-sr-ssl-version": "NotChecked", + "x-ssl-bypass": "No", + "x-ssl-bypass-reason": "-", + "x-ssl-policy-action": "Decrypt", + "x-ssl-policy-categories": "-", + "x-ssl-policy-dst-host": "-", + "x-ssl-policy-dst-host-source": "Unknown", + "x-ssl-policy-dst-ip": "-", + "x-ssl-policy-name": "-", + "x-ssl-policy-src-ip": "-", + "x-transaction-id": "1615432978285898071", + "x-type": "http_transaction" + } +] diff --git a/x-pack/filebeat/input/azureblobstorage/types.go b/x-pack/filebeat/input/azureblobstorage/types.go index 67012198423e..3a5f39487fdb 100644 --- a/x-pack/filebeat/input/azureblobstorage/types.go +++ b/x-pack/filebeat/input/azureblobstorage/types.go @@ -20,6 +20,7 @@ type Source struct { PollInterval time.Duration TimeStampEpoch *int64 FileSelectors []fileSelectorConfig + ReaderConfig readerConfig ExpandEventListFromField string }