Panic: fatal error: concurrent map iteration and map write

[mkorolyov]

Filebeat Version: 6.4.2
OS: Docker image based from alpine:3.6
ForumURL: link
Steps to Reproduce: Separate filebeat container per application.
StackTrace:

fatal error: concurrent map iteration and map write

    goroutine 60 [running]:
    runtime.throw(0x15dee00, 0x26)
           	/usr/local/go/src/runtime/panic.go:616 +0x81 fp=0xc420663640 sp=0xc420663620 pc=0x9b1ab1
    runtime.mapiternext(0xc420663708)
           	/usr/local/go/src/runtime/hashmap.go:747 +0x55c fp=0xc4206636d0 sp=0xc420663640 pc=0x98fa7c
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.foldMapInterface(0xc420339470, 0x1444680, 0xc4203c0660, 0xc4203c0660, 0x160fbc0)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold_map.go:34 +0xea fp=0xc420663778 sp=0xc4206636d0 pc=0xda1aba
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.foldInterfaceValue(0xc420339470, 0x1543160, 0xc4203c0660, 0x0, 0x0)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold.go:89 +0x15b fp=0xc4206637f0 sp=0xc420663778 pc=0xd9fb4b
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.foldMapInlineInterface(0xc420339470, 0x1543160, 0xc42047b758, 0x95, 0x1543160, 0xc42047b758)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold_map_inline.generated.go:44 +0x153 fp=0xc4206638a8 sp=0xc4206637f0 pc=0xda2643
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.makeFieldInlineFold.func1(0xc420339470, 0x14a9d00, 0xc42047b740, 0x99, 0x0, 0x0)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold_reflect.go:292 +0x86 fp=0xc4206638f8 sp=0xc4206638a8 pc=0xdf5d16
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.makeFieldsFold.func1(0xc420339470, 0x14a9d00, 0xc42047b740, 0x99, 0x0, 0x14a9d00)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold_reflect.go:177 +0x88 fp=0xc420663950 sp=0xc4206638f8 pc=0xdf5b38
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.makeStructFold.func1(0xc420339470, 0x14a9d00, 0xc42047b740, 0x99, 0x0, 0x0)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold_reflect.go:167 +0x95 fp=0xc420663998 sp=0xc420663950 pc=0xdf5a35
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.foldAnyReflect(0xc420339470, 0x14a9d00, 0xc42047b740, 0x99, 0x99, 0xc42005d000)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold_reflect.go:511 +0xb9 fp=0xc4206639d8 sp=0xc420663998 pc=0xda7799
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.foldInterfaceValue(0xc420339470, 0x14a9d00, 0xc42047b740, 0xc42047b740, 0xc42047b740)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold.go:92 +0x1c9 fp=0xc420663a50 sp=0xc4206639d8 pc=0xd9fbb9
    github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype.(*Iterator).Fold(0xc420339470, 0x14a9d00, 0xc42047b740, 0xc42047b740, 0x0)
           	/go/src/github.com/elastic/beats/vendor/github.com/elastic/go-structform/gotype/fold.go:69 +0x41 fp=0xc420663a88 sp=0xc420663a50 pc=0xd9f9c1
    github.com/elastic/beats/libbeat/outputs/elasticsearch.(*jsonEncoder).AddRaw(0xc420332d60, 0x148db00, 0xc42054b200, 0x0, 0x0)
           	/go/src/github.com/elastic/beats/libbeat/outputs/elasticsearch/enc.go:119 +0x300 fp=0xc420663b88 sp=0xc420663a88 pc=0xe23d40
    github.com/elastic/beats/libbeat/outputs/elasticsearch.(*jsonEncoder).Add(0xc420332d60, 0x147c860, 0xc420193c00, 0x148db00, 0xc42054b200, 0xc420193c00, 0x0)
           	/go/src/github.com/elastic/beats/libbeat/outputs/elasticsearch/enc.go:139 +0x8b fp=0xc420663be8 sp=0xc420663b88 pc=0xe23dfb
    github.com/elastic/beats/libbeat/outputs/elasticsearch.bulkEncodePublishRequest(0x7fad51c7bbd0, 0xc420332d60, 0x1684760, 0xc420192240, 0x0, 0xc42054b080, 0x32, 0x63e, 0xc4203b00c0, 0xc42008c3c0, ...)
           	/go/src/github.com/elastic/beats/libbeat/outputs/elasticsearch/client.go:373 +0x1ac fp=0xc420663ce8 sp=0xc420663be8 pc=0xe1dbcc
    github.com/elastic/beats/libbeat/outputs/elasticsearch.(*Client).publishEvents(0xc4201e6420, 0xc42054b080, 0x32, 0x63e, 0x0, 0x0, 0x0, 0x0, 0x0)
           	/go/src/github.com/elastic/beats/libbeat/outputs/elasticsearch/client.go:304 +0x14e fp=0xc420663e28 sp=0xc420663ce8 pc=0xe1d35e
    github.com/elastic/beats/libbeat/outputs/elasticsearch.(*Client).Publish(0xc4201e6420, 0x169f080, 0xc42040dc80, 0xc4200a4840, 0xc420663f18)
           	/go/src/github.com/elastic/beats/libbeat/outputs/elasticsearch/client.go:271 +0x43 fp=0xc420663e90 sp=0xc420663e28 pc=0xe1d173
    github.com/elastic/beats/libbeat/outputs.(*backoffClient).Publish(0xc420332f20, 0x169f080, 0xc42040dc80, 0x0, 0x0)
           	/go/src/github.com/elastic/beats/libbeat/outputs/backoff.go:60 +0x4b fp=0xc420663ed8 sp=0xc420663e90 pc=0xd6d3fb
    github.com/elastic/beats/libbeat/publisher/pipeline.(*netClientWorker).run(0xc420192780)
           	/go/src/github.com/elastic/beats/libbeat/publisher/pipeline/output.go:119 +0x324 fp=0xc420663fd8 sp=0xc420663ed8 pc=0xe8fe54
    runtime.goexit()
           	/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc420663fe0 sp=0xc420663fd8 pc=0x9e1891
    created by github.com/elastic/beats/libbeat/publisher/pipeline.makeClientWorker
           	/go/src/github.com/elastic/beats/libbeat/publisher/pipeline/output.go:48 +0xf0

Config:

  filebeat.inputs:
    - prospector.type: log
      paths:
            - "/home/engine/log/app/stderr.log*"
      fields_under_root: true
      fields:
        host.ip_address: ${LOCAL_IP}
    - prospector.type: log
      paths:
            - "/home/engine/log/app/stdout.log*"
      close_removed: true
      close_renamed: true
      json.keys_under_root: true
      json.add_error_key: true
      fields_under_root: true
      fields:
        host.ip_address: ${LOCAL_IP}
    name: ${BEATNAME}
    setup.template.name: "filebeat-6.4.2"
    setup.template.pattern: "filebeat-6.4.2-*"
    setup.template.json.enabled: true
    setup.template.json.name: "filebeat-6.4.2"
    setup.template.json.path: "filebeat-6.4.2.template.json"
    setup.template.overwrite: false
    output.elasticsearch:
      hosts: ["http://${ELASTICSEARCH}:9300/"]
      index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"

[urso]

For 7.x this will be address in #10801