Merge remote-tracking branch 'upstream/master' into feature/windows-7-32

* upstream/master: (25 commits)
  [Elastic Agent] Send checkin payload to Fleet (elastic#19857)
  [Ingest Manager] Fixed tests across agent elastic#19877
  [Ingest Manager] Fix serialization test  elastic#19876
  Fix service start type mapping in windows/service metricset (elastic#19551)
  ci: Change comment trigger detection method (elastic#19827)
  Add 21 autogenerated filesets from rsa2elk devices (elastic#19713)
  [Ingest Manager] Agent config cleanup (elastic#19848)
  libbeat/publisher/pipeline: fix data races (elastic#19821)
  Update monitoring-internal-collection.asciidoc (elastic#19422) (elastic#19697)
  [Elastic Agent] Trust exchange endpoint must bind to 127.0.0.1 (elastic#19861)
  Specify an ECS version in Auditbeat/Packetbeat/Winlogbeat (elastic#19159)
  Add azure billing metricset (elastic#19207)
  Add support for appinsights in the metricbeat azure module (elastic#18940)
  Add MySQL query metricset with lightweight module and SQL helper (elastic#18955)
  [Ingest Manager] Refuse invalid stream values in configuration (elastic#19587)
  Do not use vendor during integration tests (elastic#19839)
  LIBBEAT: Enhancement Convert dissected values from String to other basic data types and IP (elastic#18683)
  [Elastic Agent] Remove support for "logs" and only support logfile (elastic#19761)
  [CI] support windows-2012 (elastic#19773)
  Do not update go.mod during packaging and testing (elastic#19823)
  ...

.ci/apm-beats-update.groovy

@@ -34,15 +34,18 @@ pipeline {
       agent { label 'ubuntu && immutable' }
       when {
         beforeAgent true
-        expression {
-          def ret = isCommentTrigger() || isUserTrigger()
-          if(!ret){
-            currentBuild.result = 'NOT_BUILT'
-            currentBuild.description = "The build has been skipped"
-            currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
-            echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, and GitHub comment")
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
           }
-          return ret
         }
       }
       /**

.ci/jobs/apm-beats-update.yml

@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'

.ci/jobs/beats.yml

@@ -7,7 +7,6 @@
     concurrent: true
     project-type: multibranch
     prune-dead-branches: true
-    number-to-keep: 10
     days-to-keep: 30
     script-path: 'Jenkinsfile'
     triggers: []
@@ -18,12 +17,9 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
-        property-strategies:
-          all-branches:
-          - suppress-scm-triggering: true
         repo: 'beats'
         repo-owner: 'elastic'
         credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
@@ -32,7 +28,7 @@
         build-strategies:
         - tags:
             ignore-tags-older-than: -1
-            ignore-tags-newer-than: -1
+            ignore-tags-newer-than: 365
         - change-request:
             ignore-target-only-changes: true
         - named-branches:

.ci/jobs/packaging.yml

@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'

.ci/packaging.groovy

@@ -37,15 +37,18 @@ pipeline {
       agent { label 'ubuntu && immutable' }
       when {
         beforeAgent true
-        expression {
-          def ret = isCommentTrigger() || isUserTrigger() || isUpstreamTrigger()
-          if(!ret){
-            currentBuild.result = 'NOT_BUILT'
-            currentBuild.description = "The build has been skipped"
-            currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
-            echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+        anyOf {
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
           }
-          return ret
         }
       }
       stages {

.ci/windows.groovy

@@ -13,10 +13,10 @@ import groovy.transform.Field
  List of supported windows versions to be tested with
  NOTE:
    - 'windows-10' is too slow
-   - 'windows-2012-r2', 'windows-2008-r2', 'windows-7' are disabled
+   - 'windows-2008-r2', 'windows-7' are disabled
       since we are working on releasing each windows version incrementally.
 */
-@Field def windowsVersions = ['windows-2019', 'windows-2016', 'windows-7-32-bit']
+@Field def windowsVersions = ['windows-2019', 'windows-2016', 'windows-2012-r2', 'windows-7-32-bit']
 
 pipeline {
   agent { label 'ubuntu && immutable' }
@@ -349,9 +349,10 @@ def mageTargetWin(String context, String directory, String target, String label)
     log(level: 'INFO', text: "context=${context} directory=${directory} target=${target} os=${label}")
     def immutable = label.equals('windows-7-32-bit') ? 'windows-immutable-32-bit' : 'windows-immutable'
 
-    // NOTE: skip filebeat with windows-2016 since there are some test failures.
-    if (directory.equals('filebeat') && label.equals('windows-2016')) {
-      log(level: 'WARN', text: "Skipped stage for the 'filebeat' with 'windows-2016' as long as there are test failures to be analysed.")
+    // NOTE: skip filebeat with windows-2016/2012-r2 since there are some test failures.
+    //       See https://github.com/elastic/beats/issues/19787 https://github.com/elastic/beats/issues/19641
+    if (directory.equals('filebeat') && (label.equals('windows-2016') || label.equals('windows-2012-r2') || label.equals('windows-7-32-bit'))) {
+      log(level: 'WARN', text: "Skipped stage for the 'filebeat' with '${label}' as long as there are test failures to be analysed.")
     } else {
       node("${immutable} && ${label}"){
         withBeatsEnvWin() {

CHANGELOG-developer.next.asciidoc

@@ -95,4 +95,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add IP* fields to `fields.yml` generator script in Filebeat. {issue}17998[17998] {pull}18256[18256]
 - Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606]
 - Remove vendor folder from repository. {pull}18655[18655]
+- Added SQL helper that can be used from any Metricbeat module {pull}18955[18955]
 - Update Go version to 1.14.4. {pull}19753[19753]

CHANGELOG.next.asciidoc

@@ -267,9 +267,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Stop counterCache only when already started {pull}19103[19103]
 - Set tags correctly if the dimension value is ARN {issue}19111[19111] {pull}19433[19433]
 - Fix bug incorrect parsing of float numbers as integers in Couchbase module {issue}18949[18949] {pull}19055[19055]
+- Fix mapping of service start type in the service metricset, windows module. {pull}19551[19551]
 - Fix config example in the perfmon configuration files. {pull}19539[19539]
 - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
 - Fix k8s scheduler compatibility issue. {pull}19699[19699]
+- Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
 
 *Packetbeat*
 
@@ -324,6 +326,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817]
 - Add support for multiple sets of hints on autodiscover {pull}18883[18883]
 - Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181]
+- Add data type conversion in `dissect` processor for converting string values to other basic data types. {pull}18683[18683]
 - Add the `ignore_failure` configuration option to the dissect processor. {pull}19464[19464]
 - Add the `overwrite_keys` configuration option to the dissect processor. {pull}19464[19464]
 - Add support to trim captured values in the dissect processor. {pull}19464[19464]
@@ -438,6 +441,28 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346]
 - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379]
 - Improve ECS categorization field mappings in azure module. {issue}16155[16155] {pull}19376[19376]
+- Add text & flattened versions of fields with unknown subfields in aws cloudtrail fileset. {issue}18866[18866] {pull}19121[19121]
+- Add experimental dataset tomcat/log for Apache TomCat logs {pull}19713[19713]
+- Add experimental dataset netscout/sightline for Netscout Arbor Sightline logs {pull}19713[19713]
+- Add experimental dataset barracuda/waf for Barracuda Web Application Firewall logs {pull}19713[19713]
+- Add experimental dataset f5/bigipapm for F5 Big-IP Access Policy Manager logs {pull}19713[19713]
+- Add experimental dataset bluecoat/director for Bluecoat Director logs {pull}19713[19713]
+- Add experimental dataset cisco/nexus for Cisco Nexus logs {pull}19713[19713]
+- Add experimental dataset citrix/virtualapps for Citrix Virtual Apps logs {pull}19713[19713]
+- Add experimental dataset cylance/protect for Cylance Protect logs {pull}19713[19713]
+- Add experimental dataset f5/firepass for F5 FirePass SSL VPN logs {pull}19713[19713]
+- Add experimental dataset fortinet/clientendpoint for Fortinet FortiClient Endpoint Protection logs {pull}19713[19713]
+- Add experimental dataset imperva/securesphere for Imperva Secure Sphere logs {pull}19713[19713]
+- Add experimental dataset infoblox/nios for Infoblox Network Identity Operating System logs {pull}19713[19713]
+- Add experimental dataset juniper/junos for Juniper Junos OS logs {pull}19713[19713]
+- Add experimental dataset kaspersky/av for Kaspersky Anti-Virus logs {pull}19713[19713]
+- Add experimental dataset microsoft/dhcp for Microsoft DHCP Server logs {pull}19713[19713]
+- Add experimental dataset tenable/nessus_security for Tenable Nessus Security Scanner logs {pull}19713[19713]
+- Add experimental dataset rapid7/nexpose for Rapid7 Nexpose logs {pull}19713[19713]
+- Add experimental dataset radware/defensepro for Radware DefensePro logs {pull}19713[19713]
+- Add experimental dataset sonicwall/firewall for Sonicwall Firewalls logs {pull}19713[19713]
+- Add experimental dataset squid/log for Squid Proxy Server logs {pull}19713[19713]
+- Add experimental dataset zscaler/zia for Zscaler Internet Access logs {pull}19713[19713]
 
 *Heartbeat*
 
@@ -539,7 +564,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423]
 - Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660]
 - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703]
+- Added `performance` and `query` metricsets to `mysql` module. {pull}18955[18955]
 - The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706]
+- Adds support for app insights metrics in the azure module. {issue}18570[18570] {pull}18940[18940]
 
 *Packetbeat*
 

NOTICE.txt

@@ -5703,11 +5703,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.5.0/LICE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-client/v7
-Version: v7.0.0-20200601155656-d6a9eb4f6d07
+Version: v7.0.0-20200709172729-d43b7ad5833a
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20200601155656-d6a9eb4f6d07/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.0.0-20200709172729-d43b7ad5833a/LICENSE.txt:
 
 ELASTIC LICENSE AGREEMENT
 
@@ -13238,6 +13238,61 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+--------------------------------------------------------------------------------
+Dependency : github.com/shopspring/decimal
+Version: v1.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/shopspring/decimal@v1.2.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Spring, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+- Based on https://github.com/oguzbilgic/fpd, which has the following license:
+"""
+The MIT License (MIT)
+
+Copyright (c) 2013 Oguz Bilgic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/spf13/cobra
 Version: v0.0.3
@@ -35549,6 +35604,36 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+--------------------------------------------------------------------------------
+Dependency : github.com/satori/go.uuid
+Version: v1.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/satori/go.uuid@v1.2.0/LICENSE:
+
+Copyright (C) 2013-2018 by Maxim Bublis <b@codemonkey.ru>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/sergi/go-diff
 Version: v1.1.0

auditbeat/cmd/root.go

@@ -24,12 +24,19 @@ import (
 	"github.com/elastic/beats/v7/auditbeat/core"
 	"github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/publisher/processing"
 	"github.com/elastic/beats/v7/metricbeat/beater"
 	"github.com/elastic/beats/v7/metricbeat/mb/module"
 )
 
-// Name of the beat (auditbeat).
-const Name = "auditbeat"
+const (
+	// Name of the beat (auditbeat).
+	Name = "auditbeat"
+
+	// ecsVersion specifies the version of ECS that Auditbeat is implementing.
+	ecsVersion = "1.5.0"
+)
 
 // RootCmd for running auditbeat.
 var RootCmd *cmd.BeatsRootCmd
@@ -40,6 +47,13 @@ var ShowCmd = &cobra.Command{
 	Short: "Show modules information",
 }
 
+// withECSVersion is a modifier that adds ecs.version to events.
+var withECSVersion = processing.WithFields(common.MapStr{
+	"ecs": common.MapStr{
+		"version": ecsVersion,
+	},
+})
+
 func init() {
 	create := beater.Creator(
 		beater.WithModuleOptions(
@@ -51,6 +65,7 @@ func init() {
 		RunFlags:      runFlags,
 		Name:          Name,
 		HasDashboards: true,
+		Processing:    processing.MakeDefaultSupport(true, withECSVersion, processing.WithHost, processing.WithAgentMeta()),
 	}
 	RootCmd = cmd.GenRootCmdWithSettings(create, settings)
 	RootCmd.AddCommand(ShowCmd)

auditbeat/main.go

@@ -22,10 +22,6 @@ import (
 
 	"github.com/elastic/beats/v7/auditbeat/cmd"
 
-	// Register modules.
-	_ "github.com/elastic/beats/v7/auditbeat/module/auditd"
-	_ "github.com/elastic/beats/v7/auditbeat/module/file_integrity"
-
 	// Register includes.
 	_ "github.com/elastic/beats/v7/auditbeat/include"
 )

dev-tools/mage/crossbuild.go

@@ -263,6 +263,7 @@ func (b GolangCrossBuilder) Build() error {
 
 	args = append(args,
 		"--rm",
+		"--env", "GOFLAGS=-mod=readonly",
 		"--env", "MAGEFILE_VERBOSE="+verbose,
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
 		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),

dev-tools/mage/fields.go

@@ -102,6 +102,7 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
 	}
 
 	cmd := []string{"run",
+		"-mod=readonly",
 		filepath.Join(beatsDir, globalFieldsCmdPath),
 		"-es_beats_path", beatsDir,
 		"-beat_path", baseDir,
@@ -127,6 +128,7 @@ func GenerateFieldsGo(fieldsYML, out string) error {
 	}
 
 	cmd := []string{"run",
+		"-mod=readonly",
 		filepath.Join(beatsDir, assetCmdPath),
 		"-pkg", "include",
 		"-in", fieldsYML,

dev-tools/mage/integtest.go

@@ -236,6 +236,7 @@ func initRunner(tester IntegrationTester, dir string, passInEnv map[string]strin
 	// Create the custom env for the runner.
 	env := map[string]string{
 		insideIntegrationTestEnvVar: "true",
+		"GOFLAGS":                   "-mod=readonly",
 	}
 	for name, value := range passInEnv {
 		env[name] = value