Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…
Browse files Browse the repository at this point in the history
…ne-2.0

* upstream/master: (44 commits)
  Update users.asciidoc (elastic#20802) (elastic#21108)
  Fix docker provider builder. (elastic#21118)
  [Elastic Agent] Add docker composable dynamic provider. (elastic#20842)
  Add new modules/filesets from rsa2elk for 7.10 (elastic#20820)
  Fix broken links to external websites (elastic#21061)
  [docs] typo in the command line (elastic#20799)
  [Filebeat] add panos type and sub_type (elastic#20912)
  Move the `compute_vm_scalset` to  a light metricset and map the cloud metadata (elastic#21038)
  [Filebeat] Add support for Cloudtrail digest files (elastic#21086)
  Add metrics collection from cost explorer into aws/billing metricset (elastic#20527)
  Add vendoring to Google Cloud Functions again (elastic#21070)
  [Elastic Agent] Add fleet.host.id for sending to endpoint. (elastic#21042)
  Do not need Google credentials before using it (elastic#21072)
  [Filebeat][New Module] Zoom webhook module (elastic#20414)
  Add support for GMT timezone offset in decode_cef (elastic#20993)
  Filebeat: Fix random error on harvester close (elastic#21048)
  Add ingress controller dashboards (elastic#21052)
  Fix loggers in composable module. (elastic#21047)
  [Ingest Manager] Increase kibana client timeout to 5 minutes (elastic#21037)
  Add changelog. (elastic#21041)
  ...
  • Loading branch information
v1v committed Sep 17, 2020
2 parents b4a1faf + fd0fe73 commit 4c7e8da
Show file tree
Hide file tree
Showing 513 changed files with 392,768 additions and 31,331 deletions.
5 changes: 3 additions & 2 deletions .ci/apm-beats-update.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
@Library('apm@current') _

pipeline {
agent none
agent { label 'master' }
environment {
REPO = 'apm-server'
BASE_DIR = "src/github.com/elastic/${env.REPO}"
Expand Down Expand Up @@ -31,7 +31,7 @@ pipeline {
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
agent { label 'ubuntu-18 && immutable' }
when {
beforeAgent true
anyOf {
Expand All @@ -53,6 +53,7 @@ pipeline {
Checkout the code and stash it, to use it on other stages.
*/
stage('Checkout') {
options { skipDefaultCheckout() }
steps {
deleteDir()
gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
Expand Down
46 changes: 46 additions & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,52 @@
:issue: https://github.com/elastic/beats/issues/
:pull: https://github.com/elastic/beats/pull/

[[release-notes-7.9.1]]
=== Beats version 7.9.1
https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits]

==== Breaking changes

*Affecting all Beats*

- Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]

==== Bugfixes

*Affecting all Beats*

- Update replicaset group to apps/v1 {pull}15854[15854]
- Rename cloud.provider `az` value to `azure` inside the add_cloud_metadata processor. {pull}20689[20689]
- Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811]

*Filebeat*

- Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
- Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
- Update documentation in the azure module filebeat. {pull}20815[20815]

*Heartbeat*

- Stop rescheduling tasks of stopped monitors. {pull}20570[20570]

*Metricbeat*

- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
- Add fallback for PdhExpandWildCardPathW failing in perfmon metricset. {issue}20139[20139] {pull}20630[20630]
- Fix resource tags in aws cloudwatch metricset {issue}20326[20326] {pull}20385[20385]
- Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]

*Winlogbeat*

- Fix duplicated field error when exporting index-pattern with migration.6_to_7.enabled. {issue}20521[20521] {pull}20540[20540]
- Fix `event.outcome` in the security module for non-English languages. {issue}20079[20079] {pull}20564[20564]

==== Added

*Affecting all Beats*

- Added support for more message types for Cisco ASA and FTD. {pull}20565[20565]

[[release-notes-7.9.0]]
=== Beats version 7.9.0
https://github.com/elastic/beats/compare/v7.8.1...v7.9.0[View commits]
Expand Down
46 changes: 36 additions & 10 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Tracking session end reason in panw module. {pull}18705[18705]
- Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
- Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
- Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]

*Heartbeat*

Expand Down Expand Up @@ -259,6 +260,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix long registry migration times. {pull}20717[20717] {issue}20705[20705]
- Fix event types and categories in auditd module to comply with ECS {pull}20652[20652]
- Update documentation in the azure module filebeat. {pull}20815[20815]
- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908]
- Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]

*Heartbeat*

Expand All @@ -267,6 +272,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add support for new `service_name` option to all monitors. {pull}19932[19932].
- Stop rescheduling tasks of stopped monitors. {pull}20570[20570]

*Heartbeat*


*Journalbeat*


Expand Down Expand Up @@ -309,7 +317,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix "ID" event generator of Google Cloud module {issue}17160[17160] {pull}17608[17608]
- Add privileged option for Auditbeat in Openshift {pull}17637[17637]
- Fix storage metricset to allow config without region/zone. {issue}17623[17623] {pull}17624[17624]
- Add a switch to the driver definition on SQL module to use pretty names. {pull}17378[17378]
- Fix overflow on Prometheus rates when new buckets are added on the go. {pull}17753[17753]
- Remove specific win32 api errors from events in perfmon. {issue}18292[18292] {pull}18361[18361]
- Fix application_pool metricset after pdh changes. {pull}18477[18477]
Expand All @@ -330,15 +337,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
- Fix k8s scheduler compatibility issue. {pull}19699[19699]
- Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
- Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
- Add required option for `metrics` in app_insights. {pull}20406[20406]
- Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
- Add fallback for PdhExpandWildCardPathW failing in perfmon metricset. {issue}20139[20139] {pull}20630[20630]
- Fix resource tags in aws cloudwatch metricset {issue}20326[20326] {pull}20385[20385]
- Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
- Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
- Update fields.yml in the azure module, missing metrics field. {pull}20918[20918]
- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
- Disable Kafka metricsets based on Jolokia by default. They require a different configuration. {pull}20989[20989]

*Packetbeat*

Expand All @@ -350,12 +353,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d

- Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436]
- Fields from Winlogbeat modules were not being included in index templates and patterns. {pull}18983[18983]
- Fix `event.outcome` in the security module for non-English languages. {issue}20079[20079] {pull}20564[20564]
- Fix duplicated field error when exporting index-pattern with migration.6_to_7.enabled. {issue}20521[20521] {pull}20540[20540]

*Functionbeat*

- Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287]
- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]

==== Added

Expand Down Expand Up @@ -407,8 +410,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
- Add leader election for Kubernetes autodiscover. {pull}20281[20281]
- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
- Added support for more message types for Cisco ASA and FTD. {pull}20565[20565]
- Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464]
- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
- Add ingress controller dashboards. {pull}21052[21052]
- Added experimental `citrix` module. {pull}20820[20820]
- Added experimental `cyberark` module. {pull}20820[20820]
- Added experimental `proofpoint` module. {pull}20820[20820]
- Added experimental `snort` module. {pull}20820[20820]
- Added experimental `symantec` module. {pull}20820[20820]
- Added experimental dataset `barracuda/spamfirewall`. {pull}20820[20820]
- Added experimental dataset `cisco/meraki`. {pull}20820[20820]
- Added experimental dataset `f5/bigipafm`. {pull}20820[20820]
- Added experimental dataset `fortinet/fortimail`. {pull}20820[20820]
- Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820]
- Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
- Added experimental dataset `sophos/utm`. {pull}20820[20820]

*Auditbeat*

Expand Down Expand Up @@ -554,6 +570,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Convert httpjson to v2 input {pull}20226[20226]
- Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
- Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927]
- Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958]
- Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983]
- Improve Santa module with `x509` ECS mappings {pull}20976[20976]
- Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]
- Added new module for Zoom webhooks {pull}20414[20414]
- Add type and sub_type to panw panos fileset {pull}20912[20912]

*Heartbeat*

Expand Down Expand Up @@ -669,9 +691,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649]
- Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
- Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
- Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103]
- Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889]
- Request prometheus endpoints to be gzipped by default {pull}20766[20766]
- Release all kubernetes `state` metricsets as GA {pull}20901[20901]
- Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738]
- Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
- Sanitize `event.host`. {pull}21022[21022]

*Packetbeat*

Expand Down
4 changes: 4 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ pipeline {
JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
}
options {
timeout(time: 2, unit: 'HOURS')
Expand All @@ -61,6 +62,7 @@ pipeline {
string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
}
stages {
/**
Expand Down Expand Up @@ -1125,6 +1127,7 @@ def isChangedOSSCode(patterns) {
def allPatterns = [
"^Jenkinsfile",
"^go.mod",
"^pytest.ini",
"^libbeat/.*",
"^testing/.*",
"^dev-tools/.*",
Expand All @@ -1138,6 +1141,7 @@ def isChangedXPackCode(patterns) {
def allPatterns = [
"^Jenkinsfile",
"^go.mod",
"^pytest.ini",
"^libbeat/.*",
"^dev-tools/.*",
"^testing/.*",
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ notice:
.PHONY: python-env
python-env:
@test -d $(PYTHON_ENV) || ${PYTHON_EXE} -m venv $(VENV_PARAMS) $(PYTHON_ENV)
@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 pylint==2.4.4
@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.5.4 pylint==2.4.4
@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'

Expand Down
68 changes: 34 additions & 34 deletions NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2182,36 +2182,6 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/auto
limitations under the License.


--------------------------------------------------------------------------------
Dependency : github.com/Masterminds/semver
Version: v1.4.2
Licence type (autodetected): MIT
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/!masterminds/semver@v1.4.2/LICENSE.txt:

The Masterminds
Copyright (C) 2014-2015, Matt Butcher and Matt Farina

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.


--------------------------------------------------------------------------------
Dependency : github.com/Microsoft/go-winio
Version: v0.4.15-0.20190919025122-fc70bd9a86b5
Expand Down Expand Up @@ -2555,11 +2525,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

--------------------------------------------------------------------------------
Dependency : github.com/antlr/antlr4
Version: v0.0.0-20200225173536-225249fdaef5
Version: v0.0.0-20200820155224-be881fa6b91d
Licence type (autodetected): MIT
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/antlr/antlr4@v0.0.0-20200225173536-225249fdaef5/LICENSE.txt:
Contents of probable licence file $GOMODCACHE/github.com/antlr/antlr4@v0.0.0-20200820155224-be881fa6b91d/LICENSE.txt:

[The "BSD 3-clause license"]
Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
Expand Down Expand Up @@ -13971,11 +13941,11 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/scram@v0.0.0-201808

--------------------------------------------------------------------------------
Dependency : go.elastic.co/apm
Version: v1.8.1-0.20200902013556-b34fe04da73f
Version: v1.8.1-0.20200909061013-2aef45b9cf4b
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.8.1-0.20200902013556-b34fe04da73f/LICENSE:
Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.8.1-0.20200909061013-2aef45b9cf4b/LICENSE:

Apache License
Version 2.0, January 2004
Expand Down Expand Up @@ -19451,6 +19421,36 @@ Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/xgb@v0.0.0
// such litigation is filed.


--------------------------------------------------------------------------------
Dependency : github.com/Masterminds/semver
Version: v1.4.2
Licence type (autodetected): MIT
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/!masterminds/semver@v1.4.2/LICENSE.txt:

The Masterminds
Copyright (C) 2014-2015, Matt Butcher and Matt Farina

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.


--------------------------------------------------------------------------------
Dependency : github.com/Microsoft/hcsshim
Version: v0.8.7
Expand Down
2 changes: 1 addition & 1 deletion auditbeat/tests/system/test_file_integrity.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def file_events(objs, path, expected):
evts = set()
for obj in objs:
if 'file.path' in obj and 'event.action' in obj and obj['file.path'].lower() == path.lower():
if type(obj['event.action']) == list:
if isinstance(obj['event.action'], list):
evts = evts.union(set(obj['event.action']))
else:
evts.add(obj['event.action'])
Expand Down
2 changes: 1 addition & 1 deletion dev-tools/cmd/dashboards/export_5x_dashboards.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def ExportDashboards(es, regex, kibana_index, output_directory):

try:
reg_exp = re.compile(regex, re.IGNORECASE)
except:
except BaseException:
print("Wrong regex {}".format(regex))
return

Expand Down
19 changes: 19 additions & 0 deletions dev-tools/mage/gotool/go.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) {
return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
}

// ListDepsLocation calls `go list -dep` for every package spec given.
func ListDepsLocation(pkg string) (map[string]string, error) {
const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}`

lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg))
if err != nil {
return nil, err
}
deps := make(map[string]string, len(lines))
for _, l := range lines {
parts := strings.Split(l, ";")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid number of parts")
}
deps[parts[0]] = parts[1]
}
return deps, nil
}

// ListTestFiles lists all go and cgo test files available in a package.
func ListTestFiles(pkg string) ([]string, error) {
const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` +
Expand Down
4 changes: 2 additions & 2 deletions dev-tools/mage/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func InstallVendored(importPath string) error {

// InstallGoLicenser target installs go-licenser
func InstallGoLicenser() error {
return gotool.Get(
gotool.Get.Package(GoLicenserImportPath),
return gotool.Install(
gotool.Install.Package(GoLicenserImportPath),
)
}
Loading

0 comments on commit 4c7e8da

Please sign in to comment.