diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index cec790b06bff..b6d572e5e559 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -42,6 +42,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -16944,6 +16945,63 @@ IBM MQ module +[[exported-fields-iis]] +== iis fields + +iis module + + + +[float] +=== iis + + + + +[float] +=== application_pool + +application_pool + + + +*`iis.application_pool.name`*:: ++ +-- +application pool name + + +type: keyword + +-- + +*`iis.webserver.*.*`*:: ++ +-- +webserver + + +type: object + +-- + +[float] +=== website + +website + + + +*`iis.website.name`*:: ++ +-- +website name + + +type: keyword + +-- + [[exported-fields-istio]] == istio fields diff --git a/metricbeat/docs/modules/iis.asciidoc b/metricbeat/docs/modules/iis.asciidoc new file mode 100644 index 000000000000..50dae38e10fc --- /dev/null +++ b/metricbeat/docs/modules/iis.asciidoc @@ -0,0 +1,99 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-iis]] +== iis module + +beta[] + +This is the iis module. + +IIS (Internet Information Services) is a secure, reliable, and scalable Web server that provides an easy to manage platform for developing and hosting Web applications and services. + +The `iis` module will periodically retrieve IIS related metrics using performance counters such as: + + - System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process). + - IIS performance counters like Web Service: Bytes Received/Sec, Web Service: Bytes Sent/Sec, etc, which are helpful to track to identify potential spikes in traffic. + - Web Service Cache counters in order to monitor user mode cache and output cache. + + +The `iis` module mericsets are `webserver`, `website` and `application_pool`. + +[source,yaml] +---- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] +---- + +[float] +== Metricsets + +[float] +=== `webserver` +A light metricset using the windows perfmon metricset as the base metricset. +This metricset allows users to retrieve aggregated metrics for the entire webserver, + +[float] +=== `website` +A light metricset using the windows perfmon metricset as the base metricset. +This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered. + +[float] +=== `application_pool` +This metricset will collect metrics of specific application pools, users can configure which websites they want to monitor, else, all are considered. + + +[float] +=== Module-specific configuration notes + +`application_pool.name`:: []string, users can specify the application pools they would like to monitor. + + + +[float] +=== Example configuration + +The iis module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +include::iis/application_pool.asciidoc[] + +include::iis/webserver.asciidoc[] + +include::iis/website.asciidoc[] + diff --git a/metricbeat/docs/modules/iis/application_pool.asciidoc b/metricbeat/docs/modules/iis/application_pool.asciidoc new file mode 100644 index 000000000000..3b1d78ab13f8 --- /dev/null +++ b/metricbeat/docs/modules/iis/application_pool.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-iis-application_pool]] +=== iis application_pool metricset + +beta[] + +include::../../../module/iis/application_pool/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/application_pool/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/iis/webserver.asciidoc b/metricbeat/docs/modules/iis/webserver.asciidoc new file mode 100644 index 000000000000..4352ab768f01 --- /dev/null +++ b/metricbeat/docs/modules/iis/webserver.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-iis-webserver]] +=== iis webserver metricset + +beta[] + +include::../../../module/iis/webserver/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/webserver/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/iis/website.asciidoc b/metricbeat/docs/modules/iis/website.asciidoc new file mode 100644 index 000000000000..1132d876ad49 --- /dev/null +++ b/metricbeat/docs/modules/iis/website.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-iis-website]] +=== iis website metricset + +beta[] + +include::../../../module/iis/website/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/iis/website/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 5587b923345e..b90ed67a8bea 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -111,6 +111,10 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> beta[] +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.3+| .3+| |<> beta[] +|<> beta[] +|<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .5+| .5+| |<> beta[] |<> beta[] @@ -274,6 +278,7 @@ include::modules/graphite.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/http.asciidoc[] include::modules/ibmmq.asciidoc[] +include::modules/iis.asciidoc[] include::modules/istio.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] diff --git a/metricbeat/helper/windows/pdh/pdh_query_windows.go b/metricbeat/helper/windows/pdh/pdh_query_windows.go index c7d8f0d82bf0..898d5573a60a 100644 --- a/metricbeat/helper/windows/pdh/pdh_query_windows.go +++ b/metricbeat/helper/windows/pdh/pdh_query_windows.go @@ -80,7 +80,7 @@ func (q *Query) AddCounter(counterPath string, instance string, format string, w var instanceName string // Extract the instance name from the counterPath. if instance == "" || wildcard { - instanceName, err = matchInstanceName(counterPath) + instanceName, err = MatchInstanceName(counterPath) if err != nil { return err } @@ -178,6 +178,18 @@ func (q *Query) GetFormattedCounterValues() (map[string][]CounterValue, error) { return rtn, nil } +// GetCountersAndInstances returns a list of counters and instances for a given object +func (q *Query) GetCountersAndInstances(objectName string) ([]string, []string, error) { + counters, instances, err := PdhEnumObjectItems(objectName) + if err != nil { + return nil, nil, errors.Wrapf(err, "Unable to retrieve counter and instance list for %s", objectName) + } + if len(counters) == 0 && len(instances) == 0 { + return nil, nil, errors.Errorf("Unable to retrieve counter and instance list for %s", objectName) + } + return UTF16ToStringArray(counters), UTF16ToStringArray(instances), nil +} + // ExpandWildCardPath examines local computer and returns those counter paths that match the given counter path which contains wildcard characters. func (q *Query) ExpandWildCardPath(wildCardPath string) ([]string, error) { if wildCardPath == "" { @@ -209,8 +221,8 @@ func (q *Query) Close() error { return PdhCloseQuery(q.Handle) } -// matchInstanceName will check first for instance and then for any objects names. -func matchInstanceName(counterPath string) (string, error) { +// MatchInstanceName will check first for instance and then for any objects names. +func MatchInstanceName(counterPath string) (string, error) { matches := instanceNameRegexp.FindStringSubmatch(counterPath) if len(matches) != 2 { matches = objectNameRegexp.FindStringSubmatch(counterPath) diff --git a/metricbeat/helper/windows/pdh/pdh_windows.go b/metricbeat/helper/windows/pdh/pdh_windows.go index bccca2c5784d..ccfb2ea50f38 100644 --- a/metricbeat/helper/windows/pdh/pdh_windows.go +++ b/metricbeat/helper/windows/pdh/pdh_windows.go @@ -41,6 +41,7 @@ import ( //sys _PdhExpandWildCardPath(dataSource *uint16, wildcardPath *uint16, expandedPathList *uint16, pathListLength *uint32) (errcode error) [failretval!=0] = pdh.PdhExpandWildCardPathW //sys _PdhExpandCounterPath(wildcardPath *uint16, expandedPathList *uint16, pathListLength *uint32) (errcode error) [failretval!=0] = pdh.PdhExpandCounterPathW //sys _PdhGetCounterInfo(counter PdhCounterHandle, text uint16, size *uint32, lpBuffer *byte) (errcode error) [failretval!=0] = pdh.PdhGetCounterInfoW +//sys _PdhEnumObjectItems(dataSource uint16, machineName uint16, objectName *uint16, counterList *uint16, counterListSize *uint32, instanceList *uint16, instanceListSize *uint32, detailLevel uint32, flags uint32) (errcode error) [failretval!=0] = pdh.PdhEnumObjectItemsW type PdhQueryHandle uintptr @@ -50,6 +51,9 @@ type PdhCounterHandle uintptr var InvalidCounterHandle = ^PdhCounterHandle(0) +// PerformanceDetailWizard is the counter detail level +const PerformanceDetailWizard = 400 + // PdhCounterInfo struct contains the performance counter details type PdhCounterInfo struct { DwLength uint32 @@ -247,6 +251,48 @@ func PdhCloseQuery(query PdhQueryHandle) error { return nil } +// PdhEnumObjectItems returns the counters and instance info for given object +func PdhEnumObjectItems(objectName string) ([]uint16, []uint16, error) { + var ( + cBuff = make([]uint16, 1) + cBuffSize = uint32(0) + iBuff = make([]uint16, 1) + iBuffSize = uint32(0) + ) + obj := windows.StringToUTF16Ptr(objectName) + if err := _PdhEnumObjectItems( + 0, + 0, + obj, + &cBuff[0], + &cBuffSize, + &iBuff[0], + &iBuffSize, + PerformanceDetailWizard, + 0); err != nil { + if PdhErrno(err.(syscall.Errno)) != PDH_MORE_DATA { + return nil, nil, PdhErrno(err.(syscall.Errno)) + } + cBuff = make([]uint16, cBuffSize) + iBuff = make([]uint16, iBuffSize) + + if err = _PdhEnumObjectItems( + 0, + 0, + obj, + &cBuff[0], + &cBuffSize, + &iBuff[0], + &iBuffSize, + PerformanceDetailWizard, + 0); err != nil { + return nil, nil, err + } + return cBuff, iBuff, nil + } + return nil, nil, nil +} + // Error returns a more explicit error message. func (e PdhErrno) Error() string { // If the value is not one of the known PDH errors then assume its a diff --git a/metricbeat/helper/windows/pdh/zpdh_windows.go b/metricbeat/helper/windows/pdh/zpdh_windows.go index 8d2891b73cba..15473a7f5a23 100644 --- a/metricbeat/helper/windows/pdh/zpdh_windows.go +++ b/metricbeat/helper/windows/pdh/zpdh_windows.go @@ -66,6 +66,7 @@ var ( procPdhExpandWildCardPathW = modpdh.NewProc("PdhExpandWildCardPathW") procPdhExpandCounterPathW = modpdh.NewProc("PdhExpandCounterPathW") procPdhGetCounterInfoW = modpdh.NewProc("PdhGetCounterInfoW") + procPdhEnumObjectItemsW = modpdh.NewProc("PdhEnumObjectItemsW") ) func _PdhOpenQuery(dataSource *uint16, userData uintptr, query *PdhQueryHandle) (errcode error) { @@ -181,3 +182,11 @@ func _PdhGetCounterInfo(counter PdhCounterHandle, text uint16, size *uint32, lpB } return } + +func _PdhEnumObjectItems(dataSource uint16, machineName uint16, objectName *uint16, counterList *uint16, counterListSize *uint32, instanceList *uint16, instanceListSize *uint32, detailLevel uint32, flags uint32) (errcode error) { + r0, _, _ := syscall.Syscall9(procPdhEnumObjectItemsW.Addr(), 9, uintptr(dataSource), uintptr(machineName), uintptr(unsafe.Pointer(objectName)), uintptr(unsafe.Pointer(counterList)), uintptr(unsafe.Pointer(counterListSize)), uintptr(unsafe.Pointer(instanceList)), uintptr(unsafe.Pointer(instanceListSize)), uintptr(detailLevel), uintptr(flags)) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 503ce16ef21f..eb96b2f25c11 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -82,6 +82,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/http" _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/http/server" + _ "github.com/elastic/beats/metricbeat/module/iis" + _ "github.com/elastic/beats/metricbeat/module/iis/application_pool" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3bb9ce6b7878..834e103021b2 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -346,6 +346,18 @@ metricbeat.modules: # fields: # added to the the response in root. overwrites existing fields # key: "value" +#--------------------------------- Iis Module --------------------------------- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] + #------------------------------- Jolokia Module ------------------------------- - module: jolokia #metricsets: ["jmx"] diff --git a/metricbeat/module/iis/_meta/config.yml b/metricbeat/module/iis/_meta/config.yml new file mode 100644 index 000000000000..a2ea8fc2e7d8 --- /dev/null +++ b/metricbeat/module/iis/_meta/config.yml @@ -0,0 +1,10 @@ +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] diff --git a/metricbeat/module/iis/_meta/docs.asciidoc b/metricbeat/module/iis/_meta/docs.asciidoc new file mode 100644 index 000000000000..309b37aeee77 --- /dev/null +++ b/metricbeat/module/iis/_meta/docs.asciidoc @@ -0,0 +1,50 @@ +This is the iis module. + +IIS (Internet Information Services) is a secure, reliable, and scalable Web server that provides an easy to manage platform for developing and hosting Web applications and services. + +The `iis` module will periodically retrieve IIS related metrics using performance counters such as: + + - System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process). + - IIS performance counters like Web Service: Bytes Received/Sec, Web Service: Bytes Sent/Sec, etc, which are helpful to track to identify potential spikes in traffic. + - Web Service Cache counters in order to monitor user mode cache and output cache. + + +The `iis` module mericsets are `webserver`, `website` and `application_pool`. + +[source,yaml] +---- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] +---- + +[float] +== Metricsets + +[float] +=== `webserver` +A light metricset using the windows perfmon metricset as the base metricset. +This metricset allows users to retrieve aggregated metrics for the entire webserver, + +[float] +=== `website` +A light metricset using the windows perfmon metricset as the base metricset. +This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered. + +[float] +=== `application_pool` +This metricset will collect metrics of specific application pools, users can configure which websites they want to monitor, else, all are considered. + + +[float] +=== Module-specific configuration notes + +`application_pool.name`:: []string, users can specify the application pools they would like to monitor. + diff --git a/metricbeat/module/iis/_meta/fields.yml b/metricbeat/module/iis/_meta/fields.yml new file mode 100644 index 000000000000..9a7b776c59f6 --- /dev/null +++ b/metricbeat/module/iis/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: iis + title: "iis" + description: > + iis module + release: beta + + fields: + - name: iis + type: group + description: > + fields: diff --git a/metricbeat/module/iis/application_pool/_meta/data.json b/metricbeat/module/iis/application_pool/_meta/data.json new file mode 100644 index 000000000000..b1c084bfa2ee --- /dev/null +++ b/metricbeat/module/iis/application_pool/_meta/data.json @@ -0,0 +1,28 @@ +{ + "@timestamp" : "2020-02-13T14:55:27.000Z", + "metricset" : { + "name" : "application_pool", + "period" : 10000 + }, + "service" : { + "type" : "iis" + }, + "process" : { + "pid" : 7664 + }, + "iis" : { + "application_pool" : { + "net_clr" : { + "total_exceptions_thrown" : 0 + }, + "name" : "test.local", + "process" : { + "thread_count" : 32, + "private_byte" : 3.4684928E7, + "handle_count" : 743, + "virtual_bytes" : 6.49535488E8, + "working_set" : 4.2012672E7 + } + } + } +} diff --git a/metricbeat/module/iis/application_pool/_meta/docs.asciidoc b/metricbeat/module/iis/application_pool/_meta/docs.asciidoc new file mode 100644 index 000000000000..e85da0ee7e89 --- /dev/null +++ b/metricbeat/module/iis/application_pool/_meta/docs.asciidoc @@ -0,0 +1,14 @@ +This is the application_pool metricset of the module iis. + +This metricset allows users to retrieve relevant metrics for the application pools running on IIS. + +Metric values are divided in several groups: + +The `process` object contains System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process). + +The `net_clr` object which returns ASP.NET error rate counter values. +Users can specify the application pools they would like to monitor using the configuration option `application_pool.name`, else, all application pools are considered. + + + + diff --git a/metricbeat/module/iis/application_pool/_meta/fields.yml b/metricbeat/module/iis/application_pool/_meta/fields.yml new file mode 100644 index 000000000000..48b70c26101e --- /dev/null +++ b/metricbeat/module/iis/application_pool/_meta/fields.yml @@ -0,0 +1,10 @@ +- name: application_pool + type: group + release: beta + description: > + application_pool + fields: + - name: name + type: keyword + description: > + application pool name diff --git a/metricbeat/module/iis/application_pool/application_pool.go b/metricbeat/module/iis/application_pool/application_pool.go new file mode 100644 index 000000000000..913ac03eea1f --- /dev/null +++ b/metricbeat/module/iis/application_pool/application_pool.go @@ -0,0 +1,107 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package application_pool + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("iis", "application_pool", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + log *logp.Logger + reader *Reader +} + +// Config for the iis website metricset. +type Config struct { + Names []string `config:"application_pool.name"` +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The iis application_pool metricset is beta.") + var config Config + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + // instantiate reader object + reader, err := newReader() + if err != nil { + return nil, err + } + ms := &MetricSet{ + BaseMetricSet: base, + log: logp.NewLogger("application pool"), + reader: reader, + } + if err := ms.reader.initCounters(config.Names); err != nil { + return ms, err + } + return ms, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + var config Config + if err := m.Module().UnpackConfig(&config); err != nil { + return nil + } + + events, err := m.reader.fetch(config.Names) + if err != nil { + return errors.Wrap(err, "failed reading counters") + } + + for _, event := range events { + isOpen := report.Event(event) + if !isOpen { + break + } + } + return nil +} + +// Close will be called when metricbeat is stopped, should close the query. +func (m *MetricSet) Close() error { + err := m.reader.close() + if err != nil { + return errors.Wrap(err, "failed to close pdh query") + } + return nil +} diff --git a/metricbeat/module/iis/application_pool/application_pool_test.go b/metricbeat/module/iis/application_pool/application_pool_test.go new file mode 100644 index 000000000000..f118d135c1f2 --- /dev/null +++ b/metricbeat/module/iis/application_pool/application_pool_test.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package application_pool + +import ( + "testing" + "time" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestMetricsetNoErrors(t *testing.T) { + config := map[string]interface{}{ + "module": "iis", + "metricsets": []string{"application_pool"}, + } + + ms := mbtest.NewReportingMetricSetV2Error(t, config) + mbtest.ReportingFetchV2Error(ms) + time.Sleep(60 * time.Millisecond) + + _, errs := mbtest.ReportingFetchV2Error(ms) + if len(errs) > 0 { + t.Fatal(errs) + } +} diff --git a/metricbeat/module/iis/application_pool/doc.go b/metricbeat/module/iis/application_pool/doc.go new file mode 100644 index 000000000000..3bc21201b3e7 --- /dev/null +++ b/metricbeat/module/iis/application_pool/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package application_pool diff --git a/metricbeat/module/iis/application_pool/reader.go b/metricbeat/module/iis/application_pool/reader.go new file mode 100644 index 000000000000..11651d5e510b --- /dev/null +++ b/metricbeat/module/iis/application_pool/reader.go @@ -0,0 +1,277 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package application_pool + +import ( + "strings" + + "github.com/elastic/go-sysinfo" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper/windows/pdh" + "github.com/elastic/beats/metricbeat/mb" +) + +// Reader strucr will contain the pdh query and config options +type Reader struct { + Query pdh.Query // PDH Query + ApplicationPools []ApplicationPool // Mapping of counter path to key used for the label (e.g. processor.name) + log *logp.Logger // logger + hasRun bool // will check if the reader has run a first time + WorkerProcesses map[string]string +} + +// ApplicationPool struct contains the list of applications and their worker processes +type ApplicationPool struct { + Name string + WorkerProcessIds []int + counters map[string]string +} + +// WorkerProcess struct contains the worker process details +type WorkerProcess struct { + ProcessId int + InstanceName string +} + +const ecsProcessId = "process.pid" + +var appPoolCounters = map[string]string{ + "process.pid": "\\Process(w3wp*)\\ID Process", + "process.cpu_usage_perc": "\\Process(w3wp*)\\% Processor Time", + "process.handle_count": "\\Process(w3wp*)\\Handle Count", + "process.thread_count": "\\Process(w3wp*)\\Thread Count", + "process.working_set": "\\Process(w3wp*)\\Working Set", + "process.private_byte": "\\Process(w3wp*)\\Private Bytes", + "process.virtual_bytes": "\\Process(w3wp*)\\Virtual Bytes", + "process.page_faults_per_sec": "\\Process(w3wp*)\\Page Faults/sec", + "process.io_read_operations_per_sec": "\\Process(w3wp*)\\IO Read Operations/sec", + "process.io_write_operations_per_sec": "\\Process(w3wp*)\\IO Write Operations/sec", + "net_clr.total_exceptions_thrown": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown", + "net_clr.exceptions_thrown_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Exceps Thrown / sec", + "net_clr.filters_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Filters / sec", + "net_clr.finallys_per_sec": "\\.NET CLR Exceptions(w3wp*)\\# of Finallys / sec", + "net_clr.throw_to_catch_depth_per_sec": "\\.NET CLR Exceptions(w3wp*)\\Throw To Catch Depth / sec", +} + +// newReader creates a new instance of Reader. +func newReader() (*Reader, error) { + var query pdh.Query + if err := query.Open(); err != nil { + return nil, err + } + reader := &Reader{ + Query: query, + log: logp.NewLogger("website"), + } + + return reader, nil +} + +// initCounters func retrieves the running application worker processes and adds the counters to the pdh query +func (re *Reader) initCounters(filtered []string) error { + apps, err := getApplicationPools(filtered) + if err != nil { + return errors.Wrap(err, "failed retrieving running worker processes") + } + if len(apps) == 0 { + re.log.Info("no running application pools found") + return nil + } + re.ApplicationPools = apps + re.WorkerProcesses = make(map[string]string) + var newQueries []string + for key, value := range appPoolCounters { + counters, err := re.Query.ExpandWildCardPath(value) + if err != nil { + re.log.Error(err, `failed to expand counter path (query="%v")`, value) + continue + } + for _, count := range counters { + if err = re.Query.AddCounter(count, "", "float", true); err != nil { + return errors.Wrapf(err, `failed to add counter (query="%v")`, count) + } + newQueries = append(newQueries, count) + re.WorkerProcesses[count] = key + } + } + err = re.Query.RemoveUnusedCounters(newQueries) + if err != nil { + return errors.Wrap(err, "failed removing unused counter values") + } + return nil +} + +// fetch executes collects the query data and maps the counter values to events. +func (re *Reader) fetch(names []string) ([]mb.Event, error) { + // refresh performance counter list + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + // A flag is set if the second call has been executed else refresh will fail (reader.executed) + if re.hasRun || len(re.Query.Counters) == 0 { + err := re.initCounters(names) + if err != nil { + return nil, errors.Wrap(err, "failed retrieving counters") + } + } + // if the ignore_non_existent_counters flag is set and no valid counter paths are found the Read func will still execute, a check is done before + if len(re.Query.Counters) == 0 { + return nil, nil + } + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if err := re.Query.CollectData(); err != nil { + return nil, errors.Wrap(err, "failed querying counter values") + } + + // Get the values. + values, err := re.Query.GetFormattedCounterValues() + if err != nil { + return nil, errors.Wrap(err, "failed formatting counter values") + } + + workers := getProcessIds(values) + events := make(map[string]mb.Event) + for _, appPool := range re.ApplicationPools { + events[appPool.Name] = mb.Event{ + MetricSetFields: common.MapStr{ + "name": appPool.Name, + }, + RootFields: common.MapStr{}, + } + for counterPath, value := range values { + for _, val := range value { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.hasRun { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("website"), "query", counterPath) + continue + } + if val.Instance == appPool.Name { + events[appPool.Name].MetricSetFields.Put(appPool.counters[counterPath], val.Measurement) + } else if hasWorkerProcess(val.Instance, workers, appPool.WorkerProcessIds) { + if re.WorkerProcesses[counterPath] == ecsProcessId { + events[appPool.Name].RootFields.Put(re.WorkerProcesses[counterPath], val.Measurement) + } else { + events[appPool.Name].MetricSetFields.Put(re.WorkerProcesses[counterPath], val.Measurement) + } + } + } + + } + } + + re.hasRun = true + results := make([]mb.Event, 0, len(events)) + for _, val := range events { + results = append(results, val) + } + return results, nil +} + +// Close will close the PDH query for now. +func (re *Reader) close() error { + return re.Query.Close() +} + +// getApplicationPools method retrieves the w3wp.exe processes and the application pool name, also filters on the application pool names configured by users +func getApplicationPools(names []string) ([]ApplicationPool, error) { + processes, err := getw3wpProceses() + if err != nil { + return nil, err + } + var appPools = make(map[string][]int) + for key, value := range processes { + appPools[value] = append(appPools[value], key) + } + var applicationPools []ApplicationPool + for key, value := range appPools { + applicationPools = append(applicationPools, ApplicationPool{Name: key, WorkerProcessIds: value}) + } + if len(names) == 0 { + return applicationPools, nil + } + var filtered []ApplicationPool + for _, n := range names { + for _, w3 := range applicationPools { + if n == w3.Name { + filtered = append(filtered, w3) + } + } + } + return filtered, nil +} + +// getw3wpProceses func retrieves the running w3wp process ids. +// A worker process is a windows process (w3wp.exe) which runs Web applications, +// and is responsible for handling requests sent to a Web Server for a specific application pool. +func getw3wpProceses() (map[int]string, error) { + processes, err := sysinfo.Processes() + if err != nil { + return nil, err + } + wps := make(map[int]string) + for _, p := range processes { + info, err := p.Info() + if err != nil { + continue + } + if info.Name == "w3wp.exe" { + if len(info.Args) > 0 { + for i, ar := range info.Args { + if ar == "-ap" && len(info.Args) > i+1 { + wps[info.PID] = info.Args[i+1] + break + } + } + } + } + } + return wps, nil +} + +// getProcessIds func maps the process ids from the counter values to worker process obj +func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess { + var workers []WorkerProcess + for key, values := range counterValues { + if strings.Contains(key, "\\ID Process") { + workers = append(workers, WorkerProcess{InstanceName: values[0].Instance, ProcessId: int(values[0].Measurement.(float64))}) + } + } + return workers +} + +// hasWorkerProcess func checks if workerprocess list contains the process id +func hasWorkerProcess(instance string, workers []WorkerProcess, pids []int) bool { + for _, worker := range workers { + if worker.InstanceName == instance { + for _, pid := range pids { + if pid == worker.ProcessId { + return true + } + } + } + } + return false +} diff --git a/metricbeat/module/iis/application_pool/reader_test.go b/metricbeat/module/iis/application_pool/reader_test.go new file mode 100644 index 000000000000..3fddd416dd98 --- /dev/null +++ b/metricbeat/module/iis/application_pool/reader_test.go @@ -0,0 +1,73 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package application_pool + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/metricbeat/helper/windows/pdh" +) + +// TestNewReaderValid should successfully instantiate the reader. +func TestNewReaderValid(t *testing.T) { + reader, err := newReader() + assert.Nil(t, err) + assert.NotNil(t, reader) + assert.NotNil(t, reader.Query) + assert.NotNil(t, reader.Query.Handle) + assert.NotNil(t, reader.Query.Counters) + assert.Zero(t, len(reader.Query.Counters)) + defer reader.close() +} + +// TestInitCounters should successfully instantiate the reader counters. +func TestInitCounters(t *testing.T) { + reader, err := newReader() + assert.NotNil(t, reader) + assert.Nil(t, err) + + err = reader.initCounters([]string{}) + assert.Nil(t, err) + // if iis is not enabled, the reader.ApplicationPools is empty + if len(reader.ApplicationPools) > 0 { + assert.NotZero(t, len(reader.Query.Counters)) + assert.NotZero(t, len(reader.WorkerProcesses)) + } + defer reader.close() +} + +func TestGetProcessIds(t *testing.T) { + var key = "\\Process(w3wp#1)\\ID Process" + var counters = []pdh.CounterValue{ + { + Instance: "w3wp#1", + Measurement: 124.00, + Err: nil, + }, + } + counterList := make(map[string][]pdh.CounterValue) + counterList[key] = counters + workerProcesses := getProcessIds(counterList) + assert.NotZero(t, len(workerProcesses)) + assert.Equal(t, float64(workerProcesses[0].ProcessId), counters[0].Measurement.(float64)) + assert.Equal(t, workerProcesses[0].InstanceName, counters[0].Instance) +} diff --git a/metricbeat/module/iis/doc.go b/metricbeat/module/iis/doc.go new file mode 100644 index 000000000000..3cd0f406ee2a --- /dev/null +++ b/metricbeat/module/iis/doc.go @@ -0,0 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package iis is a Metricbeat module that contains MetricSets. + +package iis diff --git a/metricbeat/module/iis/fields.go b/metricbeat/module/iis/fields.go new file mode 100644 index 000000000000..20cec62b75e3 --- /dev/null +++ b/metricbeat/module/iis/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package iis + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "iis", asset.ModuleFieldsPri, AssetIis); err != nil { + panic(err) + } +} + +// AssetIis returns asset data. +// This is the base64 encoded gzipped contents of ../metricbeat/module/iis. +func AssetIis() string { + return "eJy0kbGOgzAQRHt/xYgSKfkAF/cryMAm2ovBlu1cxN+ffAYOiEE02YJidzTzzFzwoEGC2QsgcNAkUTD7QgAt+caxDWx6iS8BIOrQmfapSQCONClPEjUFJQRwY9Ktl3/KC3rV0eQcJwyWJO7OPO24yQSsTZZGylrNjYriyhqjZ0HOOc4a73+fTU1zkLHFWqLF7+owIT1oeBnXbm4HABsIRIi1/ZT5otqT+yF3La/liVcnIFN/UxMW67So0vWmjdo5Vp2ylvv7qCzK4twfnTHnbfYlHOgTfb5bv9e4W+J+hYcFjpnJ7zcAAP//navhqA==" +} diff --git a/metricbeat/module/iis/module.yml b/metricbeat/module/iis/module.yml new file mode 100644 index 000000000000..48b01480e3dd --- /dev/null +++ b/metricbeat/module/iis/module.yml @@ -0,0 +1,4 @@ +name: iis +metricsets: +- webserver +- website diff --git a/metricbeat/module/iis/webserver/_meta/data.json b/metricbeat/module/iis/webserver/_meta/data.json new file mode 100644 index 000000000000..e01bfc68c05c --- /dev/null +++ b/metricbeat/module/iis/webserver/_meta/data.json @@ -0,0 +1,57 @@ +{ + "@timestamp" : "2020-02-12T15:40:39.727Z", + "service" : { + "type" : "iis" + }, + "iis" : { + "webserver" : { + "process" : { + "worker_process_count" : 4, + "virtual_bytes" : 2.699976704E9, + "io_read_operations_per_sec" : 0, + "page_faults_per_Sec" : 0, + "private_byte" : 6.5712128E7, + "working_set" : 1396736.0, + "handle_count" : 0, + "cpu_usage_perc" : 0, + "io_write_operations_per_sec" : 0, + "thread_count" : 0 + }, + "network" : { + "total_delete_requests" : 0, + "bytes_received_per_sec" : 0, + "total_post_requests" : 0, + "delete_requests_per_sec" : 0, + "total_connection_attempts" : 73, + "bytes_sent_per_sec" : 0, + "get_requests_per_sec" : 0, + "maximum_connections" : 11, + "total_bytes_received" : 68170, + "service_uptime" : 588930, + "total_get_requests" : 162, + "total_bytes_sent" : 2898527.0, + "post_requests_per_sec" : 0 + }, + "cache" : { + "total_uris_cached" : 24, + "uri_cache_misses" : 161, + "file_cache_hits" : 21, + "maximum_file_cache_memory_usage" : 696, + "output_cache_current_memory_usage" : 0, + "total_files_cached" : 4, + "current_file_cache_memory_usage" : 0, + "uri_cache_hits" : 21, + "file_cache_misses" : 45, + "current_files_cached" : 0, + "output_cache_total_hits" : 0, + "output_cache_current_items" : 0, + "output_cache_total_misses" : 182, + "current_uris_cached" : 0 + } + } + }, + "metricset" : { + "name" : "webserver", + "period" : 10000 + } +} diff --git a/metricbeat/module/iis/webserver/_meta/docs.asciidoc b/metricbeat/module/iis/webserver/_meta/docs.asciidoc new file mode 100644 index 000000000000..41e1658e075d --- /dev/null +++ b/metricbeat/module/iis/webserver/_meta/docs.asciidoc @@ -0,0 +1,20 @@ +This is the webserver metricset of the module iis. + + +This metricset allows users to retrieve relevant metrics from IIS. + +Metric values are divided in several groups: + +The `process` object contains System/Process counters like the the overall server and CPU usage for the IIS Worker Processes and memory (currently used and available memory for the IIS Worker Processes). + +The `network` object contains the IIS Performance counters like: +Web Service: Bytes Received/Sec (helpful to track to identify potential spikes in traffic), Web Service: Bytes Sent/Sec (helpful to track to identify potential spikes in traffic), +Web Service: Current Connections (through experience with their apps app, users can identify what is a normal value for this) and others. + +The `cache` object contains metrics from the user mode cache and output cache. + + + + + + diff --git a/metricbeat/module/iis/webserver/_meta/fields.yml b/metricbeat/module/iis/webserver/_meta/fields.yml new file mode 100644 index 000000000000..b31375bdbb21 --- /dev/null +++ b/metricbeat/module/iis/webserver/_meta/fields.yml @@ -0,0 +1,8 @@ +- name: webserver.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + webserver + diff --git a/metricbeat/module/iis/webserver/doc.go b/metricbeat/module/iis/webserver/doc.go new file mode 100644 index 000000000000..b772274c9236 --- /dev/null +++ b/metricbeat/module/iis/webserver/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package webserver diff --git a/metricbeat/module/iis/webserver/manifest.yml b/metricbeat/module/iis/webserver/manifest.yml new file mode 100644 index 000000000000..c8d148a21c13 --- /dev/null +++ b/metricbeat/module/iis/webserver/manifest.yml @@ -0,0 +1,163 @@ +default: true +input: + module: windows + metricset: perfmon + defaults: + perfmon.group_measurements_by_instance: true + perfmon.ignore_non_existent_counters: true + perfmon.group_all_counter: "worker_process_count" + perfmon.counters: + #network + - instance_label: '' + measurement_label: network.total_bytes_received + query: '\Web Service(_Total)\Total Bytes Received' + - instance_label: '' + measurement_label: network.total_bytes_sent + query: '\Web Service(_Total)\Total Bytes Sent' + - instance_label: '' + measurement_label: network.bytes_sent_per_sec + query: '\Web Service(_Total)\Bytes Sent/sec' + - instance_label: '' + measurement_label: network.bytes_received_per_sec + query: '\Web Service(_Total)\Bytes Received/sec' + - instance_label: '' + measurement_label: network.current_connections + query: '\Web Service(_Total)\Current Connections' + - instance_label: '' + measurement_label: network.maximum_connections + query: '\Web Service(_Total)\Maximum Connections' + - instance_label: '' + measurement_label: network.total_connection_attempts + query: '\Web Service(_Total)\Total Connection Attempts (all instances)' + - instance_label: '' + measurement_label: network.total_get_requests + query: '\Web Service(_Total)\Total Get Requests' + - instance_label: '' + measurement_label: network.get_requests_per_sec + query: '\Web Service(_Total)\Get Requests/sec' + - instance_label: '' + measurement_label: network.total_post_requests + query: '\Web Service(_Total)\Total Post Requests' + - instance_label: '' + measurement_label: network.post_requests_per_sec + query: '\Web Service(_Total)\Post Requests/sec' + - instance_label: '' + measurement_label: network.total_delete_requests + query: '\Web Service(_Total)\Total Delete Requests' + - instance_label: '' + measurement_label: network.delete_requests_per_sec + query: '\Web Service(_Total)\Delete Requests/sec' + - instance_label: '' + measurement_label: network.service_uptime + query: '\Web Service(_Total)\Service Uptime' + - instance_label: '' + measurement_label: network.current_anonymous_users + query: '\Web Service(_Total)\Current Anonymous Users' + - instance_label: '' + measurement_label: network.current_nonanonymous_users + query: '\Web Service(_Total)\Current NonAnonymous Users' + - instance_label: '' + measurement_label: network.total_anonymous_users + query: '\Web Service(_Total)\Total Anonymous Users' + - instance_label: '' + measurement_label: network.anonymous_users_per_sec + query: '\Web Service(_Total)\Anonymous Users/sec' + - instance_label: '' + measurement_label: network.total_nonanonymous_users + query: '\Web Service(_Total)\Total NonAnonymous Users' + + #asp.net + - instance_label: '' + measurement_label: asp_net_application.errors_per_sec + query: '\ASP.NET Applications(__Total__)\Errors Total/Sec' + - instance_label: '' + measurement_label: asp_net_application.pipeline_instance_count + query: '\ASP.NET Applications(__Total__)\Pipeline Instance Count' + - instance_label: '' + measurement_label: asp_net_application.requests_executing + query: '\ASP.NET Applications(__Total__)\Requests Executing' + - instance_label: '' + measurement_label: asp_net_application.requests_in_application_queue + query: '\ASP.NET Applications(__Total__)\Requests in Application Queue' + format: 'large' + - instance_label: '' + measurement_label: asp_net_application.requests_per_sec + query: '\ASP.NET Applications(__Total__)\Requests/Sec' + - instance_label: '' + measurement_label: asp_net.application_restarts + query: '\ASP.NET\Application Restarts' + - instance_label: '' + measurement_label: asp_net.request_wait_time + query: '\ASP.NET\Request Wait Time' + + #cache + - instance_label: '' + measurement_label: cache.current_files_cached + query: '\Web Service Cache\Current Files Cached' + - instance_label: '' + measurement_label: cache.total_files_cached + query: '\Web Service Cache\Total Files Cached' + - instance_label: '' + measurement_label: cache.file_cache_hits + query: '\Web Service Cache\File Cache Hits' + - instance_label: '' + measurement_label: cache.file_cache_misses + query: '\Web Service Cache\File Cache Misses' + - instance_label: '' + measurement_label: cache.current_file_cache_memory_usage + query: '\Web Service Cache\Current File Cache Memory Usage' + - instance_label: '' + measurement_label: cache.maximum_file_cache_memory_usage + query: '\Web Service Cache\Maximum File Cache Memory Usage' + - instance_label: '' + measurement_label: cache.current_uris_cached + query: '\Web Service Cache\Current URIs Cached' + - instance_label: '' + measurement_label: cache.total_uris_cached + query: '\Web Service Cache\Total URIs Cached' + - instance_label: '' + measurement_label: cache.uri_cache_hits + query: '\Web Service Cache\URI Cache Hits' + - instance_label: '' + measurement_label: cache.uri_cache_misses + query: '\Web Service Cache\URI Cache Misses' + - instance_label: '' + measurement_label: cache.output_cache_current_memory_usage + query: '\Web Service Cache\Output Cache Current Memory Usage' + - instance_label: '' + measurement_label: cache.output_cache_current_items + query: '\Web Service Cache\Output Cache Current Items' + - instance_label: '' + measurement_label: cache.output_cache_total_hits + query: '\Web Service Cache\Output Cache Total Hits' + - instance_label: '' + measurement_label: cache.output_cache_total_misses + query: '\Web Service Cache\Output Cache Total Misses' + #process + - instance_label: '' + measurement_label: process.cpu_usage_perc + query: '\Process(w3wp*)\% Processor Time' + - instance_label: '' + measurement_label: process.handle_count + query: '\Process(w3wp*)\Handle Count' + - instance_label: '' + measurement_label: process.thread_count + query: '\Process(w3wp*)\Thread Count' + - instance_label: '' + measurement_label: process.working_set + query: '\Process(w3wp*)\Working Set' + - instance_label: '' + measurement_label: process.private_byte + query: '\Process(w3wp*)\Private Bytes' + - instance_label: '' + measurement_label: process.virtual_bytes + query: '\Process(w3wp*)\Virtual Bytes' + - instance_label: '' + measurement_label: process.page_faults_per_Sec + query: '\Process(w3wp*)\Page Faults/sec' + - instance_label: '' + measurement_label: process.io_read_operations_per_sec + query: '\Process(w3wp*)\IO Read Operations/sec' + - instance_label: '' + measurement_label: process.io_write_operations_per_sec + query: '\Process(w3wp*)\IO Write Operations/sec' diff --git a/metricbeat/module/iis/webserver/webserver_integration_test.go b/metricbeat/module/iis/webserver/webserver_integration_test.go new file mode 100644 index 000000000000..643b883bd835 --- /dev/null +++ b/metricbeat/module/iis/webserver/webserver_integration_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package webserver + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/metricbeat/module/windows" + _ "github.com/elastic/beats/metricbeat/module/windows/perfmon" +) + +func TestData(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + m.WriteEvents(t, "") +} + +func TestFetch(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + events, errs := m.FetchEvents() + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "iis", + "metricsets": []string{"webserver"}, + } +} diff --git a/metricbeat/module/iis/webserver/webserver_test.go b/metricbeat/module/iis/webserver/webserver_test.go new file mode 100644 index 000000000000..4d639aadbe55 --- /dev/null +++ b/metricbeat/module/iis/webserver/webserver_test.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package webserver + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/metricbeat/module/iis/website/_meta/data.json b/metricbeat/module/iis/website/_meta/data.json new file mode 100644 index 000000000000..b4bbb9c25ba9 --- /dev/null +++ b/metricbeat/module/iis/website/_meta/data.json @@ -0,0 +1,33 @@ +{ + "@timestamp" : "2020-02-05T12:47:14.651Z", + "metricset" : { + "name" : "website", + "period" : 10000 + }, + "service" : { + "type" : "iis" + }, + "iis" : { + "website" : { + "name" : "Default Web Site", + "total_bytes_received" : 23122, + "current_connections" : 0, + "service_uptime" : 1195668.0, + "maximum_connections" : 1, + "total_post_requests" : 0, + "total_bytes_sent" : 11350, + "total_connection_attempts" : 33, + "bytes_sent_per_sec" : 0, + "post_requests_per_sec" : 0, + "delete_requests_per_sec" : 0, + "get_requests_per_sec" : 0, + "total_delete_requests" : 0, + "bytes_received_per_sec" : 0, + "total_get_requests" : 39 + } + }, + "event" : { + "dataset" : "iis.website", + "module" : "iis" + } +} diff --git a/metricbeat/module/iis/website/_meta/docs.asciidoc b/metricbeat/module/iis/website/_meta/docs.asciidoc new file mode 100644 index 000000000000..e4e6a5e9dc99 --- /dev/null +++ b/metricbeat/module/iis/website/_meta/docs.asciidoc @@ -0,0 +1,12 @@ +This is the website metricset of the module iis. + +This metricset allows users to retrieve relevant metrics for the websites running on IIS. + +The metrics contain the IIS Performance counter values like: + Web Service: Bytes Received/Sec (helpful to track to identify potential spikes in traffic), Web Service: Bytes Sent/Sec (helpful to track to identify potential spikes in traffic), + Web Service: Current Connections (through experience with their apps app, users can identify what is a normal value for this) and others. + + + + + diff --git a/metricbeat/module/iis/website/_meta/fields.yml b/metricbeat/module/iis/website/_meta/fields.yml new file mode 100644 index 000000000000..74be20459303 --- /dev/null +++ b/metricbeat/module/iis/website/_meta/fields.yml @@ -0,0 +1,10 @@ +- name: website + type: group + release: beta + description: > + website + fields: + - name: name + type: keyword + description: > + website name diff --git a/metricbeat/module/iis/website/doc.go b/metricbeat/module/iis/website/doc.go new file mode 100644 index 000000000000..81e02fe557b4 --- /dev/null +++ b/metricbeat/module/iis/website/doc.go @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package website diff --git a/metricbeat/module/iis/website/manifest.yml b/metricbeat/module/iis/website/manifest.yml new file mode 100644 index 000000000000..564ad6db86ce --- /dev/null +++ b/metricbeat/module/iis/website/manifest.yml @@ -0,0 +1,61 @@ +default: true +input: + module: windows + metricset: perfmon + defaults: + perfmon.group_measurements_by_instance: true + perfmon.ignore_non_existent_counters: true + perfmon.counters: + #network + - instance_label: 'name' + measurement_label: total_bytes_received + query: '\Web Service(*)\Total Bytes Received' + - instance_label: 'name' + measurement_label: total_bytes_sent + query: '\Web Service(*)\Total Bytes Sent' + - instance_label: 'name' + measurement_label: bytes_sent_per_sec + query: '\Web Service(*)\Bytes Sent/sec' + - instance_label: 'name' + measurement_label: bytes_received_per_sec + query: '\Web Service(*)\Bytes Received/sec' + - instance_label: 'name' + measurement_label: current_connections + query: '\Web Service(*)\Current Connections' + - instance_label: 'name' + measurement_label: maximum_connections + query: '\Web Service(*)\Maximum Connections' + - instance_label: 'name' + measurement_label: total_connection_attempts + query: '\Web Service(*)\Total Connection Attempts (all instances)' + - instance_label: 'name' + measurement_label: total_get_requests + query: '\Web Service(*)\Total Get Requests' + - instance_label: 'name' + measurement_label: get_requests_per_sec + query: '\Web Service(*)\Get Requests/sec' + - instance_label: 'name' + measurement_label: total_post_requests + query: '\Web Service(*)\Total Post Requests' + - instance_label: 'name' + measurement_label: post_requests_per_sec + query: '\Web Service(*)\Post Requests/sec' + - instance_label: 'name' + measurement_label: total_delete_requests + query: '\Web Service(*)\Total Delete Requests' + - instance_label: 'name' + measurement_label: delete_requests_per_sec + query: '\Web Service(*)\Delete Requests/sec' + - instance_label: 'name' + measurement_label: service_uptime + query: '\Web Service(*)\Service Uptime' + - instance_label: 'name' + measurement_label: total_put_requests + query: '\Web Service(*)\Total PUT Requests' + - instance_label: 'name' + measurement_label: put_requests_per_sec + query: '\Web Service(*)\PUT Requests/sec' + +processors: +- drop_event.when.equals: + iis.website.name: '_Total' diff --git a/metricbeat/module/iis/website/website_integration_test.go b/metricbeat/module/iis/website/website_integration_test.go new file mode 100644 index 000000000000..3b8bea6a5f37 --- /dev/null +++ b/metricbeat/module/iis/website/website_integration_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package website + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/metricbeat/module/windows" + _ "github.com/elastic/beats/metricbeat/module/windows/perfmon" +) + +func TestData(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + m.WriteEvents(t, "") +} + +func TestFetch(t *testing.T) { + m := mbtest.NewFetcher(t, getConfig()) + events, errs := m.FetchEvents() + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "iis", + "metricsets": []string{"website"}, + } +} diff --git a/metricbeat/module/iis/website/website_test.go b/metricbeat/module/iis/website/website_test.go new file mode 100644 index 000000000000..f9618fa3bfbb --- /dev/null +++ b/metricbeat/module/iis/website/website_test.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package website + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index 61d03b5d5696..4c90de92fdd8 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -57,8 +57,7 @@ values as shown below. "%[measurement_label]": , ---- -*`instance_label`*:: The label used to identify the counter instance. This -field is required. +*`instance_label`*:: The label used to identify the counter instance. *`instance_name`*:: The instance name to use in the event when the counter's path (`query`) does not include an instance or when you want to override the diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index a3d048ebcf73..fffd5d6cfb59 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -22,6 +22,8 @@ package perfmon import ( "strings" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -31,7 +33,7 @@ import ( // CounterConfig for perfmon counters. type CounterConfig struct { - InstanceLabel string `config:"instance_label" validate:"required"` + InstanceLabel string `config:"instance_label"` InstanceName string `config:"instance_name"` MeasurementLabel string `config:"measurement_label" validate:"required"` Query string `config:"query" validate:"required"` @@ -40,13 +42,16 @@ type CounterConfig struct { // Config for the windows perfmon metricset. type Config struct { - IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` - GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` - CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` + IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` + GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` + CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` + GroupAllCountersTo string `config:"perfmon.group_all_counter"` } +const metricsetName = "perfmon" + func init() { - mb.Registry.MustAddMetricSet("windows", "perfmon", New) + mb.Registry.MustAddMetricSet("windows", metricsetName, New, mb.WithHostParser(parse.EmptyHostParser)) } type MetricSet struct { @@ -83,7 +88,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, reader: reader, - log: logp.NewLogger("perfmon"), + log: logp.NewLogger(metricsetName), }, nil } @@ -115,6 +120,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { break } } + return nil } diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 51eef6b7e957..22b6b5f89ddd 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -20,6 +20,7 @@ package perfmon import ( + "fmt" "regexp" "strconv" "strings" @@ -33,9 +34,9 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -var ( - processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) -) +var processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) + +const instanceCountLabel = ":count" // Reader will contain the config options type Reader struct { @@ -94,21 +95,20 @@ func NewReader(config Config) (*Reader, error) { r.measurement[v] = counter.MeasurementLabel } } - return r, nil } // RefreshCounterPaths will recheck for any new instances and add them to the counter list -func (r *Reader) RefreshCounterPaths() error { +func (re *Reader) RefreshCounterPaths() error { var newCounters []string - for _, counter := range r.config.CounterConfig { - childQueries, err := r.query.GetCounterPaths(counter.Query) + for _, counter := range re.config.CounterConfig { + childQueries, err := re.query.GetCounterPaths(counter.Query) if err != nil { - if r.config.IgnoreNECounters { + if re.config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: - r.log.Infow("Ignoring non existent counter", "error", err, + re.log.Infow("Ignoring non existent counter", "error", err, logp.Namespace("perfmon"), "query", counter.Query) continue } @@ -120,15 +120,15 @@ func (r *Reader) RefreshCounterPaths() error { // there are cases when the ExpandWildCardPath will retrieve a successful status but not an expanded query so we need to check for the size of the list if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { - if err := r.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { + if err := re.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } - r.instanceLabel[v] = counter.InstanceLabel - r.measurement[v] = counter.MeasurementLabel + re.instanceLabel[v] = counter.InstanceLabel + re.measurement[v] = counter.MeasurementLabel } } } - err := r.query.RemoveUnusedCounters(newCounters) + err := re.query.RemoveUnusedCounters(newCounters) if err != nil { return errors.Wrap(err, "failed removing unused counter values") } @@ -137,33 +137,45 @@ func (r *Reader) RefreshCounterPaths() error { } // Read executes a query and returns those values in an event. -func (r *Reader) Read() ([]mb.Event, error) { +func (re *Reader) Read() ([]mb.Event, error) { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if err := r.query.CollectData(); err != nil { + if err := re.query.CollectData(); err != nil { return nil, errors.Wrap(err, "failed querying counter values") } // Get the values. - values, err := r.query.GetFormattedCounterValues() + values, err := re.query.GetFormattedCounterValues() if err != nil { return nil, errors.Wrap(err, "failed formatting counter values") } + var events []mb.Event + // GroupAllCountersTo config option where counters for all instances are aggregated and instance count is added in the event under the string value provided by this option. + if re.config.GroupAllCountersTo != "" { + event := re.groupToEvent(values) + events = append(events, event) + } else { + events = re.groupToEvents(values) + } + re.executed = true + return events, nil +} +func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { eventMap := make(map[string]*mb.Event) - for counterPath, values := range values { + for counterPath, values := range counters { for ind, val := range values { // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !r.executed { - r.log.Debugw("Ignoring the first measurement because the data isn't ready", + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) continue } var eventKey string - if r.config.GroupMeasurements && val.Err == nil { + if re.config.GroupMeasurements && val.Err == nil { // Send measurements with the same instance label as part of the same event eventKey = val.Instance } else { @@ -178,37 +190,78 @@ func (r *Reader) Read() ([]mb.Event, error) { MetricSetFields: common.MapStr{}, Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), } - if val.Instance != "" { + if val.Instance != "" && re.instanceLabel[counterPath] != "" { //will ignore instance counter if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], match) + eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], match) } else { - eventMap[eventKey].MetricSetFields.Put(r.instanceLabel[counterPath], val.Instance) + eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], val.Instance) } } } event := eventMap[eventKey] if val.Measurement != nil { - event.MetricSetFields.Put(r.measurement[counterPath], val.Measurement) + event.MetricSetFields.Put(re.measurement[counterPath], val.Measurement) } else { - event.MetricSetFields.Put(r.measurement[counterPath], 0) + event.MetricSetFields.Put(re.measurement[counterPath], 0) } } } - // Write the values into the map. events := make([]mb.Event, 0, len(eventMap)) for _, val := range eventMap { events = append(events, *val) } + return events +} - r.executed = true - return events, nil +func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { + event := mb.Event{ + MetricSetFields: common.MapStr{}, + } + measurements := make(map[string]float64, 0) + for counterPath, values := range counters { + for _, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + var counterVal float64 + switch val.Measurement.(type) { + case int64: + counterVal = float64(val.Measurement.(int64)) + default: + counterVal = val.Measurement.(float64) + } + if _, ok := measurements[re.measurement[counterPath]]; !ok { + measurements[re.measurement[counterPath]] = counterVal + measurements[re.measurement[counterPath]+instanceCountLabel] = 1 + } else { + measurements[re.measurement[counterPath]+instanceCountLabel] = measurements[re.measurement[counterPath]+instanceCountLabel] + 1 + measurements[re.measurement[counterPath]] = measurements[re.measurement[counterPath]] + counterVal + } + } + } + for key, val := range measurements { + if strings.Contains(key, instanceCountLabel) { + if val == 1 { + continue + } else { + event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], re.config.GroupAllCountersTo), val) + } + } else { + event.MetricSetFields.Put(key, val) + } + } + return event } // Close will close the PDH query for now. -func (r *Reader) Close() error { - return r.query.Close() +func (re *Reader) Close() error { + return re.query.Close() } // matchParentProcess will try to get the parent process name diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index dd5a58aa5225..7f925d21a623 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -50,13 +50,14 @@ func TestNewReaderWithValidQueryPath(t *testing.T) { CounterConfig: []CounterConfig{counter}, } reader, err := NewReader(config) + defer reader.Close() assert.Nil(t, err) assert.NotNil(t, reader) assert.NotNil(t, reader.query) assert.NotNil(t, reader.query.Handle) assert.NotNil(t, reader.query.Counters) assert.NotZero(t, len(reader.query.Counters)) - defer reader.Close() + } // TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. diff --git a/metricbeat/modules.d/iis.yml.disabled b/metricbeat/modules.d/iis.yml.disabled new file mode 100644 index 000000000000..f81d67eedffa --- /dev/null +++ b/metricbeat/modules.d/iis.yml.disabled @@ -0,0 +1,13 @@ +# Module: iis +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-iis.html + +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ed863b7d970c..976a85ae5034 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -498,6 +498,18 @@ metricbeat.modules: # the options for this metricset are also available here. metrics_path: /metrics +#--------------------------------- Iis Module --------------------------------- +- module: iis + metricsets: + - webserver + - website + - application_pool + enabled: true + period: 10s + + # filter on application pool names + # application_pool.name: [] + #-------------------------------- Istio Module -------------------------------- # Istio mesh. To collect all Mixer-generated metrics - module: istio