Add metricbeat iis module (elastic#15059)

* created iis module

* work in progress

* iis changes

* add iis module

* light

* work on build

* work on build

* build

* fmt update

* temp

* work on website

* temp

* temp

* manifest changes

* temp

* temp

* work on wp

* adding application pool metricset

* wmi option

* test

* work on apppool

* work on app pool

* work on website metricset

* work on tests

* Work on website

* work on website

* work on website

* perfmon fix

* work on websie

* update config

* feedback

* work on feedback

* temp

* ecs

* temp

* add counters

metricbeat/docs/fields.asciidoc

@@ -42,6 +42,7 @@ grouped in the following categories:
 * <<exported-fields-host-processor>>
 * <<exported-fields-http>>
 * <<exported-fields-ibmmq>>
+* <<exported-fields-iis>>
 * <<exported-fields-istio>>
 * <<exported-fields-jolokia>>
 * <<exported-fields-jolokia-autodiscover>>
@@ -16951,6 +16952,63 @@ IBM MQ module
 
 
 
+[[exported-fields-iis]]
+== iis fields
+
+iis module
+
+
+
+[float]
+=== iis
+
+
+
+
+[float]
+=== application_pool
+
+application_pool
+
+
+
+*`iis.application_pool.name`*::
++
+--
+application pool name
+
+
+type: keyword
+
+--
+
+*`iis.webserver.*.*`*::
++
+--
+webserver
+
+
+type: object
+
+--
+
+[float]
+=== website
+
+website
+
+
+
+*`iis.website.name`*::
++
+--
+website name
+
+
+type: keyword
+
+--
+
 [[exported-fields-istio]]
 == istio fields
 

metricbeat/docs/modules/iis.asciidoc

@@ -0,0 +1,99 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-module-iis]]
+== iis module
+
+beta[]
+
+This is the iis module.
+
+IIS (Internet Information Services) is a secure, reliable, and scalable Web server that provides an easy to manage platform for developing and hosting Web applications and services.
+
+The `iis` module will periodically retrieve IIS related metrics using performance counters such as:
+
+ - System/Process counters like the the overall server and CPU usage for the IIS Worker Process and memory (currently used and available memory for the IIS Worker Process).
+ - IIS performance counters like Web Service: Bytes Received/Sec, Web Service: Bytes Sent/Sec, etc, which are helpful to track to identify potential spikes in traffic.
+ - Web Service Cache counters in order to monitor user mode cache and output cache.
+
+
+The `iis` module mericsets are `webserver`, `website` and `application_pool`.
+
+[source,yaml]
+----
+- module: iis
+  metricsets:
+    - webserver
+    - website
+    - application_pool
+  enabled: true
+  period: 10s
+
+ # filter on application pool names
+ # application_pool.name: []
+----
+
+[float]
+== Metricsets
+
+[float]
+=== `webserver`
+A light metricset using the windows perfmon metricset as the base metricset.
+This metricset allows users to retrieve aggregated metrics for the entire webserver,
+
+[float]
+=== `website`
+A light metricset using the windows perfmon metricset as the base metricset.
+This metricset will collect metrics of specific sites, users can configure which websites they want to monitor, else, all are considered.
+
+[float]
+=== `application_pool`
+This metricset will collect metrics of specific application pools, users can configure which websites they want to monitor, else, all are considered.
+
+
+[float]
+=== Module-specific configuration notes
+
+`application_pool.name`:: []string, users can specify the application pools they would like to monitor.
+
+
+
+[float]
+=== Example configuration
+
+The iis module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: iis
+  metricsets:
+    - webserver
+    - website
+    - application_pool
+  enabled: true
+  period: 10s
+
+ # filter on application pool names
+ # application_pool.name: []
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-iis-application_pool,application_pool>>
+
+* <<metricbeat-metricset-iis-webserver,webserver>>
+
+* <<metricbeat-metricset-iis-website,website>>
+
+include::iis/application_pool.asciidoc[]
+
+include::iis/webserver.asciidoc[]
+
+include::iis/website.asciidoc[]
+

metricbeat/docs/modules/iis/application_pool.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-iis-application_pool]]
+=== iis application_pool metricset
+
+beta[]
+
+include::../../../module/iis/application_pool/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-iis,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/iis/application_pool/_meta/data.json[]
+----

metricbeat/docs/modules/iis/webserver.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-iis-webserver]]
+=== iis webserver metricset
+
+beta[]
+
+include::../../../module/iis/webserver/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-iis,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/iis/webserver/_meta/data.json[]
+----

metricbeat/docs/modules/iis/website.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-iis-website]]
+=== iis website metricset
+
+beta[]
+
+include::../../../module/iis/website/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-iis,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/iis/website/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -112,6 +112,10 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-http-server,server>>   
 |<<metricbeat-module-ibmmq,IBM MQ>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-ibmmq-qmgr,qmgr>> beta[]  
+|<<metricbeat-module-iis,iis>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
+.3+| .3+|  |<<metricbeat-metricset-iis-application_pool,application_pool>> beta[]  
+|<<metricbeat-metricset-iis-webserver,webserver>> beta[]  
+|<<metricbeat-metricset-iis-website,website>> beta[]  
 |<<metricbeat-module-istio,istio>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .5+| .5+|  |<<metricbeat-metricset-istio-citadel,citadel>> beta[]  
 |<<metricbeat-metricset-istio-galley,galley>> beta[]  
@@ -278,6 +282,7 @@ include::modules/graphite.asciidoc[]
 include::modules/haproxy.asciidoc[]
 include::modules/http.asciidoc[]
 include::modules/ibmmq.asciidoc[]
+include::modules/iis.asciidoc[]
 include::modules/istio.asciidoc[]
 include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]

metricbeat/helper/windows/pdh/pdh_query_windows.go

@@ -80,7 +80,7 @@ func (q *Query) AddCounter(counterPath string, instance string, format string, w
 	var instanceName string
 	// Extract the instance name from the counterPath.
 	if instance == "" || wildcard {
-		instanceName, err = matchInstanceName(counterPath)
+		instanceName, err = MatchInstanceName(counterPath)
 		if err != nil {
 			return err
 		}
@@ -178,6 +178,18 @@ func (q *Query) GetFormattedCounterValues() (map[string][]CounterValue, error) {
 	return rtn, nil
 }
 
+// GetCountersAndInstances returns a list of counters and instances for a given object
+func (q *Query) GetCountersAndInstances(objectName string) ([]string, []string, error) {
+	counters, instances, err := PdhEnumObjectItems(objectName)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "Unable to retrieve counter and instance list for %s", objectName)
+	}
+	if len(counters) == 0 && len(instances) == 0 {
+		return nil, nil, errors.Errorf("Unable to retrieve counter and instance list for %s", objectName)
+	}
+	return UTF16ToStringArray(counters), UTF16ToStringArray(instances), nil
+}
+
 // ExpandWildCardPath  examines local computer and returns those counter paths that match the given counter path which contains wildcard characters.
 func (q *Query) ExpandWildCardPath(wildCardPath string) ([]string, error) {
 	if wildCardPath == "" {
@@ -209,8 +221,8 @@ func (q *Query) Close() error {
 	return PdhCloseQuery(q.Handle)
 }
 
-// matchInstanceName will check first for instance and then for any objects names.
-func matchInstanceName(counterPath string) (string, error) {
+// MatchInstanceName will check first for instance and then for any objects names.
+func MatchInstanceName(counterPath string) (string, error) {
 	matches := instanceNameRegexp.FindStringSubmatch(counterPath)
 	if len(matches) != 2 {
 		matches = objectNameRegexp.FindStringSubmatch(counterPath)

metricbeat/helper/windows/pdh/pdh_windows.go

@@ -41,6 +41,7 @@ import (
 //sys _PdhExpandWildCardPath(dataSource *uint16, wildcardPath *uint16, expandedPathList *uint16, pathListLength *uint32) (errcode error) [failretval!=0] = pdh.PdhExpandWildCardPathW
 //sys _PdhExpandCounterPath(wildcardPath *uint16, expandedPathList *uint16, pathListLength *uint32) (errcode error) [failretval!=0] = pdh.PdhExpandCounterPathW
 //sys _PdhGetCounterInfo(counter PdhCounterHandle, text uint16, size *uint32, lpBuffer *byte) (errcode error) [failretval!=0] = pdh.PdhGetCounterInfoW
+//sys _PdhEnumObjectItems(dataSource uint16, machineName uint16, objectName *uint16, counterList *uint16, counterListSize *uint32, instanceList *uint16, instanceListSize *uint32, detailLevel uint32, flags uint32) (errcode error) [failretval!=0] = pdh.PdhEnumObjectItemsW
 
 type PdhQueryHandle uintptr
 
@@ -50,6 +51,9 @@ type PdhCounterHandle uintptr
 
 var InvalidCounterHandle = ^PdhCounterHandle(0)
 
+// PerformanceDetailWizard is the counter detail level
+const PerformanceDetailWizard = 400
+
 // PdhCounterInfo struct contains the performance counter details
 type PdhCounterInfo struct {
 	DwLength         uint32
@@ -247,6 +251,48 @@ func PdhCloseQuery(query PdhQueryHandle) error {
 	return nil
 }
 
+// PdhEnumObjectItems returns the counters and instance info for given object
+func PdhEnumObjectItems(objectName string) ([]uint16, []uint16, error) {
+	var (
+		cBuff     = make([]uint16, 1)
+		cBuffSize = uint32(0)
+		iBuff     = make([]uint16, 1)
+		iBuffSize = uint32(0)
+	)
+	obj := windows.StringToUTF16Ptr(objectName)
+	if err := _PdhEnumObjectItems(
+		0,
+		0,
+		obj,
+		&cBuff[0],
+		&cBuffSize,
+		&iBuff[0],
+		&iBuffSize,
+		PerformanceDetailWizard,
+		0); err != nil {
+		if PdhErrno(err.(syscall.Errno)) != PDH_MORE_DATA {
+			return nil, nil, PdhErrno(err.(syscall.Errno))
+		}
+		cBuff = make([]uint16, cBuffSize)
+		iBuff = make([]uint16, iBuffSize)
+
+		if err = _PdhEnumObjectItems(
+			0,
+			0,
+			obj,
+			&cBuff[0],
+			&cBuffSize,
+			&iBuff[0],
+			&iBuffSize,
+			PerformanceDetailWizard,
+			0); err != nil {
+			return nil, nil, err
+		}
+		return cBuff, iBuff, nil
+	}
+	return nil, nil, nil
+}
+
 // Error returns a more explicit error message.
 func (e PdhErrno) Error() string {
 	// If the value is not one of the known PDH errors then assume its a