Disable default prospector and adjust short configs (#4105)

This does two changes:

* Adds `enabled: false` in the default prospector in the short and long configs.
  (which fixes #3442)
* Updates the short config files of the modules to include the path definitions.
  I think this is better for a "module first" experience, where it gives a bit of
  context on how they work.

This means that the default configuration starts and gives no errors, but also doesn't
publish anything. IMO, this behaviour is fine considering configuration reloading.

* Print error in case all prospectors are disabled and config reloading is off

Also, add the `auth` fileset to the config samples, it was missing.

filebeat/_meta/common.full.p2.yml

@@ -16,6 +16,9 @@ filebeat.prospectors:
 #------------------------------ Log prospector --------------------------------
 - input_type: log
 
+  # Change to true to enable this prospector configuration.
+  enabled: false
+
   # Paths that should be crawled and fetched. Glob based paths.
   # To fetch all ".log" files from a specific level of subdirectories
   # /var/log/*/*.log can be used.

filebeat/_meta/common.p2.yml

@@ -11,6 +11,9 @@ filebeat.prospectors:
 
 - input_type: log
 
+  # Change to true to enable this prospector configuration.
+  enabled: false
+
   # Paths that should be crawled and fetched. Glob based paths.
   paths:
     - /var/log/*.log

filebeat/beater/filebeat.go

@@ -57,8 +57,16 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 	// Add prospectors created by the modules
 	config.Prospectors = append(config.Prospectors, moduleProspectors...)
 
-	if !config.ConfigProspector.Enabled() && len(config.Prospectors) == 0 {
-		return nil, errors.New("No prospectors defined. What files do you want me to watch?")
+	haveEnabledProspectors := false
+	for _, prospector := range config.Prospectors {
+		if prospector.Enabled() {
+			haveEnabledProspectors = true
+			break
+		}
+	}
+
+	if !config.ConfigProspector.Enabled() && !haveEnabledProspectors {
+		return nil, errors.New("No modules or prospectors enabled and configuration reloading disabled. What files do you want me to watch?")
 	}
 
 	if *once && config.ConfigProspector.Enabled() {

filebeat/filebeat.full.yml

@@ -25,17 +25,24 @@ filebeat.modules:
     # can be added under this section.
     #prospector:
 
+  # Authorization logs
+  #auth:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:
+
 #------------------------------- Apache2 Module ------------------------------
 #- module: apache2
   # Access logs
   #access:
     #enabled: true
 
-    # Ingest Node pipeline to use. Options are `with_plugins` (default)
-    # and `no_plugins`. Use `no_plugins` if you don't have the geoip or
-    # the user agent Node ingest plugins installed.
-    #var.pipeline: with_plugins
-
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
@@ -139,11 +146,6 @@ filebeat.modules:
   #access:
     #enabled: true
 
-    # Ingest Node pipeline to use. Options are `with_plugins` (default)
-    # and `no_plugins`. Use `no_plugins` if you don't have the geoip or
-    # the user agent Node ingest plugins installed.
-    #var.pipeline: with_plugins
-
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
@@ -183,6 +185,9 @@ filebeat.prospectors:
 #------------------------------ Log prospector --------------------------------
 - input_type: log
 
+  # Change to true to enable this prospector configuration.
+  enabled: false
+
   # Paths that should be crawled and fetched. Glob based paths.
   # To fetch all ".log" files from a specific level of subdirectories
   # /var/log/*/*.log can be used.

filebeat/filebeat.yml

@@ -13,19 +13,85 @@ filebeat.modules:
 
 #------------------------------- System Module -------------------------------
 #- module: system
+  # Syslog
+  #syslog:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Authorization logs
+  #auth:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+#------------------------------- Apache2 Module ------------------------------
+#- module: apache2
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
 
 #------------------------------- Auditd Module -------------------------------
 #- module: auditd
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 
 #-------------------------------- MySQL Module -------------------------------
 #- module: mysql
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Slow logs
+  #slowlog:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
 
 #-------------------------------- Nginx Module -------------------------------
 #- module: nginx
-  # Ingest Node pipeline to use. Options are `with_plugins` (default)
-  # and `no_plugins`. Use `no_plugins` if you don't have the geoip or
-  # the user agent Node ingest plugins installed.
-  #access.var.pipeline: with_plugins
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
 
 
 # For more available modules and options, please see the filebeat.full.yml sample
@@ -41,6 +107,9 @@ filebeat.prospectors:
 
 - input_type: log
 
+  # Change to true to enable this prospector configuration.
+  enabled: false
+
   # Paths that should be crawled and fetched. Glob based paths.
   paths:
     - /var/log/*.log

filebeat/module/apache2/_meta/config.full.yml

@@ -3,11 +3,6 @@
   #access:
     #enabled: true
 
-    # Ingest Node pipeline to use. Options are `with_plugins` (default)
-    # and `no_plugins`. Use `no_plugins` if you don't have the geoip or
-    # the user agent Node ingest plugins installed.
-    #var.pipeline: with_plugins
-
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:

filebeat/module/apache2/_meta/config.yml

@@ -0,0 +1,16 @@
+#- module: apache2
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/auditd/_meta/config.yml

@@ -1 +1,8 @@
 #- module: auditd
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+

filebeat/module/icinga/_meta/config.yml

@@ -1 +1,24 @@
 #- module: icinga
+  # Main logs
+  #main:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Debug logs
+  #debug:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Startup logs
+  #startup:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/mysql/_meta/config.yml

@@ -1 +1,16 @@
 #- module: mysql
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Slow logs
+  #slowlog:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/nginx/_meta/config.full.yml

@@ -3,11 +3,6 @@
   #access:
     #enabled: true
 
-    # Ingest Node pipeline to use. Options are `with_plugins` (default)
-    # and `no_plugins`. Use `no_plugins` if you don't have the geoip or
-    # the user agent Node ingest plugins installed.
-    #var.pipeline: with_plugins
-
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:

filebeat/module/nginx/_meta/config.yml

@@ -1,5 +1,16 @@
 #- module: nginx
-  # Ingest Node pipeline to use. Options are `with_plugins` (default)
-  # and `no_plugins`. Use `no_plugins` if you don't have the geoip or
-  # the user agent Node ingest plugins installed.
-  #access.var.pipeline: with_plugins
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/system/_meta/config.full.yml

@@ -10,3 +10,15 @@
     # Prospector configuration (advanced). Any prospector configuration option
     # can be added under this section.
     #prospector:
+
+  # Authorization logs
+  #auth:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+    # Prospector configuration (advanced). Any prospector configuration option
+    # can be added under this section.
+    #prospector:

filebeat/module/system/_meta/config.yml

@@ -1 +1,16 @@
 #- module: system
+  # Syslog
+  #syslog:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Authorization logs
+  #auth:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/tests/system/test_prospector.py

@@ -276,14 +276,9 @@ def test_shutdown_no_prospectors(self):
 
         filebeat = self.start_beat()
 
-        # wait for first  "Start next scan" log message
         self.wait_until(
             lambda: self.log_contains(
-                "No prospectors defined"),
-            max_timeout=10)
-
-        self.wait_until(
-            lambda: self.log_contains("No prospectors defined"),
+                "No modules or prospectors enabled"),
             max_timeout=10)
 
         filebeat.check_wait(exit_code=1)