Fix failing tests

- Add virtualenv for running Ansible in a reproducible manner.
- Change regex for matching Metricbeat output from the system-cpu metricset

  The order of the JSON keys in the `metricset` object is different in the output than before (I haven't determined why.)

- Update the win_copy dest argument to include the file name and not just the destination directory.
- Refactor the Makefile and update the README.
- Temporarily disable Filebeat testing on Windows while elastic/beats#6145 is resolved.

.gitignore

@@ -4,6 +4,7 @@
 # Directories
 /.vagrant
 /.idea
+/ve
 
 # Files
 .DS_Store

Makefile

@@ -1,18 +1,37 @@
+RUN_SETTINGS?=staging
+# Control ansible's verbosity by setting this with -v or -vvvvv (more verbose).
+ANSIBLE_VERBOSE?=
+# Exluding localhost, as localhost is assumed to be Mac OS X with ssh enabled.
+ANSIBLE_LIMIT?=all:!localhost
+# Extra flags to pass to Ansible (e.g. --skip-tags filebeat).
+ANSIBLE_EXTRA_FLAGS?=
 
+# TODO (2018-01-22 andrewkroh): Remove this after Filebeat is fixed.
+# https://github.com/elastic/beats/issues/6145
+ANSIBLE_EXTRA_FLAGS=--skip-tags windows_filebeat
+
+# Create a virtualenv to run Ansible.
+ve: ve/bin/activate
+ve/bin/activate: requirements.txt
+	@test -d ve || virtualenv ve
+	@ve/bin/pip install -Ur requirements.txt
+	@touch ve/bin/activate
+
+# Setup the VMs and the SSH config needed for Ansible to communicate to them.
 setup:
 	vagrant up
 	vagrant ssh-config > ssh_config
 
-# Run build based on RUN_SETTINGS name
-run: setup
-	# Exluding localhost, as localhost is assumed to be Mac OS X with ssh enabled
-	ansible-playbook -i hosts --ssh-extra-args '-F ssh_config' -e @run-settings-${RUN_SETTINGS}.yml --limit 'all:!localhost' site.yml
-
-1.1.0:
-	RUN_SETTINGS=1.1.0-latest make run
-
-1.2.0:
-	RUN_SETTINGS=1.2.0-latest make run
+# Run Ansible from the virtualenv.
+run: ve
+	ve/bin/ansible-playbook \
+	${ANSIBLE_VERBOSE} \
+	-i hosts \
+	--ssh-extra-args '-F ssh_config' \
+	-e @run-settings-${RUN_SETTINGS}.yml \
+	--limit "${ANSIBLE_LIMIT}" \
+	${ANSIBLE_EXTRA_FLAGS} \
+	site.yml
 
 # This destroys all vagrant machines and removes the vagrant related data
 clean:

README.md

@@ -1,38 +1,51 @@
 # The Beats Tester
 
 Vagrant + Ansible setup for testing the OS packages and basic e2e tests for all
-the [Beats](https://www.elastic.co/products/beats)
+the [Beats](https://www.elastic.co/products/beats).
 
 
 ## Execute
 
 First, you need to bring the machines up:
 
-    vagrant up
-
-and make them known by name to SSH:
-
-    vagrant ssh-config >> ~/.ssh/config
+    make setup
 
 Then you can use Ansible to run the tests. Because they involve lots of VMs and
 commands executed over SSH, these tests are slow (currently 15 minutes in
-total). However, while creating tests or checking something quickly, you can
-use different Ansible commands to execute only a subset.
+total). However, while creating tests or checking something quickly, you can use
+different Ansible commands to execute only a subset.
 
 Here are some execution examples:
 
-* Test the most recent nightly builds (excluding OS X):
+* Test the most recent [nightly builds](https://internal-ci.elastic.co/job/elastic+release-manager+master+unified-snapshot/) (excluding OS X):
 
-        make nightlies
+        RUN_SETTINGS=snapshot make run
 
 * All tests, all platforms, a particular release:
 
-        ansible-playbook -i hosts -e @run-settings-1.0.0-beta3.yml site.yml
+        # Edit version contained in run-settings-released.yml.
+        RUN_SETTINGS=released make run
 
 * Only a particular Beat, Packetbeat in the example:
 
-        ansible-playbook -i hosts -e @run-settings-nightly.yml --tags packetbeat site.yml
+        export ANSIBLE_EXTRA_FLAGS="--tags packetbeat"
+        make run
 
 * Only a particular OS, Debian 6 amd64 in the example:
 
-        ansible-playbook -i hosts -e @run-settings-nightly.yml --limit tester-debian6-64 site.yml
+        # Instead using 'make setup' launch vagrant machines manually.
+        vagrant up tester-debian6-64
+        vagrant ssh-config > ssh_config
+        export ANSIBLE_LIMIT="tester-debian9-64"
+        make run
+
+* Enable Ansible debug.
+
+        export ANSIBLE_VERBOSE="-vvv"
+        make run
+
+## Cleanup
+
+You need to tear down the VMs when you are finished.
+
+    make clean

requirements.txt

@@ -0,0 +1,3 @@
+ansible==2.4.2.0
+pywinrm>=0.2.2
+markupsafe

roles/common/tasks/main.yml

@@ -27,7 +27,7 @@
   set_fact:
     beat_url: "{{url_base}}/{{beat_name}}/{{beat_name}}-{{version}}-windows-{{win_arch}}.zip"
     beat_pkg: "{{beat_name}}-{{version}}-windows-{{win_arch}}.zip"
-    beat_cfg: c:\\users\\vagrant\\{{beat_name}}-{{version}}-windows\\{{beat_name}}.yml
+    beat_cfg: 'c:\users\vagrant\{{beat_name}}-{{version}}-windows-{{win_arch}}\{{beat_name}}.yml'
     workdir: c:\\users\\vagrant
     installdir: c:\\users\\vagrant\\{{beat_name}}-{{version}}-windows-{{win_arch}}
   when: ansible_os_family == "Windows"

roles/test-metricbeat-file/tasks/main.yml

@@ -10,13 +10,14 @@
   when: ansible_os_family in ["Debian", "RedHat", "Suse"]
 
 - name: Start Metricbeat (darwin)
-  shell: chdir={{installdir}} ./{{beat_name}} -E output.elasticsearch.enabled=false -E output.file.path={{workdir}}/output -E metricbeat.max_start_delay=0s
-
+  shell: './{{ beat_name }} -E output.elasticsearch.enabled=false -E output.file.path={{ workdir }}/output -E metricbeat.max_start_delay=0s'
+  args:
+    chdir: '{{ installdir }}'
   when: ansible_os_family == "Darwin"
   async: 45
   poll: 0  # run in bg
 
 - name: Wait for the output file to be created, should contain cpu stats
   wait_for: >
     path={{workdir}}/output/metricbeat timeout=15
-    search_regex='"module":"system","name":"cpu"'
+    search_regex='"name":"cpu","module":"system"'

roles/test-win-filebeat-file/tasks/main.yml

@@ -5,7 +5,7 @@
   win_file: path=c:\windows\syswow64\.filebeat state=absent
 
 - name: Replace configuration file (windows)
-  win_copy: src={{beat_name}}.yml dest={{installdir}}
+  win_copy: src={{beat_name}}.yml dest='{{ beat_cfg }}'
 
 - name: Install Filebeat service
   script: run_script.ps1 -script {{installdir}}\\install-service-{{beat_name}}.ps1

roles/test-win-heartbeat-file/tasks/main.yml

@@ -2,7 +2,7 @@
   win_file: path={{workdir}}\\output state=absent
 
 - name: Replace configuration file (windows)
-  win_copy: src={{beat_name}}.yml dest={{installdir}}
+  win_copy: src={{beat_name}}.yml dest='{{ beat_cfg }}'
 
 - name: Install service
   script: run_script.ps1 -script {{installdir}}\\install-service-{{beat_name}}.ps1

roles/test-win-metricbeat-file/tasks/main.yml

@@ -2,7 +2,7 @@
   win_file: path={{workdir}}\\output state=absent
 
 - name: Replace configuration file (windows)
-  win_copy: src=metricbeat.yml dest={{installdir}}
+  win_copy: src=metricbeat.yml dest='{{ beat_cfg }}'
 
 - name: Install Metricbeat service
   script: run_script.ps1 -script {{installdir}}\\install-service-metricbeat.ps1

roles/test-win-packetbeat-file/tasks/main.yml

@@ -2,7 +2,7 @@
   win_file: path={{workdir}}\\output state=absent
 
 - name: Replace configuration file (windows)
-  win_copy: src=packetbeat.yml dest={{installdir}}
+  win_copy: src=packetbeat.yml dest='{{ beat_cfg }}'
 
 - name: Install Packetbeat service
   script: run_script.ps1 -script {{installdir}}\\install-service-packetbeat.ps1
@@ -33,4 +33,3 @@
     that:
       - "output_stat.stat.exists"
       - "output_stat.stat.size > 800"
-

roles/test-win-winlogbeat-file/tasks/main.yml

@@ -5,7 +5,7 @@
   script: run_script.ps1 -script {{ installdir }}\\install-service-{{ beat_name }}.ps1
 
 - name: Replace configuration file with template (windows)
-  win_template: src={{beat_name}}.yml dest={{installdir}}
+  win_template: src={{beat_name}}.yml dest='{{ beat_cfg }}'
 
 - name: Start Winlogbeat
   win_service: name={{ beat_name }} state=started

run-settings-released.yml

@@ -0,0 +1,3 @@
+# Released artifacts.
+url_base: https://artifacts.elastic.co/downloads/beats
+version:  6.1.2

run-settings-snapshot.yml

@@ -0,0 +1,4 @@
+# Snapshot builds.
+# Published by: https://internal-ci.elastic.co/job/elastic+release-manager+master+unified-snapshot/
+url_base: https://snapshots.elastic.co/downloads/beats
+version: 7.0.0-alpha1-SNAPSHOT

run-settings-staging.yml

@@ -1,2 +1,3 @@
-url_base: https://staging.elastic.co/6.0.0-rc2-ba5a2827/downloads/beats
-version: 6.0.0-rc2
+# Staged releases.
+url_base: https://staging.elastic.co/6.2.0-0557a3c6/downloads/beats
+version: 6.2.0

site.yml

@@ -137,6 +137,7 @@
   tags:
     - filebeat
     - windows
+    - windows_filebeat
   vars:
     - beat_name: filebeat
     - win_arch: x86
@@ -152,6 +153,7 @@
   tags:
     - filebeat
     - windows
+    - windows_filebeat
   vars:
     - beat_name: filebeat
     - win_arch: x86_64