Add staging-with-rebase-focal to CI and fix testinfra tests (#5638)

* Fixes #5636 uses right path of dhclient in Focal

`sbin/dhclient` string is the correct path in Xenial and on Focal.

In Xenial the file is at `/sbin/dhclient`.
In Focal the file is at `/usr/sbin/dhclient`

* Adds CI job staging-test-with-rebase-focal

This will run the Focal staging job in the CI.

* Updates test strings to verify gpg2 output

* Adds systemd service file ossec and DNS fix

Disables systemd-resolved and hardcoded DNS for Focal
In Focal by default we have systemd-resolved trying to do DNS
resolution. This does not pick up our DNS configuration via resolvconf.
So, we can have hardcoded /etc/resolv.conf via template file.

Enables and restarts OSSEC service properly via systemd module
Now instead of a command, we are enabling and restarting the service
using the Ansible systemd module.

We also remove the old /etc/init.d/ossec file as only we want
only our service to exists.

* Update listing port test

Now uses testinfra primitives instead of lsof workaround

* Adds ossec.service file while building the ossec-server package

We remove the /etc/init.d/ossec file and using the systemd
service file in the ossec-server package.

* Uses ossec.service only for Focal build

* Adds new testinfra test for ossec

We verify that Xenial uses sysv script, and Focal is using
the ossec.service file to start the service in the mon server.

* Consolidates resolvconf config for dual distros

Under Focal, we were writing the nameserver info to two (2) files, but
only testing one of them. Using a vars-based approach now, and the test
logic now looks in the correct spot for Focal.

* Adds systemd service check for ossec-agent

Same as we've done for ossec-server, let's make sure that ossec-agent is
also managed via systemd when running under Focal.

* Use service file ossec pacakges

* Adds ossec.service file for the agent

OSSEC server and agent requires two different service files.
Details at https://kushaldas.in/posts/story-of-debugging-exit-0.html

Co-authored-by: mickael e <mickael@freedom.press>
Co-authored-by: Conor Schaefer <conor@freedom.press>

.circleci/config.yml

@@ -336,6 +336,37 @@ jobs:
       - store_artifacts:
           path: ~/sd/junit
 
+  staging-test-with-rebase-focal:
+    machine:
+      enabled: true
+
+    working_directory: ~/sd
+    steps:
+      - checkout
+      - *rebaseontarget
+      - *installenchant
+
+      - run:
+          name: Run Staging tests on GCE
+          command: |
+            BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n)")
+            if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
+            BASE_OS=focal make ci-go
+          no_output_timeout: 35m
+
+      - run:
+          name: Ensure environment torn down
+          # Always report true, since env should will destroyed already
+          # if all tests passed.
+          command: make ci-teardown || true
+          when: always
+
+      - store_test_results:
+          path: ~/sd/junit
+
+      - store_artifacts:
+          path: ~/sd/junit
+
   deb-tests:
     docker:
       - image: gcr.io/cloud-builders/docker
@@ -415,6 +446,13 @@ workflows:
                 - /i18n-.*/
           requires:
             - lint
+      - staging-test-with-rebase-focal:
+          filters:
+            branches:
+              ignore:
+                - /i18n-.*/
+          requires:
+            - lint
       - translation-tests:
           requires:
             - lint

devops/gce-nested/ci-go.sh

@@ -12,6 +12,7 @@ set -e
 set -u
 set -o pipefail
 
+export BASE_OS="${BASE_OS:-xenial}"
 
 ./devops/gce-nested/gce-start.sh
 ./devops/gce-nested/gce-runner.sh

devops/gce-nested/gce-runner.sh

@@ -4,6 +4,7 @@
 # for storage as artifacts on the build, so devs can review via web.
 set -e
 set -u
+BASE_OS="${BASE_OS:-xenial}"
 
 
 TOPLEVEL="$(git rev-parse --show-toplevel)"
@@ -51,9 +52,15 @@ function copy_securedrop_repo() {
 
 # Main logic
 copy_securedrop_repo
-ssh_gce "make build-debs-notest"
 
 # The test results should be collected regardless of pass/fail,
 # so register a trap to ensure the fetch always runs.
 trap fetch_junit_test_results EXIT
-ssh_gce "make staging"
+if [ "${BASE_OS:-'xenial'}" = "xenial" ]
+then
+	ssh_gce "make build-debs-notest"
+	ssh_gce "make staging"
+else
+	ssh_gce "make build-debs-notest-focal"
+	ssh_gce "make staging-focal"
+fi

install_files/ansible-base/roles/build-ossec-deb-pkg/files/ossec-agent.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=OSSEC service for agent
+
+[Service]
+Type=forking
+ExecStart=/var/ossec/bin/ossec-control start
+ExecStop=/var/ossec/bin/ossec-control stop
+RemainAfterExit=True
+SuccessExitStatus=1
+
+[Install]
+WantedBy=multi-user.target

install_files/ansible-base/roles/build-ossec-deb-pkg/files/ossec.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=OSSEC service
+
+[Service]
+Type=forking
+ExecStart=/var/ossec/bin/ossec-control start
+ExecStop=/var/ossec/bin/ossec-control stop
+
+[Install]
+WantedBy=multi-user.target

install_files/ansible-base/roles/build-ossec-deb-pkg/tasks/main.yml

@@ -86,6 +86,7 @@
 - name: Run OSSEC installer script on extracted source.
   command: /tmp/ossec-hids-{{ ossec_version }}/install.sh
 
+
 - name: Create OSSEC build directory.
   file:
     state: directory
@@ -116,6 +117,38 @@
     - etc
     - usr
 
+- name: Delete the old service file
+  file:
+    state: absent
+    path: "{{ ossec_build_dir }}/etc/init.d/ossec"
+  when:
+    - ansible_host.find("-sd-generic-ossec") != -1
+    - ansible_distribution_release == "focal"
+
+- name: Create directory for our systemd based service file
+  file:
+    state: directory
+    path: "{{ ossec_build_dir }}/etc/systemd/system"
+  when:
+    - ansible_host.find("-sd-generic-ossec") != -1
+    - ansible_distribution_release == "focal"
+
+- name: Copy our systemd based service file for ossec-server
+  copy:
+    src: ossec.service
+    dest: "{{ ossec_build_dir }}/etc/systemd/system/ossec.service"
+  when:
+    - ansible_host.endswith("-sd-generic-ossec-server")
+    - ansible_distribution_release == "focal"
+
+- name: Copy our systemd based service file for ossec-agent
+  copy:
+    src: ossec-agent.service
+    dest: "{{ ossec_build_dir }}/etc/systemd/system/ossec.service"
+  when:
+    - ansible_host.endswith("-sd-generic-ossec-agent")
+    - ansible_distribution_release == "focal"
+
 - name: Build SecureDrop OSSEC deb package.
   command: dpkg-deb --build {{ ossec_build_dir }}
 

install_files/ansible-base/roles/common/tasks/harden_dns.yml

@@ -2,7 +2,17 @@
 - name: Configure DNS server IP.
   template:
     src: dns_base
-    dest: /etc/resolvconf/resolv.conf.d/base
+    dest: "{{ resolvconf_target_filepath }}"
+  tags:
+    - dns
+    - hardening
+
+- name: Disable systemd-resolved on Focal
+  systemd:
+    name: systemd-resolved
+    state: stopped
+    enabled: no
+  when: ansible_distribution_release == 'focal'
   tags:
     - dns
     - hardening

install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml

@@ -3,3 +3,5 @@ securedrop_kernel_packages_to_remove:
   - linux-virtual
   - linux-generic
   - 'linux-image-.*generic'
+
+resolvconf_target_filepath: /etc/resolv.conf

install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml

@@ -7,3 +7,5 @@ securedrop_kernel_packages_to_remove:
   - linux-image-generic-lts-xenial
   - 'linux-image-.*generic'
   - 'linux-headers-.*'
+
+resolvconf_target_filepath: /etc/resolvconf/resolv.conf.d/base

install_files/ansible-base/roles/ossec/handlers/main.yml

@@ -1,6 +1,6 @@
 ---
 # Single handler to operate on *both* OSSEC hosts, server & client.
 - name: restart ossec
-  service:
+  systemd:
     name: ossec
     state: restarted

install_files/ansible-base/roles/ossec/tasks/configure_client.yml

@@ -6,3 +6,10 @@
   when: not install_local_packages
   tags:
     - apt
+
+- name: Enable the OSSEC service
+  systemd:
+    name: ossec
+    daemon_reload: yes
+    enabled: yes
+    masked: no

install_files/ansible-base/roles/ossec/tasks/configure_server.yml

@@ -113,3 +113,10 @@
     creates: /var/ossec/etc/sslmanager.cert
   tags:
     - ossec_auth
+
+- name: Enable the OSSEC service
+  systemd:
+    name: ossec
+    daemon_reload: yes
+    enabled: yes
+    masked: no

molecule/testinfra/app/test_appenv.py

@@ -72,7 +72,8 @@ def test_gpg_key_in_keyring(host):
     with host.sudo(sdvars.securedrop_user):
         c = host.run("gpg --homedir /var/lib/securedrop/keys "
                      "--list-keys 28271441")
-        assert "pub   4096R/28271441 2013-10-12" in c.stdout
+        assert "2013-10-12" in c.stdout
+        assert "28271441" in c.stdout
 
 
 def test_ensure_logo(host):

molecule/testinfra/app/test_ossec_agent.py

@@ -21,6 +21,20 @@ def test_hosts_files(host):
                                                                     mon_host))
 
 
+def test_ossec_service_start_style(host):
+    """
+    Ensure that the OSSEC services are managed by systemd under Focal,
+    but by sysv under Xenial.
+    """
+    if host.system_info.codename == "focal":
+        value = "/etc/systemd/system/ossec.service"
+    else:
+        value = "/etc/init.d/ossec"
+    with host.sudo():
+        c = host.check_output("systemctl status ossec")
+        assert value in c
+
+
 def test_hosts_duplicate(host):
     """ Regression test for duplicate entries """
     assert host.check_output("uniq --repeated /etc/hosts") == ""

molecule/testinfra/common/test_fpf_apt_repo.py

@@ -45,11 +45,7 @@ def test_fpf_apt_repo_fingerprint(host):
 
     c = host.run('apt-key finger')
 
-    fpf_gpg_pub_key_info = """/etc/apt/trusted.gpg.d/securedrop-keyring.gpg
----------------------------------------------
-pub   4096R/00F4AD77 2016-10-20 [expires: 2021-06-30]
-      Key fingerprint = 2224 5C81 E3BA EB41 38B3  6061 310F 5612 00F4 AD77
-uid                  SecureDrop Release Signing Key"""
+    fpf_gpg_pub_key_info = "2224 5C81 E3BA EB41 38B3  6061 310F 5612 00F4 AD77"
 
     assert c.rc == 0
     assert fpf_gpg_pub_key_info in c.stdout

molecule/testinfra/common/test_system_hardening.py

@@ -40,7 +40,11 @@ def test_dns_setting(host):
     """
     Ensure DNS service is hard-coded in resolv.conf config.
     """
-    f = host.file('/etc/resolvconf/resolv.conf.d/base')
+    if host.system_info.codename == "focal":
+        fpath = "/etc/resolv.conf"
+    else:
+        fpath = "/etc/resolvconf/resolv.conf.d/base"
+    f = host.file(fpath)
     assert f.is_file
     assert f.user == "root"
     assert f.group == "root"

molecule/testinfra/mon/test_mon_network.py

@@ -66,16 +66,5 @@ def test_listening_ports(host, ossec_service):
     """
     socket = "{proto}://{host}:{port}".format(**ossec_service)
     with host.sudo():
-        # Really hacky work-around for bug found in testinfra 1.12.0
-        # https://github.com/philpep/testinfra/issues/311
-        if "udp" in socket:
-            lsof_socket = "{proto}@{host}:{port}".format(**ossec_service)
-            udp_check = host.run("lsof -n -i"+lsof_socket)
-
-            if ossec_service['listening']:
-                assert udp_check.rc == 0
-            else:
-                assert udp_check.rc == 1
-        else:
-            assert (host.socket(socket).is_listening ==
-                    ossec_service['listening'])
+        assert (host.socket(socket).is_listening ==
+                ossec_service['listening'])

molecule/testinfra/mon/test_ossec_server.py

@@ -5,6 +5,7 @@
 
 securedrop_test_vars = testutils.securedrop_test_vars
 testinfra_hosts = [securedrop_test_vars.monitor_hostname]
+python_version = securedrop_test_vars.python_version
 
 
 def test_ossec_connectivity(host):
@@ -21,6 +22,20 @@ def test_ossec_connectivity(host):
         assert c == desired_output
 
 
+def test_ossec_service_start_style(host):
+    """
+    Ensure that the OSSEC services are managed by systemd under Focal,
+    but by sysv under Xenial.
+    """
+    if host.system_info.codename == "focal":
+        value = "/etc/systemd/system/ossec.service"
+    else:
+        value = "/etc/init.d/ossec"
+    with host.sudo():
+        c = host.check_output("systemctl status ossec")
+        assert value in c
+
+
 # Permissions don't match between Ansible and OSSEC deb packages postinst.
 @pytest.mark.xfail
 @pytest.mark.parametrize('keyfile', [

molecule/testinfra/vars/app-prod.yml

@@ -20,7 +20,7 @@ app_directories:
   - /var/lib/securedrop/tmp
 
 apparmor_enforce:
-  - "/sbin/dhclient"
+  - "sbin/dhclient"
   - "/usr/lib/NetworkManager/nm-dhcp-client.action"
   - "/usr/lib/connman/scripts/dhclient-script"
   - "/usr/sbin/ntpd"

molecule/testinfra/vars/app-staging.yml

@@ -38,7 +38,7 @@ pip_deps:
 apparmor_complain: []
 
 apparmor_enforce:
-  - "/sbin/dhclient"
+  - "sbin/dhclient"
   - "/usr/lib/NetworkManager/nm-dhcp-client.action"
   - "/usr/lib/connman/scripts/dhclient-script"
   - "/usr/sbin/ntpd"

molecule/testinfra/vars/staging.yml

@@ -38,7 +38,7 @@ pip_deps:
 apparmor_complain: []
 
 apparmor_enforce:
-  - "/sbin/dhclient"
+  - "sbin/dhclient"
   - "/usr/lib/NetworkManager/nm-dhcp-client.action"
   - "/usr/lib/connman/scripts/dhclient-script"
   - "/usr/sbin/ntpd"