Bundle mod_wsgi in securedrop-app-code package

To move to Python 3, we need to remove the dependency on Ubuntu's
libapache2-mod-wsgi package. Since the libapache2-mod-wsgi-py3 package
is in the universe section, this change bundles mod_wsgi in our
securedrop-app-code package.

For now, we have to simply drop libapache2-mod-wsgi as a dependency of
securedrop-app-code. Actually uninstalling it disables the Apache
module, breaking the Apache configuration. So we're leaving it
installed and modifying /etc/apache2/mods-available/wsgi.load to point
to our bundled mod_wsgi. This should be safe even if
libapache2-mod-wsgi is updated, as we're specifying --force-confdef
and --force-confold for cron-apt upgrades.

At some point when we establish more control over the state of /etc/
on SecureDrop servers and can ensure our bundled module stays
configured, we'll be able to remove the libapache2-mod-wsgi package.

While testing these changes in prod VMC, I ran into difficulties with
direct SSH, so I tweaked the Vagrantfile to support that better.

To run a mix of Python 2 and 3, we need to omit backport requirements
when running under 3, so I've added environment markers to enum34 and
ipaddress.

Also, --generate-hashes for develop and test requirements too.

.circleci/config.yml

@@ -63,7 +63,9 @@ jobs:
 
       - run:
           name: Install development dependencies
-          command: pip install -U -r securedrop/requirements/develop-requirements.txt
+          command: |
+            pip install -U pip
+            pip install -U -r securedrop/requirements/develop-requirements.txt
 
       - run:
           name: Run code linting

Makefile

@@ -103,16 +103,16 @@ safety: ## Runs `safety check` to check python dependencies for vulnerabilities
 .PHONY: bandit
 bandit: ## Run bandit with medium level excluding test-related folders
 	pip install --upgrade pip && \
-        pip install --upgrade bandit!=1.6.0 && \
+		pip install --upgrade bandit!=1.6.0 && \
 	bandit --recursive . --exclude admin/.tox,admin/.venv,admin/.eggs,molecule,testinfra,securedrop/tests,.tox,.venv -ll
 
 .PHONY: update-pip-requirements
 update-pip-requirements: ## Updates all Python requirements files via pip-compile.
 	make -C admin update-pip-requirements
-	pip-compile --output-file securedrop/requirements/develop-requirements.txt \
+	pip-compile --generate-hashes --output-file securedrop/requirements/develop-requirements.txt \
 		admin/requirements-ansible.in \
 		securedrop/requirements/develop-requirements.in
-	pip-compile --output-file securedrop/requirements/test-requirements.txt \
+	pip-compile --generate-hashes --output-file securedrop/requirements/test-requirements.txt \
 		securedrop/requirements/test-requirements.in
 	pip-compile --generate-hashes --output-file securedrop/requirements/securedrop-app-code-requirements.txt \
 		securedrop/requirements/securedrop-app-code-requirements.in

Vagrantfile

@@ -11,16 +11,16 @@ Vagrant.configure("2") do |config|
   # so the key insertion feature should be disabled.
   config.ssh.insert_key = false
 
+  config.ssh.port = 22
+
   # The staging hosts are just like production but allow non-tor access
   # for the web interfaces and ssh.
   config.vm.define 'mon-staging', autostart: false do |staging|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("mon-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
-    elsif ARGV[0] == "ssh"
-      config.ssh.host = "10.0.1.3"
-      config.ssh.port = 22
+      staging.ssh.host = find_ssh_aths("mon-ssh-aths")
+      staging.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      staging.ssh.host = "10.0.1.3"
     end
     staging.vm.hostname = "mon-staging"
     staging.vm.box = "bento/ubuntu-16.04"
@@ -30,12 +30,10 @@ Vagrant.configure("2") do |config|
 
   config.vm.define 'app-staging', autostart: false do |staging|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("app-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
-    elsif ARGV[0] == "ssh"
-      config.ssh.host = "10.0.1.2"
-      config.ssh.port = 22
+      staging.ssh.host = find_ssh_aths("app-ssh-aths")
+      staging.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      staging.ssh.host = "10.0.1.2"
     end
     staging.vm.hostname = "app-staging"
     staging.vm.box = "bento/ubuntu-16.04"
@@ -62,9 +60,10 @@ Vagrant.configure("2") do |config|
   # All access to SSH and the web interfaces is only over Tor.
   config.vm.define 'mon-prod', autostart: false do |prod|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("mon-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
+      prod.ssh.host = find_ssh_aths("mon-ssh-aths")
+      prod.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      prod.ssh.host = "10.0.1.5"
     end
     prod.vm.hostname = "mon-prod"
     prod.vm.box = "bento/ubuntu-16.04"
@@ -74,9 +73,10 @@ Vagrant.configure("2") do |config|
 
   config.vm.define 'app-prod', autostart: false do |prod|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("app-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
+      prod.ssh.host = find_ssh_aths("app-ssh-aths")
+      prod.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      prod.ssh.host = "10.0.1.4"
     end
     prod.vm.hostname = "app-prod"
     prod.vm.box = "bento/ubuntu-16.04"

admin/requirements.txt

@@ -105,7 +105,8 @@ ipaddress==1.0.19 \
     # via cryptography
 jinja2==2.10.1 \
     --hash=sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013 \
-    --hash=sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b
+    --hash=sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b \
+    # via ansible
 markupsafe==1.0 \
     --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 \
     # via jinja2

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2

@@ -72,6 +72,7 @@
   /lib/x86_64-linux-gnu/libc-*.so mr,
   /lib/x86_64-linux-gnu/libz.so.* mr,
   /proc/ r,
+  /proc/*/fd/ r,
   /proc/*/mounts r,
   /proc/*/stat r,
   /proc/*/status r,
@@ -96,6 +97,8 @@
   /usr/lib{,32,64}/** mr,
   /usr/local/lib/python2.7/dist-packages/ r,
   /usr/local/lib/python2.7/dist-packages/** rwm,
+  /usr/local/lib/python3.5/dist-packages/ r,
+  /usr/local/lib/python3.5/dist-packages/** rwm,
   /usr/share/file/magic r,
   /usr/share/file/magic.mgc r,
   /usr/share/pyshared/supervisor-*-nspkg.pth r,
@@ -131,6 +134,8 @@
   /var/www/* rw,
   /var/www/journalist.wsgi r,
   /var/www/securedrop/ r,
+  /var/www/securedrop/__pycache__/ rw,
+  /var/www/securedrop/__pycache__/** rw,
   /var/www/securedrop/config.py r,
   /var/www/securedrop/config.pyc rw,
   /var/www/securedrop/crypto_util.py r,
@@ -141,6 +146,7 @@
   /var/www/securedrop/dictionaries/nouns.txt r,
   /var/www/securedrop/journalist.py r,
   /var/www/securedrop/journalist.pyc rw,
+  /var/www/securedrop/journalist_app/ r,
   /var/www/securedrop/journalist_app/__init__.py r,
   /var/www/securedrop/journalist_app/__init__.pyc rw,
   /var/www/securedrop/journalist_app/account.py r,
@@ -159,6 +165,7 @@
   /var/www/securedrop/journalist_app/main.pyc rw,
   /var/www/securedrop/journalist_app/utils.py r,
   /var/www/securedrop/journalist_app/utils.pyc rw,
+  /var/www/securedrop/journalist_app/__pycache__/ rw,
   /var/www/securedrop/journalist_app/__pycache__/** rw,
   /var/www/securedrop/journalist_templates/account_edit_hotp_secret.html r,
   /var/www/securedrop/journalist_templates/account_new_two_factor.html r,
@@ -194,8 +201,10 @@
   /var/www/securedrop/sdconfig.pyc rw,
   /var/www/securedrop/source.py r,
   /var/www/securedrop/source.pyc rw,
+  /var/www/securedrop/source_app/ r,
   /var/www/securedrop/source_app/__init__.py r,
   /var/www/securedrop/source_app/__init__.pyc rw,
+  /var/www/securedrop/source_app/__pycache__/ rw,
   /var/www/securedrop/source_app/__pycache__/** rw,
   /var/www/securedrop/source_app/api.py r,
   /var/www/securedrop/source_app/api.pyc rw,
@@ -267,7 +276,7 @@
   /var/www/securedrop/static/i/font-awesome/trash-black.png r,
   /var/www/securedrop/static/i/font-awesome/black/guard.svg r,
   /var/www/securedrop/static/i/font-awesome/white/guard.svg r,
-  /var/www/securedrop/static/i/font-awesome/white/exclamation-circle.svg r,  
+  /var/www/securedrop/static/i/font-awesome/white/exclamation-circle.svg r,
   /var/www/securedrop/static/i/hand_with_fingerprint.png r,
   /var/www/securedrop/static/i/success_checkmark.png r,
   /var/www/securedrop/static/i/server_upload.png r,

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/build_securedrop_app_code_deb.yml

@@ -16,9 +16,21 @@
 
 - include: sass.yml
 
+- name: Install Debian packages required to build wheels
+  apt:
+    name:
+      - apache2-dev
+      - python3-dev
+      - python3-pip
+      - python3-wheel
+    state: present
+
+- name: Upgrade pip
+  shell: pip3 install --upgrade pip
+
 - name: Create pip wheel archive for Debian package requirements.
   shell: |
-    pip wheel \
+    pip3 wheel --verbose \
       --no-binary :all: \
       --require-hashes \
       --requirement {{ securedrop_pip_requirements }} \
@@ -41,13 +53,14 @@
 # integrity, we can defer to a later time.
 - name: Install built pip wheels for SecureDrop.
   shell: |
-    pip install {{ securedrop_wheelhouse }}/*.whl
+    ls -al {{ securedrop_wheelhouse }}
+    pip3 install {{ securedrop_wheelhouse }}/*.whl
   tags:
     - pip
 
 - name: Create new requirements based on build/installed wheels without hashes
   shell:
-    pip freeze > {{ securedrop_pip_requirements_generated }}
+    pip3 freeze > {{ securedrop_pip_requirements_generated }}
   tags:
     - pip
 

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/translations.yml

@@ -4,7 +4,7 @@
   shell: >-
     cp config.py.example config.py ;
     trap 'rm config.py' EXIT ;
-    ./i18n_tool.py --verbose translate-messages --compile
+    python3 ./i18n_tool.py --verbose translate-messages --compile
   args:
     chdir: "{{ securedrop_code_filtered }}"
   environment:

install_files/securedrop-app-code/debian/control

@@ -6,5 +6,5 @@ Homepage: https://securedrop.org
 
 Package: securedrop-app-code
 Architecture: amd64
-Depends: python-pip,apparmor-utils,gnupg2,haveged,python,secure-delete,sqlite3,${dist:Depends},libapache2-mod-wsgi,libapache2-mod-xsendfile,redis-server,supervisor,securedrop-keyring,securedrop-config,libpython2.7-dev
+Depends: python3-pip,apparmor-utils,gnupg2,haveged,python3,secure-delete,sqlite3,${dist:Depends},libapache2-mod-xsendfile,redis-server,supervisor,securedrop-keyring,securedrop-config,devscripts
 Description: Packages the SecureDrop application code pip dependencies and apparmor profiles. This package will put the apparmor profiles in enforce mode. This package does use pip to install the pip wheelhouse

install_files/securedrop-app-code/debian/postinst

@@ -124,7 +124,7 @@ case "$1" in
 
     chown -R www-data:www-data /var/lib/securedrop /var/www/securedrop
 
-    pip install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
+    pip3 install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
       -r /var/www/securedrop/requirements/securedrop-app-code-requirements-packaged.txt
 
     chown -R www-data:www-data /var/www/securedrop
@@ -142,6 +142,9 @@ case "$1" in
     # the ability to send signals to unconfined peers.
     service apache2 stop
 
+    # Point Apache to our bundled mod_wsgi
+    mod_wsgi-express module-config > /etc/apache2/mods-available/wsgi.load
+
     # If the profile was disabled enabled it.
     if [ -e "/etc/apparmor.d/disable/usr.sbin.apache2" ]; then
         rm /etc/apparmor.d/disable/usr.sbin.apache2

molecule/builder-xenial/Dockerfile

@@ -6,6 +6,8 @@ LABEL org="Freedom of the Press"
 LABEL image_name="xenial-sd-builder-app"
 
 RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
+        apache2 \
+        apache2-dev \
         aptitude \
         debhelper \
         devscripts \
@@ -19,10 +21,10 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         make \
         ntp \
         paxctl \
-        python \
-        python-dev \
-        python-pip \
-        python-wheel \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-wheel \
         rsync \
         ruby \
         secure-delete \

molecule/testinfra/staging/app-code/test_securedrop_app_code.py

@@ -15,15 +15,21 @@ def test_apache_default_docroot_is_absent(host):
 
 
 @pytest.mark.parametrize('package', [
-  'apparmor-utils',
-  'gnupg2',
-  'haveged',
-  'python',
-  'python-pip',
-  'redis-server',
-  'secure-delete',
-  'sqlite3',
-  'supervisor',
+    "apache2",
+    "apparmor-utils",
+    "devscripts",
+    "gnupg2",
+    "haveged",
+    "libapache2-mod-xsendfile",
+    "paxctld",
+    "python3",
+    "python3-pip",
+    "redis-server",
+    "secure-delete",
+    "securedrop-config",
+    "securedrop-keyring",
+    "sqlite3",
+    "supervisor",
 ])
 def test_securedrop_application_apt_dependencies(host, package):
     """
@@ -34,6 +40,16 @@ def test_securedrop_application_apt_dependencies(host, package):
     assert host.package(package).is_installed
 
 
+def test_mod_wsgi_is_ours(host):
+    mod_wsgi_load = host.file("/etc/apache2/mods-enabled/wsgi.load").content_string
+    assert mod_wsgi_load == (
+        '''LoadModule wsgi_module '''
+        '''"/usr/local/lib/python3.5/dist-packages/mod_wsgi/server/'''
+        '''mod_wsgi-py35.cpython-35m-x86_64-linux-gnu.so"\n'''
+        '''WSGIPythonHome "/usr"'''
+    )
+
+
 def test_securedrop_application_test_locale(host):
     """
     Ensure both SecureDrop DEFAULT_LOCALE and SUPPORTED_LOCALES are present.

molecule/testinfra/staging/app/test_appenv.py

@@ -7,7 +7,7 @@
 @pytest.mark.parametrize('exp_pip_pkg', sdvars.pip_deps)
 def test_app_pip_deps(host, exp_pip_pkg):
     """ Ensure pip dependencies are installed """
-    pip = host.pip_package.get_packages()
+    pip = host.pip_package.get_packages(pip_path="pip3")
     assert pip[exp_pip_pkg['name']]['version'] == exp_pip_pkg['version']
 
 

securedrop/dockerfiles/xenial/python2/Dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && \
     apt-get install -y paxctl && \
     { apt-get install -y libgtk2.0 || echo 'libgtk2.0 was not installed'; } && \
     paxctl -cm /usr/bin/mono-sgen && dpkg-reconfigure mono-runtime-sgen && \
-    apt-get install -y devscripts vim \
+    apt-get install -y apache2-dev devscripts vim \
                        python-pip libpython2.7-dev libssl-dev secure-delete \
                        gnupg2 ruby redis-server git xvfb haveged curl wget \
                        gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo \

securedrop/dockerfiles/xenial/python3/Dockerfile

@@ -9,7 +9,7 @@ ENV USER_ID ${USER_ID:-0}
 RUN apt-get update && apt-get install -y paxctl && \
     { apt-get install -y libgtk2.0 || echo 'libgtk2.0 was not installed'; } && \
     paxctl -cm /usr/bin/mono-sgen && dpkg-reconfigure mono-runtime-sgen && \
-    apt-get install -y devscripts vim \
+    apt-get install -y apache2-dev devscripts vim \
                        python3-pip libpython3.5-dev libssl-dev secure-delete \
                        gnupg2 ruby redis-server git xvfb haveged curl wget \
                        gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo \

securedrop/requirements/develop-requirements.in

@@ -5,13 +5,16 @@ boto3
 docker-py
 # Needed for dig ansible lookup
 dnspython
+enum34==1.1.6; python_version < "3.4"
 flake8
+futures; python_version < "3.0"
 html-linter
+ipaddress==1.0.22; python_version < "3.3"
 molecule>2.13
 # Needed for ansible network filter
 # http://docs.ansible.com/ansible/latest/playbooks_filters_ipaddr.html
 netaddr
-pip-tools>=3.5.0,<4
+pip-tools>=3.8.0,<4
 pyenchant
 pylint
 pytest-xdist