Bundle mod_wsgi in securedrop-app-code package

.circleci/config.yml

@@ -63,7 +63,9 @@ jobs:
 
       - run:
           name: Install development dependencies
-          command: pip install -U -r securedrop/requirements/develop-requirements.txt
+          command: |
+            pip install -U pip
+            pip install -U -r securedrop/requirements/develop-requirements.txt
 
       - run:
           name: Run code linting

.gitignore

@@ -50,6 +50,7 @@ __pycache__/
 .Python
 env/
 build/*.deb
+build/xenial
 develop-eggs/
 dist/
 downloads/
@@ -99,7 +100,7 @@ docs/_build/
 .env
 
 # virtualenv
-.venv/
+.venv*/
 venv/
 ENV/
 

Makefile

@@ -103,19 +103,20 @@ safety: ## Runs `safety check` to check python dependencies for vulnerabilities
 .PHONY: bandit
 bandit: ## Run bandit with medium level excluding test-related folders
 	pip install --upgrade pip && \
-        pip install --upgrade bandit!=1.6.0 && \
+		pip install --upgrade bandit!=1.6.0 && \
 	bandit --recursive . --exclude admin/.tox,admin/.venv,admin/.eggs,molecule,testinfra,securedrop/tests,.tox,.venv -ll
 
 .PHONY: update-pip-requirements
 update-pip-requirements: ## Updates all Python requirements files via pip-compile.
 	make -C admin update-pip-requirements
-	pip-compile --output-file securedrop/requirements/develop-requirements.txt \
+	pip-compile --generate-hashes --output-file securedrop/requirements/develop-requirements.txt \
 		admin/requirements-ansible.in \
 		securedrop/requirements/develop-requirements.in
-	pip-compile --output-file securedrop/requirements/test-requirements.txt \
+	pip-compile --generate-hashes --output-file securedrop/requirements/test-requirements.txt \
 		securedrop/requirements/test-requirements.in
-	pip-compile --generate-hashes --output-file securedrop/requirements/securedrop-app-code-requirements.txt \
-		securedrop/requirements/securedrop-app-code-requirements.in
+	PYTHON_VERSION=3 securedrop/bin/dev-shell pip-compile --generate-hashes \
+		 --output-file requirements/securedrop-app-code-requirements.txt \
+		requirements/securedrop-app-code-requirements.in
 
 .PHONY: libvirt-share
 libvirt-share: ## Configure ACLs to allow RWX for libvirt VM (e.g. Admin Workstation)

Vagrantfile

@@ -11,16 +11,16 @@ Vagrant.configure("2") do |config|
   # so the key insertion feature should be disabled.
   config.ssh.insert_key = false
 
+  config.ssh.port = 22
+
   # The staging hosts are just like production but allow non-tor access
   # for the web interfaces and ssh.
   config.vm.define 'mon-staging', autostart: false do |staging|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("mon-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
-    elsif ARGV[0] == "ssh"
-      config.ssh.host = "10.0.1.3"
-      config.ssh.port = 22
+      staging.ssh.host = find_ssh_aths("mon-ssh-aths")
+      staging.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      staging.ssh.host = "10.0.1.3"
     end
     staging.vm.hostname = "mon-staging"
     staging.vm.box = "bento/ubuntu-16.04"
@@ -30,12 +30,10 @@ Vagrant.configure("2") do |config|
 
   config.vm.define 'app-staging', autostart: false do |staging|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("app-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
-    elsif ARGV[0] == "ssh"
-      config.ssh.host = "10.0.1.2"
-      config.ssh.port = 22
+      staging.ssh.host = find_ssh_aths("app-ssh-aths")
+      staging.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      staging.ssh.host = "10.0.1.2"
     end
     staging.vm.hostname = "app-staging"
     staging.vm.box = "bento/ubuntu-16.04"
@@ -62,9 +60,10 @@ Vagrant.configure("2") do |config|
   # All access to SSH and the web interfaces is only over Tor.
   config.vm.define 'mon-prod', autostart: false do |prod|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("mon-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
+      prod.ssh.host = find_ssh_aths("mon-ssh-aths")
+      prod.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      prod.ssh.host = "10.0.1.5"
     end
     prod.vm.hostname = "mon-prod"
     prod.vm.box = "bento/ubuntu-16.04"
@@ -74,9 +73,10 @@ Vagrant.configure("2") do |config|
 
   config.vm.define 'app-prod', autostart: false do |prod|
     if ENV['SECUREDROP_SSH_OVER_TOR']
-      config.ssh.host = find_ssh_aths("app-ssh-aths")
-      config.ssh.proxy_command = tor_ssh_proxy_command
-      config.ssh.port = 22
+      prod.ssh.host = find_ssh_aths("app-ssh-aths")
+      prod.ssh.proxy_command = tor_ssh_proxy_command
+    elsif ARGV[0].start_with? "ssh"
+      prod.ssh.host = "10.0.1.4"
     end
     prod.vm.hostname = "app-prod"
     prod.vm.box = "bento/ubuntu-16.04"

admin/requirements.txt

@@ -105,7 +105,8 @@ ipaddress==1.0.19 \
     # via cryptography
 jinja2==2.10.1 \
     --hash=sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013 \
-    --hash=sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b
+    --hash=sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b \
+    # via ansible
 markupsafe==1.0 \
     --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 \
     # via jinja2

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2

@@ -72,6 +72,7 @@
   /lib/x86_64-linux-gnu/libc-*.so mr,
   /lib/x86_64-linux-gnu/libz.so.* mr,
   /proc/ r,
+  /proc/*/fd/ r,
   /proc/*/mounts r,
   /proc/*/stat r,
   /proc/*/status r,
@@ -96,6 +97,8 @@
   /usr/lib{,32,64}/** mr,
   /usr/local/lib/python2.7/dist-packages/ r,
   /usr/local/lib/python2.7/dist-packages/** rwm,
+  /usr/local/lib/python3.5/dist-packages/ r,
+  /usr/local/lib/python3.5/dist-packages/** rwm,
   /usr/share/file/magic r,
   /usr/share/file/magic.mgc r,
   /usr/share/pyshared/supervisor-*-nspkg.pth r,
@@ -131,6 +134,8 @@
   /var/www/* rw,
   /var/www/journalist.wsgi r,
   /var/www/securedrop/ r,
+  /var/www/securedrop/__pycache__/ rw,
+  /var/www/securedrop/__pycache__/** rw,
   /var/www/securedrop/config.py r,
   /var/www/securedrop/config.pyc rw,
   /var/www/securedrop/crypto_util.py r,
@@ -141,6 +146,7 @@
   /var/www/securedrop/dictionaries/nouns.txt r,
   /var/www/securedrop/journalist.py r,
   /var/www/securedrop/journalist.pyc rw,
+  /var/www/securedrop/journalist_app/ r,
   /var/www/securedrop/journalist_app/__init__.py r,
   /var/www/securedrop/journalist_app/__init__.pyc rw,
   /var/www/securedrop/journalist_app/account.py r,
@@ -159,6 +165,7 @@
   /var/www/securedrop/journalist_app/main.pyc rw,
   /var/www/securedrop/journalist_app/utils.py r,
   /var/www/securedrop/journalist_app/utils.pyc rw,
+  /var/www/securedrop/journalist_app/__pycache__/ rw,
   /var/www/securedrop/journalist_app/__pycache__/** rw,
   /var/www/securedrop/journalist_templates/account_edit_hotp_secret.html r,
   /var/www/securedrop/journalist_templates/account_new_two_factor.html r,
@@ -194,8 +201,10 @@
   /var/www/securedrop/sdconfig.pyc rw,
   /var/www/securedrop/source.py r,
   /var/www/securedrop/source.pyc rw,
+  /var/www/securedrop/source_app/ r,
   /var/www/securedrop/source_app/__init__.py r,
   /var/www/securedrop/source_app/__init__.pyc rw,
+  /var/www/securedrop/source_app/__pycache__/ rw,
   /var/www/securedrop/source_app/__pycache__/** rw,
   /var/www/securedrop/source_app/api.py r,
   /var/www/securedrop/source_app/api.pyc rw,
@@ -267,7 +276,7 @@
   /var/www/securedrop/static/i/font-awesome/trash-black.png r,
   /var/www/securedrop/static/i/font-awesome/black/guard.svg r,
   /var/www/securedrop/static/i/font-awesome/white/guard.svg r,
-  /var/www/securedrop/static/i/font-awesome/white/exclamation-circle.svg r,  
+  /var/www/securedrop/static/i/font-awesome/white/exclamation-circle.svg r,
   /var/www/securedrop/static/i/hand_with_fingerprint.png r,
   /var/www/securedrop/static/i/success_checkmark.png r,
   /var/www/securedrop/static/i/server_upload.png r,

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/build_securedrop_app_code_deb.yml

@@ -16,14 +16,22 @@
 
 - include: sass.yml
 
+- name: Install Debian packages required to build wheels
+  apt:
+    name:
+      - apache2-dev
+      - python3-dev
+      - python3-pip
+      - python3-wheel
+    state: present
+
 - name: Create pip wheel archive for Debian package requirements.
   shell: |
-    pip wheel \
+    pip3 wheel \
       --no-binary :all: \
       --require-hashes \
       --requirement {{ securedrop_pip_requirements }} \
-      --wheel-dir {{ securedrop_wheelhouse }} 2>&1 | tee /tmp/w.out
-      ! grep -i --quiet 'Failed to build' /tmp/w.out
+      --wheel-dir {{ securedrop_wheelhouse }} 2>&1
   register: wheel_build_output
   tags: pip
 
@@ -40,14 +48,13 @@
 # built wheels change. Since the deb package itself is signed (providing
 # integrity, we can defer to a later time.
 - name: Install built pip wheels for SecureDrop.
-  shell: |
-    pip install {{ securedrop_wheelhouse }}/*.whl
+  shell: pip3 install {{ securedrop_wheelhouse }}/*.whl
   tags:
     - pip
 
 - name: Create new requirements based on build/installed wheels without hashes
   shell:
-    pip freeze > {{ securedrop_pip_requirements_generated }}
+    grep -v -e '--hash' -e '^\s*#' {{ securedrop_pip_requirements }} | awk '{print $1}' > {{ securedrop_pip_requirements_generated }}
   tags:
     - pip
 

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/translations.yml

@@ -4,7 +4,7 @@
   shell: >-
     cp config.py.example config.py ;
     trap 'rm config.py' EXIT ;
-    ./i18n_tool.py --verbose translate-messages --compile
+    python3 ./i18n_tool.py --verbose translate-messages --compile
   args:
     chdir: "{{ securedrop_code_filtered }}"
   environment:

install_files/securedrop-app-code/debian/control

@@ -6,5 +6,5 @@ Homepage: https://securedrop.org
 
 Package: securedrop-app-code
 Architecture: amd64
-Depends: python-pip,apparmor-utils,gnupg2,haveged,python,secure-delete,sqlite3,${dist:Depends},libapache2-mod-wsgi,libapache2-mod-xsendfile,redis-server,supervisor,securedrop-keyring,securedrop-config,libpython2.7-dev
+Depends: ${dist:Depends}, apparmor-utils, gnupg2, haveged, libapache2-mod-xsendfile, libapache2-mod-wsgi (= 4.3.0-1.1build1), python3, python3-pip, redis-server, secure-delete, securedrop-config, securedrop-keyring, sqlite3, supervisor
 Description: Packages the SecureDrop application code pip dependencies and apparmor profiles. This package will put the apparmor profiles in enforce mode. This package does use pip to install the pip wheelhouse

install_files/securedrop-app-code/debian/postinst

@@ -124,7 +124,7 @@ case "$1" in
 
     chown -R www-data:www-data /var/lib/securedrop /var/www/securedrop
 
-    pip install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
+    pip3 install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
       -r /var/www/securedrop/requirements/securedrop-app-code-requirements-packaged.txt
 
     chown -R www-data:www-data /var/www/securedrop
@@ -142,6 +142,9 @@ case "$1" in
     # the ability to send signals to unconfined peers.
     service apache2 stop
 
+    # Point Apache to our bundled mod_wsgi
+    mod_wsgi-express module-config > /etc/apache2/mods-available/wsgi.load
+
     # If the profile was disabled enabled it.
     if [ -e "/etc/apparmor.d/disable/usr.sbin.apache2" ]; then
         rm /etc/apparmor.d/disable/usr.sbin.apache2

install_files/securedrop-config/etc/cron-apt/action.d/0-app-code

@@ -0,0 +1,2 @@
+update
+install -y securedrop-app-code

molecule/builder-xenial/Dockerfile

@@ -6,6 +6,8 @@ LABEL org="Freedom of the Press"
 LABEL image_name="xenial-sd-builder-app"
 
 RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
+        apache2 \
+        apache2-dev \
         aptitude \
         debhelper \
         devscripts \
@@ -19,10 +21,10 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         make \
         ntp \
         paxctl \
-        python \
-        python-dev \
-        python-pip \
-        python-wheel \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-wheel \
         rsync \
         ruby \
         secure-delete \

molecule/testinfra/staging/app-code/test_securedrop_app_code.py

@@ -15,15 +15,21 @@ def test_apache_default_docroot_is_absent(host):
 
 
 @pytest.mark.parametrize('package', [
-  'apparmor-utils',
-  'gnupg2',
-  'haveged',
-  'python',
-  'python-pip',
-  'redis-server',
-  'secure-delete',
-  'sqlite3',
-  'supervisor',
+    "apache2",
+    "apparmor-utils",
+    "devscripts",
+    "gnupg2",
+    "haveged",
+    "libapache2-mod-xsendfile",
+    "paxctld",
+    "python3",
+    "python3-pip",
+    "redis-server",
+    "secure-delete",
+    "securedrop-config",
+    "securedrop-keyring",
+    "sqlite3",
+    "supervisor",
 ])
 def test_securedrop_application_apt_dependencies(host, package):
     """
@@ -34,6 +40,16 @@ def test_securedrop_application_apt_dependencies(host, package):
     assert host.package(package).is_installed
 
 
+def test_mod_wsgi_is_ours(host):
+    mod_wsgi_load = host.file("/etc/apache2/mods-enabled/wsgi.load").content_string
+    assert mod_wsgi_load == (
+        '''LoadModule wsgi_module '''
+        '''"/usr/local/lib/python3.5/dist-packages/mod_wsgi/server/'''
+        '''mod_wsgi-py35.cpython-35m-x86_64-linux-gnu.so"\n'''
+        '''WSGIPythonHome "/usr"'''
+    )
+
+
 def test_securedrop_application_test_locale(host):
     """
     Ensure both SecureDrop DEFAULT_LOCALE and SUPPORTED_LOCALES are present.

molecule/testinfra/staging/app/test_appenv.py

@@ -7,7 +7,7 @@
 @pytest.mark.parametrize('exp_pip_pkg', sdvars.pip_deps)
 def test_app_pip_deps(host, exp_pip_pkg):
     """ Ensure pip dependencies are installed """
-    pip = host.pip_package.get_packages()
+    pip = host.pip_package.get_packages(pip_path="pip3")
     assert pip[exp_pip_pkg['name']]['version'] == exp_pip_pkg['version']
 
 

molecule/upgrade/side_effect.yml

@@ -1,12 +1,12 @@
 ---
-- name: Perform apt upgrades
+- name: Simulate cron-apt upgrade
   hosts: securedrop
   become: yes
   tasks:
-    - name: Perform safe upgrade
-      apt:
-        update_cache: yes
-        upgrade: yes
+    - name: Run cron-apt
+      shell: cron-apt -i -s
+    - name: Update apt cache
+      shell: apt update
 
 - name: Lay out app testing deps
   hosts: securedrop_application_server

securedrop/dockerfiles/xenial/python2/Dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && \
     apt-get install -y paxctl && \
     { apt-get install -y libgtk2.0 || echo 'libgtk2.0 was not installed'; } && \
     paxctl -cm /usr/bin/mono-sgen && dpkg-reconfigure mono-runtime-sgen && \
-    apt-get install -y devscripts vim \
+    apt-get install -y apache2-dev devscripts vim \
                        python-pip libpython2.7-dev libssl-dev secure-delete \
                        gnupg2 ruby redis-server git xvfb haveged curl wget \
                        gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo \

securedrop/dockerfiles/xenial/python3/Dockerfile

@@ -9,7 +9,7 @@ ENV USER_ID ${USER_ID:-0}
 RUN apt-get update && apt-get install -y paxctl && \
     { apt-get install -y libgtk2.0 || echo 'libgtk2.0 was not installed'; } && \
     paxctl -cm /usr/bin/mono-sgen && dpkg-reconfigure mono-runtime-sgen && \
-    apt-get install -y devscripts vim \
+    apt-get install -y apache2-dev devscripts vim \
                        python3-pip libpython3.5-dev libssl-dev secure-delete \
                        gnupg2 ruby redis-server git xvfb haveged curl wget \
                        gettext paxctl x11vnc enchant libffi-dev sqlite3 gettext sudo \