Requirements pinning for app code build

Makefile

@@ -114,7 +114,7 @@ update-pip-requirements: ## Updates all Python requirements files via pip-compil
 		securedrop/requirements/develop-requirements.in
 	pip-compile --output-file securedrop/requirements/test-requirements.txt \
 		securedrop/requirements/test-requirements.in
-	pip-compile --output-file securedrop/requirements/securedrop-app-code-requirements.txt \
+	pip-compile --generate-hashes --output-file securedrop/requirements/securedrop-app-code-requirements.txt \
 		securedrop/requirements/securedrop-app-code-requirements.in
 
 .PHONY: libvirt-share

admin/Dockerfile

@@ -16,5 +16,5 @@ RUN python bootstrap.py -v
 ENV VIRTUAL_ENV /opt/.venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY requirements-dev.txt .
-RUN pip install -r requirements-dev.txt
+RUN pip install --require-hashes -r requirements-dev.txt
 RUN chown -R $USER_NAME /opt

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml

@@ -8,6 +8,10 @@ securedrop_code_filtered: "{{ securedrop_app_code_deb_dir }}/var/www/securedrop"
 
 securedrop_pip_requirements: "{{ securedrop_code_filtered }}/requirements/securedrop-app-code-requirements.txt"
 
+securedrop_pip_requirements_generated: "{{ securedrop_code_filtered }}/requirements/securedrop-app-code-requirements-packaged.txt"
+
+securedrop_wheelhouse: "{{ securedrop_app_code_deb_dir }}/var/securedrop/wheelhouse"
+
 securedrop_app_rsync_opts:
   - "--chmod=u=rwX,g=rX,o=rX"
   - "--chown=root:root"

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/build_securedrop_app_code_deb.yml

@@ -1,9 +1,4 @@
 ---
-- name: Install pip wheel.
-  pip:
-    name: wheel
-  tags: pip
-
 - name: Copy install_files/securedrop-app-code dir to build path.
   synchronize:
     src: "{{ role_path }}/../../../securedrop-app-code/"
@@ -24,11 +19,38 @@
 - name: Create pip wheel archive for Debian package requirements.
   shell: |
     pip wheel \
-      -r {{ securedrop_code_filtered }}/requirements/securedrop-app-code-requirements.txt \
-      -w {{ securedrop_app_code_deb_dir }}/var/securedrop/wheelhouse 2>&1 | tee /tmp/w.out
+      --no-binary :all: \
+      --require-hashes \
+      --requirement {{ securedrop_pip_requirements }} \
+      --wheel-dir {{ securedrop_wheelhouse }} 2>&1 | tee /tmp/w.out
       ! grep -i --quiet 'Failed to build' /tmp/w.out
+  register: wheel_build_output
   tags: pip
 
+- name: Ensure source hash sums matched at wheel build-time
+  fail: msg="Source hash sum mismatch, build cannot continue."
+  failed_when: "'THESE PACKAGES DO NOT MATCH THE HASHES' in wheel_build_output.stdout"
+
+# Here we need to regenerate a new securedrop-app-code-requirements.txt file
+# *without* hashes to bundle in the .deb package, for reasons that follow:
+# If requirements hashes are in the requirments file, pip will automatically
+# use hash checking mode to install the dependencies (which is done in postinst
+# of securedrop-app-code package).
+# dpkg-buildpackage modifies the wheel archive such that the hashes of those
+# built wheels change. Since the deb package itself is signed (providing
+# integrity, we can defer to a later time.
+- name: Install built pip wheels for SecureDrop.
+  shell: |
+    pip install {{ securedrop_wheelhouse }}/*.whl
+  tags:
+    - pip
+
+- name: Create new requirements based on build/installed wheels without hashes
+  shell:
+    pip freeze > {{ securedrop_pip_requirements_generated }}
+  tags:
+    - pip
+
 - include: translations.yml
 
 - name: Create apparmor.d directory in build path.

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/translations.yml

@@ -1,9 +1,4 @@
 ---
-- name: Install pip dependencies for SecureDrop.
-  pip:
-    requirements: "{{ securedrop_code_filtered }}/requirements/securedrop-app-code-requirements.txt"
-  tags:
-    - pip
 
 - name: Compile PO to MO.
   shell: >-

install_files/securedrop-app-code/debian/postinst

@@ -125,7 +125,7 @@ case "$1" in
     chown -R www-data:www-data /var/lib/securedrop /var/www/securedrop
 
     pip install --no-index --find-links=/var/securedrop/wheelhouse --upgrade \
-      -r /var/www/securedrop/requirements/securedrop-app-code-requirements.txt
+      -r /var/www/securedrop/requirements/securedrop-app-code-requirements-packaged.txt
 
     chown -R www-data:www-data /var/www/securedrop
     chown www-data:www-data /var/www/journalist.wsgi

molecule/builder-xenial/Dockerfile

@@ -22,6 +22,7 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         python \
         python-dev \
         python-pip \
+        python-wheel \
         rsync \
         ruby \
         secure-delete \

molecule/builder-xenial/image_hash

@@ -1,2 +1,2 @@
-# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2019_04_25
-79d8b8c8068f486eb914ff8340beaa4d1f17782a16cf00f0d63b4eb021379a66
+# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2019_05_15
+20c4aa2c6a01a135379ff439ae439ce0ad80ec9e41db8c2b242f54310c536a92

securedrop/dockerfiles/xenial/python2/Dockerfile

@@ -50,7 +50,7 @@ RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckod
 
 COPY requirements requirements
 
-RUN pip install -r requirements/securedrop-app-code-requirements.txt && \
+RUN pip install --require-hashes -r requirements/securedrop-app-code-requirements.txt && \
     pip install -r requirements/test-requirements.txt && \
     pip install --upgrade setuptools  # Fixes #4036 pybabel requires latest version of setuptools
 

securedrop/dockerfiles/xenial/python3/Dockerfile

@@ -48,7 +48,7 @@ RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckod
     paxctl -cm /bin/geckodriver
 
 COPY requirements requirements
-RUN pip3 install -r requirements/securedrop-app-code-requirements.txt && \
+RUN pip3 install --require-hashes -r requirements/securedrop-app-code-requirements.txt && \
     pip3 install -r requirements/test-requirements.txt && \
     pip3 install --upgrade setuptools  # Fixes #4036 pybabel requires latest version of setuptools