Merge pull request #382 from freedomofpress/reorg

Reorganize how wheels are stored, stop special-casing bootstrap

Makefile

@@ -1,12 +1,13 @@
 DEFAULT_GOAL: help
+SHELL := /bin/bash
 
 .PHONY: securedrop-proxy
 securedrop-proxy: ## Builds Debian package for securedrop-proxy code
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
 
 .PHONY: securedrop-client
 securedrop-client: ## Builds Debian package for securedrop-client code
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-client" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-client" ./scripts/build-debianpackage
 
 .PHONY: securedrop-workstation-config
 securedrop-workstation-config: ## Builds Debian metapackage for Qubes Workstation base dependencies
@@ -22,11 +23,11 @@ securedrop-workstation-viewer: ## Builds Debian metapackage for Disposable VM de
 
 .PHONY: securedrop-export
 securedrop-export: ## Builds Debian package for Qubes Workstation export scripts
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-export" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-export" ./scripts/build-debianpackage
 
 .PHONY: securedrop-log
 securedrop-log: ## Builds Debian package for Qubes Workstation securedrop-log scripts
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-log" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-log" ./scripts/build-debianpackage
 
 .PHONY: securedrop-keyring
 securedrop-keyring: ## Builds Debian package containing the release key
@@ -46,9 +47,9 @@ requirements: ## Creates requirements files for the Python projects
 
 .PHONY: build-wheels
 build-wheels: ## Builds the wheels and adds them to the localwheels directory
-	./scripts/verify-sha256sum-signature
-	./scripts/build-sync-wheels -p ${PKG_DIR}
-	./scripts/sync-sha256sums
+	./scripts/verify-sha256sum-signature $$(basename ${PKG_DIR})
+	./scripts/build-sync-wheels
+	./scripts/sync-sha256sums $$(basename ${PKG_DIR})
 	@printf "Done! Now please follow the instructions in\n"
 	@printf "https://github.com/freedomofpress/securedrop-debian-packaging-guide/"
 	@printf "to push these changes to the FPF PyPI index\n"
@@ -57,10 +58,6 @@ build-wheels: ## Builds the wheels and adds them to the localwheels directory
 test: ## Run simple test suite (skips reproducibility checks)
 	pytest -v tests/test_update_requirements.py
 
-.PHONY: clean
-clean: ## Removes all non-version controlled packaging artifacts
-	rm -rf localwheels/*
-
 .PHONY: reprotest
 reprotest: ## Runs only reproducibility tests, for .deb and .whl files
 	pytest -vvs tests/test_reproducible_*.py

README.md

@@ -40,20 +40,21 @@ If we have to update the tool, use the following steps
 rm -rf .venv && python3 -m venv .venv
 source .venv/bin/activate
 # Then install pip-tools, from pinned dependencies
-python3 -m pip install -r requirements.txt
+python3 -m pip install -r workstation-bootstrap/requirements.txt
 # Then update the requirements.in file as required
-pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in
+pip-compile --allow-unsafe --generate-hashes \
+    --output-file=workstation-bootstrap/requirements.txt workstation-bootstrap/requirements.in
 # Now we are ready for bootstrapping
-./scripts/build-sync-wheels --cache ./bootstrap -p $PWD
+./scripts/build-sync-wheels --project workstation-bootstrap --pkg-dir ./workstation-bootstrap --requirements .
 # Here we have the new wheels ready
 # Now let us recreate our new sha256sums for bootstrapping
-BOOTSTRAP=true ./scripts/sync-sha256sums
+./scripts/sync-sha256sums ./workstation-bootstrap
 # now let us sign the list of sha256sums
-gpg --armor --output bootstrap-sha256sums.txt.asc --detach-sig  bootstrap-sha256sums.txt
+gpg --armor --output workstation-bootstrap/sha256sums.txt.asc --detach-sig  workstation-bootstrap/sha256sums.txt
 # We can even verify if we want
-BOOTSTRAP=true ./scripts/verify-sha256sum-signature
+./scripts/verify-sha256sum-signature ./workstation-bootstrap/
 # Update the build-requirements.txt file
-PKG_DIR=$PWD BOOTSTRAP=true ./scripts/update-requirements
+./scripts/update-requirements --pkg-dir ./workstation-bootstrap/ --project workstation-bootstrap --requirements .
 ```
 
 Make sure that your GPG public key is stored in `pubkeys/`, so CI can verify the signatures.
@@ -73,9 +74,8 @@ to do the following (we are taking `securedrop-client` project as example):
 You can create a fresh virtualenv and install the build tools from our bootstrapped wheels.
 
 ```
-python3 -m venv .venv
-source .venv/bin/activate
-python3 -m pip install --require-hashes --no-index --no-deps --no-cache-dir -r build-requirements.txt --find-links ./bootstrap/
+rm -rf .venv
+make install-deps
 ```
 
 Remember that the following steps needs to be done from the same virtual environment.
@@ -104,9 +104,8 @@ Also update the index HTML files accordingly commit your changes.
 After these steps, please rerun the command again.
 ```
 
-The next step is to build the wheels. To do this step, you will need an owner
-of the SecureDrop release key to build the wheel and sign the updated sha256sums file
-with the release key. If you're not sure who to ask, ping @redshiftzero for a pointer.
+The next step is to build the wheels. To do this step, you will need a maintainer
+to build the wheels and sign the updated sha256sums file with your individual key.
 
 ### 2. Build wheels
 
@@ -120,12 +119,6 @@ This above command will let you know about any new wheels + sources. It will
 build/download sources from PyPI (by verifying it against the sha256sums from
 the `requirements.txt` of the project).
 
-Then navigate back to the project's code directory and run the following command.
-
-```bash
-python3 setup.py sdist
-```
-
 ### 3. Commit changes to the localwheels directory (if only any update of wheels)
 
 Now add these built artifacts to version control:

scripts/build-debianpackage

@@ -12,15 +12,26 @@ set -e
 set -u
 set -o pipefail
 
+
+# Validate required args.
+if [[ -z "${PKG_NAME:-}" ]]; then
+    echo "Set PKG_NAME of the build";
+    exit 1
+fi
+
 # Store root of repo, since we'll change dirs several times.
 CUR_DIR="$(git rev-parse --show-toplevel)"
 VERSION_CODENAME=$("${CUR_DIR}/scripts/codename")
 
-# Verify sha256sums.txt in the git repo
-"${CUR_DIR}/scripts/verify-sha256sum-signature"
-
 # Disable use of pip cache during debhelper build actions.
 export DH_PIP_EXTRA_ARGS="--no-cache-dir --require-hashes"
+# Point dh-virtualenv/pip to our prebuilt wheels
+export WHEELS_DIR="${CUR_DIR}/${PKG_NAME}/wheels"
+
+if [[ -d "${WHEELS_DIR}" ]]; then
+    # Verify sha256sums.txt in the git repo if we have dependencies
+    "${CUR_DIR}/scripts/verify-sha256sum-signature" "${PKG_NAME}"
+fi
 
 # Declare general packaging building workspace; subdirs will
 # be created within, to build specific packages.
@@ -29,12 +40,6 @@ mkdir -p "$TOP_BUILDDIR"
 rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}"
 mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}"
 
-# Validate required args.
-if [[ -z "${PKG_NAME:-}" ]]; then
-    echo "Set PKG_NAME of the build";
-    exit 1
-fi
-
 
 # Look up most recent release from GitHub repo
 function find_latest_version() {
@@ -115,8 +120,10 @@ if [[ "${PKG_NAME}" =~ ^(securedrop-client|securedrop-proxy|securedrop-export|se
 
     # Hop into the package build dir, to run dpkg-buildpackage
     cd "$TOP_BUILDDIR/$PKG_NAME/"
-    # Verify all the hashes from the verified sha256sums.txt
-    "${CUR_DIR}/scripts/verify-hashes" "${CUR_DIR}/sha256sums.txt"
+    if [[ -d "${WHEELS_DIR}" ]]; then
+        # Verify all the hashes from the verified sha256sums.txt if we have dependencies
+        "${CUR_DIR}/scripts/verify-hashes" "${CUR_DIR}/${PKG_NAME}/sha256sums.txt"
+    fi
 
     echo "All hashes verified."
 else
@@ -158,7 +165,7 @@ export SOURCE_DATE_EPOCH
 # Build the package
 dpkg-buildpackage -us -uc
 
-# Tell the user the path of the files buillt
+# Tell the user the path of the files built
 pkg_path="$(find "$TOP_BUILDDIR" -type f -iname "${PKG_NAME}_${PKG_VERSION}*.deb" | head -n1)"
 if [[ -f "$pkg_path" ]]; then
     echo "Package location: $pkg_path"