From 629dc347f930b65fb963d7d372dae9368783a64b Mon Sep 17 00:00:00 2001
From: Nick Muerdter <nick.muerdter@nrel.gov>
Date: Mon, 14 Dec 2015 17:08:50 -0700
Subject: [PATCH] Fix some packaging issues.

- Rework all the before/after install/remove scripts for the RPM and DEB
  packages so that upgrades and uninstalls are more properly handled.
  Most notably, upgrades of DEBs could lead to a broken installation
  with some missing symlinks for /usr/bin/api-umbrella. This also
  resolves a number of idiosyncrasies between how these scripts get
  called in different order and with different arguments depending on
  RPM vs DEB and upgrades vs fresh installs.
- Fix the "before-remove" script accidentally referencing the
  script intended for "after-remove".
- Fix the RPM packages leaving a bunch of empty directories in
  /opt/api-umbrella/embedded after uninstall.
- Add test coverage to our serverspec suite to actually test package
  uninstall, package purge (deb only), and the upgrade process between
  older versions of API Umbrella and the current version. These tests
  aren't fast, but given all the combinations of platforms and upgrade
  possibilities, this helps ensure we have a clean upgrade path when
  building new packages.
- Allow the "api-umbrella status" and "api-umbrella stop" commands to
  work properly against a legacy installation of the NodeJS app. This is
  needed so we can cleanly handle package upgrades.
- Better error handling for if you try to start api-umbrella when it's
  already started or stop api-umbrella when it's already stopped.
- Better error handling for the "run_command" function in case the shell
  gets killed early.
---
 Makefile                                      | 108 ++++-
 build/package/build                           |  18 +-
 build/package/build_and_verify_with_docker    |   2 +-
 build/package/files/etc/init.d/api-umbrella   |   7 +-
 build/package/publish                         |  12 +-
 build/package/scripts/after-install           | 156 +++---
 build/package/scripts/after-remove            | 106 ++--
 build/package/scripts/before-remove           |  51 +-
 build/verify_package/run                      |   9 +-
 build/verify_package/run_with_docker          |   2 +-
 .../spec/localhost/service_spec.rb            | 453 +++++++++++++++---
 src/api-umbrella/cli/reload.lua               |   2 +-
 src/api-umbrella/cli/run.lua                  |  11 +
 src/api-umbrella/cli/status.lua               |  33 ++
 src/api-umbrella/cli/stop.lua                 | 124 ++++-
 src/api-umbrella/utils/run_command.lua        |  13 +-
 src/api-umbrella/version.lua                  |   2 +-
 17 files changed, 839 insertions(+), 270 deletions(-)

diff --git a/Makefile b/Makefile
index 6000373f8..551d22752 100644
--- a/Makefile
+++ b/Makefile
@@ -299,6 +299,7 @@ LUACHECK_VERSION:=0.11.1-1
 	test \
 	check_shared_objects \
 	download_deps \
+	download_verify_package_deps \
 	package \
 	verify_package \
 	package_docker_centos6 \
@@ -1125,47 +1126,104 @@ clean:
 check_shared_objects:
 	find $(EMBEDDED_DIR) -type f | xargs ldd 2>&1 | grep " => " | grep -o "^[^(]*" | sort | uniq
 
+$(DEPS_DIR)/verify_package/centos-6/api-umbrella-0.8.0-1.el6.x86_64.rpm:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ http://sourceforge.net/projects/api-umbrella/files/el/6/api-umbrella-0.8.0-1.el6.x86_64.rpm/download
+
+$(DEPS_DIR)/verify_package/centos-6/api-umbrella-0.9.0-1.el6.x86_64.rpm:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ https://bintray.com/artifact/download/nrel/api-umbrella-el6/api-umbrella-0.9.0-1.el6.x86_64.rpm
+
+$(DEPS_DIR)/verify_package/centos-7/api-umbrella-0.8.0-1.el7.x86_64.rpm:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ http://sourceforge.net/projects/api-umbrella/files/el/7/api-umbrella-0.8.0-1.el7.x86_64.rpm/download
+
+$(DEPS_DIR)/verify_package/centos-7/api-umbrella-0.9.0-1.el7.x86_64.rpm:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ https://bintray.com/artifact/download/nrel/api-umbrella-el7/api-umbrella-0.9.0-1.el7.x86_64.rpm
+
+$(DEPS_DIR)/verify_package/ubuntu-12.04/api-umbrella_0.8.0-1_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ http://sourceforge.net/projects/api-umbrella/files/ubuntu/12.04/api-umbrella_0.8.0-1_amd64.deb/download
+
+$(DEPS_DIR)/verify_package/ubuntu-12.04/api-umbrella_0.9.0-1~precise_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ https://bintray.com/artifact/download/nrel/api-umbrella-ubuntu/pool/main/a/api-umbrella/api-umbrella_0.9.0-1%7Eprecise_amd64.deb
+
+$(DEPS_DIR)/verify_package/ubuntu-14.04/api-umbrella_0.8.0-1_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ http://sourceforge.net/projects/api-umbrella/files/ubuntu/14.04/api-umbrella_0.8.0-1_amd64.deb/download
+
+$(DEPS_DIR)/verify_package/ubuntu-14.04/api-umbrella_0.9.0-1~trusty_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ https://bintray.com/artifact/download/nrel/api-umbrella-ubuntu/pool/main/a/api-umbrella/api-umbrella_0.9.0-1%7Etrusty_amd64.deb
+
+$(DEPS_DIR)/verify_package/debian-7/api-umbrella_0.8.0-1_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ http://sourceforge.net/projects/api-umbrella/files/debian/7/api-umbrella_0.8.0-1_amd64.deb/download
+
+$(DEPS_DIR)/verify_package/debian-7/api-umbrella_0.9.0-1~wheezy_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ https://bintray.com/artifact/download/nrel/api-umbrella-debian/pool/main/a/api-umbrella/api-umbrella_0.9.0-1%7Ewheezy_amd64.deb
+
+$(DEPS_DIR)/verify_package/debian-8/api-umbrella_0.9.0-1~jessie_amd64.deb:
+	mkdir -p $(shell dirname $@)
+	curl -L -o $@ https://bintray.com/artifact/download/nrel/api-umbrella-debian/pool/main/a/api-umbrella/api-umbrella_0.9.0-1%7Ejessie_amd64.deb
+
+download_verify_package_deps: \
+	$(DEPS_DIR)/verify_package/centos-6/api-umbrella-0.8.0-1.el6.x86_64.rpm \
+	$(DEPS_DIR)/verify_package/centos-6/api-umbrella-0.9.0-1.el6.x86_64.rpm \
+	$(DEPS_DIR)/verify_package/centos-7/api-umbrella-0.8.0-1.el7.x86_64.rpm \
+	$(DEPS_DIR)/verify_package/centos-7/api-umbrella-0.9.0-1.el7.x86_64.rpm \
+	$(DEPS_DIR)/verify_package/ubuntu-12.04/api-umbrella_0.8.0-1_amd64.deb \
+	$(DEPS_DIR)/verify_package/ubuntu-12.04/api-umbrella_0.9.0-1~precise_amd64.deb \
+	$(DEPS_DIR)/verify_package/ubuntu-14.04/api-umbrella_0.8.0-1_amd64.deb \
+	$(DEPS_DIR)/verify_package/ubuntu-14.04/api-umbrella_0.9.0-1~trusty_amd64.deb \
+	$(DEPS_DIR)/verify_package/debian-7/api-umbrella_0.8.0-1_amd64.deb \
+	$(DEPS_DIR)/verify_package/debian-7/api-umbrella_0.9.0-1~wheezy_amd64.deb \
+	$(DEPS_DIR)/verify_package/debian-8/api-umbrella_0.9.0-1~jessie_amd64.deb
+
 package:
 	$(BUILD_DIR)/package/build
 
-verify_package:
+verify_package: download_verify_package_deps
 	$(BUILD_DIR)/verify_package/run
 
-package_docker_centos6: download_deps
-	DIST=centos:6 $(BUILD_DIR)/package/build_and_verify_with_docker
+package_docker_centos6: download_deps download_verify_package_deps
+	DIST=centos-6 $(BUILD_DIR)/package/build_and_verify_with_docker
 
-verify_package_docker_centos6: download_deps
-	DIST=centos:6 $(BUILD_DIR)/verify_package/run_with_docker
+verify_package_docker_centos6: download_deps download_verify_package_deps
+	DIST=centos-6 $(BUILD_DIR)/verify_package/run_with_docker
 
-package_docker_centos7: download_deps
-	DIST=centos:7 $(BUILD_DIR)/package/build_and_verify_with_docker
+package_docker_centos7: download_deps download_verify_package_deps
+	DIST=centos-7 $(BUILD_DIR)/package/build_and_verify_with_docker
 
-verify_package_docker_centos7: download_deps
-	DIST=centos:7 $(BUILD_DIR)/verify_package/run_with_docker
+verify_package_docker_centos7: download_deps download_verify_package_deps
+	DIST=centos-7 $(BUILD_DIR)/verify_package/run_with_docker
 
-package_docker_ubuntu1204: download_deps
-	DIST=ubuntu:12.04 $(BUILD_DIR)/package/build_and_verify_with_docker
+package_docker_ubuntu1204: download_deps download_verify_package_deps
+	DIST=ubuntu-12.04 $(BUILD_DIR)/package/build_and_verify_with_docker
 
-verify_package_docker_ubuntu1204: download_deps
-	DIST=ubuntu:12.04 $(BUILD_DIR)/verify_package/run_with_docker
+verify_package_docker_ubuntu1204: download_deps download_verify_package_deps
+	DIST=ubuntu-12.04 $(BUILD_DIR)/verify_package/run_with_docker
 
-package_docker_ubuntu1404: download_deps
-	DIST=ubuntu:14.04 $(BUILD_DIR)/package/build_and_verify_with_docker
+package_docker_ubuntu1404: download_deps download_verify_package_deps
+	DIST=ubuntu-14.04 $(BUILD_DIR)/package/build_and_verify_with_docker
 
-verify_package_docker_ubuntu1404: download_deps
-	DIST=ubuntu:14.04 $(BUILD_DIR)/verify_package/run_with_docker
+verify_package_docker_ubuntu1404: download_deps download_verify_package_deps
+	DIST=ubuntu-14.04 $(BUILD_DIR)/verify_package/run_with_docker
 
-package_docker_debian7: download_deps
-	DIST=debian:7 $(BUILD_DIR)/package/build_and_verify_with_docker
+package_docker_debian7: download_deps download_verify_package_deps
+	DIST=debian-7 $(BUILD_DIR)/package/build_and_verify_with_docker
 
-verify_package_docker_debian7: download_deps
-	DIST=debian:7 $(BUILD_DIR)/verify_package/run_with_docker
+verify_package_docker_debian7: download_deps download_verify_package_deps
+	DIST=debian-7 $(BUILD_DIR)/verify_package/run_with_docker
 
-package_docker_debian8: download_deps
-	DIST=debian:8 $(BUILD_DIR)/package/build_and_verify_with_docker
+package_docker_debian8: download_deps download_verify_package_deps
+	DIST=debian-8 $(BUILD_DIR)/package/build_and_verify_with_docker
 
-verify_package_docker_debian8: download_deps
-	DIST=debian:8 $(BUILD_DIR)/verify_package/run_with_docker
+verify_package_docker_debian8: download_deps download_verify_package_deps
+	DIST=debian-8 $(BUILD_DIR)/verify_package/run_with_docker
 
 all_packages: \
 	download_deps \
diff --git a/build/package/build b/build/package/build
index a851d1ec9..e641f8191 100755
--- a/build/package/build
+++ b/build/package/build
@@ -14,7 +14,7 @@ else
   ITERATION="0.1.$PRE"
 fi
 
-FPM_ARGS=""
+FPM_ARGS=()
 
 if command -v yum &> /dev/null; then
   PACKAGE_TYPE="rpm"
@@ -71,7 +71,8 @@ if command -v yum &> /dev/null; then
     xz
   )
   RPM_DIST=$(rpm --query centos-release | grep -o "el[0-9]")
-  FPM_ARGS="$FPM_ARGS --rpm-dist $RPM_DIST --rpm-compression xz"
+  FPM_ARGS+=("--rpm-dist" "$RPM_DIST")
+  FPM_ARGS+=("--rpm-compression" "xz")
 
   if [ "$RPM_DIST" == "el6" ]; then
     if [ ! -f /etc/yum.repos.d/wandisco-git.repo ]; then
@@ -130,9 +131,10 @@ elif command -v dpkg &> /dev/null; then
     unzip
     xz-utils
   )
-  FPM_ARGS="$FPM_ARGS --deb-compression xz"
+  FPM_ARGS+=("--deb-compression" "xz")
+  FPM_ARGS+=("--deb-no-default-config-files")
 
-  if [ "$DIST" == "debian:7" ]; then
+  if [ "$DIST" == "debian-7" ]; then
     PACKAGE_DEPENDENCIES+=("libffi5")
   else
     PACKAGE_DEPENDENCIES+=("libffi6")
@@ -174,7 +176,7 @@ cd $TMP_ROOT_DIR
 make install DESTDIR=$TMP_WORK_DIR/package/root
 
 for DEP in "${PACKAGE_DEPENDENCIES[@]}"; do
-  FPM_ARGS="$FPM_ARGS -d $DEP"
+  FPM_ARGS+=("-d" "$DEP")
 done
 
 # Use FPM to build the binary package.
@@ -195,9 +197,11 @@ bundle exec fpm \
   --iteration "$ITERATION" \
   --config-files etc/api-umbrella/api-umbrella.yml \
   --after-install $TMP_ROOT_DIR/build/package/scripts/after-install \
-  --before-remove $TMP_ROOT_DIR/build/package/scripts/after-remove \
+  --before-remove $TMP_ROOT_DIR/build/package/scripts/before-remove \
   --after-remove $TMP_ROOT_DIR/build/package/scripts/after-remove \
-  $FPM_ARGS \
+  --directories /etc/api-umbrella \
+  --directories /opt/api-umbrella \
+  ${FPM_ARGS[*]} \
   .
 
 rm -rf $ORIGINAL_ROOT_DIR/build/package/dist/$DIST
diff --git a/build/package/build_and_verify_with_docker b/build/package/build_and_verify_with_docker
index 8224159da..2088281f7 100755
--- a/build/package/build_and_verify_with_docker
+++ b/build/package/build_and_verify_with_docker
@@ -3,7 +3,7 @@
 set -e -u -x
 
 ROOT_DIR="$(dirname $(dirname $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)))"
-DOCKER_IMAGE=$DIST
+DOCKER_IMAGE=${DIST/-/:}
 CONTAINER_NAME="api-umbrella-package-$DIST"
 CONTAINER_NAME=${CONTAINER_NAME//[^a-zA-Z0-9_.-]/}
 
diff --git a/build/package/files/etc/init.d/api-umbrella b/build/package/files/etc/init.d/api-umbrella
index 82a06bb6d..ead458397 100755
--- a/build/package/files/etc/init.d/api-umbrella
+++ b/build/package/files/etc/init.d/api-umbrella
@@ -114,6 +114,10 @@ status() {
   return $RETVAL
 }
 
+status_quiet() {
+  status > /dev/null 2>&1
+}
+
 case "$1" in
   start)
     start
@@ -131,7 +135,8 @@ case "$1" in
     reload
     ;;
   condrestart)
-    status > /dev/null 2>&1 && restart || :
+    status_quiet || exit 0
+    restart
     ;;
   *)
     echo "Usage: $NAME {start|stop|status|reload|restart|condrestart}"
diff --git a/build/package/publish b/build/package/publish
index 70eec2d9d..0cba5d44d 100755
--- a/build/package/publish
+++ b/build/package/publish
@@ -12,12 +12,12 @@ else
 fi
 
 PACKAGES=(
-  build/package/dist/centos:6/api-umbrella-$VERSION-$ITERATION.el6.x86_64.rpm
-  build/package/dist/centos:7/api-umbrella-$VERSION-$ITERATION.el7.x86_64.rpm
-  build/package/dist/debian:7/api-umbrella_$VERSION-$ITERATION~wheezy_amd64.deb
-  build/package/dist/debian:8/api-umbrella_$VERSION-$ITERATION~jessie_amd64.deb
-  build/package/dist/ubuntu:12.04/api-umbrella_$VERSION-$ITERATION~precise_amd64.deb
-  build/package/dist/ubuntu:14.04/api-umbrella_$VERSION-$ITERATION~trusty_amd64.deb
+  build/package/dist/centos-6/api-umbrella-$VERSION-$ITERATION.el6.x86_64.rpm
+  build/package/dist/centos-7/api-umbrella-$VERSION-$ITERATION.el7.x86_64.rpm
+  build/package/dist/debian-7/api-umbrella_$VERSION-$ITERATION~wheezy_amd64.deb
+  build/package/dist/debian-8/api-umbrella_$VERSION-$ITERATION~jessie_amd64.deb
+  build/package/dist/ubuntu-12.04/api-umbrella_$VERSION-$ITERATION~precise_amd64.deb
+  build/package/dist/ubuntu-14.04/api-umbrella_$VERSION-$ITERATION~trusty_amd64.deb
 )
 
 for PACKAGE in "${PACKAGES[@]}"; do
diff --git a/build/package/scripts/after-install b/build/package/scripts/after-install
index bc42ee883..79abd3bb9 100755
--- a/build/package/scripts/after-install
+++ b/build/package/scripts/after-install
@@ -1,63 +1,105 @@
 #!/bin/bash
-#
-# Perform necessary api-umbrella setup steps
-# after package is installed.
-#
-
-function error_exit {
-  echo "api-umbrella: ${1:-"Unknown Error"}" 1>&2
-  exit 1
-}
-
-USER=api-umbrella
-GROUP=api-umbrella
-DEPLOY_USER=api-umbrella-deploy
-DEPLOY_GROUP=api-umbrella-deploy
-
-getent group $GROUP > /dev/null || groupadd -r $GROUP
-getent passwd $USER > /dev/null || \
-  useradd -r -g $GROUP -s /sbin/nologin \
-    -d /opt/api-umbrella -c "API Umbrella user" $USER
-
-getent group $DEPLOY_GROUP > /dev/null || groupadd -r $DEPLOY_GROUP
-getent passwd $DEPLOY_USER > /dev/null || \
-  useradd -r -g $DEPLOY_GROUP -s /bin/bash \
-    -d /home/$DEPLOY_USER -c "API Umbrella deployment user" $DEPLOY_USER
-
-# Add the deploy user to the app group, so the deploy user can read config
-# files.
-if ! groups $DEPLOY_USER | grep -q -E "\s$GROUP(\s|$)"; then
-  usermod -a -G $GROUP $DEPLOY_USER
-fi
 
-# Fix previously created deploy user that couldn't actually login.
-if getent passwd $DEPLOY_USER | grep -q "/sbin/nologin"; then
-  usermod -d /home/$DEPLOY_USER -s /bin/bash $DEPLOY_USER
-fi
+set -e -u
 
-# Create an empty .ssh/authorized_keys file with proper permissions if it
-# doesn't already exist.
-if [ ! -f /home/$DEPLOY_USER/.ssh/authorized_keys ]; then
-  mkdir -p /home/$DEPLOY_USER/.ssh
-  touch /home/$DEPLOY_USER/.ssh/authorized_keys
-  chown -R $DEPLOY_USER:$DEPLOY_GROUP /home/$DEPLOY_USER
-  chmod 700 /home/$DEPLOY_USER
-  chmod 700 /home/$DEPLOY_USER/.ssh
-  chmod 600 /home/$DEPLOY_USER/.ssh/authorized_keys
-fi
+CONFIGURE=false
+RESTART_SERVICE=false
+
+case "$1" in
+  # dpkg
+  configure)
+    CONFIGURE=true
+
+    # Upgrade
+    if [ -n $2 ]; then
+      RESTART_SERVICE=true
+    fi
+
+    ;;
+  abort-deconfigure|abort-remove|abort-upgrade)
+    ;;
+
+  # rpm
+  1) # install
+    CONFIGURE=true
+    ;;
+  2) # upgrade
+    CONFIGURE=true
+    ;;
+esac
+
+if [ "$CONFIGURE" = "true" ]; then
+  USER=api-umbrella
+  GROUP=api-umbrella
+  DEPLOY_USER=api-umbrella-deploy
+  DEPLOY_GROUP=api-umbrella-deploy
 
-chown -R $USER:$GROUP /opt/api-umbrella/etc /opt/api-umbrella/var
-chown -R $DEPLOY_USER:$DEPLOY_GROUP /opt/api-umbrella/embedded/apps
-chown -R $USER:$DEPLOY_GROUP /opt/api-umbrella/embedded/apps/core/shared/src/api-umbrella/web-app/tmp
-chmod 775 /opt/api-umbrella/embedded/apps/core/shared/src/api-umbrella/web-app/tmp
-
-# Install service, but don't activate.
-if command -v chkconfig > /dev/null 2>&1; then
-  chkconfig --add api-umbrella || error_exit "Cannot enable api-umbrella service"
-elif command -v update-rc.d > /dev/null 2>&1; then
-  update-rc.d api-umbrella defaults 85 15 || error_exit "Cannot enable api-umbrella service"
-else
-  error_exit "No supported init tool found."
+  # Create the main user & group.
+  if ! getent group $GROUP > /dev/null; then
+    groupadd -r $GROUP
+  fi
+  if ! getent passwd $USER > /dev/null; then
+    useradd -r -g $GROUP -s /sbin/nologin \
+      -d /opt/api-umbrella -c "API Umbrella user" $USER
+  fi
+
+  # Create the deploy user & group.
+  if ! getent group $DEPLOY_GROUP > /dev/null; then
+    groupadd -r $DEPLOY_GROUP
+  fi
+  if ! getent passwd $DEPLOY_USER > /dev/null; then
+    useradd -r -g $DEPLOY_GROUP -s /bin/bash \
+      -d /home/$DEPLOY_USER -c "API Umbrella deployment user" $DEPLOY_USER
+  fi
+
+  # Add the deploy user to the app group, so the deploy user can read config
+  # files.
+  if ! groups $DEPLOY_USER | grep -q -E "\s$GROUP(\s|$)"; then
+    usermod -a -G $GROUP $DEPLOY_USER
+  fi
+
+  # Fix previously created deploy user that couldn't actually login.
+  if getent passwd $DEPLOY_USER | grep -q "/sbin/nologin"; then
+    usermod -d /home/$DEPLOY_USER -s /bin/bash $DEPLOY_USER
+  fi
+
+  # Create an empty .ssh/authorized_keys file with proper permissions if it
+  # doesn't already exist.
+  if [ ! -f /home/$DEPLOY_USER/.ssh/authorized_keys ]; then
+    mkdir -p /home/$DEPLOY_USER/.ssh
+    touch /home/$DEPLOY_USER/.ssh/authorized_keys
+    chown -R $DEPLOY_USER:$DEPLOY_GROUP /home/$DEPLOY_USER
+    chmod 700 /home/$DEPLOY_USER
+    chmod 700 /home/$DEPLOY_USER/.ssh
+    chmod 600 /home/$DEPLOY_USER/.ssh/authorized_keys
+  fi
+
+  # Set file permissions
+  chown -R $USER:$GROUP /opt/api-umbrella/etc /opt/api-umbrella/var
+  chown -R $DEPLOY_USER:$DEPLOY_GROUP /opt/api-umbrella/embedded/apps
+  chown -R $USER:$DEPLOY_GROUP /opt/api-umbrella/embedded/apps/core/shared/src/api-umbrella/web-app/tmp
+  chmod 775 /opt/api-umbrella/embedded/apps/core/shared/src/api-umbrella/web-app/tmp
+
+  # Re-create symlinks that may have inadvertently been cleaned up by the API
+  # Umbrella v0.8 and v0.9 after-remove scripts during upgrades (this should be
+  # fixed by the v0.10 after-remove script, so at some point, we can probably
+  # remove this logic).
+  if [ ! -f /usr/bin/api-umbrella ]; then
+    cd /usr/bin && ln -snf ../../opt/api-umbrella/bin/api-umbrella ./api-umbrella
+  fi
+  if [ ! -f /var/log/api-umbrella ]; then
+    cd /var/log && ln -snf ../../opt/api-umbrella/var/log ./api-umbrella
+  fi
+
+  # Install service, but don't activate.
+  if command -v chkconfig > /dev/null 2>&1; then
+    chkconfig --add api-umbrella
+  elif command -v update-rc.d > /dev/null 2>&1; then
+    update-rc.d api-umbrella defaults 85 15 > /dev/null
+  fi
 fi
 
-exit 0
+if [ "$RESTART_SERVICE" = "true" ]; then
+  # On upgrades, restart the service if it's currently running.
+  /etc/init.d/api-umbrella condrestart
+fi
diff --git a/build/package/scripts/after-remove b/build/package/scripts/after-remove
index 753ee15db..14f6e1ad1 100755
--- a/build/package/scripts/after-remove
+++ b/build/package/scripts/after-remove
@@ -1,42 +1,74 @@
 #!/bin/bash
-#
-# Perform necessary api-umbrella removal steps
-# after package is uninstalled.
-#
-
-function error_exit {
-  echo "api-umbrella: ${1:-"Unknown Error"}" 1>&2
-  exit 1
-}
-
-function remove_service {
-  if command -v chkconfig > /dev/null 2>&1; then
-    chkconfig --del api-umbrella > /dev/null 2>&1
-  elif command -v update-rc.d > /dev/null 2>&1; then
-    update-rc.d api-umbrella remove > /dev/null 2>&1
+
+set -e -u
+
+REMOVE_ACCOUNTS=false
+REMOVE_DATA=false
+REMOVE_FILES=false
+REMOVE_SERVICE=false
+RESTART_SERVICE=false
+
+case "$1" in
+  # dpkg
+  remove)
+    REMOVE_FILES=true
+    REMOVE_SERVICE=true
+    ;;
+  purge)
+    REMOVE_ACCOUNTS=true
+    REMOVE_DATA=true
+    ;;
+  abort-install|abort-upgrade|disappear|failed-upgrade|upgrade)
+    ;;
+
+  # rpm
+  0) # uninstall
+    REMOVE_FILES=true
+    ;;
+  1) # upgrade
+    RESTART_SERVICE=true
+    ;;
+esac
+
+if [ "$REMOVE_FILES" = "true" ]; then
+  rm -f /usr/bin/api-umbrella /var/log/api-umbrella
+  rmdir /etc/api-umbrella > /dev/null 2>&1 || true # Delete directory if empty.
+fi
+
+if [ "$REMOVE_SERVICE" = "true" ]; then
+  if command -v update-rc.d > /dev/null 2>&1; then
+    update-rc.d api-umbrella remove > /dev/null
   fi
+fi
 
-  rm -f /usr/bin/api-umbrella
-  rm -f /var/log/api-umbrella
-  rmdir /etc/api-umbrella > /dev/null 2>&1 # Delete directory if empty.
-}
-
-# On RedHat RPM systems, when an upgrade is performed, the old package is
-# removed after the newer one is installed. Only perform cleanup on full
-# removals.
-#
-# See: http://tickets.opscode.com/browse/CHEF-3022
-if [ ! -f /etc/redhat-release -a ! -f /etc/fedora-release -a ! -f /etc/system-release ]; then
-  # Non-RPM system: Go ahead with removal
-  remove_service
-elif [ "$1" -eq "0" ]; then
-  # RPM system: Uninstalling (not upgrading)
-  remove_service
-elif [ "$1" -eq "1" ]; then
-  # RPM system: Upgrading (not uninstalling)
-  # Restart, but only if started.
-  /etc/init.d/api-umbrella status > /dev/null 2>&1 || exit 0
-  /etc/init.d/api-umbrella restart > /dev/null 2>&1 || error_exit "Cannot restart api-umbrella service"
+if [ "$REMOVE_DATA" = "true" ]; then
+  rm -rf /etc/api-umbrella /opt/api-umbrella
 fi
 
-exit 0
+if [ "$REMOVE_ACCOUNTS" = "true" ]; then
+  USER=api-umbrella
+  GROUP=api-umbrella
+  DEPLOY_USER=api-umbrella-deploy
+  DEPLOY_GROUP=api-umbrella-deploy
+
+  if getent passwd $DEPLOY_USER > /dev/null; then
+    userdel $DEPLOY_USER
+  fi
+
+  if getent passwd $USER > /dev/null; then
+    userdel $USER
+  fi
+
+  if getent group $DEPLOY_GROUP > /dev/null; then
+    groupdel $DEPLOY_GROUP
+  fi
+
+  if getent group $GROUP > /dev/null; then
+    groupdel $GROUP
+  fi
+fi
+
+if [ "$RESTART_SERVICE" = "true" ]; then
+  # On upgrades, restart the service if it's currently running.
+  /etc/init.d/api-umbrella condrestart
+fi
diff --git a/build/package/scripts/before-remove b/build/package/scripts/before-remove
index 6bcb8bb2b..f9ea243de 100755
--- a/build/package/scripts/before-remove
+++ b/build/package/scripts/before-remove
@@ -1,30 +1,33 @@
 #!/bin/bash
-#
-# Perform necessary api-umbrella setup steps
-# prior to installing package.
-#
 
-function error_exit {
-  echo "api-umbrella: ${1:-"Unknown Error"}" 1>&2
-  exit 1
-}
+set -e -u
 
-function stop_service {
-  /etc/init.d/api-umbrella status > /dev/null 2>&1 || return 0
-  /etc/init.d/api-umbrella stop > /dev/null 2>&1 || error_exit "Cannot stop api-umbrella service"
-}
+STOP_SERVICE=false
+REMOVE_SERVICE=false
 
-# On RedHat RPM systems, when an upgrade is performed, the old package is
-# removed after the newer one is installed. Only perform stops on full
-# removals.
-#
-# See: http://tickets.opscode.com/browse/CHEF-3022
-if [ ! -f /etc/redhat-release -a ! -f /etc/fedora-release -a ! -f /etc/system-release ]; then
-  # Non-RPM system: Stop before removal
-  stop_service
-elif [ "$1" -eq "0" ]; then
-  # RPM system: Uninstalling (not upgrading)
-  stop_service
+case "$1" in
+  # dpkg
+  remove)
+    STOP_SERVICE=true
+    ;;
+  deconfigure|failed-upgrade|upgrade)
+    ;;
+
+  # rpm
+  0) # uninstall
+    STOP_SERVICE=true
+    REMOVE_SERVICE=true
+    ;;
+  1) # upgrade
+    ;;
+esac
+
+if [ "$STOP_SERVICE" = "true" ]; then
+  /etc/init.d/api-umbrella stop || true
 fi
 
-exit 0
+if [ "$REMOVE_SERVICE" = "true" ]; then
+  if command -v chkconfig > /dev/null 2>&1; then
+    chkconfig --del api-umbrella
+  fi
+fi
diff --git a/build/verify_package/run b/build/verify_package/run
index 5afa49e90..e361f45e5 100755
--- a/build/verify_package/run
+++ b/build/verify_package/run
@@ -9,7 +9,6 @@ if command -v yum > /dev/null 2>&1; then
   yum -y install $ROOT_DIR/build/package/dist/$DIST/*
 elif command -v dpkg > /dev/null 2>&1; then
   apt-get update
-  apt-get -y install uuid-dev
   dpkg -i $ROOT_DIR/build/package/dist/$DIST/* || apt-get install -y -f
 else
   echo "Unknown build system"
@@ -24,6 +23,10 @@ fi
 api-umbrella health --wait-for-status green
 
 # Install the test package dependencies.
+#
+# Note: We do this after installing and starting API Umbrella to ensure that
+# API Umbrella can start on its own (and we're not accidentally depending on
+# any of these test dependencies).
 if command -v yum > /dev/null 2>&1; then
   yum -y install epel-release
   yum -y install \
@@ -36,7 +39,7 @@ elif command -v dpkg > /dev/null 2>&1; then
     ruby \
     socat \
     sudo
-  if [ "$DIST" == "ubuntu:12.04" ] || [ "$DIST" == "debian:7" ]; then
+  if [ "$DIST" == "ubuntu-12.04" ] || [ "$DIST" == "debian-7" ]; then
     apt-get -y install rubygems
   fi
 fi
@@ -48,4 +51,4 @@ cd $ROOT_DIR/build/verify_package
 bundle
 
 # Run the serverspec suite.
-bundle exec rake spec
+env ROOT_DIR=$ROOT_DIR DIST=$DIST bundle exec rake spec
diff --git a/build/verify_package/run_with_docker b/build/verify_package/run_with_docker
index aee28ea3c..b48442486 100755
--- a/build/verify_package/run_with_docker
+++ b/build/verify_package/run_with_docker
@@ -3,7 +3,7 @@
 set -e -u -x
 
 ROOT_DIR="$(dirname $(dirname $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)))"
-DOCKER_IMAGE=$DIST
+DOCKER_IMAGE=${DIST/-/:}
 CONTAINER_NAME="api-umbrella-package-$DIST-verify"
 CONTAINER_NAME=${CONTAINER_NAME//[^a-zA-Z0-9_.-]/}
 
diff --git a/build/verify_package/spec/localhost/service_spec.rb b/build/verify_package/spec/localhost/service_spec.rb
index c976bbac8..6a642944e 100644
--- a/build/verify_package/spec/localhost/service_spec.rb
+++ b/build/verify_package/spec/localhost/service_spec.rb
@@ -6,19 +6,245 @@
 
 MultiJson.use(:ok_json)
 
-describe "api-umbrella" do
+RSpec.shared_examples("installed") do
   it "installs the package" do
     expect(package("api-umbrella")).to be_installed
   end
 
-  it "runs the service" do
-    expect(service("api-umbrella")).to be_running.under(:init)
-  end
-
   it "enables the service" do
     expect(service("api-umbrella")).to be_enabled
   end
 
+  it "symlinks the main api-umbrella binary" do
+    subject = file("/usr/bin/api-umbrella")
+    expect(subject).to be_symlink
+    expect(subject).to be_owned_by("root")
+    expect(subject).to be_grouped_into("root")
+    expect(subject).to be_linked_to("../../opt/api-umbrella/bin/api-umbrella")
+  end
+
+  it "installs a init.d file" do
+    subject = file("/etc/init.d/api-umbrella")
+    expect(subject).to be_file
+    expect(subject).to be_mode(755)
+    expect(subject).to be_owned_by("root")
+    expect(subject).to be_grouped_into("root")
+  end
+
+  it "installs a logrotate.d file" do
+    subject = file("/etc/logrotate.d/api-umbrella")
+    expect(subject).to be_file
+    expect(subject).to be_mode(644)
+    expect(subject).to be_owned_by("root")
+    expect(subject).to be_grouped_into("root")
+  end
+
+  it "installs a sudoers.d file" do
+    subject = file("/etc/sudoers.d/api-umbrella")
+    expect(subject).to be_file
+    expect(subject).to be_mode(440)
+    expect(subject).to be_owned_by("root")
+    expect(subject).to be_grouped_into("root")
+  end
+
+  it "installs a api-umbrella.yml file" do
+    subject = file("/etc/api-umbrella/api-umbrella.yml")
+    expect(subject).to be_file
+    expect(subject).to be_mode(644)
+    expect(subject).to be_owned_by("root")
+    expect(subject).to be_grouped_into("root")
+  end
+
+  it "symlinks the log directory" do
+    subject = file("/var/log/api-umbrella")
+    expect(subject).to be_symlink
+    expect(subject).to be_owned_by("root")
+    expect(subject).to be_grouped_into("root")
+    expect(subject).to be_linked_to("../../opt/api-umbrella/var/log")
+  end
+
+  it "sets up the api-umbrella user" do
+    subject = user("api-umbrella")
+    expect(subject).to exist
+    expect(subject).to belong_to_group("api-umbrella")
+    expect(subject).to have_home_directory("/opt/api-umbrella")
+    expect(subject).to have_login_shell("/sbin/nologin")
+  end
+
+  it "sets up the api-umbrella-deploy user's home directory and empty ssh keys file" do
+    subject = user("api-umbrella-deploy")
+    expect(subject).to exist
+    expect(subject).to belong_to_group("api-umbrella-deploy")
+    expect(subject).to have_home_directory("/home/api-umbrella-deploy")
+    expect(subject).to have_login_shell("/bin/bash")
+
+    subject = file("/home/api-umbrella-deploy")
+    expect(subject).to be_directory
+    expect(subject).to be_mode(700)
+    expect(subject).to be_owned_by("api-umbrella-deploy")
+    expect(subject).to be_grouped_into("api-umbrella-deploy")
+
+    subject = file("/home/api-umbrella-deploy/.ssh")
+    expect(subject).to be_directory
+    expect(subject).to be_mode(700)
+    expect(subject).to be_owned_by("api-umbrella-deploy")
+    expect(subject).to be_grouped_into("api-umbrella-deploy")
+
+    subject = file("/home/api-umbrella-deploy/.ssh/authorized_keys")
+    expect(subject).to be_file
+    expect(subject).to be_mode(600)
+    expect(subject).to be_owned_by("api-umbrella-deploy")
+    expect(subject).to be_grouped_into("api-umbrella-deploy")
+    expect(subject.content).to eql("")
+  end
+end
+
+RSpec.shared_examples("package upgrade") do |package_version|
+  def ensure_uninstalled
+    command_result = command("/etc/init.d/api-umbrella stop")
+    command_result.exit_status
+
+    case(os[:family])
+    when "redhat"
+      command_result = command("yum -y remove api-umbrella")
+    when "ubuntu", "debian"
+      command_result = command("dpkg --purge api-umbrella")
+    end
+    command_result.exit_status
+
+    expect(package("api-umbrella")).to_not be_installed
+    FileUtils.rm_rf("/opt/api-umbrella")
+    FileUtils.rm_rf("/etc/api-umbrella")
+  end
+
+  def install_package(version)
+    if(version == :current)
+      package_path = "#{ENV["ROOT_DIR"]}/build/package/dist/#{ENV["DIST"]}/*"
+    else
+      package_path = "#{ENV["ROOT_DIR"]}/build/work/deps/verify_package/#{ENV["DIST"]}/*#{version}*"
+    end
+
+    case(os[:family])
+    when "redhat"
+      command_result = command("yum -y install #{package_path}")
+    when "ubuntu", "debian"
+      command_result = command("dpkg -i #{package_path} || apt-get install -y -f")
+    end
+    expect(command_result.exit_status).to eql(0)
+
+    # We may get some warnings during upgrades (due to non-empty directories),
+    # but make sure we don't have any other unexpected STDERR output.
+    stderr = command_result.stderr
+    stderr.gsub!(/^dpkg: warning:.*/, "")
+    stderr.strip!
+    expect(stderr).to eql("")
+
+    expect(package("api-umbrella")).to be_installed
+  end
+
+  describe "from v#{package_version}" do
+    describe "service stopped before upgrade" do
+      before(:all) do
+        ensure_uninstalled
+        install_package(package_version)
+      end
+
+      after(:all) do
+        ensure_uninstalled
+      end
+
+      it "is not running before upgrade" do
+        expect(service("api-umbrella")).to_not be_running.under(:init)
+      end
+
+      it "upgrades the package" do
+        expect(package("api-umbrella").version.version).to start_with(package_version)
+        install_package(:current)
+        expect(package("api-umbrella").version.version).to_not start_with(package_version)
+      end
+
+      it "is not running after upgrade" do
+        expect(service("api-umbrella")).to_not be_running.under(:init)
+      end
+
+      it "can start the service" do
+        command_result = command("/etc/init.d/api-umbrella start")
+        expect(command_result.exit_status).to eql(0)
+        expect(command_result.stderr).to eql("")
+
+        expect(service("api-umbrella")).to be_running.under(:init)
+      end
+
+      it_behaves_like "installed"
+    end
+
+    describe "service running before upgrade" do
+      before(:all) do
+        ensure_uninstalled
+        install_package(package_version)
+      end
+
+      after(:all) do
+        ensure_uninstalled
+      end
+
+      it "starts the service before the upgrade" do
+        command_result = command("/etc/init.d/api-umbrella start")
+        expect(command_result.exit_status).to eql(0)
+        expect(command_result.stderr).to eql("")
+        expect(service("api-umbrella")).to be_running.under(:init)
+
+        command_result = command("/etc/init.d/api-umbrella status")
+        expect(command_result.stdout).to match(/pid \d+/)
+        @pre_upgrade_pid = command_result.stdout.match(/pid (\d+)/)[1]
+      end
+
+      it "upgrades the package" do
+        expect(package("api-umbrella").version.version).to start_with(package_version)
+        install_package(:current)
+        expect(package("api-umbrella").version.version).to_not start_with(package_version)
+      end
+
+      # Due to a bug in the prerm script in v0.8.0, API Umbrella will always be
+      # stopped during upgrades from v0.8 on Ubuntu & Debian. There's not a
+      # very clean solution, so we'll just ensure this doesn't happen for
+      # future upgrades.
+      if(["ubuntu", "debian"].include?(os[:family]) && package_version == "0.8.0")
+        it "is not running after upgrade (due to v0.8.0 prerm script bug)" do
+          expect(service("api-umbrella")).to_not be_running.under(:init)
+        end
+
+        it "can start the service" do
+          command_result = command("/etc/init.d/api-umbrella start")
+          expect(command_result.exit_status).to eql(0)
+          expect(command_result.stderr).to eql("")
+
+          expect(service("api-umbrella")).to be_running.under(:init)
+        end
+      else
+        it "restarts the service during the upgrade" do
+          expect(service("api-umbrella")).to be_running.under(:init)
+
+          command_result = command("/etc/init.d/api-umbrella status")
+          expect(command_result.stdout).to match(/pid \d+/)
+          post_upgrade_pid = command_result.stdout.match(/pid (\d+)/)[1]
+
+          expect(post_upgrade_pid).to_not eql(@pre_upgrade_pid)
+        end
+      end
+
+      it_behaves_like "installed"
+    end
+  end
+end
+
+describe "api-umbrella" do
+  it_behaves_like "installed"
+
+  it "runs the service" do
+    expect(service("api-umbrella")).to be_running.under(:init)
+  end
+
   it "reports green from the health api endpoint" do
     response = RestClient::Request.execute(:method => :get, :url => "https://localhost/api-umbrella/v1/health.json", :verify_ssl => false)
     data = MultiJson.load(response.body)
@@ -88,6 +314,23 @@
     expect(command("sudo -u api-umbrella-deploy sudo -n /etc/init.d/api-umbrella status").stdout).to include("is running")
   end
 
+  it "exits immediately if start is called when already started" do
+    command_result = command("/etc/init.d/api-umbrella start")
+    expect(command_result.exit_status).to eql(0)
+    expect(command_result.stdout).to include("api-umbrella is already running")
+    expect(command_result.stderr).to eql("")
+  end
+
+  it "accepts a reload command" do
+    command_result = command("/etc/init.d/api-umbrella reload")
+    expect(command_result.exit_status).to eql(0)
+    expect(command_result.stderr).to eql("")
+
+    # Wait for API Umbrella to become fully started and health again, to ensure
+    # subsequent tests don't fail.
+    expect(command("api-umbrella health --wait-for-status green").exit_status).to eql(0)
+  end
+
   it "can be stopped and started again" do
     # Stop
     command_result = command("/etc/init.d/api-umbrella stop")
@@ -96,7 +339,30 @@
 
     # Check status
     expect(service("api-umbrella")).to_not be_running.under(:init)
-    expect(command("/etc/init.d/api-umbrella status").stdout).to include("is stopped")
+    command_result = command("/etc/init.d/api-umbrella status")
+    expect(command_result.exit_status).to eql(3)
+    expect(command_result.stdout).to include("api-umbrella is stopped")
+    expect(command_result.stderr).to eql("")
+
+    # Run some extra tests while we have API Umbrella in the stopped state:
+    #
+    # Verify behavior of stop command after already stopped.
+    command_result = command("/etc/init.d/api-umbrella stop")
+    expect(command_result.exit_status).to eql(0)
+    expect(command_result.stdout).to include("api-umbrella is already stopped")
+    expect(command_result.stderr).to eql("")
+
+    # Verify behavior of reload command when stopped.
+    command_result = command("/etc/init.d/api-umbrella reload")
+    expect(command_result.exit_status).to eql(7)
+    expect(command_result.stdout).to include("api-umbrella is stopped")
+    expect(command_result.stderr).to eql("")
+
+    # Verify behavior of condrestart command when stopped.
+    command_result = command("/etc/init.d/api-umbrella condrestart")
+    expect(command_result.exit_status).to eql(0)
+    expect(command_result.stdout).to eql("")
+    expect(command_result.stderr).to eql("")
 
     # Start again
     command_result = command("/etc/init.d/api-umbrella start")
@@ -126,86 +392,115 @@
     expect(command("api-umbrella health --wait-for-status green").exit_status).to eql(0)
   end
 
-  it "symlinks the main api-umbrella binary" do
-    subject = file("/usr/bin/api-umbrella")
-    expect(subject).to be_symlink
-    expect(subject).to be_owned_by("root")
-    expect(subject).to be_grouped_into("root")
-    expect(subject).to be_linked_to("../../opt/api-umbrella/bin/api-umbrella")
-  end
-
-  it "installs a init.d file" do
-    subject = file("/etc/init.d/api-umbrella")
-    expect(subject).to be_file
-    expect(subject).to be_mode(755)
-    expect(subject).to be_owned_by("root")
-    expect(subject).to be_grouped_into("root")
-  end
-
-  it "installs a logrotate.d file" do
-    subject = file("/etc/logrotate.d/api-umbrella")
-    expect(subject).to be_file
-    expect(subject).to be_mode(644)
-    expect(subject).to be_owned_by("root")
-    expect(subject).to be_grouped_into("root")
-  end
+  describe "uninstall" do
+    before(:all) do
+      expect(package("api-umbrella")).to be_installed
+
+      # For RPMs modify the config file (append a new line), so we can make
+      # sure modified files get preserved on uninstall.
+      if(os[:family] == "redhat")
+        File.open("/etc/api-umbrella/api-umbrella.yml", "a") do |f|
+          f.puts "\n"
+        end
+      end
+
+      case(os[:family])
+      when "redhat"
+        command_result = command("yum -y remove api-umbrella")
+      when "ubuntu", "debian"
+        command_result = command("dpkg -r api-umbrella")
+      end
+      expect(command_result.exit_status).to eql(0)
+      expect(command_result.stderr).to eql("")
+    end
 
-  it "installs a sudoers.d file" do
-    subject = file("/etc/sudoers.d/api-umbrella")
-    expect(subject).to be_file
-    expect(subject).to be_mode(440)
-    expect(subject).to be_owned_by("root")
-    expect(subject).to be_grouped_into("root")
-  end
+    it "uninstalls the package" do
+      expect(package("api-umbrella")).to_not be_installed
+    end
 
-  it "installs a api-umbrella.yml file" do
-    subject = file("/etc/api-umbrella/api-umbrella.yml")
-    expect(subject).to be_file
-    expect(subject).to be_mode(644)
-    expect(subject).to be_owned_by("root")
-    expect(subject).to be_grouped_into("root")
-  end
+    it "stops the service" do
+      expect(service("api-umbrella")).to_not be_running.under(:init)
+    end
 
-  it "sets up the api-umbrella user" do
-    subject = user("api-umbrella")
-    expect(subject).to exist
-    expect(subject).to belong_to_group("api-umbrella")
-    expect(subject).to have_home_directory("/opt/api-umbrella")
-    expect(subject).to have_login_shell("/sbin/nologin")
-  end
+    it "disables the service" do
+      expect(service("api-umbrella")).to_not be_enabled
+    end
 
-  it "sets up the api-umbrella-deploy user's home directory and empty ssh keys file" do
-    subject = user("api-umbrella-deploy")
-    expect(subject).to exist
-    expect(subject).to belong_to_group("api-umbrella-deploy")
-    expect(subject).to have_home_directory("/home/api-umbrella-deploy")
-    expect(subject).to have_login_shell("/bin/bash")
+    [
+      "/etc/init.d/api-umbrella",
+      "/etc/logrotate.d/api-umbrella",
+      "/etc/sudoers.d/api-umbrella",
+      "/opt/api-umbrella/embedded",
+      "/usr/bin/api-umbrella",
+      "/var/log/api-umbrella",
+    ].each do |path|
+      it "removes #{path}" do
+        subject = file(path)
+        expect(subject).to_not exist
+      end
+
+      if(os[:family] == "redhat")
+        it "removes #{path}.rpmsave" do
+          subject = file("#{path}.rpmsave")
+          expect(subject).to_not exist
+        end
+      end
+    end
 
-    subject = file("/home/api-umbrella-deploy")
-    expect(subject).to be_directory
-    expect(subject).to be_mode(700)
-    expect(subject).to be_owned_by("api-umbrella-deploy")
-    expect(subject).to be_grouped_into("api-umbrella-deploy")
+    [
+      "/etc/api-umbrella/api-umbrella.yml",
+    ].each do |path|
+      if(os[:family] == "redhat")
+        it "removes #{path}" do
+          subject = file(path)
+          expect(subject).to_not exist
+        end
+
+        it "keeps #{path}.rpmsave" do
+          subject = file("#{path}.rpmsave")
+          expect(subject).to exist
+        end
+      else
+        it "keeps #{path}" do
+          subject = file(path)
+          expect(subject).to exist
+        end
+      end
+    end
 
-    subject = file("/home/api-umbrella-deploy/.ssh")
-    expect(subject).to be_directory
-    expect(subject).to be_mode(700)
-    expect(subject).to be_owned_by("api-umbrella-deploy")
-    expect(subject).to be_grouped_into("api-umbrella-deploy")
+    [
+      "/opt/api-umbrella/var/log",
+      "/opt/api-umbrella/var/db",
+    ].each do |path|
+      it "keeps #{path}" do
+        subject = file(path)
+        expect(subject).to exist
+      end
+    end
 
-    subject = file("/home/api-umbrella-deploy/.ssh/authorized_keys")
-    expect(subject).to be_file
-    expect(subject).to be_mode(600)
-    expect(subject).to be_owned_by("api-umbrella-deploy")
-    expect(subject).to be_grouped_into("api-umbrella-deploy")
-    expect(subject.content).to eql("")
+    if(["ubuntu", "debian"].include?(os[:family]))
+      describe "purge" do
+        before(:all) do
+          expect(package("api-umbrella")).to_not be_installed
+
+          command_result = command("dpkg --purge api-umbrella")
+          expect(command_result.exit_status).to eql(0)
+          expect(command_result.stderr).to eql("")
+        end
+
+        [
+          "/etc/api-umbrella",
+          "/opt/api-umbrella",
+        ].each do |path|
+          it "removes #{path}" do
+            subject = file(path)
+            expect(subject).to_not exist
+          end
+        end
+      end
+    end
   end
 
-  it "symlinks the log directory" do
-    subject = file("/var/log/api-umbrella")
-    expect(subject).to be_symlink
-    expect(subject).to be_owned_by("root")
-    expect(subject).to be_grouped_into("root")
-    expect(subject).to be_linked_to("../../opt/api-umbrella/var/log")
-  end
+  it_behaves_like "package upgrade", "0.8.0"
+  it_behaves_like "package upgrade", "0.9.0"
 end
diff --git a/src/api-umbrella/cli/reload.lua b/src/api-umbrella/cli/reload.lua
index 694da3336..3640ec216 100644
--- a/src/api-umbrella/cli/reload.lua
+++ b/src/api-umbrella/cli/reload.lua
@@ -60,7 +60,7 @@ return function(options)
   local running = status()
   if not running then
     print("api-umbrella is stopped")
-    os.exit(1)
+    os.exit(7)
   end
 
   local config = setup()
diff --git a/src/api-umbrella/cli/run.lua b/src/api-umbrella/cli/run.lua
index 57bb5e90e..26aca6006 100644
--- a/src/api-umbrella/cli/run.lua
+++ b/src/api-umbrella/cli/run.lua
@@ -1,8 +1,19 @@
 local path = require "pl.path"
 local setup = require "api-umbrella.cli.setup"
+local status = require "api-umbrella.cli.status"
 local unistd = require "posix.unistd"
 
 local function start_perp(config, options)
+  local running, pid = status()
+  if running then
+    print "api-umbrella is already running"
+    if options and options["background"] then
+      os.exit(0)
+    else
+      os.exit(1)
+    end
+  end
+
   local perp_base = path.join(config["etc_dir"], "perp")
   local args = {
     "-0", "api-umbrella",
diff --git a/src/api-umbrella/cli/status.lua b/src/api-umbrella/cli/status.lua
index 3a5014452..401dd25dd 100644
--- a/src/api-umbrella/cli/status.lua
+++ b/src/api-umbrella/cli/status.lua
@@ -2,6 +2,33 @@ local file = require "pl.file"
 local path = require "pl.path"
 local read_config = require "api-umbrella.cli.read_config"
 local run_command = require "api-umbrella.utils.run_command"
+local stringx = require "pl.stringx"
+
+-- Get the status of the legacy nodejs version (v0.8 and before) of the app.
+--
+-- This is present here so that we can cleanly perform package upgrades from
+-- the legacy version and restart after the upgrade. At some point we can
+-- probably remove this (once we no longer need to support upgrades from v0.8).
+local function legacy_status()
+  -- Only perform the legacy status if it appears as though it might be
+  -- present.
+  if not path.exists("/opt/api-umbrella/var/run/forever") then
+    return nil
+  end
+
+  local _, output, err = run_command("pgrep -f '^/opt/api-umbrella/embedded/bin/node.*api-umbrella run'")
+  if err and output ~= "" then
+    return nil
+  elseif err and output == "" then
+    -- If the legacy process isn't running, then go ahead and remove the legacy
+    -- directory so we know we don't need to do this again for an upgraded box.
+    run_command("rm -rf /opt/api-umbrella/var/run/forever")
+    return nil
+  end
+
+  local pid = tonumber(stringx.strip(output))
+  return true, pid
+end
 
 local function perp_status(config)
   local running = false
@@ -29,5 +56,11 @@ end
 
 return function()
   local config = read_config()
+
+  local legacy_running, legacy_pid = legacy_status()
+  if legacy_running ~= nil then
+    return legacy_running, legacy_pid
+  end
+
   return perp_status(config)
 end
diff --git a/src/api-umbrella/cli/stop.lua b/src/api-umbrella/cli/stop.lua
index 9e5fdbb91..bc4820ce3 100644
--- a/src/api-umbrella/cli/stop.lua
+++ b/src/api-umbrella/cli/stop.lua
@@ -1,40 +1,118 @@
+local path = require "pl.path"
 local run_command = require "api-umbrella.utils.run_command"
 local status = require "api-umbrella.cli.status"
+local stringx = require "pl.stringx"
 local time = require "posix.time"
 
+-- Stop the legacy nodejs version (v0.8 and before) of the app.
+--
+-- This is present here so that we can cleanly perform package upgrades from
+-- the legacy version and restart after the upgrade. At some point we can
+-- probably remove this (once we no longer need to support upgrades from v0.8).
+local function stop_legacy()
+  -- Only perform the legacy stop if it appears as though it might be present.
+  if not path.exists("/opt/api-umbrella/var/run/forever") then
+    return true
+  end
+
+  local running, _ = status()
+  if not running then
+    print "api-umbrella is already stopped"
+    return true
+  end
+
+  -- Stop the various processes tied to the old service.
+  local _, output, err = run_command("pkill -TERM -f '^/opt/api-umbrella/embedded/bin/node.*forever/bin/monitor.*api-umbrella'")
+  if err and output ~= "" then
+    return false, err
+  end
+
+  _, output, err = run_command("pkill -TERM -f '^/opt/api-umbrella/embedded/bin/node.*api-umbrella run'")
+  if err and output ~= "" then
+    return false, err
+  end
+
+  _, output, err = run_command("pgrep -f '^/opt/api-umbrella/embedded/bin/python.*supervisord.*api-umbrella'")
+  if err and output ~= "" then
+    return false, err
+  elseif err and output == "" then
+    return true
+  end
+
+  local supervisord_pid = stringx.strip(output)
+  _, _, err = run_command("kill -s TERM " .. supervisord_pid)
+  if err then
+    return false, err
+  end
+
+  -- Wait until the old processes have completely shut down.
+  local stopped = false
+  for _ = 1, 200 do
+    local supervisord_status = run_command("pgrep -f '/opt/api-umbrella/embedded/bin/python.*supervisord.*api-umbrella'")
+    if supervisord_status == 0 then
+      -- Sleep for 0.2 seconds.
+      time.nanosleep({ tv_sec = 0, tv_nsec = 200000000 })
+    else
+      stopped = true
+      break
+    end
+  end
+
+  if not stopped then
+    return false, "failed to stop"
+  end
+
+  -- Remove the legacy directory so we know we don't need to do this again for
+  -- an upgraded box.
+  _, _, err = run_command("rm -rf /opt/api-umbrella/var/run/forever")
+  if err then
+    return false, err
+  end
+
+  return true
+end
+
 local function stop_perp()
   local running, pid = status()
-  if running then
-    local _, _, err = run_command("kill -s TERM " .. pid)
-    if err then
-      return false, err
-    end
+  if not running then
+    print "api-umbrella is already stopped"
+    return true
+  end
 
-    -- After sending the kill command, wait for everything to actually
-    -- shutdown. This allows us to more easily handle restarts (since we can
-    -- ensure everything is stopped before calling start).
-    --
-    -- Wait up to 40 seconds.
-    local stopped = false
-    for _ = 1, 200 do
-      running, _ = status()
-      if running then
-        -- Sleep for 0.2 seconds.
-        time.nanosleep({ tv_sec = 0, tv_nsec = 200000000 })
-      else
-        stopped = true
-        break
-      end
-    end
+  local _, _, err = run_command("kill -s TERM " .. pid)
+  if err then
+    return false, err
+  end
 
-    if not stopped then
-      return false, "failed to stop"
+  -- After sending the kill command, wait for everything to actually
+  -- shutdown. This allows us to more easily handle restarts (since we can
+  -- ensure everything is stopped before calling start).
+  --
+  -- Wait up to 40 seconds.
+  local stopped = false
+  for _ = 1, 200 do
+    running, _ = status()
+    if running then
+      -- Sleep for 0.2 seconds.
+      time.nanosleep({ tv_sec = 0, tv_nsec = 200000000 })
+    else
+      stopped = true
+      break
     end
   end
 
+  if not stopped then
+    return false, "failed to stop"
+  end
+
   return true
 end
 
 return function()
+  local ok, err = stop_legacy()
+  if not ok then
+    return ok, err
+  end
+
   return stop_perp()
 end
diff --git a/src/api-umbrella/utils/run_command.lua b/src/api-umbrella/utils/run_command.lua
index bbe79e1ad..4d4e4374a 100644
--- a/src/api-umbrella/utils/run_command.lua
+++ b/src/api-umbrella/utils/run_command.lua
@@ -14,11 +14,16 @@ return function(command)
   handle:close()
 
   local output, status = string.match(all_output, "^(.*)===STATUS_CODE:(%d+)\n$")
-  status = tonumber(status)
-
   local err = nil
-  if not status or status ~= 0 then
-    err = "Executing command failed: " .. command .. "\n\n" .. output
+  if output == nil and status == nil then
+    -- This means we never got the "STATUS_CODE" output, so the entire
+    -- sub-processes must have gotten killed off.
+    err = "Executing command failed: " .. command .. "\n\nCommand exited prematurely. Was it killed by an external process?"
+  else
+    status = tonumber(status)
+    if not status or status ~= 0 then
+      err = "Executing command failed: " .. command .. "\n\n" .. output
+    end
   end
 
   return status, output, err
diff --git a/src/api-umbrella/version.lua b/src/api-umbrella/version.lua
index 777f79583..002666b1a 100644
--- a/src/api-umbrella/version.lua
+++ b/src/api-umbrella/version.lua
@@ -1 +1 @@
-return "0.9.0"
+return "0.10.0-pre1"