Source Upload override and independent stage support (#60)

* Add executable permissions to common_sa_build.sh * Add the ability to override the default list of source files to upload * Reformatting readme * Sourcing common_sa_build to maintain the previous behavior. * Add prepare_sources_to_upload.sh, ros_bootstrap.sh, and small refactor to reduce duplication * Simplify UPLOAD_SOURCES option to only allow enable/disable
aws-robotics · Oct 3, 2019 · 7ad2d9a · 7ad2d9a
1 parent 715bdba
commit 7ad2d9a
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 17 deletions.
diff --git a/README.md b/README.md
@@ -3,6 +3,26 @@
 ## Overview
 This repository contains scripts used in Travis CIs for [AWS RoboMaker] sample applications and [AWS RoboMaker] ROS Cloud Extensions.
 
+## Configuration Options
+
+Configuration is done via environment variables. When adding a new option, make sure to also pass that variable into the docker container used for the build (see `ce_build.sh`).
+
+### Common
+
+* `ROS_VERSION`, `ROS_DISTRO`, `GAZEBO_VERSION`: determine which build flavour and docker image to use
+
+### Sample Applications
+
+* `WORKSPACES`: which workspaces should be built.
+* `SA_PACKAGE_NAME`: controls which package's manifest file would determine the version of the application bundle that's going to be uploaded to S3.
+* `UPLOAD_SOURCES`: by default, the source files for `${WORKSPACES}` will be uploaded (along with LICENSE, NOTICE, README and roboMakerSettings.json files). You may override the default behavior.
+  * `UPLOAD_SOURCES=false`: Skip source upload
+  * Note: when workspaces are built separately (i.e. one build job for each workspace), it is recommended to disable source upload for the build jobs and add a dedicated stage for it using `prepare_sources_to_upload.sh`.
+
+### Cloud Extensions
+
+* `NO_TEST`: unit tests are enabled by default, specify NO_TESTS=true to override.
+
 ## Testing locally
 
 ### Testing Sample Applications
@@ -36,4 +56,4 @@ cleanup_artifacts && bash -x ros1_sa_build.sh &> sa_build.log
 
 This library is licensed under the Apache 2.0 License. 
 
-[AWS RoboMaker]: https://github.com/aws-robotics
+[AWS RoboMaker]: https://github.com/aws-robotics
diff --git a/ce_build.sh b/ce_build.sh
@@ -38,6 +38,7 @@ docker run -v "${PWD}/shared:/shared" \
   -e GAZEBO_VERSION="${GAZEBO_VERSION:-7}" \
   -e DOCKER_BUILD_SCRIPT="${DOCKER_BUILD_SCRIPT}" \
   -e WORKSPACES="${WORKSPACES}" \
+  -e UPLOAD_SOURCES="${UPLOAD_SOURCES}" \
   --name "${ROS_DISTRO}-container" \
   --network=host \
   -dit "ros:${ROS_DISTRO}-ros-core" /bin/bash

diff --git a/common_sa_build.sh b/common_sa_build.sh
@@ -26,11 +26,15 @@ do
   fi
 done
 
-# Create archive of all sources files
-SOURCES_INCLUDES="${WORKSPACES} LICENSE* NOTICE* README* roboMakerSettings.json"
-cd /${ROS_DISTRO}_ws/src/${BUILD_DIR_NAME}/
-/usr/bin/zip -r /shared/sources.zip $SOURCES_INCLUDES
-tar cvzf /shared/sources.tar.gz $SOURCES_INCLUDES
+# Create archive of relevant source files (unless UPLOAD_SOURCES is false)
+if [ ! -z "$UPLOAD_SOURCES" ] && [ "$UPLOAD_SOURCES" == "false" ]; then
+  echo "Skipping source upload for this build job"
+else
+  SOURCES_INCLUDES="${WORKSPACES} LICENSE* NOTICE* README* roboMakerSettings.json"
+  cd /${ROS_DISTRO}_ws/src/${BUILD_DIR_NAME}/
+  /usr/bin/zip -r /shared/sources.zip $SOURCES_INCLUDES
+  tar cvzf /shared/sources.tar.gz $SOURCES_INCLUDES
+fi
 
 for WS in $WORKSPACES
 do

diff --git a/prepare_sources_to_upload.sh b/prepare_sources_to_upload.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# This is a standalone script that is meant to be run from Travis directly, outside of a docker container.
+set -xe
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# Bootstrap to get basic ROS dependencies (rosinstall)
+ROS_BOOTSTRAP_SCRIPT="${SCRIPT_DIR}/ros_bootstrap.sh"
+sudo ROS_VERSION=${ROS_VERSION} "${ROS_BOOTSTRAP_SCRIPT}"
+
+if [ -z "$WORKSPACES" ]; then
+  WORKSPACES="robot_ws simulation_ws"
+fi
+
+for WS in ${WORKSPACES}
+do
+  WS_DIR="${TRAVIS_BUILD_DIR}/${WS}"
+  echo "looking for ${WS}, ${WS_DIR}"
+  if [ -d "${WS_DIR}" ]; then
+    echo "WS ${WS_DIR} found, running rosws update"
+    rosws update -t "${WS_DIR}"
+  fi
+done
+
+SOURCES_INCLUDES="${WORKSPACES} LICENSE* NOTICE* README* roboMakerSettings.json"
+mkdir shared 2>/dev/null
+/usr/bin/zip -r shared/sources.zip $SOURCES_INCLUDES
+tar cvzf shared/sources.tar.gz $SOURCES_INCLUDES
diff --git a/ros1_sa_build.sh b/ros1_sa_build.sh
@@ -4,11 +4,9 @@ set -xe
 export SCRIPT_DIR=$(dirname ${DOCKER_BUILD_SCRIPT})
 
 # install dependencies
-apt-get update && apt-get install -q -y dirmngr gnupg2 lsb-release zip python3-pip python3-apt dpkg
-pip3 install -U setuptools
-sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
-apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654
-apt-get update && apt-get install --no-install-recommends -y python-rosdep python-rosinstall python3-colcon-common-extensions ros-$ROS_DISTRO-ros-base
+ROS_BOOTSTRAP_SCRIPT=${SCRIPT_DIR}/ros_bootstrap.sh
+"${ROS_BOOTSTRAP_SCRIPT}"
+apt-get update && apt-get install --no-install-recommends -y python3-colcon-common-extensions ros-$ROS_DISTRO-ros-base
 pip3 install colcon-bundle colcon-ros-bundle
 
 COMMON_SA_BUILD_SCRIPT=${SCRIPT_DIR}/common_sa_build.sh

diff --git a/ros2_sa_build.sh b/ros2_sa_build.sh
@@ -4,12 +4,9 @@ set -xe
 export SCRIPT_DIR=$(dirname ${DOCKER_BUILD_SCRIPT})
 
 # install dependencies
-apt-get update && apt-get install -q -y dirmngr curl gnupg2 lsb-release zip python3-pip python3-apt dpkg
-pip3 install -U setuptools
-
-curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | apt-key add -
-sh -c 'echo "deb [arch=amd64,arm64] http://packages.ros.org/ros2/ubuntu `lsb_release -cs` main" > /etc/apt/sources.list.d/ros2-latest.list'
-apt-get update && apt-get install --no-install-recommends -y python3-rosdep python3-rosinstall python3-colcon-common-extensions ros-$ROS_DISTRO-ros-base
+ROS_BOOTSTRAP_SCRIPT=${SCRIPT_DIR}/ros_bootstrap.sh
+"${ROS_BOOTSTRAP_SCRIPT}"
+apt-get update && apt-get install --no-install-recommends -y python3-colcon-common-extensions ros-$ROS_DISTRO-ros-base
 apt list --upgradable 2>/dev/null | awk {'print $1'} | sed 's/\/.*//g' | grep $ROS_DISTRO | xargs apt install -y
 pip3 install colcon-bundle colcon-ros-bundle
 

diff --git a/ros_bootstrap.sh b/ros_bootstrap.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Set up ROS APT and install basic dependencies (rosdep, rosinstall). Must have ROS_VERSION set when called.
+set -xe
+
+apt-get update && apt-get install -q -y dirmngr curl gnupg2 lsb-release zip python3-pip python3-apt dpkg
+pip3 install -U setuptools
+
+if [ "${ROS_VERSION}" == "1" ]; then
+  sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
+  apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654
+  apt-get update && apt-get install -y python-rosdep python-rosinstall
+elif [ "${ROS_VERSION}" == "2" ]; then
+  curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | apt-key add -
+  sh -c 'echo "deb [arch=amd64,arm64] http://packages.ros.org/ros2/ubuntu `lsb_release -cs` main" > /etc/apt/sources.list.d/ros2-latest.list'
+  apt-get update && apt-get install -y python3-rosdep python3-rosinstall
+else
+  echo "ROS_VERSION not defined or recognized"
+  exit 1
+fi