Make unit tests run in a Docker container

CMakeLists.txt

@@ -13,7 +13,7 @@ find_package(Sanitizers)
 # Include Boost as an imported target
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_USE_STATIC_LIBS ON)
-find_package(Boost 1.60.0 REQUIRED COMPONENTS thread system chrono date_time atomic)
+find_package(Boost 1.58.0 REQUIRED COMPONENTS thread system chrono date_time atomic)
 add_library(boost INTERFACE IMPORTED)
 set_property(TARGET boost PROPERTY
     INTERFACE_INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS})

ClipperLibBaseDockerfile

@@ -1,40 +1,17 @@
-FROM frolvlad/alpine-gxx
+FROM ubuntu:16.04
 
-RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
-
-RUN apk add --no-cache git bash make boost-dev cmake libev-dev hiredis-dev zeromq-dev \
-    libtool automake autoconf-archive@testing lz4-dev xz-dev snappy-dev zlib-dev \
-    glog-dev@testing scons openssl-dev libevent-dev
-
-# Set up Folly
-
-## Install DoubleConversion
-RUN git clone https://github.com/google/double-conversion \
-    && cd double-conversion \
-    && git checkout tags/v3.0.0 \
-    && cmake -DBUILD_SHARED_LIBS=ON . \
-    && make \
-    && make install \
-    && ln -s src double-conversion
-
-ENV DOUBLE_CONVERSION_HOME=/double-conversion/double-conversion/
-
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$DOUBLE_CONVERSION_HOME
-
-## Install GFLags
-RUN git clone https://github.com/gflags/gflags \
-    && cd gflags \
-    && git checkout tags/v2.2.1 \
-    && mkdir build \
-    && cd build \
-    && cmake -DBUILD_SHARED_LIBS=ON .. \
-    && make install
+RUN apt-get update && apt-get install -y g++ automake autoconf autoconf-archive libtool libboost-all-dev \
+    libevent-dev libdouble-conversion-dev libgoogle-glog-dev libgflags-dev liblz4-dev \
+    liblzma-dev libsnappy-dev make zlib1g-dev binutils-dev libjemalloc-dev libssl-dev \
+    pkg-config libiberty-dev git cmake libev-dev libhiredis-dev libzmq5 libzmq5-dev
 
 ## Install Folly
 RUN git clone https://github.com/facebook/folly \
     && cd folly/folly \
     && git checkout tags/v2017.08.14.00 \
     && autoreconf -ivf \
-    && ./configure LDFLAGS=-L$DOUBLE_CONVERSION_HOME/ CPPFLAGS=-I$DOUBLE_CONVERSION_HOME \
-    && make \
+    && ./configure \
+    && make -j4 \
     && make install
+
+# vim: set filetype=dockerfile:

ClipperTestsDockerfile

@@ -0,0 +1,61 @@
+ARG CODE_VERSION=develop
+FROM clipper/lib_base:${CODE_VERSION}
+
+# install docker
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install --fix-missing -y \
+      wget apt-transport-https ca-certificates curl software-properties-common \
+      openjdk-8-jdk clang-format \
+      && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+      && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
+      && apt-get update \
+      && apt-get install -y docker-ce
+
+RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \
+    && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \
+    && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \
+    && rm /tmp/anaconda.sh
+ENV PATH "/opt/conda/bin:$PATH"
+RUN conda install -y libgcc pyzmq
+
+RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml docker \
+      kubernetes pyspark docker-compose findspark
+
+RUN mkdir /spark && cd /spark \
+        && curl -o spark.tgz https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz \
+        && tar zxf spark.tgz && rm spark.tgz
+
+ENV SPARK_HOME /spark/spark-2.1.1-bin-hadoop2.7
+
+# Install maven
+ARG MAVEN_VERSION=3.5.0
+ARG SHA=beb91419245395bd69a4a6edad5ca3ec1a8b64e41457672dc687c173a495f034
+ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
+  && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-$MAVEN_VERSION-bin.tar.gz \
+  && echo "${SHA}  /tmp/apache-maven.tar.gz" | sha256sum -c - \
+  && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \
+  && rm -f /tmp/apache-maven.tar.gz \
+  && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \
+  && git clone https://github.com/zeromq/jzmq.git /root/jzmq \
+  && cd /root/jzmq/jzmq-jni \
+  && ./autogen.sh \
+  && ./configure \
+  && make \
+  && make install
+
+ENV MAVEN_HOME /usr/share/maven
+ENV MAVEN_CONFIG /root/.m2
+ENV JZMQ_HOME /usr/local/lib/
+
+
+COPY ./ /clipper
+
+# Patch Clipper
+RUN cd /clipper/src/libs/spdlog \
+    && git apply ../patches/make_spdlog_compile_linux.patch
+
+
+ENTRYPOINT ["/clipper/bin/ci_checks.sh"]
+
+# vim: set filetype=dockerfile:

ManagementFrontendDockerfile

@@ -3,8 +3,7 @@ FROM clipper/lib_base:${CODE_VERSION}
 
 COPY ./ /clipper
 
-RUN apk add --no-cache git bash make boost-dev cmake libev-dev hiredis-dev zeromq-dev \
-    && cd /clipper/src/libs/spdlog \
+RUN cd /clipper/src/libs/spdlog \
     && git apply ../patches/make_spdlog_compile_linux.patch \
     && cd /clipper \
     && ./configure --cleanup-quiet \

bin/build_docker_images.sh

@@ -20,6 +20,7 @@ tag=$(<VERSION.txt)
 time docker build -t clipper/lib_base:$tag -f ./ClipperLibBaseDockerfile ./
 time docker build --build-arg CODE_VERSION=$tag -t clipper/query_frontend:$tag -f QueryFrontendDockerfile ./
 time docker build --build-arg CODE_VERSION=$tag -t clipper/management_frontend:$tag -f ManagementFrontendDockerfile ./
+time docker build --build-arg CODE_VERSION=$tag -t clipper/unittests:$tag -f ClipperTestsDockerfile ./
 cd -
 
 # Build Spark JVM Container

bin/ci_checks.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+set -o pipefail
+
+unset CDPATH
+# one-liner from http://stackoverflow.com/a/246128
+# Determines absolute path of the directory containing
+# the script.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Let the user start this script from anywhere in the filesystem.
+$DIR/check_format.sh
+$DIR/run_unittests.sh

bin/run_ci.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+set -u
+
+# TODO: Remove the /tmp volume mount in the Clipper admin refactor PR
+docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp clipper/unittests:develop