From db12183f6005f9139914a56fc3625aeb5e8bc38b Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Tue, 29 Aug 2017 13:53:27 -0700 Subject: [PATCH 1/7] Added unit tests docker file --- ClipperTestsDockerfile | 32 ++++++++++++++++++++++++++++++++ bin/build_docker_images.sh | 2 ++ 2 files changed, 34 insertions(+) create mode 100644 ClipperTestsDockerfile diff --git a/ClipperTestsDockerfile b/ClipperTestsDockerfile new file mode 100644 index 000000000..b6ca3b2d7 --- /dev/null +++ b/ClipperTestsDockerfile @@ -0,0 +1,32 @@ +ARG CODE_VERSION=develop +FROM clipper/lib_base:${CODE_VERSION} + + +# Install Anaconda and Python deps +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ + && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ + && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \ + && rm /tmp/anaconda.sh +ENV PATH "/opt/conda/bin:$PATH" +RUN conda install -y libgcc pyzmq +RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml docker kubernetes pyspark + +# install docker +RUN apt-get update && apt-get install -y \ + apt-transport-https ca-certificates curl software-properties-common \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - \ + && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + && apt-get update \ + && apt-get install -yqq docker-ce + +# RUN gpasswd -a $NB_USER docker + +COPY ./ /clipper + +# Patch Clipper +RUN cd /clipper/src/libs/spdlog \ + && git apply ../patches/make_spdlog_compile_linux.patch + +ENTRYPOINT ["/clipper/bin/check_format.sh && /clipper/bin/run_unittests.sh -a"] + +# vim: set filetype=dockerfile: diff --git a/bin/build_docker_images.sh b/bin/build_docker_images.sh index 89649e7b2..518d607fd 100755 --- a/bin/build_docker_images.sh +++ b/bin/build_docker_images.sh @@ -20,6 +20,8 @@ tag=$( Date: Wed, 30 Aug 2017 04:12:54 +0000 Subject: [PATCH 2/7] more updates to docker files --- ClipperLibBaseDockerfile | 10 ++++--- ClipperTestsDockerfile | 58 +++++++++++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/ClipperLibBaseDockerfile b/ClipperLibBaseDockerfile index 9401c82a2..926d596ab 100644 --- a/ClipperLibBaseDockerfile +++ b/ClipperLibBaseDockerfile @@ -1,4 +1,6 @@ -FROM frolvlad/alpine-gxx +FROM frolvlad/alpine-glibc + +RUN apk add --no-cache gcc g++ musl-dev RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories @@ -13,7 +15,7 @@ RUN git clone https://github.com/google/double-conversion \ && cd double-conversion \ && git checkout tags/v3.0.0 \ && cmake -DBUILD_SHARED_LIBS=ON . \ - && make \ + && make -j4 \ && make install \ && ln -s src double-conversion @@ -36,5 +38,7 @@ RUN git clone https://github.com/facebook/folly \ && git checkout tags/v2017.08.14.00 \ && autoreconf -ivf \ && ./configure LDFLAGS=-L$DOUBLE_CONVERSION_HOME/ CPPFLAGS=-I$DOUBLE_CONVERSION_HOME \ - && make \ + && make -j4 \ && make install + +# vim: set filetype=dockerfile: diff --git a/ClipperTestsDockerfile b/ClipperTestsDockerfile index b6ca3b2d7..2b12a3d76 100644 --- a/ClipperTestsDockerfile +++ b/ClipperTestsDockerfile @@ -1,23 +1,57 @@ ARG CODE_VERSION=develop FROM clipper/lib_base:${CODE_VERSION} + + +ENV CONDA_DIR="/opt/conda" +ENV PATH="$CONDA_DIR/bin:$PATH" + +# Install conda +# From https://github.com/frol/docker-alpine-miniconda3/blob/017a9c9dc134e98729a075de105b19b89343b892/Dockerfile +RUN CONDA_VERSION="4.3.14" && \ + apk add --no-cache --virtual=.build-dependencies wget ca-certificates bash && \ + \ + mkdir -p "$CONDA_DIR" && \ + wget "http://repo.continuum.io/miniconda/Miniconda2-${CONDA_VERSION}-Linux-x86_64.sh" -O miniconda.sh && \ + bash miniconda.sh -f -b -p "$CONDA_DIR" && \ + echo "export PATH=$CONDA_DIR/bin:\$PATH" > /etc/profile.d/conda.sh && \ + rm miniconda.sh && \ + # conda update && \ + # conda update --all --yes && \ + # conda config --set auto_update_conda False && \ + # rm -r "$CONDA_DIR/pkgs/" && \ + # \ + # apk del --purge .build-dependencies && \ + # \ + mkdir -p "$CONDA_DIR/locks" && \ + chmod 777 "$CONDA_DIR/locks" + + # Install Anaconda and Python deps -RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ - && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ - && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \ - && rm /tmp/anaconda.sh +# RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ +# && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ +# && bash /tmp/anaconda.sh -b -p /opt/conda \ +# && rm /tmp/anaconda.sh ENV PATH "/opt/conda/bin:$PATH" -RUN conda install -y libgcc pyzmq +RUN conda install -y pyzmq RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml docker kubernetes pyspark -# install docker -RUN apt-get update && apt-get install -y \ - apt-transport-https ca-certificates curl software-properties-common \ - && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - \ - && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ - && apt-get update \ - && apt-get install -yqq docker-ce +# RUN echo "https://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories && \ +# echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ +# echo "@community https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories + +RUN echo "@community https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories + +RUN apk add --update docker@community + +# # install docker +# RUN apt-get update && apt-get install -y \ +# apt-transport-https ca-certificates curl software-properties-common \ +# && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - \ +# && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ +# && apt-get update \ +# && apt-get install -yqq docker-ce # RUN gpasswd -a $NB_USER docker From c4ec591c29135af2f388a26eda0885e328f8c89c Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Tue, 29 Aug 2017 23:36:50 -0700 Subject: [PATCH 3/7] updated dockerfiles --- CMakeLists.txt | 2 +- ClipperLibBaseDockerfile | 40 +++++------------------ ClipperTestsDockerfile | 61 ++++++++---------------------------- ManagementFrontendDockerfile | 3 +- bin/build_docker_images.sh | 4 +-- 5 files changed, 25 insertions(+), 85 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f405c2636..0489f9da9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ find_package(Sanitizers) # Include Boost as an imported target set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_LIBS ON) -find_package(Boost 1.60.0 REQUIRED COMPONENTS thread system chrono date_time atomic) +find_package(Boost 1.58.0 REQUIRED COMPONENTS thread system chrono date_time atomic) add_library(boost INTERFACE IMPORTED) set_property(TARGET boost PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${Boost_INCLUDE_DIRS}) diff --git a/ClipperLibBaseDockerfile b/ClipperLibBaseDockerfile index 926d596ab..ad405fa77 100644 --- a/ClipperLibBaseDockerfile +++ b/ClipperLibBaseDockerfile @@ -1,43 +1,19 @@ -FROM frolvlad/alpine-glibc +FROM ubuntu:16.04 -RUN apk add --no-cache gcc g++ musl-dev +# TODO: add this back +# ENV DEBIAN_FRONTEND noninteractive -RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories - -RUN apk add --no-cache git bash make boost-dev cmake libev-dev hiredis-dev zeromq-dev \ - libtool automake autoconf-archive@testing lz4-dev xz-dev snappy-dev zlib-dev \ - glog-dev@testing scons openssl-dev libevent-dev - -# Set up Folly - -## Install DoubleConversion -RUN git clone https://github.com/google/double-conversion \ - && cd double-conversion \ - && git checkout tags/v3.0.0 \ - && cmake -DBUILD_SHARED_LIBS=ON . \ - && make -j4 \ - && make install \ - && ln -s src double-conversion - -ENV DOUBLE_CONVERSION_HOME=/double-conversion/double-conversion/ - -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$DOUBLE_CONVERSION_HOME - -## Install GFLags -RUN git clone https://github.com/gflags/gflags \ - && cd gflags \ - && git checkout tags/v2.2.1 \ - && mkdir build \ - && cd build \ - && cmake -DBUILD_SHARED_LIBS=ON .. \ - && make install +RUN apt-get update && apt-get install -y g++ automake autoconf autoconf-archive libtool libboost-all-dev \ + libevent-dev libdouble-conversion-dev libgoogle-glog-dev libgflags-dev liblz4-dev \ + liblzma-dev libsnappy-dev make zlib1g-dev binutils-dev libjemalloc-dev libssl-dev \ + pkg-config libiberty-dev git cmake libev-dev libhiredis-dev libzmq5 libzmq5-dev ## Install Folly RUN git clone https://github.com/facebook/folly \ && cd folly/folly \ && git checkout tags/v2017.08.14.00 \ && autoreconf -ivf \ - && ./configure LDFLAGS=-L$DOUBLE_CONVERSION_HOME/ CPPFLAGS=-I$DOUBLE_CONVERSION_HOME \ + && ./configure \ && make -j4 \ && make install diff --git a/ClipperTestsDockerfile b/ClipperTestsDockerfile index 2b12a3d76..cc68e4779 100644 --- a/ClipperTestsDockerfile +++ b/ClipperTestsDockerfile @@ -1,58 +1,23 @@ ARG CODE_VERSION=develop FROM clipper/lib_base:${CODE_VERSION} - - -ENV CONDA_DIR="/opt/conda" -ENV PATH="$CONDA_DIR/bin:$PATH" - -# Install conda -# From https://github.com/frol/docker-alpine-miniconda3/blob/017a9c9dc134e98729a075de105b19b89343b892/Dockerfile -RUN CONDA_VERSION="4.3.14" && \ - apk add --no-cache --virtual=.build-dependencies wget ca-certificates bash && \ - \ - mkdir -p "$CONDA_DIR" && \ - wget "http://repo.continuum.io/miniconda/Miniconda2-${CONDA_VERSION}-Linux-x86_64.sh" -O miniconda.sh && \ - bash miniconda.sh -f -b -p "$CONDA_DIR" && \ - echo "export PATH=$CONDA_DIR/bin:\$PATH" > /etc/profile.d/conda.sh && \ - rm miniconda.sh && \ - # conda update && \ - # conda update --all --yes && \ - # conda config --set auto_update_conda False && \ - # rm -r "$CONDA_DIR/pkgs/" && \ - # \ - # apk del --purge .build-dependencies && \ - # \ - mkdir -p "$CONDA_DIR/locks" && \ - chmod 777 "$CONDA_DIR/locks" - - - -# Install Anaconda and Python deps -# RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ -# && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ -# && bash /tmp/anaconda.sh -b -p /opt/conda \ -# && rm /tmp/anaconda.sh +# install docker +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install --fix-missing -y \ + wget apt-transport-https ca-certificates curl software-properties-common \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ + && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + && apt-get update \ + && apt-get install --fix-missing -y docker-ce + +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ + && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ + && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \ + && rm /tmp/anaconda.sh ENV PATH "/opt/conda/bin:$PATH" RUN conda install -y pyzmq + RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml docker kubernetes pyspark -# RUN echo "https://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories && \ -# echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ -# echo "@community https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories - -RUN echo "@community https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories - -RUN apk add --update docker@community - -# # install docker -# RUN apt-get update && apt-get install -y \ -# apt-transport-https ca-certificates curl software-properties-common \ -# && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - \ -# && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ -# && apt-get update \ -# && apt-get install -yqq docker-ce - # RUN gpasswd -a $NB_USER docker COPY ./ /clipper diff --git a/ManagementFrontendDockerfile b/ManagementFrontendDockerfile index b1df230c3..8214d40b2 100644 --- a/ManagementFrontendDockerfile +++ b/ManagementFrontendDockerfile @@ -3,8 +3,7 @@ FROM clipper/lib_base:${CODE_VERSION} COPY ./ /clipper -RUN apk add --no-cache git bash make boost-dev cmake libev-dev hiredis-dev zeromq-dev \ - && cd /clipper/src/libs/spdlog \ +RUN cd /clipper/src/libs/spdlog \ && git apply ../patches/make_spdlog_compile_linux.patch \ && cd /clipper \ && ./configure --cleanup-quiet \ diff --git a/bin/build_docker_images.sh b/bin/build_docker_images.sh index 518d607fd..67d1f9b84 100755 --- a/bin/build_docker_images.sh +++ b/bin/build_docker_images.sh @@ -18,8 +18,8 @@ tag=$( Date: Wed, 30 Aug 2017 07:31:49 +0000 Subject: [PATCH 4/7] Format check is working --- ClipperLibBaseDockerfile | 3 --- ClipperTestsDockerfile | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ClipperLibBaseDockerfile b/ClipperLibBaseDockerfile index ad405fa77..ca250a8a5 100644 --- a/ClipperLibBaseDockerfile +++ b/ClipperLibBaseDockerfile @@ -1,8 +1,5 @@ FROM ubuntu:16.04 -# TODO: add this back -# ENV DEBIAN_FRONTEND noninteractive - RUN apt-get update && apt-get install -y g++ automake autoconf autoconf-archive libtool libboost-all-dev \ libevent-dev libdouble-conversion-dev libgoogle-glog-dev libgflags-dev liblz4-dev \ liblzma-dev libsnappy-dev make zlib1g-dev binutils-dev libjemalloc-dev libssl-dev \ diff --git a/ClipperTestsDockerfile b/ClipperTestsDockerfile index cc68e4779..a4aac5748 100644 --- a/ClipperTestsDockerfile +++ b/ClipperTestsDockerfile @@ -20,12 +20,15 @@ RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml doc # RUN gpasswd -a $NB_USER docker +RUN DEBIAN_FRONTEND=noninteractive apt-get install --fix-missing -y clang-format + COPY ./ /clipper # Patch Clipper RUN cd /clipper/src/libs/spdlog \ && git apply ../patches/make_spdlog_compile_linux.patch -ENTRYPOINT ["/clipper/bin/check_format.sh && /clipper/bin/run_unittests.sh -a"] + +ENTRYPOINT ["/clipper/bin/check_format.sh"] # vim: set filetype=dockerfile: From 53258a6674a628f64dfb2b1471f6fd672d551449 Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Wed, 30 Aug 2017 09:17:09 +0000 Subject: [PATCH 5/7] pretty sure dockerized tests are passing --- ClipperTestsDockerfile | 41 +++++++++++++++++++++++++++++++------- bin/build_docker_images.sh | 5 ++--- bin/ci_checks.sh | 15 ++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) create mode 100755 bin/ci_checks.sh diff --git a/ClipperTestsDockerfile b/ClipperTestsDockerfile index a4aac5748..1bba1fe54 100644 --- a/ClipperTestsDockerfile +++ b/ClipperTestsDockerfile @@ -4,23 +4,50 @@ FROM clipper/lib_base:${CODE_VERSION} # install docker RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install --fix-missing -y \ wget apt-transport-https ca-certificates curl software-properties-common \ + openjdk-8-jdk clang-format \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ && apt-get update \ - && apt-get install --fix-missing -y docker-ce + && apt-get install -y docker-ce RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \ && rm /tmp/anaconda.sh ENV PATH "/opt/conda/bin:$PATH" -RUN conda install -y pyzmq +RUN conda install -y libgcc pyzmq -RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml docker kubernetes pyspark +RUN pip install requests subprocess32 scikit-learn numpy scipy fabric pyyaml docker \ + kubernetes pyspark docker-compose findspark + +RUN mkdir /spark && cd /spark \ + && curl -o spark.tgz https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz \ + && tar zxf spark.tgz && rm spark.tgz + +ENV SPARK_HOME /spark/spark-2.1.1-bin-hadoop2.7 + +# Install maven +ARG MAVEN_VERSION=3.5.0 +ARG SHA=beb91419245395bd69a4a6edad5ca3ec1a8b64e41457672dc687c173a495f034 +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries + +RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ + && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-$MAVEN_VERSION-bin.tar.gz \ + && echo "${SHA} /tmp/apache-maven.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ + && git clone https://github.com/zeromq/jzmq.git /root/jzmq \ + && cd /root/jzmq/jzmq-jni \ + && ./autogen.sh \ + && ./configure \ + && make \ + && make install + +ENV MAVEN_HOME /usr/share/maven +ENV MAVEN_CONFIG /root/.m2 +ENV JZMQ_HOME /usr/local/lib/ -# RUN gpasswd -a $NB_USER docker - -RUN DEBIAN_FRONTEND=noninteractive apt-get install --fix-missing -y clang-format COPY ./ /clipper @@ -29,6 +56,6 @@ RUN cd /clipper/src/libs/spdlog \ && git apply ../patches/make_spdlog_compile_linux.patch -ENTRYPOINT ["/clipper/bin/check_format.sh"] +ENTRYPOINT ["/clipper/bin/ci_checks.sh"] # vim: set filetype=dockerfile: diff --git a/bin/build_docker_images.sh b/bin/build_docker_images.sh index 67d1f9b84..467a73ead 100755 --- a/bin/build_docker_images.sh +++ b/bin/build_docker_images.sh @@ -18,10 +18,9 @@ tag=$( Date: Wed, 30 Aug 2017 09:45:43 -0700 Subject: [PATCH 6/7] move CI docker command to script --- bin/run_ci.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 bin/run_ci.sh diff --git a/bin/run_ci.sh b/bin/run_ci.sh new file mode 100755 index 000000000..d83487426 --- /dev/null +++ b/bin/run_ci.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e +set -u + +docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp clipper/unittests:develop From 09170df895b666adc7a8de3255e674cc8cf49a5f Mon Sep 17 00:00:00 2001 From: Dan Crankshaw Date: Wed, 30 Aug 2017 10:36:38 -0700 Subject: [PATCH 7/7] added todo --- bin/run_ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/run_ci.sh b/bin/run_ci.sh index d83487426..662125e44 100755 --- a/bin/run_ci.sh +++ b/bin/run_ci.sh @@ -3,4 +3,5 @@ set -e set -u +# TODO: Remove the /tmp volume mount in the Clipper admin refactor PR docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp clipper/unittests:develop