build: external deps build on Windows

bazel/README.md

@@ -27,16 +27,15 @@ up-to-date with the latest security patches. See
 for how to update or override dependencies.
 
 1. Install the latest version of [Bazel](https://bazel.build/versions/master/docs/install.html) in your environment.
-2. Install external dependencies libtool, cmake, and realpath libraries separately.
+2. Install external dependencies libtool, cmake, ninja, and realpath libraries separately.
 On Ubuntu, run the following commands:
 ```
  apt-get install libtool
  apt-get install cmake
  apt-get install realpath
  apt-get install clang-format-5.0
- apt-get install autoconf
  apt-get install automake
- apt-get install pkg-config
+ apt-get install ninja-build
 ```
 
 On Fedora (maybe also other red hat distros), run the following:
@@ -52,9 +51,8 @@ brew install cmake
 brew install libtool
 brew install go
 brew install bazel
-brew install autoconf
 brew install automake
-brew install pkg-config
+brew install ninja
 ```
 
 Envoy compiles and passes tests with the version of clang installed by XCode 9.3.0:

ci/build_container/build_container_centos.sh

@@ -9,7 +9,7 @@ curl -L -o /etc/yum.repos.d/alonid-llvm-5.0.0-epel-7.repo \
 
 # dependencies for bazel and build_recipes
 yum install -y java-1.8.0-openjdk-devel unzip which openssl rpm-build \
-               cmake3 devtoolset-4-gcc-c++ git golang libtool make patch rsync wget \
+               cmake3 devtoolset-4-gcc-c++ git golang libtool make ninja-build patch rsync wget \
                clang-5.0.0 devtoolset-4-libatomic-devel llvm-5.0.0 python-virtualenv bc
 yum clean all
 

ci/build_container/build_container_ubuntu.sh

@@ -6,7 +6,7 @@ set -e
 apt-get update
 export DEBIAN_FRONTEND=noninteractive
 apt-get install -y wget software-properties-common make cmake git python python-pip \
-  bc libtool autoconf automake zip time golang g++ gdb strace wireshark tshark
+  bc libtool ninja-build automake zip time golang g++ gdb strace wireshark tshark
 # clang head (currently 5.0)
 wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main"

ci/build_container/build_recipes/benchmark.sh

@@ -8,11 +8,17 @@ git clone https://github.com/google/benchmark.git
 mkdir build
 
 cd build
-cmake -G "Unix Makefiles" ../benchmark \
+cmake -G "Ninja" ../benchmark \
   -DCMAKE_BUILD_TYPE=RELEASE \
   -DBENCHMARK_ENABLE_GTEST_TESTS=OFF
-make
-cp src/libbenchmark.a "$THIRDPARTY_BUILD"/lib
+ninja
+
+benchmark_lib="libbenchmark.a"
+if [[ "${OS}" == "Windows_NT" ]]; then
+  benchmark_lib="benchmark.lib"
+fi
+
+cp "src/$benchmark_lib" "$THIRDPARTY_BUILD"/lib
 cd ../benchmark
 
 INCLUDE_DIR="$THIRDPARTY_BUILD/include/testing/base/public"

ci/build_container/build_recipes/cares.sh

@@ -10,10 +10,31 @@ VERSION=cares-1_14_0
 CPPFLAGS="$(for f in $CXXFLAGS; do if [[ $f =~ -D.* ]]; then echo $f; fi; done | tr '\n' ' ')"
 CFLAGS="$(for f in $CXXFLAGS; do if [[ ! $f =~ -D.* ]]; then echo $f; fi; done | tr '\n' ' ')"
 
-wget -O c-ares-"$VERSION".tar.gz https://github.com/c-ares/c-ares/archive/"$VERSION".tar.gz
+curl https://github.com/c-ares/c-ares/archive/"$VERSION".tar.gz -sLo c-ares-"$VERSION".tar.gz
 tar xf c-ares-"$VERSION".tar.gz
 cd c-ares-"$VERSION"
-./buildconf
-./configure --prefix="$THIRDPARTY_BUILD" --enable-shared=no --enable-lib-only \
-  --enable-debug --enable-optimize
-make V=1 install
+
+mkdir build
+cd build
+
+build_type=RelWithDebInfo
+if [[ "${OS}" == "Windows_NT" ]]; then
+  # On Windows, every object file in the final executable needs to be compiled to use the
+  # same version of the C Runtime Library. If Envoy is built with '-c dbg', then it will
+  # use the Debug C Runtime Library. Setting CMAKE_BUILD_TYPE to Debug will cause c-ares
+  # to use the debug version as well
+  # TODO: when '-c fastbuild' and '-c opt' work for Windows builds, set this appropriately
+  build_type=Debug
+fi
+
+cmake -G "Ninja" -DCMAKE_INSTALL_PREFIX="$THIRDPARTY_BUILD" \
+  -DCARES_SHARED=no \
+  -DCARES_STATIC=on \
+  -DCMAKE_BUILD_TYPE="$build_type" \
+  ..
+ninja
+ninja install
+
+if [[ "${OS}" == "Windows_NT" ]]; then
+  cp "CMakeFiles/c-ares.dir/c-ares.pdb" "$THIRDPARTY_BUILD/lib/c-ares.pdb"
+fi

ci/build_container/build_recipes/gperftools.sh

@@ -2,9 +2,13 @@
 
 set -e
 
+if [[ "${OS}" == "Windows_NT" ]]; then
+  exit 0
+fi
+
 VERSION=2.7
 
-wget -O gperftools-"$VERSION".tar.gz https://github.com/gperftools/gperftools/releases/download/gperftools-"$VERSION"/gperftools-"$VERSION".tar.gz
+curl https://github.com/gperftools/gperftools/releases/download/gperftools-"$VERSION"/gperftools-"$VERSION".tar.gz -sLo gperftools-"$VERSION".tar.gz
 tar xf gperftools-"$VERSION".tar.gz
 cd gperftools-"$VERSION"
 

ci/build_container/build_recipes/libevent.sh

@@ -4,8 +4,33 @@ set -e
 
 VERSION=2.1.8-stable
 
-wget -O libevent-"$VERSION".tar.gz https://github.com/libevent/libevent/releases/download/release-"$VERSION"/libevent-"$VERSION".tar.gz
-tar xf libevent-"$VERSION".tar.gz
-cd libevent-"$VERSION"
-./configure --prefix="$THIRDPARTY_BUILD" --enable-shared=no --disable-libevent-regress --disable-openssl
-make V=1 install
+curl https://github.com/libevent/libevent/archive/release-"$VERSION".tar.gz -sLo libevent-release-"$VERSION".tar.gz
+tar xf libevent-release-"$VERSION".tar.gz
+cd libevent-release-"$VERSION"
+
+mkdir build
+cd build
+
+# libevent defaults CMAKE_BUILD_TYPE to Release
+build_type=Release
+if [[ "${OS}" == "Windows_NT" ]]; then
+  # On Windows, every object file in the final executable needs to be compiled to use the
+  # same version of the C Runtime Library. If Envoy is built with '-c dbg', then it will
+  # use the Debug C Runtime Library. Setting CMAKE_BUILD_TYPE to Debug will cause libevent
+  # to use the debug version as well
+  # TODO: when '-c fastbuild' and '-c opt' work for Windows builds, set this appropriately
+  build_type=Debug
+fi
+
+cmake -G "Ninja" \
+  -DCMAKE_INSTALL_PREFIX="$THIRDPARTY_BUILD" \
+  -DEVENT__DISABLE_OPENSSL:BOOL=on \
+  -DEVENT__DISABLE_REGRESS:BOOL=on \
+  -DCMAKE_BUILD_TYPE="$build_type" \
+  ..
+ninja
+ninja install
+
+if [[ "${OS}" == "Windows_NT" ]]; then
+  cp "CMakeFiles/event.dir/event.pdb" "$THIRDPARTY_BUILD/lib/event.pdb"
+fi

ci/build_container/build_recipes/luajit.sh

@@ -4,7 +4,7 @@ set -e
 
 VERSION=2.0.5
 
-wget -O LuaJIT-"$VERSION".tar.gz https://github.com/LuaJIT/LuaJIT/archive/v"$VERSION".tar.gz
+curl https://github.com/LuaJIT/LuaJIT/archive/v"$VERSION".tar.gz -sLo LuaJIT-"$VERSION".tar.gz
 tar xf LuaJIT-"$VERSION".tar.gz
 cd LuaJIT-"$VERSION"
 
@@ -46,15 +46,26 @@ index f7f81a4..e698517 100644
  # Disable the JIT compiler, i.e. turn LuaJIT into a pure interpreter.
  #XCFLAGS+= -DLUAJIT_DISABLE_JIT
 @@ -564,7 +564,7 @@ endif
- 
+
  Q= @
  E= @echo
 -#Q=
 +Q=
  #E= @:
- 
+
  ##############################################################################
 EOF
-patch -p1 < ../luajit_make.diff
 
-DEFAULT_CC=${CC} TARGET_CFLAGS=${CFLAGS} TARGET_LDFLAGS=${CFLAGS} CFLAGS="" make V=1 PREFIX="$THIRDPARTY_BUILD" install
+if [[ "${OS}" == "Windows_NT" ]]; then
+  cd src
+  ./msvcbuild.bat debug
+
+  mkdir -p "$THIRDPARTY_BUILD/include/luajit-2.0"
+  cp *.h* "$THIRDPARTY_BUILD/include/luajit-2.0"
+  cp luajit.lib "$THIRDPARTY_BUILD/lib"
+  cp *.pdb "$THIRDPARTY_BUILD/lib"
+else
+  patch -p1 < ../luajit_make.diff
+
+  DEFAULT_CC=${CC} TARGET_CFLAGS=${CFLAGS} TARGET_LDFLAGS=${CFLAGS} CFLAGS="" make V=1 PREFIX="$THIRDPARTY_BUILD" install
+fi

ci/build_container/build_recipes/nghttp2.sh

@@ -7,11 +7,41 @@ set -e
 # TODO(PiotrSikora): switch back to releases once v1.33.0 is out.
 VERSION=e5b3f9addd49bca27e2f99c5c65a564eb5c0cf6d  # 2018-06-09
 
-wget -O nghttp2-"$VERSION".tar.gz https://github.com/nghttp2/nghttp2/archive/"$VERSION".tar.gz
+curl https://github.com/nghttp2/nghttp2/archive/"$VERSION".tar.gz -sLo nghttp2-"$VERSION".tar.gz
 tar xf nghttp2-"$VERSION".tar.gz
 cd nghttp2-"$VERSION"
-autoreconf -i
-automake
-autoconf
-./configure --prefix="$THIRDPARTY_BUILD" --enable-shared=no --enable-lib-only
-make V=1 install
+
+# Allow nghttp2 to build as static lib on Windows
+# TODO: remove once https://github.com/nghttp2/nghttp2/pull/1198 is merged
+cat > nghttp2_cmakelists.diff << 'EOF'
+diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
+index 17e422b2..e58070f5 100644
+--- a/lib/CMakeLists.txt
++++ b/lib/CMakeLists.txt
+@@ -56,6 +56,7 @@ if(HAVE_CUNIT OR ENABLE_STATIC_LIB)
+     COMPILE_FLAGS "${WARNCFLAGS}"
+     VERSION ${LT_VERSION} SOVERSION ${LT_SOVERSION}
+     ARCHIVE_OUTPUT_NAME nghttp2
++    ARCHIVE_OUTPUT_DIRECTORY static
+   )
+   target_compile_definitions(nghttp2_static PUBLIC "-DNGHTTP2_STATICLIB")
+   if(ENABLE_STATIC_LIB)
+EOF
+
+if [[ "${OS}" == "Windows_NT" ]]; then
+  git apply nghttp2_cmakelists.diff
+fi
+
+mkdir build
+cd build
+
+cmake -G "Ninja" -DCMAKE_INSTALL_PREFIX="$THIRDPARTY_BUILD" \
+  -DENABLE_STATIC_LIB=on \
+  -DENABLE_LIB_ONLY=on \
+  ..
+ninja
+ninja install
+
+if [[ "${OS}" == "Windows_NT" ]]; then
+  cp "lib/CMakeFiles/nghttp2_static.dir/nghttp2_static.pdb" "$THIRDPARTY_BUILD/lib/nghttp2_static.pdb"
+fi

ci/build_container/build_recipes/yaml-cpp.sh

@@ -4,11 +4,31 @@ set -e
 
 VERSION=0.6.2
 
-wget -O yaml-cpp-"$VERSION".tar.gz https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-"$VERSION".tar.gz
+curl https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-"$VERSION".tar.gz -sLo yaml-cpp-"$VERSION".tar.gz
 tar xf yaml-cpp-"$VERSION".tar.gz
 cd yaml-cpp-yaml-cpp-"$VERSION"
-cmake -DCMAKE_INSTALL_PREFIX:PATH="$THIRDPARTY_BUILD" \
+
+mkdir build
+cd build
+
+build_type=RelWithDebInfo
+if [[ "${OS}" == "Windows_NT" ]]; then
+  # On Windows, every object file in the final executable needs to be compiled to use the
+  # same version of the C Runtime Library. If Envoy is built with '-c dbg', then it will
+  # use the Debug C Runtime Library. Setting CMAKE_BUILD_TYPE to Debug will cause yaml-cpp
+  # to use the debug version as well
+  # TODO: when '-c fastbuild' and '-c opt' work for Windows builds, set this appropriately
+  build_type=Debug
+fi
+
+cmake -G "Ninja" -DCMAKE_INSTALL_PREFIX:PATH="$THIRDPARTY_BUILD" \
   -DCMAKE_CXX_FLAGS:STRING="${CXXFLAGS} ${CPPFLAGS}" \
   -DCMAKE_C_FLAGS:STRING="${CFLAGS} ${CPPFLAGS}" \
-  -DCMAKE_BUILD_TYPE=RelWithDebInfo .
-make VERBOSE=1 install
+  -DYAML_CPP_BUILD_TESTS=off \
+  -DCMAKE_BUILD_TYPE="$build_type" \
+  ..
+ninja install
+
+if [[ "${OS}" == "Windows_NT" ]]; then
+  cp "CMakeFiles/yaml-cpp.dir/yaml-cpp.pdb" "$THIRDPARTY_BUILD/lib/yaml-cpp.pdb"
+fi

ci/build_container/build_recipes/zlib.sh

@@ -4,8 +4,15 @@ set -e
 
 VERSION=1.2.11
 
-wget -O zlib-"$VERSION".tar.gz https://github.com/madler/zlib/archive/v"$VERSION".tar.gz
+curl https://github.com/madler/zlib/archive/v"$VERSION".tar.gz -sLo zlib-"$VERSION".tar.gz
 tar xf zlib-"$VERSION".tar.gz
 cd zlib-"$VERSION"
-./configure --prefix="$THIRDPARTY_BUILD"
-make V=1 install
+mkdir build
+cd build
+cmake -G "Ninja" -DCMAKE_INSTALL_PREFIX:PATH="$THIRDPARTY_BUILD" ..
+ninja
+ninja install
+
+if [[ "${OS}" == "Windows_NT" ]]; then
+  cp "CMakeFiles/zlibstatic.dir/zlibstatic.pdb" "$THIRDPARTY_BUILD/lib/zlibstatic.pdb"
+fi

ci/mac_ci_setup.sh

@@ -21,7 +21,7 @@ if ! brew update; then
     exit 1
 fi
 
-DEPS="automake bazel cmake coreutils go libtool wget"
+DEPS="automake bazel cmake coreutils go libtool wget ninja"
 for DEP in ${DEPS}
 do
     is_installed "${DEP}" || install "${DEP}"

ci/prebuilt/BUILD

@@ -24,8 +24,12 @@ cc_library(
 )
 
 cc_library(
+    # TODO: Remove 'event_pthreads' once no BUIlD files use it
+    # glob here is due to the fact that ninja doesn't produce a libevent_pthreads.a file -
+    # all the symbols are included in libevent.a. This is a hack so that CI can pass before
+    # https://github.com/envoyproxy/envoy/pull/3909 is merged as well
     name = "event_pthreads",
-    srcs = ["thirdparty_build/lib/libevent_pthreads.a"],
+    srcs = glob(["thirdparty_build/lib/libevent_pthreads.a"]),
     deps = [":event"],
 )