WIP - Add x86_64-musl as a host architecture

.travis.yml

@@ -17,7 +17,7 @@ matrix:
   include:
     # Images used in testing PR and try-build should be run first.
     - env: IMAGE=x86_64-gnu-llvm-6.0 RUST_BACKTRACE=1
-      if: type = pull_request OR branch = auto
+      if: branch = auto
 
     - env: IMAGE=dist-x86_64-linux DEPLOY=1
       if: branch = try OR branch = auto
@@ -159,7 +159,7 @@ matrix:
     - env: IMAGE=dist-x86_64-freebsd DEPLOY=1
       if: branch = auto
     - env: IMAGE=dist-x86_64-musl DEPLOY=1
-      if: branch = auto
+#      if: branch = auto
     - env: IMAGE=dist-x86_64-netbsd DEPLOY=1
       if: branch = auto
     - env: IMAGE=asmjs
@@ -185,7 +185,7 @@ matrix:
     - env: IMAGE=x86_64-gnu-distcheck
       if: branch = auto
     - env: IMAGE=mingw-check
-      if: type = pull_request OR branch = auto
+      if: branch = auto
 
     - stage: publish toolstate
       if: branch = master AND type = push

src/ci/docker/dist-x86_64-musl/Dockerfile

@@ -4,6 +4,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   make \
   file \
+  wget \
   curl \
   ca-certificates \
   python2.7 \
@@ -18,19 +19,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 WORKDIR /build/
 
-COPY scripts/musl.sh /build/
+COPY scripts/musl-toolchain.sh /build/
 # We need to mitigate rust-lang/rust#34978 when compiling musl itself as well
-RUN CC=gcc \
-    CFLAGS="-Wa,-mrelax-relocations=no" \
-    CXX=g++ \
-    CXXFLAGS="-Wa,-mrelax-relocations=no" \
-    bash musl.sh x86_64 && rm -rf /build
+# TODO: Check what this issue is and if we can ignore it
+
+RUN bash musl-toolchain.sh x86_64-linux-musl && rm -rf build
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV RUST_CONFIGURE_ARGS \
-      --musl-root-x86_64=/musl-x86_64 \
+      --musl-root-x86_64=/usr/local/x86_64-linux-musl \
       --enable-extended \
       --disable-docs
 
@@ -39,8 +38,15 @@ ENV RUST_CONFIGURE_ARGS \
 # way to produce "super compatible" binaries.
 #
 # See: https://github.com/rust-lang/rust/issues/34978
-ENV CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no
+#ENV CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no
+
+ENV HOSTS=x86_64-unknown-linux-musl \
+    CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc \
+    CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++
+
+#    CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_LINKER=musl-gcc \
+#    CARGO_TARGET_ARM_UNKNOWN_LINUX_MUSLEABIHF_RUNNER="qemu-arm -L /musl-arm"
 
 ENV SCRIPT \
-      python2.7 ../x.py test --target x86_64-unknown-linux-musl && \
-      python2.7 ../x.py dist --target x86_64-unknown-linux-musl
+      python2.7 ../x.py test --host $HOSTS --target $HOSTS && \
+      python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/scripts/musl-toolchain.sh

@@ -0,0 +1,77 @@
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  rm /tmp/build.log
+  set -x
+}
+
+TARGET=$1
+#ARCH=$1
+#TARGET=linux-musl-$ARCH
+ARCH=x86_64
+
+OUTPUT=/usr/local
+shift
+
+git clone https://github.com/richfelker/musl-cross-make -b v0.9.7
+cd musl-cross-make
+
+hide_output make -j$(nproc) TARGET=$TARGET
+hide_output make install TARGET=$TARGET OUTPUT=$OUTPUT
+
+cd -
+
+# Make musl binaries executable
+
+ln -s $OUTPUT/$TARGET/lib/libc.so /lib/ld-musl-$ARCH.so.1
+echo $OUTPUT/$TARGET/lib >> /etc/ld-musl-$ARCH.path
+
+
+export CC=$TARGET-gcc
+export CXX=$TARGET-g++
+export CFLAGS="-fPIC $CFLAGS"
+
+LLVM=60
+
+# may have been downloaded in a previous run
+if [ ! -d libunwind-release_$LLVM ]; then
+  curl -L https://github.com/llvm-mirror/llvm/archive/release_$LLVM.tar.gz | tar xzf -
+  curl -L https://github.com/llvm-mirror/libunwind/archive/release_$LLVM.tar.gz | tar xzf -
+fi
+
+mkdir libunwind-build
+cd libunwind-build
+cmake ../libunwind-release_$LLVM \
+          -DLLVM_PATH=/build/llvm-release_$LLVM \
+          -DLIBUNWIND_ENABLE_SHARED=0 \
+          -DCMAKE_C_COMPILER=$CC \
+          -DCMAKE_CXX_COMPILER=$CXX \
+          -DCMAKE_C_FLAGS="$CFLAGS" \
+          -DCMAKE_CXX_FLAGS="$CXXFLAGS"
+
+hide_output make -j$(nproc)
+cp lib/libunwind.a $OUTPUT/$TARGET/lib
+cd - && rm -rf libunwind-build
+

src/librustc_codegen_llvm/back/link.rs

@@ -916,13 +916,10 @@ fn link_args(cmd: &mut dyn Linker,
         let mut position_independent_executable = false;
 
         if t.options.position_independent_executables {
-            let empty_vec = Vec::new();
-            let args = sess.opts.cg.link_args.as_ref().unwrap_or(&empty_vec);
-            let more_args = &sess.opts.cg.link_arg;
-            let mut args = args.iter().chain(more_args.iter()).chain(used_link_args.iter());
 
+            let static_pie = t.options.static_position_independent_executables;
             if get_reloc_model(sess) == llvm::RelocMode::PIC
-                && !sess.crt_static() && !args.any(|x| *x == "-static") {
+                && (!sess.crt_static() || static_pie) {
                 position_independent_executable = true;
             }
         }

src/librustc_target/spec/linux_musl_base.rs

@@ -1,34 +1,34 @@
-use spec::{LinkerFlavor, TargetOptions};
+use spec::{LinkerFlavor, TargetOptions, RelroLevel};
 
 pub fn opts() -> TargetOptions {
     let mut base = super::linux_base::opts();
 
-    // Make sure that the linker/gcc really don't pull in anything, including
-    // default objects, libs, etc.
-    base.pre_link_args_crt.insert(LinkerFlavor::Gcc, Vec::new());
-    base.pre_link_args_crt.get_mut(&LinkerFlavor::Gcc).unwrap().push("-nostdlib".to_string());
+    base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-fPIC".to_string());
 
     // At least when this was tested, the linker would not add the
     // `GNU_EH_FRAME` program header to executables generated, which is required
     // when unwinding to locate the unwinding information. I'm not sure why this
     // argument is *not* necessary for normal builds, but it can't hurt!
     base.pre_link_args.get_mut(&LinkerFlavor::Gcc).unwrap().push("-Wl,--eh-frame-hdr".to_string());
 
-    // When generating a statically linked executable there's generally some
-    // small setup needed which is listed in these files. These are provided by
-    // a musl toolchain and are linked by default by the `musl-gcc` script. Note
-    // that `gcc` also does this by default, it just uses some different files.
-    //
-    // Each target directory for musl has these object files included in it so
-    // they'll be included from there.
-    base.pre_link_objects_exe_crt.push("crt1.o".to_string());
-    base.pre_link_objects_exe_crt.push("crti.o".to_string());
-    base.post_link_objects_crt.push("crtn.o".to_string());
-
     // These targets statically link libc by default
     base.crt_static_default = true;
+
     // These targets allow the user to choose between static and dynamic linking.
     base.crt_static_respected = true;
 
+    base.crt_static_allows_dylibs = true;
+
+    // Static position-independent executables are supported.
+    base.static_position_independent_executables = true;
+
+    // Defaults for dynamic linking
+    base.dynamic_linking = true;
+    base.executables = true;
+    base.has_elf_tls = true;
+    base.has_rpath = true;
+    base.position_independent_executables = true;
+    base.relro_level = RelroLevel::Full;
+
     base
 }

src/librustc_target/spec/mod.rs

@@ -580,6 +580,8 @@ pub struct TargetOptions {
     /// the functions in the executable are not randomized and can be used
     /// during an exploit of a vulnerability in any code.
     pub position_independent_executables: bool,
+    /// Support for static position independent executables.
+    pub static_position_independent_executables: bool,
     /// Determines if the target always requires using the PLT for indirect
     /// library calls or not. This controls the default value of the `-Z plt` flag.
     pub needs_plt: bool,
@@ -735,6 +737,7 @@ impl Default for TargetOptions {
             has_rpath: false,
             no_default_libraries: true,
             position_independent_executables: false,
+            static_position_independent_executables: false,
             needs_plt: false,
             relro_level: RelroLevel::None,
             pre_link_objects_exe: Vec::new(),
@@ -1035,6 +1038,7 @@ impl Target {
         key!(has_rpath, bool);
         key!(no_default_libraries, bool);
         key!(position_independent_executables, bool);
+        key!(static_position_independent_executables, bool);
         key!(needs_plt, bool);
         key!(relro_level, RelroLevel)?;
         key!(archive_format);
@@ -1246,6 +1250,7 @@ impl ToJson for Target {
         target_option_val!(has_rpath);
         target_option_val!(no_default_libraries);
         target_option_val!(position_independent_executables);
+        target_option_val!(static_position_independent_executables);
         target_option_val!(needs_plt);
         target_option_val!(relro_level);
         target_option_val!(archive_format);

src/tools/compiletest/src/runtest.rs

@@ -1566,18 +1566,15 @@ impl<'test> TestCx<'test> {
                 None
             } else if self.config.target.contains("cloudabi")
                 || self.config.target.contains("emscripten")
-                || (self.config.target.contains("musl") && !aux_props.force_host)
                 || self.config.target.contains("wasm32")
             {
                 // We primarily compile all auxiliary libraries as dynamic libraries
                 // to avoid code size bloat and large binaries as much as possible
                 // for the test suite (otherwise including libstd statically in all
                 // executables takes up quite a bit of space).
                 //
-                // For targets like MUSL or Emscripten, however, there is no support for
-                // dynamic libraries so we just go back to building a normal library. Note,
-                // however, that for MUSL if the library is built with `force_host` then
-                // it's ok to be a dylib as the host should always support dylibs.
+                // For targets like Emscripten, however, there is no support for
+                // dynamic libraries so we just go back to building a normal library.
                 Some("lib")
             } else {
                 Some("dylib")