diff --git a/.travis.yml b/.travis.yml index a1bbb8a884fef..996e5ec07b545 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -language: rust +language: minimal sudo: required dist: trusty services: @@ -20,7 +20,7 @@ matrix: - env: IMAGE=x86_64-gnu-cargotest - env: IMAGE=x86_64-gnu-debug - env: IMAGE=x86_64-gnu-nopt - - env: IMAGE=x86_64-gnu-rustbuild + - env: IMAGE=x86_64-gnu-make - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1 - env: IMAGE=x86_64-musl @@ -39,7 +39,7 @@ matrix: install: brew install ccache - env: > RUST_CHECK_TARGET=check - RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin --enable-rustbuild + RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin --disable-rustbuild SRC=. os: osx install: brew install ccache @@ -51,17 +51,16 @@ matrix: install: brew install ccache script: - - if [ -z "$ALLOW_PR" ] && [ "$TRAVIS_BRANCH" != "auto" ]; then - echo skipping, not a full build; - elif [ -z "$ENABLE_AUTO" ] then - echo skipping, not quite ready yet - elif [ "$TRAVIS_OS_NAME" = "osx" ]; then - git submodule update --init; - src/ci/run.sh; - else - git submodule update --init; - src/ci/docker/run.sh $IMAGE; - fi + - > + if [ "$ALLOW_PR" = "" ] && [ "$TRAVIS_BRANCH" != "auto" ]; then + echo skipping, not a full build; + elif [ "$TRAVIS_OS_NAME" = "osx" ]; then + git submodule update --init; + src/ci/run.sh; + else + git submodule update --init; + src/ci/docker/run.sh $IMAGE; + fi # Save tagged docker images we created and load them if they're available before_cache: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c0f93c3703a5..20a0bd2e256d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,13 +86,17 @@ benchmarks, generate documentation, install a fresh build of Rust, and more. It's your best friend when working on Rust, allowing you to compile & test your contributions before submission. -All the configuration for the build system lives in [the `mk` directory][mkdir] -in the project root. It can be hard to follow in places, as it uses some -advanced Make features which make for some challenging reading. If you have -questions on the build system internals, try asking in -[`#rust-internals`][pound-rust-internals]. +The build system lives in [the `src/bootstrap` directory][bootstrap] in the +project root. Our build system is itself written in Rust and is based on Cargo +to actually build all the compiler's crates. If you have questions on the build +system internals, try asking in [`#rust-internals`][pound-rust-internals]. -[mkdir]: https://github.com/rust-lang/rust/tree/master/mk/ +[bootstrap]: https://github.com/rust-lang/rust/tree/master/src/bootstrap/ + +> **Note**: the build system was recently rewritten from a jungle of makefiles +> to the current incarnation you'll see in `src/bootstrap`. If you experience +> bugs you can temporarily revert back to the makefiles with +> `--disable-rustbuild` passed to `./configure`. ### Configuration @@ -119,42 +123,111 @@ configuration used later in the build process. Some options to note: To see a full list of options, run `./configure --help`. -### Useful Targets - -Some common make targets are: - -- `make tips` - show useful targets, variables and other tips for working with - the build system. -- `make rustc-stage1` - build up to (and including) the first stage. For most - cases we don't need to build the stage2 compiler, so we can save time by not - building it. The stage1 compiler is a fully functioning compiler and - (probably) will be enough to determine if your change works as expected. -- `make $host/stage1/bin/rustc` - Where $host is a target triple like x86_64-unknown-linux-gnu. - This will build just rustc, without libstd. This is the fastest way to recompile after - you changed only rustc source code. Note however that the resulting rustc binary - won't have a stdlib to link against by default. You can build libstd once with - `make rustc-stage1`, rustc will pick it up afterwards. libstd is only guaranteed to - work if recompiled, so if there are any issues recompile it. -- `make check` - build the full compiler & run all tests (takes a while). This +### Building + +Although the `./configure` script will generate a `Makefile`, this is actually +just a thin veneer over the actual build system driver, `x.py`. This file, at +the root of the repository, is used to build, test, and document various parts +of the compiler. You can execute it as: + +```sh +python x.py build +``` + +On some systems you can also use the shorter version: + +```sh +./x.py build +``` + +To learn more about the driver and top-level targets, you can execute: + +```sh +python x.py --help +``` + +The general format for the driver script is: + +```sh +python x.py [] +``` + +Some example commands are `build`, `test`, and `doc`. These will build, test, +and document the specified directory. The second argument, ``, is +optional and defaults to working over the entire compiler. If specified, +however, only that specific directory will be built. For example: + +```sh +# build the entire compiler +python x.py build + +# build all documentation +python x.py doc + +# run all test suites +python x.py test + +# build only the standard library +python x.py build src/libstd + +# test only one particular test suite +python x.py test src/test/rustdoc + +# build only the stage0 libcore library +python x.py build src/libcore --stage 0 +``` + +You can explore the build system throught the various `--help` pages for each +subcommand. For example to learn more about a command you can run: + +``` +python x.py build --help +``` + +To learn about all possible rules you can execute, run: + +``` +python x.py build --help --verbose +``` + +### Useful commands + +Some common invocations of `x.py` are: + +- `x.py build --help` - show the help message and explain the subcommand +- `x.py build src/libtest --stage 1` - build up to (and including) the first + stage. For most cases we don't need to build the stage2 compiler, so we can + save time by not building it. The stage1 compiler is a fully functioning + compiler and (probably) will be enough to determine if your change works as + expected. +- `x.py build src/rustc --stage 1` - This will build just rustc, without libstd. + This is the fastest way to recompile after you changed only rustc source code. + Note however that the resulting rustc binary won't have a stdlib to link + against by default. You can build libstd once with `x.py build src/libstd`, + but it is is only guaranteed to work if recompiled, so if there are any issues + recompile it. +- `x.py test` - build the full compiler & run all tests (takes a while). This is what gets run by the continuous integration system against your pull request. You should run this before submitting to make sure your tests pass & everything builds in the correct manner. -- `make check-stage1-std NO_REBUILD=1` - test the standard library without - rebuilding the entire compiler -- `make check TESTNAME=` - Run a matching set of tests. +- `x.py test src/libstd --stage 1` - test the standard library without + recompiling stage 2. +- `x.py test src/test/run-pass --filter TESTNAME` - Run a matching set of tests. - `TESTNAME` should be a substring of the tests to match against e.g. it could be the fully qualified test name, or just a part of it. `TESTNAME=collections::hash::map::test_map::test_capacity_not_less_than_len` or `TESTNAME=test_capacity_not_less_than_len`. -- `make check-stage1-rpass TESTNAME=` - Run a single - rpass test with the stage1 compiler (this will be quicker than running the - command above as we only build the stage1 compiler, not the entire thing). - You can also leave off the `-rpass` to run all stage1 test types. -- `make check-stage1-coretest` - Run stage1 tests in `libcore`. -- `make tidy` - Check that the source code is in compliance with Rust's style - guidelines. There is no official document describing Rust's full guidelines - as of yet, but basic rules like 4 spaces for indentation and no more than 99 - characters in a single line should be kept in mind when writing code. +- `x.py test src/test/run-pass --stage 1 --filter ` - + Run a single rpass test with the stage1 compiler (this will be quicker than + running the command above as we only build the stage1 compiler, not the entire + thing). You can also leave off the directory argument to run all stage1 test + types. +- `x.py test src/libcore --stage 1` - Run stage1 tests in `libcore`. +- `x.py test src/tools/tidy` - Check that the source code is in compliance with + Rust's style guidelines. There is no official document describing Rust's full + guidelines as of yet, but basic rules like 4 spaces for indentation and no + more than 99 characters in a single line should be kept in mind when writing + code. ## Pull Requests @@ -172,19 +245,17 @@ amount of time you have to wait. You need to have built the compiler at least once before running these will work, but that’s only one full build rather than one each time. - $ make -j8 rustc-stage1 && make check-stage1 + $ python x.py test --stage 1 is one such example, which builds just `rustc`, and then runs the tests. If you’re adding something to the standard library, try - $ make -j8 check-stage1-std NO_REBUILD=1 - -This will not rebuild the compiler, but will run the tests. + $ python x.py test src/libstd --stage 1 Please make sure your pull request is in compliance with Rust's style guidelines by running - $ make tidy + $ python x.py test src/tools/tidy Make this check before every pull request (and every new commit in a pull request) ; you can add [git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) diff --git a/README.md b/README.md index 7360651095bb5..2133b17de0fbb 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,14 @@ Read ["Installing Rust"] from [The Book]. ```sh $ ./configure - $ make && make install + $ make && sudo make install ``` - > ***Note:*** You may need to use `sudo make install` if you do not - > normally have permission to modify the destination directory. The - > install locations can be adjusted by passing a `--prefix` argument - > to `configure`. Various other options are also supported – pass + > ***Note:*** Install locations can be adjusted by passing a `--prefix` + > argument to `configure`. Various other options are also supported – pass > `--help` for more information on them. - When complete, `make install` will place several programs into + When complete, `sudo make install` will place several programs into `/usr/local/bin`: `rustc`, the Rust compiler, and `rustdoc`, the API-documentation tool. This install does not include [Cargo], Rust's package manager, which you may also want to build. @@ -108,30 +106,22 @@ MSVC builds of Rust additionally require an installation of Visual Studio 2013 (or later) so `rustc` can use its linker. Make sure to check the “C++ tools” option. -With these dependencies installed, the build takes two steps: +With these dependencies installed, you can build the compiler in a `cmd.exe` +shell with: ```sh -$ ./configure -$ make && make install +> python x.py build ``` -#### MSVC with rustbuild - -The old build system, based on makefiles, is currently being rewritten into a -Rust-based build system called rustbuild. This can be used to bootstrap the -compiler on MSVC without needing to install MSYS or MinGW. All you need are -[Python 2](https://www.python.org/downloads/), -[CMake](https://cmake.org/download/), and -[Git](https://git-scm.com/downloads) in your PATH (make sure you do not use the -ones from MSYS if you have it installed). You'll also need Visual Studio 2013 or -newer with the C++ tools. Then all you need to do is to kick off rustbuild. +If you're running inside of an msys shell, however, you can run: -``` -python x.py build +```sh +$ ./configure --build=x86_64-pc-windows-msvc +$ make && make install ``` -Currently rustbuild only works with some known versions of Visual Studio. If you -have a more recent version installed that a part of rustbuild doesn't understand +Currently building Rust only works with some known versions of Visual Studio. If +you have a more recent version installed the build system doesn't understand then you may need to force rustbuild to use an older version. This can be done by manually calling the appropriate vcvars file before running the bootstrap. @@ -149,16 +139,6 @@ $ ./configure $ make docs ``` -Building the documentation requires building the compiler, so the above -details will apply. Once you have the compiler built, you can - -```sh -$ make docs NO_REBUILD=1 -``` - -To make sure you don’t re-build the compiler because you made a change -to some documentation. - The generated documentation will appear in a top-level `doc` directory, created by the `make` rule. diff --git a/appveyor.yml b/appveyor.yml index 686c48abb30cd..bf75439b74a47 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,25 +2,22 @@ environment: matrix: # 32/64 bit MSVC - MSYS_BITS: 64 - TARGET: x86_64-pc-windows-msvc - CHECK: check - CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + RUST_CHECK_TARGET: check - MSYS_BITS: 32 - TARGET: i686-pc-windows-msvc - CHECK: check - CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + RUST_CHECK_TARGET: check - # MSVC rustbuild + # MSVC makefiles - MSYS_BITS: 64 - CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions - TARGET: x86_64-pc-windows-msvc - CHECK: check + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --disable-rustbuild + RUST_CHECK_TARGET: check # MSVC cargotest - MSYS_BITS: 64 - CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions - TARGET: x86_64-pc-windows-msvc - CHECK: check-cargotest + NO_VENDOR: 1 + RUST_CHECK_TARGET: check-cargotest + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc # 32/64-bit MinGW builds. # @@ -47,24 +44,22 @@ environment: # *not* use debug assertions and llvm assertions. This is because they take # too long on appveyor and this is tested by rustbuild below. - MSYS_BITS: 32 - TARGET: i686-pc-windows-gnu - CHECK: check + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + RUST_CHECK_TARGET: check MINGW_URL: https://s3.amazonaws.com/rust-lang-ci MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z MINGW_DIR: mingw32 - MSYS_BITS: 32 - CONFIGURE_ARGS: --enable-rustbuild --enable-llvm-assertions --enable-debug-assertions - TARGET: i686-pc-windows-gnu - CHECK: check + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild + RUST_CHECK_TARGET: check MINGW_URL: https://s3.amazonaws.com/rust-lang-ci MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z MINGW_DIR: mingw32 - MSYS_BITS: 64 - CONFIGURE_ARGS: --enable-llvm-assertions --enable-debug-assertions - TARGET: x86_64-pc-windows-gnu - CHECK: check + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu MINGW_URL: https://s3.amazonaws.com/rust-lang-ci MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z MINGW_DIR: mingw64 @@ -90,15 +85,20 @@ install: - if NOT defined MINGW_URL set PATH=C:\msys64\mingw%MSYS_BITS%\bin;C:\msys64\usr\bin;%PATH% test_script: - - sh ./configure - %CONFIGURE_ARGS% - --build=%TARGET% - - bash -c "make -j$(nproc)" - - bash -c "make %CHECK% -j$(nproc)" + - git submodule update --init + - set SRC=. + - set NO_CCACHE=1 + - sh src/ci/run.sh cache: - - build/%TARGET%/llvm -> src/rustllvm/llvm-auto-clean-trigger - - "%TARGET%/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "build/i686-pc-windows-gnu/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "build/x86_64-pc-windows-gnu/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "build/i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "build/x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "i686-pc-windows-gnu/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "x86_64-pc-windows-gnu/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger" + - "x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger" branches: only: diff --git a/configure b/configure index 483471604cb2a..a287291c2809c 100755 --- a/configure +++ b/configure @@ -631,7 +631,7 @@ opt stage0-landing-pads 1 "enable landing pads during bootstrap with stage0" opt dist-host-only 0 "only install bins for the host architecture" opt inject-std-version 1 "inject the current compiler version of libstd into programs" opt llvm-version-check 1 "check if the LLVM version is supported, build anyway" -opt rustbuild 0 "use the rust and cargo based build system" +opt rustbuild 1 "use the rust and cargo based build system" opt codegen-tests 1 "run the src/test/codegen tests" opt option-checking 1 "complain about unrecognized options in this configure script" opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)" @@ -664,11 +664,11 @@ valopt armv7-linux-androideabi-ndk "" "armv7-linux-androideabi NDK standalone pa valopt aarch64-linux-android-ndk "" "aarch64-linux-android NDK standalone path" valopt nacl-cross-path "" "NaCl SDK path (Pepper Canary is recommended). Must be absolute!" valopt musl-root "/usr/local" "MUSL root installation directory (deprecated)" -valopt musl-root-x86_64 "/usr/local" "x86_64-unknown-linux-musl install directory" -valopt musl-root-i686 "/usr/local" "i686-unknown-linux-musl install directory" -valopt musl-root-arm "/usr/local" "arm-unknown-linux-musleabi install directory" -valopt musl-root-armhf "/usr/local" "arm-unknown-linux-musleabihf install directory" -valopt musl-root-armv7 "/usr/local" "armv7-unknown-linux-musleabihf install directory" +valopt musl-root-x86_64 "" "x86_64-unknown-linux-musl install directory" +valopt musl-root-i686 "" "i686-unknown-linux-musl install directory" +valopt musl-root-arm "" "arm-unknown-linux-musleabi install directory" +valopt musl-root-armhf "" "arm-unknown-linux-musleabihf install directory" +valopt musl-root-armv7 "" "armv7-unknown-linux-musleabihf install directory" valopt extra-filename "" "Additional data that is hashed and passed to the -C extra-filename flag" if [ -e ${CFG_SRC_DIR}.git ] @@ -1374,7 +1374,7 @@ then fi fi -if [ -z "$CFG_ENABLE_RUSTBUILD" ]; then +if [ -n "$CFG_DISABLE_RUSTBUILD" ]; then step_msg "making directories" @@ -1474,7 +1474,7 @@ fi step_msg "configuring submodules" # Have to be in the top of src directory for this -if [ -z "$CFG_DISABLE_MANAGE_SUBMODULES" ] && [ -z "$CFG_ENABLE_RUSTBUILD" ] +if [ -z "$CFG_DISABLE_MANAGE_SUBMODULES" ] && [ -n "$CFG_DISABLE_RUSTBUILD" ] then cd ${CFG_SRC_DIR} @@ -1546,7 +1546,7 @@ do ;; esac - if [ -n "$CFG_ENABLE_RUSTBUILD" ] + if [ -z "$CFG_DISABLE_RUSTBUILD" ] then msg "not configuring LLVM, rustbuild in use" do_reconfigure=0 @@ -1871,7 +1871,7 @@ do putvar $CFG_LLVM_INST_DIR done -if [ -n "$CFG_ENABLE_RUSTBUILD" ] +if [ -z "$CFG_DISABLE_RUSTBUILD" ] then INPUT_MAKEFILE=src/bootstrap/mk/Makefile.in else @@ -1890,5 +1890,22 @@ else step_msg "complete" fi -msg "run \`make help\`" +if [ -z "$CFG_DISABLE_RUSTBUILD" ]; then + msg "NOTE you have now configured rust to use a rewritten build system" + msg " called rustbuild, and as a result this may have bugs that " + msg " you did not see before. If you experience any issues you can" + msg " go back to the old build system with --disable-rustbuild and" + msg " please feel free to report any bugs!" + msg "" + msg "run \`python x.py --help\`" +else + warn "the makefile-based build system is deprecated in favor of rustbuild" + msg "" + msg "It is recommended you avoid passing --disable-rustbuild to get your" + msg "build working as the makefiles will be deleted on 2017-02-02. If you" + msg "encounter bugs with rustbuild please file issues against rust-lang/rust" + msg "" + msg "run \`make help\`" +fi + msg diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 24d716c11958e..d0b501e4d8969 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -66,17 +66,6 @@ The script accepts commands, flags, and filters to determine what to do: * `doc` - a command for building documentation. Like above can take arguments for what to document. -If you're more used to `./configure` and `make`, however, then you can also -configure the build system to use rustbuild instead of the old makefiles: - -``` -./configure --enable-rustbuild -make -``` - -Afterwards the `Makefile` which is generated will have a few commands like -`make check`, `make tidy`, etc. - ## Configuring rustbuild There are currently two primary methods for configuring the rustbuild build @@ -90,6 +79,13 @@ be found at `src/bootstrap/config.toml.example`, and the configuration file can also be passed as `--config path/to/config.toml` if the build system is being invoked manually (via the python script). +Finally, rustbuild makes use of the [gcc-rs crate] which has [its own +method][env-vars] of configuring C compilers and C flags via environment +variables. + +[gcc-rs crate]: https://github.com/alexcrichton/gcc-rs +[env-vars]: https://github.com/alexcrichton/gcc-rs#external-configuration-via-environment-variables + ## Build stages The rustbuild build system goes through a few phases to actually build the @@ -273,16 +269,17 @@ After that, each module in rustbuild should have enough documentation to keep you up and running. Some general areas that you may be interested in modifying are: -* Adding a new build tool? Take a look at `build/step.rs` for examples of other - tools, as well as `build/mod.rs`. +* Adding a new build tool? Take a look at `bootstrap/step.rs` for examples of + other tools. * Adding a new compiler crate? Look no further! Adding crates can be done by adding a new directory with `Cargo.toml` followed by configuring all `Cargo.toml` files accordingly. * Adding a new dependency from crates.io? We're still working on that, so hold off on that for now. -* Adding a new configuration option? Take a look at `build/config.rs` or perhaps - `build/flags.rs` and then modify the build elsewhere to read that option. -* Adding a sanity check? Take a look at `build/sanity.rs`. +* Adding a new configuration option? Take a look at `bootstrap/config.rs` or + perhaps `bootstrap/flags.rs` and then modify the build elsewhere to read that + option. +* Adding a sanity check? Take a look at `bootstrap/sanity.rs`. If you have any questions feel free to reach out on `#rust-internals` on IRC or open an issue in the bug tracker! diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index a3fabbb3e8094..0dda7f12007a4 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -30,32 +30,37 @@ def get(url, path, verbose=False): sha_path = sha_file.name try: - download(sha_path, sha_url, verbose) + download(sha_path, sha_url, False, verbose) if os.path.exists(path): if verify(path, sha_path, False): - print("using already-download file " + path) + if verbose: + print("using already-download file " + path) return else: - print("ignoring already-download file " + path + " due to failed verification") + if verbose: + print("ignoring already-download file " + path + " due to failed verification") os.unlink(path) - download(temp_path, url, verbose) - if not verify(temp_path, sha_path, True): + download(temp_path, url, True, verbose) + if not verify(temp_path, sha_path, verbose): raise RuntimeError("failed verification") - print("moving {} to {}".format(temp_path, path)) + if verbose: + print("moving {} to {}".format(temp_path, path)) shutil.move(temp_path, path) finally: - delete_if_present(sha_path) - delete_if_present(temp_path) + delete_if_present(sha_path, verbose) + delete_if_present(temp_path, verbose) -def delete_if_present(path): +def delete_if_present(path, verbose): if os.path.isfile(path): - print("removing " + path) + if verbose: + print("removing " + path) os.unlink(path) -def download(path, url, verbose): - print("downloading {} to {}".format(url, path)) +def download(path, url, probably_big, verbose): + if probably_big or verbose: + print("downloading {}".format(url)) # see http://serverfault.com/questions/301128/how-to-download if sys.platform == 'win32': run(["PowerShell.exe", "/nologo", "-Command", @@ -63,17 +68,22 @@ def download(path, url, verbose): ".DownloadFile('{}', '{}')".format(url, path)], verbose=verbose) else: - run(["curl", "-o", path, url], verbose=verbose) + if probably_big or verbose: + option = "-#" + else: + option = "-s" + run(["curl", option, "-Sf", "-o", path, url], verbose=verbose) def verify(path, sha_path, verbose): - print("verifying " + path) + if verbose: + print("verifying " + path) with open(path, "rb") as f: found = hashlib.sha256(f.read()).hexdigest() with open(sha_path, "r") as f: expected, _ = f.readline().split() verified = found == expected - if not verified and verbose: + if not verified: print("invalid checksum:\n" " found: {}\n" " expected: {}".format(found, expected)) @@ -144,6 +154,7 @@ def download_stage0(self): if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.rustc_out_of_date()): + self.print_what_it_means_to_bootstrap() if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) channel = self.stage0_rustc_channel() @@ -167,6 +178,7 @@ def download_stage0(self): if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.cargo_out_of_date()): + self.print_what_it_means_to_bootstrap() channel = self.stage0_cargo_channel() filename = "cargo-{}-{}.tar.gz".format(channel, self.build) url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date() @@ -251,7 +263,27 @@ def exe_suffix(self): else: return '' + def print_what_it_means_to_bootstrap(self): + if hasattr(self, 'printed'): + return + self.printed = True + if os.path.exists(self.bootstrap_binary()): + return + if not '--help' in sys.argv or len(sys.argv) == 1: + return + + print('info: the build system for Rust is written in Rust, so this') + print(' script is now going to download a stage0 rust compiler') + print(' and then compile the build system itself') + print('') + print('info: in the meantime you can read more about rustbuild at') + print(' src/bootstrap/README.md before the download finishes') + + def bootstrap_binary(self): + return os.path.join(self.build_dir, "bootstrap/debug/bootstrap") + def build_bootstrap(self): + self.print_what_it_means_to_bootstrap() build_dir = os.path.join(self.build_dir, "bootstrap") if self.clean and os.path.exists(build_dir): shutil.rmtree(build_dir) @@ -408,22 +440,31 @@ def main(): rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \ 'CFG_ENABLE_VENDOR' in rb.config_mk + if 'SUDO_USER' in os.environ: + if os.environ['USER'] != os.environ['SUDO_USER']: + rb.use_vendored_sources = True + print('info: looks like you are running this command under `sudo`') + print(' and so in order to preserve your $HOME this will now') + print(' use vendored sources by default. Note that if this') + print(' does not work you should run a normal build first') + print(' before running a command like `sudo make intall`') + if rb.use_vendored_sources: if not os.path.exists('.cargo'): os.makedirs('.cargo') - f = open('.cargo/config','w') - f.write(""" - [source.crates-io] - replace-with = 'vendored-sources' - registry = 'https://example.com' - - [source.vendored-sources] - directory = '{}/src/vendor' - """.format(rb.rust_root)) - f.close() + with open('.cargo/config','w') as f: + f.write(""" + [source.crates-io] + replace-with = 'vendored-sources' + registry = 'https://example.com' + + [source.vendored-sources] + directory = '{}/src/vendor' + """.format(rb.rust_root)) else: if os.path.exists('.cargo'): shutil.rmtree('.cargo') + data = stage0_data(rb.rust_root) rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1) rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1) @@ -438,7 +479,7 @@ def main(): sys.stdout.flush() # Run the bootstrap - args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")] + args = [rb.bootstrap_binary()] args.extend(sys.argv[1:]) env = os.environ.copy() env["BUILD"] = rb.build diff --git a/src/bootstrap/cc.rs b/src/bootstrap/cc.rs index e2bde4a658611..aa70e24d95222 100644 --- a/src/bootstrap/cc.rs +++ b/src/bootstrap/cc.rs @@ -51,7 +51,7 @@ pub fn find(build: &mut Build) { if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { cfg.compiler(cc); } else { - set_compiler(&mut cfg, "gcc", target, config); + set_compiler(&mut cfg, "gcc", target, config, build); } let compiler = cfg.get_compiler(); @@ -72,7 +72,7 @@ pub fn find(build: &mut Build) { if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { cfg.compiler(cxx); } else { - set_compiler(&mut cfg, "g++", host, config); + set_compiler(&mut cfg, "g++", host, config, build); } let compiler = cfg.get_compiler(); build.verbose(&format!("CXX_{} = {:?}", host, compiler.path())); @@ -83,7 +83,8 @@ pub fn find(build: &mut Build) { fn set_compiler(cfg: &mut gcc::Config, gnu_compiler: &str, target: &str, - config: Option<&Target>) { + config: Option<&Target>, + build: &Build) { match target { // When compiling for android we may have the NDK configured in the // config.toml in which case we look there. Otherwise the default @@ -119,6 +120,22 @@ fn set_compiler(cfg: &mut gcc::Config, } } + "mips-unknown-linux-musl" => { + cfg.compiler("mips-linux-musl-gcc"); + } + "mipsel-unknown-linux-musl" => { + cfg.compiler("mipsel-linux-musl-gcc"); + } + + t if t.contains("musl") => { + if let Some(root) = build.musl_root(target) { + let guess = root.join("bin/musl-gcc"); + if guess.exists() { + cfg.compiler(guess); + } + } + } + _ => {} } } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index b67eab38f5dd2..c5675fd46cbe0 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! Implementation of the various `check-*` targets of the build system. +//! Implementation of the test-related targets of the build system. //! //! This file implements the various regression test suites that we execute on //! our CI. @@ -62,6 +62,8 @@ impl fmt::Display for TestKind { pub fn linkcheck(build: &Build, stage: u32, host: &str) { println!("Linkcheck stage{} ({})", stage, host); let compiler = Compiler::new(stage, host); + + let _time = util::timeit(); build.run(build.tool_cmd(&compiler, "linkchecker") .arg(build.out.join(host).join("doc"))); } @@ -87,6 +89,7 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) { let out_dir = build.out.join("ct"); t!(fs::create_dir_all(&out_dir)); + let _time = util::timeit(); build.run(build.tool_cmd(compiler, "cargotest") .env("PATH", newpath) .arg(&build.cargo) @@ -119,7 +122,8 @@ pub fn compiletest(build: &Build, target: &str, mode: &str, suite: &str) { - println!("Check compiletest {} ({} -> {})", suite, compiler.host, target); + println!("Check compiletest suite={} mode={} ({} -> {})", + suite, mode, compiler.host, target); let mut cmd = build.tool_cmd(compiler, "compiletest"); // compiletest currently has... a lot of arguments, so let's just pass all @@ -213,6 +217,9 @@ pub fn compiletest(build: &Build, // Running a C compiler on MSVC requires a few env vars to be set, to be // sure to set them here. + // + // Note that if we encounter `PATH` we make sure to append to our own `PATH` + // rather than stomp over it. if target.contains("msvc") { for &(ref k, ref v) in build.cc[target].0.env() { if k != "PATH" { @@ -221,6 +228,7 @@ pub fn compiletest(build: &Build, } } cmd.env("RUSTC_BOOTSTRAP", "1"); + build.add_rust_test_threads(&mut cmd); cmd.arg("--adb-path").arg("adb"); cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); @@ -232,6 +240,7 @@ pub fn compiletest(build: &Build, cmd.arg("--android-cross-path").arg(""); } + let _time = util::timeit(); build.run(&mut cmd); } @@ -244,6 +253,7 @@ pub fn docs(build: &Build, compiler: &Compiler) { // Do a breadth-first traversal of the `src/doc` directory and just run // tests for all files that end in `*.md` let mut stack = vec![build.src.join("src/doc")]; + let _time = util::timeit(); while let Some(p) = stack.pop() { if p.is_dir() { @@ -272,6 +282,8 @@ pub fn error_index(build: &Build, compiler: &Compiler) { let dir = testdir(build, compiler.host); t!(fs::create_dir_all(&dir)); let output = dir.join("error-index.md"); + + let _time = util::timeit(); build.run(build.tool_cmd(compiler, "error_index_generator") .arg("markdown") .arg(&output) @@ -283,6 +295,7 @@ pub fn error_index(build: &Build, compiler: &Compiler) { fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) { let mut cmd = Command::new(build.rustdoc(compiler)); build.add_rustc_lib_path(compiler, &mut cmd); + build.add_rust_test_threads(&mut cmd); cmd.arg("--test"); cmd.arg(markdown); @@ -366,16 +379,25 @@ pub fn krate(build: &Build, dylib_path.insert(0, build.sysroot_libdir(compiler, target)); cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + if target.contains("android") { + cargo.arg("--no-run"); + } else if target.contains("emscripten") { + cargo.arg("--no-run"); + } + + cargo.arg("--"); + if build.config.quiet_tests { - cargo.arg("--"); cargo.arg("--quiet"); } + let _time = util::timeit(); + if target.contains("android") { - build.run(cargo.arg("--no-run")); + build.run(&mut cargo); krate_android(build, compiler, target, mode); } else if target.contains("emscripten") { - build.run(cargo.arg("--no-run")); + build.run(&mut cargo); krate_emscripten(build, compiler, target, mode); } else { cargo.args(&build.flags.cmd.test_args()); @@ -402,14 +424,17 @@ fn krate_android(build: &Build, target, compiler.host, test_file_name); + let quiet = if build.config.quiet_tests { "--quiet" } else { "" }; let program = format!("(cd {dir}; \ LD_LIBRARY_PATH=./{target} ./{test} \ --logfile {log} \ + {quiet} \ {args})", dir = ADB_TEST_DIR, target = target, test = test_file_name, log = log, + quiet = quiet, args = build.flags.cmd.test_args().join(" ")); let output = output(Command::new("adb").arg("shell").arg(&program)); @@ -438,18 +463,13 @@ fn krate_emscripten(build: &Build, let test_file_name = test.to_string_lossy().into_owned(); println!("running {}", test_file_name); let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured"); - let status = Command::new(nodejs) - .arg(&test_file_name) - .stderr(::std::process::Stdio::inherit()) - .status(); - match status { - Ok(status) => { - if !status.success() { - panic!("some tests failed"); - } - } - Err(e) => panic!(format!("failed to execute command: {}", e)), - }; + let mut cmd = Command::new(nodejs); + cmd.arg(&test_file_name) + .stderr(::std::process::Stdio::inherit()); + if build.config.quiet_tests { + cmd.arg("--quiet"); + } + build.run(&mut cmd); } } diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs index 75bcbfee6ee0b..e7655458aed8a 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/clean.rs @@ -46,6 +46,9 @@ fn rm_rf(build: &Build, path: &Path) { if !path.exists() { return } + if path.is_file() { + return do_op(path, "remove file", |p| fs::remove_file(p)); + } for file in t!(fs::read_dir(path)) { let file = t!(file).path(); diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index a7d80e4cdc466..7a2d56fc5d3cc 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -239,6 +239,7 @@ To learn more about a subcommand, run `./x.py -h` install: m.opt_present("install"), } } + "--help" => usage(0, &opts), cmd => { println!("unknown command: {}", cmd); usage(1, &opts); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 590c967d147f2..912b5864c818b 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -13,9 +13,56 @@ //! This module, and its descendants, are the implementation of the Rust build //! system. Most of this build system is backed by Cargo but the outer layer //! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. +//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: //! -//! More documentation can be found in each respective module below. +//! * To be an easily understandable, easily extensible, and maintainable build +//! system. +//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka +//! crates.io and Cargo. +//! * A standard interface to build across all platforms, including MSVC +//! +//! ## Architecture +//! +//! Although this build system defers most of the complicated logic to Cargo +//! itself, it still needs to maintain a list of targets and dependencies which +//! it can itself perform. Rustbuild is made up of a list of rules with +//! dependencies amongst them (created in the `step` module) and then knows how +//! to execute each in sequence. Each time rustbuild is invoked, it will simply +//! iterate through this list of steps and execute each serially in turn. For +//! each step rustbuild relies on the step internally being incremental and +//! parallel. Note, though, that the `-j` parameter to rustbuild gets forwarded +//! to appropriate test harnesses and such. +//! +//! Most of the "meaty" steps that matter are backed by Cargo, which does indeed +//! have its own parallelism and incremental management. Later steps, like +//! tests, aren't incremental and simply run the entire suite currently. +//! +//! When you execute `x.py build`, the steps which are executed are: +//! +//! * First, the python script is run. This will automatically download the +//! stage0 rustc and cargo according to `src/stage0.txt`, or using the cached +//! versions if they're available. These are then used to compile rustbuild +//! itself (using Cargo). Finally, control is then transferred to rustbuild. +//! +//! * Rustbuild takes over, performs sanity checks, probes the environment, +//! reads configuration, builds up a list of steps, and then starts executing +//! them. +//! +//! * The stage0 libstd is compiled +//! * The stage0 libtest is compiled +//! * The stage0 librustc is compiled +//! * The stage1 compiler is assembled +//! * The stage1 libstd, libtest, librustc are compiled +//! * The stage2 compiler is assembled +//! * The stage2 libstd, libtest, librustc are compiled +//! +//! Each step is driven by a separate Cargo project and rustbuild orchestrates +//! copying files between steps and otherwise preparing for Cargo to run. +//! +//! ## Further information +//! +//! More documentation can be found in each respective module below, and you can +//! also check out the `src/bootstrap/README.md` file for more information. extern crate build_helper; extern crate cmake; @@ -28,6 +75,7 @@ extern crate toml; use std::collections::HashMap; use std::env; +use std::ffi::OsString; use std::fs::{self, File}; use std::path::{Component, PathBuf, Path}; use std::process::Command; @@ -128,6 +176,7 @@ pub struct Build { cc: HashMap)>, cxx: HashMap, crates: HashMap, + is_sudo: bool, } #[derive(Debug)] @@ -187,6 +236,16 @@ impl Build { }; let local_rebuild = config.local_rebuild; + let is_sudo = match env::var_os("SUDO_USER") { + Some(sudo_user) => { + match env::var_os("USER") { + Some(user) => user != sudo_user, + None => false, + } + } + None => false, + }; + Build { flags: flags, config: config, @@ -208,6 +267,7 @@ impl Build { crates: HashMap::new(), lldb_version: None, lldb_python_dir: None, + is_sudo: is_sudo, } } @@ -414,7 +474,7 @@ impl Build { // how the actual compiler itself is called. // // These variables are primarily all read by - // src/bootstrap/{rustc,rustdoc.rs} + // src/bootstrap/bin/{rustc.rs,rustdoc.rs} cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc")) .env("RUSTC_REAL", self.compiler_path(compiler)) .env("RUSTC_STAGE", stage.to_string()) @@ -435,6 +495,7 @@ impl Build { // Enable usage of unstable features cargo.env("RUSTC_BOOTSTRAP", "1"); + self.add_rust_test_threads(&mut cargo); // Specify some various options for build scripts used throughout // the build. @@ -458,7 +519,7 @@ impl Build { if self.config.rust_optimize && cmd != "bench" { cargo.arg("--release"); } - if self.config.vendor { + if self.config.vendor || self.is_sudo { cargo.arg("--frozen"); } return cargo @@ -492,12 +553,30 @@ impl Build { fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command { let mut cmd = Command::new(self.tool(&compiler, tool)); let host = compiler.host; - let paths = vec![ + let mut paths = vec![ self.cargo_out(compiler, Mode::Libstd, host).join("deps"), self.cargo_out(compiler, Mode::Libtest, host).join("deps"), self.cargo_out(compiler, Mode::Librustc, host).join("deps"), self.cargo_out(compiler, Mode::Tool, host).join("deps"), ]; + + // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make + // mode) and that C compiler may need some extra PATH modification. Do + // so here. + if compiler.host.contains("msvc") { + let curpaths = env::var_os("PATH").unwrap_or(OsString::new()); + let curpaths = env::split_paths(&curpaths).collect::>(); + for &(ref k, ref v) in self.cc[compiler.host].0.env() { + if k != "PATH" { + continue + } + for path in env::split_paths(v) { + if !curpaths.contains(&path) { + paths.push(path); + } + } + } + } add_lib_path(paths, &mut cmd); return cmd } @@ -651,6 +730,13 @@ impl Build { add_lib_path(vec![self.rustc_libdir(compiler)], cmd); } + /// Adds the `RUST_TEST_THREADS` env var if necessary + fn add_rust_test_threads(&self, cmd: &mut Command) { + if env::var_os("RUST_TEST_THREADS").is_none() { + cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); + } + } + /// Returns the compiler's libdir where it stores the dynamic libraries that /// it itself links against. /// diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 1e73595ec9983..b165048b7b6c7 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -23,9 +23,14 @@ all: $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) -# Don’t use $(Q) here, always show how to invoke the bootstrap script directly help: - $(BOOTSTRAP) --help + $(Q)echo 'Welcome to the rustbuild build system!' + $(Q)echo + $(Q)echo This makefile is a thin veneer over the ./x.py script located + $(Q)echo in this directory. To get the full power of the build system + $(Q)echo you can run x.py directly. + $(Q)echo + $(Q)echo To learn more run \`./x.py --help\` clean: $(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS) @@ -51,15 +56,14 @@ check-cargotest: dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) install: -ifeq (root user, $(USER) $(patsubst %,user,$(SUDO_USER))) - $(Q)echo "'sudo make install' is not supported currently." -else $(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS) -endif tidy: $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) --stage 0 -check-stage2-android: - $(Q)$(BOOTSTRAP) --step check-target --target arm-linux-androideabi +check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --target arm-linux-androideabi +check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --target x86_64-unknown-linux-gnu + .PHONY: dist diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 96d1b695dd707..ffa3fe1cbf2fd 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -28,7 +28,7 @@ use cmake; use gcc; use Build; -use util::up_to_date; +use util::{self, up_to_date}; /// Compile LLVM for `target`. pub fn llvm(build: &Build, target: &str) { @@ -58,6 +58,7 @@ pub fn llvm(build: &Build, target: &str) { println!("Building LLVM for {}", target); + let _time = util::timeit(); let _ = fs::remove_dir_all(&dst.join("build")); t!(fs::create_dir_all(&dst.join("build"))); let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"}; @@ -158,6 +159,17 @@ pub fn test_helpers(build: &Build, target: &str) { println!("Building test helpers"); t!(fs::create_dir_all(&dst)); let mut cfg = gcc::Config::new(); + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform gcc of these compilers. Note, though, that + // on MSVC we still need gcc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = build.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(build.cc(target)); + } + cfg.cargo_metadata(false) .out_dir(&dst) .target(target) diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 47efa6952177c..f3fe22698bb90 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -41,10 +41,14 @@ pub fn check(build: &mut Build) { } } let have_cmd = |cmd: &OsStr| { - for path in env::split_paths(&path).map(|p| p.join(cmd)) { - if fs::metadata(&path).is_ok() || - fs::metadata(path.with_extension("exe")).is_ok() { - return Some(path); + for path in env::split_paths(&path) { + let target = path.join(cmd); + let mut cmd_alt = cmd.to_os_string(); + cmd_alt.push(".exe"); + if target.exists() || + target.with_extension("exe").exists() || + target.join(cmd_alt).exists() { + return Some(target); } } return None; diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index b8683831af122..ca169bd146ccb 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -8,6 +8,24 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +//! Definition of steps of the build system. +//! +//! This is where some of the real meat of rustbuild is located, in how we +//! define targets and the dependencies amongst them. This file can sort of be +//! viewed as just defining targets in a makefile which shell out to predefined +//! functions elsewhere about how to execute the target. +//! +//! The primary function here you're likely interested in is the `build_rules` +//! function. This will create a `Rules` structure which basically just lists +//! everything that rustbuild can do. Each rule has a human-readable name, a +//! path associated with it, some dependencies, and then a closure of how to +//! actually perform the rule. +//! +//! All steps below are defined in self-contained units, so adding a new target +//! to the build system should just involve adding the meta information here +//! along with the actual implementation elsewhere. You can find more comments +//! about how to define rules themselves below. + use std::collections::{HashMap, HashSet}; use std::mem; @@ -20,36 +38,6 @@ use install; use native; use {Compiler, Build, Mode}; -#[derive(PartialEq, Eq, Hash, Clone, Debug)] -struct Step<'a> { - name: &'a str, - stage: u32, - host: &'a str, - target: &'a str, -} - -impl<'a> Step<'a> { - fn name(&self, name: &'a str) -> Step<'a> { - Step { name: name, ..*self } - } - - fn stage(&self, stage: u32) -> Step<'a> { - Step { stage: stage, ..*self } - } - - fn host(&self, host: &'a str) -> Step<'a> { - Step { host: host, ..*self } - } - - fn target(&self, target: &'a str) -> Step<'a> { - Step { target: target, ..*self } - } - - fn compiler(&self) -> Compiler<'a> { - Compiler::new(self.stage, self.host) - } -} - pub fn run(build: &Build) { let rules = build_rules(build); let steps = rules.plan(); @@ -57,14 +45,91 @@ pub fn run(build: &Build) { } pub fn build_rules(build: &Build) -> Rules { - let mut rules: Rules = Rules::new(build); + let mut rules = Rules::new(build); + + // This is the first rule that we're going to define for rustbuild, which is + // used to compile LLVM itself. All rules are added through the `rules` + // structure created above and are configured through a builder-style + // interface. + // + // First up we see the `build` method. This represents a rule that's part of + // the top-level `build` subcommand. For example `./x.py build` is what this + // is associating with. Note that this is normally only relevant if you flag + // a rule as `default`, which we'll talk about later. + // + // Next up we'll see two arguments to this method: + // + // * `llvm` - this is the "human readable" name of this target. This name is + // not accessed anywhere outside this file itself (e.g. not in + // the CLI nor elsewhere in rustbuild). The purpose of this is to + // easily define dependencies between rules. That is, other rules + // will depend on this with the name "llvm". + // * `src/llvm` - this is the relevant path to the rule that we're working + // with. This path is the engine behind how commands like + // `./x.py build src/llvm` work. This should typically point + // to the relevant component, but if there's not really a + // path to be assigned here you can pass something like + // `path/to/nowhere` to ignore it. + // + // After we create the rule with the `build` method we can then configure + // various aspects of it. For example this LLVM rule uses `.host(true)` to + // flag that it's a rule only for host targets. In other words, LLVM isn't + // compiled for targets configured through `--target` (e.g. those we're just + // building a standard library for). + // + // Next up the `dep` method will add a dependency to this rule. The closure + // is yielded the step that represents executing the `llvm` rule itself + // (containing information like stage, host, target, ...) and then it must + // return a target that the step depends on. Here LLVM is actually + // interesting where a cross-compiled LLVM depends on the host LLVM, but + // otherwise it has no dependencies. + // + // To handle this we do a bit of dynamic dispatch to see what the dependency + // is. If we're building a LLVM for the build triple, then we don't actually + // have any dependencies! To do that we return a dependency on the "dummy" + // target which does nothing. + // + // If we're build a cross-compiled LLVM, however, we need to assemble the + // libraries from the previous compiler. This step has the same name as + // ours (llvm) but we want it for a different target, so we use the + // builder-style methods on `Step` to configure this target to the build + // triple. + // + // Finally, to finish off this rule, we define how to actually execute it. + // That logic is all defined in the `native` module so we just delegate to + // the relevant function there. The argument to the closure passed to `run` + // is a `Step` (defined below) which encapsulates information like the + // stage, target, host, etc. + rules.build("llvm", "src/llvm") + .host(true) + .dep(move |s| { + if s.target == build.config.build { + dummy(s, build) + } else { + s.target(&build.config.build) + } + }) + .run(move |s| native::llvm(build, s.target)); + + // Ok! After that example rule that's hopefully enough to explain what's + // going on here. You can check out the API docs below and also see a bunch + // more examples of rules directly below as well. + // dummy rule to do nothing, useful when a dep maps to no deps rules.build("dummy", "path/to/nowhere"); - fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { - s.name("dummy").stage(0) - .target(&build.config.build) - .host(&build.config.build) - } + + // the compiler with no target libraries ready to go + rules.build("rustc", "src/rustc") + .dep(move |s| { + if s.stage == 0 { + dummy(s, build) + } else { + s.name("librustc") + .host(&build.config.build) + .stage(s.stage - 1) + } + }) + .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); // Helper for loading an entire DAG of crates, rooted at `name` let krates = |name: &str| { @@ -85,28 +150,6 @@ pub fn build_rules(build: &Build) -> Rules { return ret }; - rules.build("rustc", "path/to/nowhere") - .dep(move |s| { - if s.stage == 0 { - dummy(s, build) - } else { - s.name("librustc") - .host(&build.config.build) - .stage(s.stage - 1) - } - }) - .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); - rules.build("llvm", "src/llvm") - .host(true) - .dep(move |s| { - if s.target == build.config.build { - dummy(s, build) - } else { - s.target(&build.config.build) - } - }) - .run(move |s| native::llvm(build, s.target)); - // ======================================================================== // Crate compilations // @@ -337,10 +380,10 @@ pub fn build_rules(build: &Build) -> Rules { .host(true) .run(move |s| check::cargotest(build, s.stage, s.target)); rules.test("check-tidy", "src/tools/tidy") - .dep(|s| s.name("tool-tidy")) + .dep(|s| s.name("tool-tidy").stage(0)) .default(true) .host(true) - .run(move |s| check::tidy(build, s.stage, s.target)); + .run(move |s| check::tidy(build, 0, s.target)); rules.test("check-error-index", "src/tools/error_index_generator") .dep(|s| s.name("libstd")) .dep(|s| s.name("tool-error-index").host(s.host)) @@ -457,16 +500,89 @@ pub fn build_rules(build: &Build) -> Rules { .run(move |s| install::install(build, s.stage, s.target)); rules.verify(); - return rules + return rules; + + fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { + s.name("dummy").stage(0) + .target(&build.config.build) + .host(&build.config.build) + } +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +struct Step<'a> { + /// Human readable name of the rule this step is executing. Possible names + /// are all defined above in `build_rules`. + name: &'a str, + + /// The stage this step is executing in. This is typically 0, 1, or 2. + stage: u32, + + /// This step will likely involve a compiler, and the target that compiler + /// itself is built for is called the host, this variable. Typically this is + /// the target of the build machine itself. + host: &'a str, + + /// The target that this step represents generating. If you're building a + /// standard library for a new suite of targets, for example, this'll be set + /// to those targets. + target: &'a str, +} + +impl<'a> Step<'a> { + /// Creates a new step which is the same as this, except has a new name. + fn name(&self, name: &'a str) -> Step<'a> { + Step { name: name, ..*self } + } + + /// Creates a new step which is the same as this, except has a new stage. + fn stage(&self, stage: u32) -> Step<'a> { + Step { stage: stage, ..*self } + } + + /// Creates a new step which is the same as this, except has a new host. + fn host(&self, host: &'a str) -> Step<'a> { + Step { host: host, ..*self } + } + + /// Creates a new step which is the same as this, except has a new target. + fn target(&self, target: &'a str) -> Step<'a> { + Step { target: target, ..*self } + } + + /// Returns the `Compiler` structure that this step corresponds to. + fn compiler(&self) -> Compiler<'a> { + Compiler::new(self.stage, self.host) + } } struct Rule<'a> { + /// The human readable name of this target, defined in `build_rules`. name: &'a str, + + /// The path associated with this target, used in the `./x.py` driver for + /// easy and ergonomic specification of what to do. path: &'a str, + + /// The "kind" of top-level command that this rule is associated with, only + /// relevant if this is a default rule. kind: Kind, + + /// List of dependencies this rule has. Each dependency is a function from a + /// step that's being executed to another step that should be executed. deps: Vec) -> Step<'a> + 'a>>, + + /// How to actually execute this rule. Takes a step with contextual + /// information and then executes it. run: Box) + 'a>, + + /// Whether or not this is a "default" rule. That basically means that if + /// you run, for example, `./x.py test` whether it's included or not. default: bool, + + /// Whether or not this is a "host" rule, or in other words whether this is + /// only intended for compiler hosts and not for targets that are being + /// generated. host: bool, } @@ -493,6 +609,8 @@ impl<'a> Rule<'a> { } } +/// Builder pattern returned from the various methods on `Rules` which will add +/// the rule to the internal list on `Drop`. struct RuleBuilder<'a: 'b, 'b> { rules: &'b mut Rules<'a>, rule: Rule<'a>, @@ -554,26 +672,35 @@ impl<'a> Rules<'a> { } } + /// Creates a new rule of `Kind::Build` with the specified human readable + /// name and path associated with it. + /// + /// The builder returned should be configured further with information such + /// as how to actually run this rule. fn build<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Build) } + /// Same as `build`, but for `Kind::Test`. fn test<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Test) } + /// Same as `build`, but for `Kind::Bench`. fn bench<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Bench) } + /// Same as `build`, but for `Kind::Doc`. fn doc<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Doc) } + /// Same as `build`, but for `Kind::Dist`. fn dist<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Dist) @@ -634,6 +761,31 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? /// Construct the top-level build steps that we're going to be executing, /// given the subcommand that our build is performing. fn plan(&self) -> Vec> { + // Ok, the logic here is pretty subtle, and involves quite a few + // conditionals. The basic idea here is to: + // + // 1. First, filter all our rules to the relevant ones. This means that + // the command specified corresponds to one of our `Kind` variants, + // and we filter all rules based on that. + // + // 2. Next, we determine which rules we're actually executing. If a + // number of path filters were specified on the command line we look + // for those, otherwise we look for anything tagged `default`. + // + // 3. Finally, we generate some steps with host and target information. + // + // The last step is by far the most complicated and subtle. The basic + // thinking here is that we want to take the cartesian product of + // specified hosts and targets and build rules with that. The list of + // hosts and targets, if not specified, come from the how this build was + // configured. If the rule we're looking at is a host-only rule the we + // ignore the list of targets and instead consider the list of hosts + // also the list of targets. + // + // Once the host and target lists are generated we take the cartesian + // product of the two and then create a step based off them. Note that + // the stage each step is associated was specified with the `--step` + // flag on the command line. let (kind, paths) = match self.build.flags.cmd { Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), @@ -664,7 +816,18 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } else { &self.build.config.target }; - let arr = if rule.host {hosts} else {targets}; + // If --target was specified but --host wasn't specified, don't run + // any host-only tests + let arr = if rule.host { + if self.build.flags.target.len() > 0 && + self.build.flags.host.len() == 0 { + &hosts[..0] + } else { + hosts + } + } else { + targets + }; hosts.iter().flat_map(move |host| { arr.iter().map(move |target| { @@ -705,6 +868,15 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } } + /// Performs topological sort of dependencies rooted at the `step` + /// specified, pushing all results onto the `order` vector provided. + /// + /// In other words, when this method returns, the `order` vector will + /// contain a list of steps which if executed in order will eventually + /// complete the `step` specified as well. + /// + /// The `added` set specified here is the set of steps that are already + /// present in `order` (and hence don't need to be added again). fn fill(&self, step: Step<'a>, order: &mut Vec>, diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index e028c52236662..cb5b456a0f2fe 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -18,6 +18,7 @@ use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::Instant; use filetime::FileTime; @@ -189,3 +190,19 @@ pub fn push_exe_path(mut buf: PathBuf, components: &[&str]) -> PathBuf { buf } + +pub struct TimeIt(Instant); + +/// Returns an RAII structure that prints out how long it took to drop. +pub fn timeit() -> TimeIt { + TimeIt(Instant::now()) +} + +impl Drop for TimeIt { + fn drop(&mut self) { + let time = self.0.elapsed(); + println!("\tfinished in {}.{:03}", + time.as_secs(), + time.subsec_nanos() / 1_000_000); + } +} diff --git a/src/ci/docker/arm-android/Dockerfile b/src/ci/docker/arm-android/Dockerfile index c5b70c227c408..121c0263cbccb 100644 --- a/src/ci/docker/arm-android/Dockerfile +++ b/src/ci/docker/arm-android/Dockerfile @@ -9,7 +9,6 @@ RUN dpkg --add-architecture i386 && \ curl \ ca-certificates \ python2.7 \ - python-minimal \ git \ cmake \ ccache \ @@ -39,8 +38,7 @@ ENV RUST_CONFIGURE_ARGS \ --arm-linux-androideabi-ndk=/android/ndk-arm-9 \ --armv7-linux-androideabi-ndk=/android/ndk-arm-9 \ --i686-linux-android-ndk=/android/ndk-x86-9 \ - --aarch64-linux-android-ndk=/android/ndk-aarch64 \ - --enable-rustbuild -ENV RUST_CHECK_TARGET check-stage2-android + --aarch64-linux-android-ndk=/android/ndk-aarch64 +ENV XPY_CHECK test --target arm-linux-androideabi RUN mkdir /tmp/obj RUN chmod 777 /tmp/obj diff --git a/src/ci/docker/cross/Dockerfile b/src/ci/docker/cross/Dockerfile index d8af878a95863..b7b23d74c9dad 100644 --- a/src/ci/docker/cross/Dockerfile +++ b/src/ci/docker/cross/Dockerfile @@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ python2.7 \ - python-minimal \ git \ cmake \ ccache \ diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index c5b1d00fb7cc1..ff5345d3aac9d 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -19,17 +19,21 @@ ci_dir="`dirname $docker_dir`" src_dir="`dirname $ci_dir`" root_dir="`dirname $src_dir`" -docker build \ +docker \ + build \ --rm \ -t rust-ci \ "`dirname "$script"`/$image" mkdir -p $HOME/.ccache mkdir -p $HOME/.cargo +mkdir -p $root_dir/obj -exec docker run \ +exec docker \ + run \ --volume "$root_dir:/checkout:ro" \ - --workdir /tmp/obj \ + --volume "$root_dir/obj:/checkout/obj" \ + --workdir /checkout/obj \ --env SRC=/checkout \ --env CCACHE_DIR=/ccache \ --volume "$HOME/.ccache:/ccache" \ diff --git a/src/ci/docker/x86_64-freebsd/Dockerfile b/src/ci/docker/x86_64-freebsd/Dockerfile index dc16c39961c45..a3a52f9e6ff95 100644 --- a/src/ci/docker/x86_64-freebsd/Dockerfile +++ b/src/ci/docker/x86_64-freebsd/Dockerfile @@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ python2.7 \ - python-minimal \ git \ cmake \ ccache \ @@ -23,7 +22,7 @@ ENV \ AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \ CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc -ENV RUST_CONFIGURE_ARGS --target=x86_64-unknown-freebsd --enable-rustbuild +ENV RUST_CONFIGURE_ARGS --target=x86_64-unknown-freebsd ENV RUST_CHECK_TARGET "" RUN mkdir /tmp/obj RUN chmod 777 /tmp/obj diff --git a/src/ci/docker/x86_64-gnu-cargotest/Dockerfile b/src/ci/docker/x86_64-gnu-cargotest/Dockerfile index 1db01f2b48d46..107e2bf8a121e 100644 --- a/src/ci/docker/x86_64-gnu-cargotest/Dockerfile +++ b/src/ci/docker/x86_64-gnu-cargotest/Dockerfile @@ -7,14 +7,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ python2.7 \ - python-minimal \ git \ cmake \ ccache \ libssl-dev \ sudo -ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-rustbuild +ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu ENV RUST_CHECK_TARGET check-cargotest +ENV NO_VENDOR 1 RUN mkdir /tmp/obj RUN chmod 777 /tmp/obj diff --git a/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile b/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile index ca06940ae5e2d..c27e3d1325fe8 100644 --- a/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile +++ b/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile @@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ python2.7 \ - python2.7-minimal \ git \ cmake \ ccache \ @@ -19,7 +18,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --enable-rustbuild \ --llvm-root=/usr/lib/llvm-3.7 ENV RUST_CHECK_TARGET check RUN mkdir /tmp/obj diff --git a/src/ci/docker/x86_64-gnu-rustbuild/Dockerfile b/src/ci/docker/x86_64-gnu-make/Dockerfile similarity index 73% rename from src/ci/docker/x86_64-gnu-rustbuild/Dockerfile rename to src/ci/docker/x86_64-gnu-make/Dockerfile index d4d0492e2a260..93229b2a0103e 100644 --- a/src/ci/docker/x86_64-gnu-rustbuild/Dockerfile +++ b/src/ci/docker/x86_64-gnu-make/Dockerfile @@ -7,14 +7,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ ca-certificates \ python2.7 \ - python-minimal \ git \ cmake \ ccache \ sudo \ gdb -ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --enable-rustbuild +ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --disable-rustbuild ENV RUST_CHECK_TARGET check RUN mkdir /tmp/obj RUN chmod 777 /tmp/obj diff --git a/src/ci/docker/x86_64-musl/Dockerfile b/src/ci/docker/x86_64-musl/Dockerfile index 1afaef2e05678..967940fb1f3ae 100644 --- a/src/ci/docker/x86_64-musl/Dockerfile +++ b/src/ci/docker/x86_64-musl/Dockerfile @@ -20,8 +20,10 @@ RUN sh /build/build-musl.sh && rm -rf /build ENV RUST_CONFIGURE_ARGS \ --target=x86_64-unknown-linux-musl \ - --musl-root=/musl-x86_64 + --musl-root-x86_64=/musl-x86_64 ENV RUST_CHECK_TARGET check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu +ENV PATH=$PATH:/musl-x86_64/bin +ENV XPY_CHECK test --target x86_64-unknown-linux-musl RUN mkdir /tmp/obj RUN chmod 777 /tmp/obj diff --git a/src/ci/run.sh b/src/ci/run.sh index da238dddecacb..10f2d15da343c 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -14,12 +14,20 @@ set -e if [ "$LOCAL_USER_ID" != "" ]; then useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user export HOME=/home/user - export LOCAL_USER_ID= - exec sudo -E -u user env PATH=$PATH "$0" + unset LOCAL_USER_ID + exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user fi if [ "$NO_LLVM_ASSERTIONS" = "" ]; then - LLVM_ASSERTIONS=--enable-llvm-assertions + ENABLE_LLVM_ASSERTIONS=--enable-llvm-assertions +fi + +if [ "$NO_VENDOR" = "" ]; then + ENABLE_VENDOR=--enable-vendor +fi + +if [ "$NO_CCACHE" = "" ]; then + ENABLE_CCACHE=--enable-ccache fi set -ex @@ -28,9 +36,9 @@ $SRC/configure \ --disable-manage-submodules \ --enable-debug-assertions \ --enable-quiet-tests \ - --enable-ccache \ - --enable-vendor \ - $LLVM_ASSERTIONS \ + $ENABLE_CCACHE \ + $ENABLE_VENDOR \ + $ENABLE_LLVM_ASSERTIONS \ $RUST_CONFIGURE_ARGS if [ "$TRAVIS_OS_NAME" = "osx" ]; then @@ -41,4 +49,8 @@ fi make -j $ncpus tidy make -j $ncpus -exec make $RUST_CHECK_TARGET -j $ncpus +if [ ! -z "$XPY_CHECK" ]; then + exec python2.7 $SRC/x.py $XPY_CHECK +else + exec make $RUST_CHECK_TARGET -j $ncpus +fi diff --git a/src/liballoc_jemalloc/build.rs b/src/liballoc_jemalloc/build.rs index 50149dfd65f78..fc849e7a50cc4 100644 --- a/src/liballoc_jemalloc/build.rs +++ b/src/liballoc_jemalloc/build.rs @@ -151,11 +151,17 @@ fn main() { cmd.arg(format!("--build={}", build_helper::gnu_target(&host))); run(&mut cmd); - run(Command::new("make") - .current_dir(&build_dir) - .arg("build_lib_static") - .arg("-j") - .arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set"))); + let mut make = Command::new("make"); + make.current_dir(&build_dir) + .arg("build_lib_static"); + + // mingw make seems... buggy? unclear... + if !host.contains("windows") { + make.arg("-j") + .arg(env::var("NUM_JOBS").expect("NUM_JOBS was not set")); + } + + run(&mut make); if target.contains("windows") { println!("cargo:rustc-link-lib=static=jemalloc"); diff --git a/src/rustllvm/llvm-auto-clean-trigger b/src/rustllvm/llvm-auto-clean-trigger index 768ba2d5b77a1..2d832fcdf2ad0 100644 --- a/src/rustllvm/llvm-auto-clean-trigger +++ b/src/rustllvm/llvm-auto-clean-trigger @@ -1,4 +1,4 @@ # If this file is modified, then llvm will be forcibly cleaned and then rebuilt. # The actual contents of this file do not matter, but to trigger a change on the # build bots then the contents should be changed so git updates the mtime. -2016-12-01 +2016-12-06 diff --git a/src/test/run-pass/no-stdio.rs b/src/test/run-pass/no-stdio.rs index ad4d56ec50ac0..85c63e184fe25 100644 --- a/src/test/run-pass/no-stdio.rs +++ b/src/test/run-pass/no-stdio.rs @@ -9,6 +9,7 @@ // except according to those terms. // ignore-emscripten +// ignore-android #![feature(libc)]