Skip to content

Commit

Permalink
mk: Bootstrap from stable instead of snapshots
Browse files Browse the repository at this point in the history
This commit removes all infrastructure from the repository for our so-called
snapshots to instead bootstrap the compiler from stable releases. Bootstrapping
from a previously stable release is a long-desired feature of distros because
they're not fans of downloading binary stage0 blobs from us. Additionally, this
makes our own CI easier as we can decommission all of the snapshot builders and
start having a regular cadence to when we update the stage0 compiler.

A new `src/etc/get-stage0.py` script was added which shares some code with
`src/bootstrap/bootstrap.py` to read a new file, `src/stage0.txt`, which lists
the current stage0 compiler as well as cargo that we bootstrap from. This script
will download the relevant `rustc` package an unpack it into `$target/stage0` as
we do today.

One problem of bootstrapping from stable releases is that we're not able to
compile unstable code (e.g. all the `#![feature]` directives in libcore/libstd).
To overcome this we employ two strategies:

* The bootstrap key of the previous compiler is hardcoded into `src/stage0.txt`
  (enabled as a result of rust-lang#32731) and exported by the build system. This enables
  nightly features in the compiler we download.
* The standard library and compiler are pinned to a specific stage0, which
  doesn't change, so we're guaranteed that we'll continue compiling as we start
  from a known fixed source.

The process for making a release will also need to be tweaked now to continue to
cadence of bootstrapping from the previous release. This process looks like:

1. Merge `beta` to `stable`
2. Produce a new stable compiler.
3. Change `master` to bootstrap from this new stable compiler.
4. Merge `master` to `beta`
5. Produce a new beta compiler
6. Change `master` to bootstrap from this new beta compiler.

Step 3 above should involve very few changes as `master` was previously
bootstrapping from `beta` which is the same as `stable` at that point in time.
Step 6, however, is where we benefit from removing lots of `#[cfg(stage0)]` and
get to use new features. This also shouldn't slow the release too much as steps
1-5 requires little work other than waiting and step 6 just needs to happen at
some point during a release cycle, it's not time sensitive.

Closes rust-lang#29555
Closes rust-lang#29557
  • Loading branch information
alexcrichton committed Apr 19, 2016
1 parent 478a33d commit 02538d4
Show file tree
Hide file tree
Showing 23 changed files with 200 additions and 2,999 deletions.
7 changes: 0 additions & 7 deletions Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -214,13 +214,6 @@ include $(CFG_SRC_DIR)mk/debuggers.mk
# Secondary makefiles, conditionalized for speed
######################################################################

# Binary snapshots
ifneq ($(strip $(findstring snap,$(MAKECMDGOALS)) \
$(findstring clean,$(MAKECMDGOALS))),)
CFG_INFO := $(info cfg: including snap rules)
include $(CFG_SRC_DIR)mk/snap.mk
endif

# The test suite
ifneq ($(strip $(findstring check,$(MAKECMDGOALS)) \
$(findstring test,$(MAKECMDGOALS)) \
Expand Down
2 changes: 1 addition & 1 deletion mk/dist.mk
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ PKG_FILES := \
rtstartup \
rustllvm \
rustc \
snapshots.txt \
stage0.txt \
rust-installer \
tools \
test) \
Expand Down
1 change: 1 addition & 0 deletions mk/main.mk
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ CFG_FILENAME_EXTRA=$(shell printf '%s' $(CFG_RELEASE)$(CFG_EXTRA_FILENAME) | $(C
# intentionally not "secure" by any definition, this is largely just a deterrent
# from users enabling unstable features on the stable compiler.
CFG_BOOTSTRAP_KEY=$(CFG_FILENAME_EXTRA)
CFG_BOOTSTRAP_KEY_STAGE0=$(shell grep 'rustc_key' $(S)src/stage0.txt | sed 's/rustc_key: '//)

ifeq ($(CFG_RELEASE_CHANNEL),stable)
# This is the normal semver version string, e.g. "0.12.0", "0.12.0-nightly"
Expand Down
2 changes: 1 addition & 1 deletion mk/reconfig.mk
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,6 @@ else
SREL_ROOT := $(SREL)
endif

config.stamp: $(S)configure $(S)Makefile.in $(S)src/snapshots.txt
config.stamp: $(S)configure $(S)Makefile.in $(S)src/stage0.txt
@$(call E, cfg: reconfiguring)
$(SREL_ROOT)configure $(CFG_CONFIGURE_ARGS)
28 changes: 0 additions & 28 deletions mk/snap.mk

This file was deleted.

9 changes: 3 additions & 6 deletions mk/stage0.mk
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,14 @@ $(HLIB0_H_$(CFG_BUILD))/:
endif

$(SNAPSHOT_RUSTC_POST_CLEANUP): \
$(S)src/snapshots.txt \
$(S)src/etc/get-snapshot.py $(MKFILE_DEPS) \
$(S)src/stage0.txt \
$(S)src/etc/get-stage0.py $(MKFILE_DEPS) \
| $(HBIN0_H_$(CFG_BUILD))/

@$(call E, fetch: $@)
# Note: the variable "SNAPSHOT_FILE" is generally not set, and so
# we generally only pass one argument to this script.
ifdef CFG_ENABLE_LOCAL_RUST
$(Q)$(S)src/etc/local_stage0.sh $(CFG_BUILD) $(CFG_LOCAL_RUST_ROOT) rustlib
else
$(Q)$(CFG_PYTHON) $(S)src/etc/get-snapshot.py $(CFG_BUILD) $(SNAPSHOT_FILE)
$(Q)$(CFG_PYTHON) $(S)src/etc/get-stage0.py $(CFG_BUILD)
endif
$(Q)if [ -e "$@" ]; then touch "$@"; else echo "ERROR: snapshot $@ not found"; exit 1; fi

Expand Down
18 changes: 16 additions & 2 deletions mk/target.mk
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ $(foreach host,$(CFG_HOST), \
# $(4) is the crate name
define RUST_TARGET_STAGE_N

ifeq ($(1),0)
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
endif

$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): CFG_COMPILER_HOST_TRIPLE = $(2)
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
$$(CRATEFILE_$(4)) \
Expand Down Expand Up @@ -113,6 +118,11 @@ endef
# $(4) - name of the tool being built
define TARGET_TOOL

ifeq ($(1),0)
$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
endif

$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
$$(TOOL_SOURCE_$(4)) \
$$(TOOL_INPUTS_$(4)) \
Expand Down Expand Up @@ -167,11 +177,15 @@ SNAPSHOT_RUSTC_POST_CLEANUP=$(HBIN0_H_$(CFG_BUILD))/rustc$(X_$(CFG_BUILD))

define TARGET_HOST_RULES

$$(TLIB$(1)_T_$(2)_H_$(3))/:
$$(TLIB$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
mkdir -p $$@

$$(TBIN$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
mkdir -p $$@

$$(TLIB$(1)_T_$(2)_H_$(3))/%: $$(RT_OUTPUT_DIR_$(2))/% \
| $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP)
$$(SNAPSHOT_RUSTC_POST_CLEANUP) \
| $$(TLIB$(1)_T_$(2)_H_$(3))/
@$$(call E, cp: $$@)
$$(Q)cp $$< $$@
endef
Expand Down
7 changes: 5 additions & 2 deletions mk/tests.mk
Original file line number Diff line number Diff line change
Expand Up @@ -241,13 +241,16 @@ cleantestlibs:
######################################################################

.PHONY: tidy
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD))
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
$(SNAPSHOT_RUSTC_POST_CLEANUP)
$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src

$(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \
$(TSREQ0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) \
$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))/stamp.std \
$(call rwildcard,$(S)src/tools/tidy/src,*.rs)
$(call rwildcard,$(S)src/tools/tidy/src,*.rs) \
$(SNAPSHOT_RUSTC_POST_CLEANUP) | \
$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))
$(STAGE0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $(S)src/tools/tidy/src/main.rs \
--out-dir $(@D) --crate-name tidy

Expand Down
151 changes: 84 additions & 67 deletions src/bootstrap/bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,22 @@ def run(args, verbose=False):
raise RuntimeError(err)
sys.exit(err)

def stage0_data(rust_root):
nightlies = os.path.join(rust_root, "src/stage0.txt")
with open(nightlies, 'r') as nightlies:
data = {}
for line in nightlies.read().split("\n"):
if line.startswith("#") or line == '':
continue
a, b = line.split(": ", 1)
data[a] = b
return data

class RustBuild:
def download_rust_nightly(self):
def download_stage0(self):
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, self.snap_rustc_date())
cargo_cache = os.path.join(cache_dst, self.snap_cargo_date())
rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
if not os.path.exists(cargo_cache):
Expand All @@ -93,41 +104,49 @@ def download_rust_nightly(self):
(not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
filename = "rust-std-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
channel = self.stage0_rustc_channel()
filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get(url + "/" + filename, tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rust-std-" + self.build,
verbose=self.verbose)

filename = "rustc-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get(url + "/" + filename, tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
with open(self.rustc_stamp(), 'w') as f:
f.write(self.snap_rustc_date())
f.write(self.stage0_rustc_date())

if self.cargo().startswith(self.bin_root()) and \
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
filename = "cargo-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/cargo-dist/" + self.snap_cargo_date()
channel = self.stage0_cargo_channel()
filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
tarball = os.path.join(cargo_cache, filename)
if not os.path.exists(tarball):
get(url + "/" + filename, tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
with open(self.cargo_stamp(), 'w') as f:
f.write(self.snap_cargo_date())
f.write(self.stage0_cargo_date())

def snap_cargo_date(self):
def stage0_cargo_date(self):
return self._cargo_date

def snap_rustc_date(self):
def stage0_cargo_channel(self):
return self._cargo_channel

def stage0_rustc_date(self):
return self._rustc_date

def stage0_rustc_channel(self):
return self._rustc_channel

def rustc_stamp(self):
return os.path.join(self.bin_root(), '.rustc-stamp')

Expand All @@ -138,13 +157,13 @@ def rustc_out_of_date(self):
if not os.path.exists(self.rustc_stamp()):
return True
with open(self.rustc_stamp(), 'r') as f:
return self.snap_rustc_date() != f.read()
return self.stage0_rustc_date() != f.read()

def cargo_out_of_date(self):
if not os.path.exists(self.cargo_stamp()):
return True
with open(self.cargo_stamp(), 'r') as f:
return self.snap_cargo_date() != f.read()
return self.stage0_cargo_date() != f.read()

def bin_root(self):
return os.path.join(self.build_dir, self.build, "stage0")
Expand Down Expand Up @@ -187,15 +206,6 @@ def exe_suffix(self):
else:
return ''

def parse_nightly_dates(self):
nightlies = os.path.join(self.rust_root, "src/nightlies.txt")
with open(nightlies, 'r') as nightlies:
rustc, cargo = nightlies.read().split("\n")[:2]
assert rustc.startswith("rustc: ")
assert cargo.startswith("cargo: ")
self._rustc_date = rustc[len("rustc: "):]
self._cargo_date = cargo[len("cargo: "):]

def build_bootstrap(self):
env = os.environ.copy()
env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
Expand Down Expand Up @@ -300,46 +310,53 @@ def build_triple(self):

return cputype + '-' + ostype

parser = argparse.ArgumentParser(description='Build rust')
parser.add_argument('--config')
parser.add_argument('-v', '--verbose', action='store_true')

args = [a for a in sys.argv if a != '-h']
args, _ = parser.parse_known_args(args)

# Configure initial bootstrap
rb = RustBuild()
rb.config_toml = ''
rb.config_mk = ''
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
rb.build_dir = os.path.join(os.getcwd(), "build")
rb.verbose = args.verbose

try:
with open(args.config or 'config.toml') as config:
rb.config_toml = config.read()
except:
pass
try:
rb.config_mk = open('config.mk').read()
except:
pass

# Fetch/build the bootstrap
rb.build = rb.build_triple()
rb.parse_nightly_dates()
rb.download_rust_nightly()
sys.stdout.flush()
rb.build_bootstrap()
sys.stdout.flush()

# Run the bootstrap
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
args.append('--src')
args.append(rb.rust_root)
args.append('--build')
args.append(rb.build)
args.extend(sys.argv[1:])
env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
rb.run(args, env)
def main():
parser = argparse.ArgumentParser(description='Build rust')
parser.add_argument('--config')
parser.add_argument('-v', '--verbose', action='store_true')

args = [a for a in sys.argv if a != '-h']
args, _ = parser.parse_known_args(args)

# Configure initial bootstrap
rb = RustBuild()
rb.config_toml = ''
rb.config_mk = ''
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
rb.build_dir = os.path.join(os.getcwd(), "build")
rb.verbose = args.verbose

try:
with open(args.config or 'config.toml') as config:
rb.config_toml = config.read()
except:
pass
try:
rb.config_mk = open('config.mk').read()
except:
pass

data = stage0_data(rb.rust_root)
rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

# Fetch/build the bootstrap
rb.build = rb.build_triple()
rb.download_stage0()
sys.stdout.flush()
rb.build_bootstrap()
sys.stdout.flush()

# Run the bootstrap
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
args.append('--src')
args.append(rb.rust_root)
args.append('--build')
args.append(rb.build)
args.extend(sys.argv[1:])
env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
rb.run(args, env)

if __name__ == '__main__':
main()
8 changes: 8 additions & 0 deletions src/bootstrap/build/channel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,4 +84,12 @@ pub fn collect(build: &mut Build) {
build.bootstrap_key = format!("{:02x}{:02x}{:02x}{:02x}",
key[0], key[1], key[2], key[3]);
env::set_var("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key);

let mut s = String::new();
t!(t!(File::open(build.src.join("src/stage0.txt"))).read_to_string(&mut s));
if let Some(line) = s.lines().find(|l| l.starts_with("rustc_key")) {
if let Some(key) = line.split(": ").nth(1) {
build.bootstrap_key_stage0 = key.to_string();
}
}
}
1 change: 0 additions & 1 deletion src/bootstrap/build/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,6 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
.env("CFG_VERSION", &build.version)
.env("CFG_BOOTSTRAP_KEY", &build.bootstrap_key)
.env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(String::new()))
.env("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key)
.env("CFG_LIBDIR_RELATIVE", "lib");

if let Some(ref ver_date) = build.ver_date {
Expand Down
Loading

0 comments on commit 02538d4

Please sign in to comment.