mk: Bootstrap from stable instead of snapshots

This commit removes all infrastructure from the repository for our so-called
snapshots to instead bootstrap the compiler from stable releases. Bootstrapping
from a previously stable release is a long-desired feature of distros because
they're not fans of downloading binary stage0 blobs from us. Additionally, this
makes our own CI easier as we can decommission all of the snapshot builders and
start having a regular cadence to when we update the stage0 compiler.

A new `src/etc/get-stage0.py` script was added which shares some code with
`src/bootstrap/bootstrap.py` to read a new file, `src/stage0.txt`, which lists
the current stage0 compiler as well as cargo that we bootstrap from. This script
will download the relevant `rustc` package an unpack it into `$target/stage0` as
we do today.

One problem of bootstrapping from stable releases is that we're not able to
compile unstable code (e.g. all the `#![feature]` directives in libcore/libstd).
To overcome this we employ two strategies:

* The bootstrap key of the previous compiler is hardcoded into `src/stage0.txt`
  (enabled as a result of #32731) and exported by the build system. This enables
  nightly features in the compiler we download.
* The standard library and compiler are pinned to a specific stage0, which
  doesn't change, so we're guaranteed that we'll continue compiling as we start
  from a known fixed source.

The process for making a release will also need to be tweaked now to continue to
cadence of bootstrapping from the previous release. This process looks like:

1. Merge `beta` to `stable`
2. Produce a new stable compiler.
3. Change `master` to bootstrap from this new stable compiler.
4. Merge `master` to `beta`
5. Produce a new beta compiler
6. Change `master` to bootstrap from this new beta compiler.

Step 3 above should involve very few changes as `master` was previously
bootstrapping from `beta` which is the same as `stable` at that point in time.
Step 6, however, is where we benefit from removing lots of `#[cfg(stage0)]` and
get to use new features. This also shouldn't slow the release too much as steps
1-5 requires little work other than waiting and step 6 just needs to happen at
some point during a release cycle, it's not time sensitive.

Closes #29555
Closes #29557
This commit is contained in:
Alex Crichton 2016-04-13 11:18:35 -07:00
parent 478a33dabc
commit 02538d463a
23 changed files with 195 additions and 2994 deletions

View File

@ -214,13 +214,6 @@ include $(CFG_SRC_DIR)mk/debuggers.mk
# Secondary makefiles, conditionalized for speed
######################################################################
# Binary snapshots
ifneq ($(strip $(findstring snap,$(MAKECMDGOALS)) \
$(findstring clean,$(MAKECMDGOALS))),)
CFG_INFO := $(info cfg: including snap rules)
include $(CFG_SRC_DIR)mk/snap.mk
endif
# The test suite
ifneq ($(strip $(findstring check,$(MAKECMDGOALS)) \
$(findstring test,$(MAKECMDGOALS)) \

View File

@ -61,7 +61,7 @@ PKG_FILES := \
rtstartup \
rustllvm \
rustc \
snapshots.txt \
stage0.txt \
rust-installer \
tools \
test) \

View File

@ -34,6 +34,7 @@ CFG_FILENAME_EXTRA=$(shell printf '%s' $(CFG_RELEASE)$(CFG_EXTRA_FILENAME) | $(C
# intentionally not "secure" by any definition, this is largely just a deterrent
# from users enabling unstable features on the stable compiler.
CFG_BOOTSTRAP_KEY=$(CFG_FILENAME_EXTRA)
CFG_BOOTSTRAP_KEY_STAGE0=$(shell grep 'rustc_key' $(S)src/stage0.txt | sed 's/rustc_key: '//)
ifeq ($(CFG_RELEASE_CHANNEL),stable)
# This is the normal semver version string, e.g. "0.12.0", "0.12.0-nightly"

View File

@ -38,6 +38,6 @@ else
SREL_ROOT := $(SREL)
endif
config.stamp: $(S)configure $(S)Makefile.in $(S)src/snapshots.txt
config.stamp: $(S)configure $(S)Makefile.in $(S)src/stage0.txt
@$(call E, cfg: reconfiguring)
$(SREL_ROOT)configure $(CFG_CONFIGURE_ARGS)

View File

@ -1,28 +0,0 @@
# Copyright 2012 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
define DEF_SNAP_FOR_STAGE_H
# $(1) stage
# $(2) triple
snap-stage$(1)-H-$(2): $$(HSREQ$(1)_H_$(2))
$(CFG_PYTHON) $(S)src/etc/make-snapshot.py stage$(1) $(2)
endef
$(foreach host,$(CFG_HOST), \
$(foreach stage,1 2 3, \
$(eval $(call DEF_SNAP_FOR_STAGE_H,$(stage),$(host)))))
snap-stage1: snap-stage1-H-$(CFG_BUILD)
snap-stage2: snap-stage2-H-$(CFG_BUILD)
snap-stage3: snap-stage3-H-$(CFG_BUILD)

View File

@ -10,17 +10,14 @@ $(HLIB0_H_$(CFG_BUILD))/:
endif
$(SNAPSHOT_RUSTC_POST_CLEANUP): \
$(S)src/snapshots.txt \
$(S)src/etc/get-snapshot.py $(MKFILE_DEPS) \
$(S)src/stage0.txt \
$(S)src/etc/get-stage0.py $(MKFILE_DEPS) \
| $(HBIN0_H_$(CFG_BUILD))/
@$(call E, fetch: $@)
# Note: the variable "SNAPSHOT_FILE" is generally not set, and so
# we generally only pass one argument to this script.
ifdef CFG_ENABLE_LOCAL_RUST
$(Q)$(S)src/etc/local_stage0.sh $(CFG_BUILD) $(CFG_LOCAL_RUST_ROOT) rustlib
else
$(Q)$(CFG_PYTHON) $(S)src/etc/get-snapshot.py $(CFG_BUILD) $(SNAPSHOT_FILE)
$(Q)$(CFG_PYTHON) $(S)src/etc/get-stage0.py $(CFG_BUILD)
endif
$(Q)if [ -e "$@" ]; then touch "$@"; else echo "ERROR: snapshot $@ not found"; exit 1; fi

View File

@ -65,6 +65,11 @@ $(foreach host,$(CFG_HOST), \
# $(4) is the crate name
define RUST_TARGET_STAGE_N
ifeq ($(1),0)
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
endif
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): CFG_COMPILER_HOST_TRIPLE = $(2)
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
$$(CRATEFILE_$(4)) \
@ -113,6 +118,11 @@ endef
# $(4) - name of the tool being built
define TARGET_TOOL
ifeq ($(1),0)
$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
endif
$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
$$(TOOL_SOURCE_$(4)) \
$$(TOOL_INPUTS_$(4)) \
@ -167,11 +177,15 @@ SNAPSHOT_RUSTC_POST_CLEANUP=$(HBIN0_H_$(CFG_BUILD))/rustc$(X_$(CFG_BUILD))
define TARGET_HOST_RULES
$$(TLIB$(1)_T_$(2)_H_$(3))/:
$$(TLIB$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
mkdir -p $$@
$$(TBIN$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
mkdir -p $$@
$$(TLIB$(1)_T_$(2)_H_$(3))/%: $$(RT_OUTPUT_DIR_$(2))/% \
| $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP)
$$(SNAPSHOT_RUSTC_POST_CLEANUP) \
| $$(TLIB$(1)_T_$(2)_H_$(3))/
@$$(call E, cp: $$@)
$$(Q)cp $$< $$@
endef

View File

@ -241,13 +241,16 @@ cleantestlibs:
######################################################################
.PHONY: tidy
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD))
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
$(SNAPSHOT_RUSTC_POST_CLEANUP)
$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src
$(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \
$(TSREQ0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) \
$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))/stamp.std \
$(call rwildcard,$(S)src/tools/tidy/src,*.rs)
$(call rwildcard,$(S)src/tools/tidy/src,*.rs) \
$(SNAPSHOT_RUSTC_POST_CLEANUP) | \
$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))
$(STAGE0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $(S)src/tools/tidy/src/main.rs \
--out-dir $(@D) --crate-name tidy

View File

@ -79,11 +79,22 @@ def run(args, verbose=False):
raise RuntimeError(err)
sys.exit(err)
def stage0_data(rust_root):
nightlies = os.path.join(rust_root, "src/stage0.txt")
with open(nightlies, 'r') as nightlies:
data = {}
for line in nightlies.read().split("\n"):
if line.startswith("#") or line == '':
continue
a, b = line.split(": ", 1)
data[a] = b
return data
class RustBuild:
def download_rust_nightly(self):
def download_stage0(self):
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, self.snap_rustc_date())
cargo_cache = os.path.join(cache_dst, self.snap_cargo_date())
rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
if not os.path.exists(cargo_cache):
@ -93,8 +104,9 @@ class RustBuild:
(not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
filename = "rust-std-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
channel = self.stage0_rustc_channel()
filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get(url + "/" + filename, tarball, verbose=self.verbose)
@ -102,32 +114,39 @@ class RustBuild:
match="rust-std-" + self.build,
verbose=self.verbose)
filename = "rustc-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get(url + "/" + filename, tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
with open(self.rustc_stamp(), 'w') as f:
f.write(self.snap_rustc_date())
f.write(self.stage0_rustc_date())
if self.cargo().startswith(self.bin_root()) and \
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
filename = "cargo-nightly-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/cargo-dist/" + self.snap_cargo_date()
channel = self.stage0_cargo_channel()
filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
tarball = os.path.join(cargo_cache, filename)
if not os.path.exists(tarball):
get(url + "/" + filename, tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
with open(self.cargo_stamp(), 'w') as f:
f.write(self.snap_cargo_date())
f.write(self.stage0_cargo_date())
def snap_cargo_date(self):
def stage0_cargo_date(self):
return self._cargo_date
def snap_rustc_date(self):
def stage0_cargo_channel(self):
return self._cargo_channel
def stage0_rustc_date(self):
return self._rustc_date
def stage0_rustc_channel(self):
return self._rustc_channel
def rustc_stamp(self):
return os.path.join(self.bin_root(), '.rustc-stamp')
@ -138,13 +157,13 @@ class RustBuild:
if not os.path.exists(self.rustc_stamp()):
return True
with open(self.rustc_stamp(), 'r') as f:
return self.snap_rustc_date() != f.read()
return self.stage0_rustc_date() != f.read()
def cargo_out_of_date(self):
if not os.path.exists(self.cargo_stamp()):
return True
with open(self.cargo_stamp(), 'r') as f:
return self.snap_cargo_date() != f.read()
return self.stage0_cargo_date() != f.read()
def bin_root(self):
return os.path.join(self.build_dir, self.build, "stage0")
@ -187,15 +206,6 @@ class RustBuild:
else:
return ''
def parse_nightly_dates(self):
nightlies = os.path.join(self.rust_root, "src/nightlies.txt")
with open(nightlies, 'r') as nightlies:
rustc, cargo = nightlies.read().split("\n")[:2]
assert rustc.startswith("rustc: ")
assert cargo.startswith("cargo: ")
self._rustc_date = rustc[len("rustc: "):]
self._cargo_date = cargo[len("cargo: "):]
def build_bootstrap(self):
env = os.environ.copy()
env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
@ -300,46 +310,53 @@ class RustBuild:
return cputype + '-' + ostype
parser = argparse.ArgumentParser(description='Build rust')
parser.add_argument('--config')
parser.add_argument('-v', '--verbose', action='store_true')
def main():
parser = argparse.ArgumentParser(description='Build rust')
parser.add_argument('--config')
parser.add_argument('-v', '--verbose', action='store_true')
args = [a for a in sys.argv if a != '-h']
args, _ = parser.parse_known_args(args)
args = [a for a in sys.argv if a != '-h']
args, _ = parser.parse_known_args(args)
# Configure initial bootstrap
rb = RustBuild()
rb.config_toml = ''
rb.config_mk = ''
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
rb.build_dir = os.path.join(os.getcwd(), "build")
rb.verbose = args.verbose
# Configure initial bootstrap
rb = RustBuild()
rb.config_toml = ''
rb.config_mk = ''
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
rb.build_dir = os.path.join(os.getcwd(), "build")
rb.verbose = args.verbose
try:
with open(args.config or 'config.toml') as config:
rb.config_toml = config.read()
except:
pass
try:
rb.config_mk = open('config.mk').read()
except:
pass
try:
with open(args.config or 'config.toml') as config:
rb.config_toml = config.read()
except:
pass
try:
rb.config_mk = open('config.mk').read()
except:
pass
# Fetch/build the bootstrap
rb.build = rb.build_triple()
rb.parse_nightly_dates()
rb.download_rust_nightly()
sys.stdout.flush()
rb.build_bootstrap()
sys.stdout.flush()
data = stage0_data(rb.rust_root)
rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
# Run the bootstrap
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
args.append('--src')
args.append(rb.rust_root)
args.append('--build')
args.append(rb.build)
args.extend(sys.argv[1:])
env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
rb.run(args, env)
# Fetch/build the bootstrap
rb.build = rb.build_triple()
rb.download_stage0()
sys.stdout.flush()
rb.build_bootstrap()
sys.stdout.flush()
# Run the bootstrap
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
args.append('--src')
args.append(rb.rust_root)
args.append('--build')
args.append(rb.build)
args.extend(sys.argv[1:])
env = os.environ.copy()
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
rb.run(args, env)
if __name__ == '__main__':
main()

View File

@ -84,4 +84,12 @@ pub fn collect(build: &mut Build) {
build.bootstrap_key = format!("{:02x}{:02x}{:02x}{:02x}",
key[0], key[1], key[2], key[3]);
env::set_var("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key);
let mut s = String::new();
t!(t!(File::open(build.src.join("src/stage0.txt"))).read_to_string(&mut s));
if let Some(line) = s.lines().find(|l| l.starts_with("rustc_key")) {
if let Some(key) = line.split(": ").nth(1) {
build.bootstrap_key_stage0 = key.to_string();
}
}
}

View File

@ -179,7 +179,6 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
.env("CFG_VERSION", &build.version)
.env("CFG_BOOTSTRAP_KEY", &build.bootstrap_key)
.env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(String::new()))
.env("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key)
.env("CFG_LIBDIR_RELATIVE", "lib");
if let Some(ref ver_date) = build.ver_date {

View File

@ -79,6 +79,7 @@ pub struct Build {
version: String,
package_vers: String,
bootstrap_key: String,
bootstrap_key_stage0: String,
// Probed tools at runtime
gdb_version: Option<String>,
@ -129,6 +130,7 @@ impl Build {
ver_date: None,
version: String::new(),
bootstrap_key: String::new(),
bootstrap_key_stage0: String::new(),
package_vers: String::new(),
cc: HashMap::new(),
cxx: HashMap::new(),
@ -402,6 +404,17 @@ impl Build {
.env("RUSTDOC_REAL", self.rustdoc(compiler))
.env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
// Set the bootstrap key depending on which stage compiler we're using.
// In stage0 we're using a previously released stable compiler, so we
// use the stage0 bootstrap key. Otherwise we use our own build's
// bootstrap key.
let bootstrap_key = if compiler.is_snapshot(self) {
&self.bootstrap_key_stage0
} else {
&self.bootstrap_key
};
cargo.env("RUSTC_BOOTSTRAP_KEY", bootstrap_key);
// Specify some various options for build scripts used throughout
// the build.
//

View File

@ -1,78 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2011-2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import tarfile
import shutil
import sys
from snapshot import *
def unpack_snapshot(triple, dl_path):
print("opening snapshot " + dl_path)
tar = tarfile.open(dl_path)
kernel = get_kernel(triple)
stagep = os.path.join(triple, "stage0")
# Remove files from prior unpackings, since snapshot rustc may not
# be able to disambiguate between multiple candidate libraries.
# (Leave dirs in place since extracting step still needs them.)
for root, _, files in os.walk(stagep):
for f in files:
print("removing " + os.path.join(root, f))
os.unlink(os.path.join(root, f))
for p in tar.getnames():
name = p.replace("rust-stage0/", "", 1)
fp = os.path.join(stagep, name)
print("extracting " + p)
tar.extract(p, download_unpack_base)
tp = os.path.join(download_unpack_base, p)
if os.path.isdir(tp) and os.path.exists(fp):
continue
shutil.move(tp, fp)
tar.close()
shutil.rmtree(download_unpack_base)
# Main
# this gets called with one or two arguments:
# The first is the O/S triple.
# The second is an optional path to the snapshot to use.
def main(argv):
triple = argv[1]
if len(argv) == 3:
dl_path = argv[2]
else:
snap = determine_curr_snapshot(triple)
dl = os.path.join(download_dir_base, snap)
url = download_url_base + "/" + snap
print("determined most recent snapshot: " + snap)
if (not os.path.exists(dl)):
get_url_to_file(url, dl)
if (snap_filename_hash_part(snap) == hash_file(dl)):
print("got download with ok hash")
else:
raise Exception("bad hash on download")
dl_path = os.path.join(download_dir_base, snap)
unpack_snapshot(triple, dl_path)
if __name__ == '__main__':
main(sys.argv)

51
src/etc/get-stage0.py Normal file
View File

@ -0,0 +1,51 @@
#!/usr/bin/env python
#
# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import shutil
import sys
import tarfile
path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../bootstrap"))
sys.path.append(path)
import bootstrap
def main(argv):
src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
triple = argv[1]
data = bootstrap.stage0_data(src_root)
channel, date = data['rustc'].split('-', 1)
dl_dir = 'dl'
if not os.path.exists(dl_dir):
os.makedirs(dl_dir)
filename_base = 'rustc-' + channel + '-' + triple
filename = filename_base + '.tar.gz'
url = 'https://static.rust-lang.org/dist/' + date + '/' + filename
dst = dl_dir + '/' + filename
if not os.path.exists(dst):
bootstrap.get(url, dst)
stage0_dst = triple + '/stage0'
if os.path.exists(stage0_dst):
for root, _, files in os.walk(stage0_dst):
for f in files:
os.unlink(os.path.join(root, f))
else:
os.makedirs(stage0_dst)
bootstrap.unpack(dst, stage0_dst, match='rustc', verbose=True)
if __name__ == '__main__':
main(sys.argv)

View File

@ -1,65 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import re
from snapshot import *
f = open(snapshotfile)
date = None
rev = None
platform = None
snap = None
i = 0
newestSet = {}
for line in f.readlines():
i += 1
parsed = parse_line(i, line)
if not parsed:
continue
if parsed["type"] == "snapshot":
if (len(newestSet) == 0 or parsed["date"] > newestSet["date"]):
newestSet["date"] = parsed["date"]
newestSet["rev"] = parsed["rev"]
newestSet["files"] = []
addingMode = True
else:
addingMode = False
elif addingMode is True and parsed["type"] == "file":
tux = re.compile("linux", re.IGNORECASE)
if (tux.match(parsed["platform"]) is not None):
ff = {}
ff["platform"] = parsed["platform"]
ff["hash"] = parsed["hash"]
newestSet["files"] += [ff]
def download_new_file(date, rev, platform, hsh):
snap = full_snapshot_name(date, rev, platform, hsh)
dl = os.path.join(download_dir_base, snap)
url = download_url_base + "/" + snap
if (not os.path.exists(dl)):
print("downloading " + url)
get_url_to_file(url, dl)
if (snap_filename_hash_part(snap) == hash_file(dl)):
print("got download with ok hash")
else:
raise Exception("bad hash on download")
for ff in newestSet["files"]:
download_new_file(newestSet["date"], newestSet["rev"],
ff["platform"], ff["hash"])

View File

@ -1,15 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2011-2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import snapshot, sys
print(snapshot.make_snapshot(sys.argv[1], sys.argv[2]))

View File

@ -1,45 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2011-2013 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
from snapshot import *
f = open(snapshotfile)
date = None
rev = None
platform = None
snap = None
i = 0
for line in f.readlines():
i += 1
parsed = parse_line(i, line)
if not parsed:
continue
if parsed["type"] == "snapshot":
date = parsed["date"]
rev = parsed["rev"]
elif rev is not None and parsed["type"] == "file":
platform = parsed["platform"]
hsh = parsed["hash"]
snap = full_snapshot_name(date, rev, platform, hsh)
dl = os.path.join(download_dir_base, snap)
url = download_url_base + "/" + snap
if (not os.path.exists(dl)):
print("downloading " + url)
get_url_to_file(url, dl)
if (snap_filename_hash_part(snap) == hash_file(dl)):
print("got download with ok hash")
else:
raise Exception("bad hash on download")

View File

@ -1,304 +0,0 @@
# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import re
import os
import sys
import glob
import tarfile
import shutil
import subprocess
import distutils.spawn
try:
import hashlib
sha_func = hashlib.sha1
except ImportError:
import sha
sha_func = sha.new
def scrub(b):
if sys.version_info >= (3,) and type(b) == bytes:
return b.decode('ascii')
else:
return b
src_dir = scrub(os.getenv("CFG_SRC_DIR"))
if not src_dir:
raise Exception("missing env var CFG_SRC_DIR")
snapshotfile = os.path.join(src_dir, "src", "snapshots.txt")
download_url_base = "https://static.rust-lang.org/stage0-snapshots"
download_dir_base = "dl"
download_unpack_base = os.path.join(download_dir_base, "unpack")
snapshot_files = {
"bitrig": ["bin/rustc"],
"dragonfly": ["bin/rustc"],
"freebsd": ["bin/rustc"],
"linux": ["bin/rustc"],
"macos": ["bin/rustc"],
"netbsd": ["bin/rustc"],
"openbsd": ["bin/rustc"],
"solaris": ["bin/rustc"],
"winnt": ["bin/rustc.exe"],
}
winnt_runtime_deps_32 = ["libgcc_s_dw2-1.dll", "libstdc++-6.dll"]
winnt_runtime_deps_64 = ["libgcc_s_seh-1.dll", "libstdc++-6.dll"]
def parse_line(n, line):
global snapshotfile
if re.match(r"\s*$", line):
return None
if re.match(r"^T\s*$", line):
return None
match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line)
if match:
return {"type": "file",
"platform": match.group(1),
"hash": match.group(2).lower()}
match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line)
if not match:
raise Exception("%s:%d:E syntax error: " % (snapshotfile, n))
return {"type": "snapshot",
"date": match.group(2),
"rev": match.group(3)}
def partial_snapshot_name(date, rev, platform):
return ("rust-stage0-%s-%s-%s.tar.bz2" %
(date, rev, platform))
def full_snapshot_name(date, rev, platform, hsh):
return ("rust-stage0-%s-%s-%s-%s.tar.bz2" %
(date, rev, platform, hsh))
def get_kernel(triple):
t = triple.split('-')
if len(t) == 2:
os_name = t[1]
else:
os_name = t[2]
if os_name == "windows":
return "winnt"
if os_name == "darwin":
return "macos"
if os_name == "freebsd":
return "freebsd"
if os_name == "dragonfly":
return "dragonfly"
if os_name == "bitrig":
return "bitrig"
if os_name == "netbsd":
return "netbsd"
if os_name == "openbsd":
return "openbsd"
return "linux"
def get_cpu(triple):
arch = triple.split('-')[0]
if arch == "i686":
return "i386"
return arch
def get_platform(triple):
return "%s-%s" % (get_kernel(triple), get_cpu(triple))
def cmd_out(cmdline):
p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
return scrub(p.communicate()[0].strip())
def local_rev_info(field):
return cmd_out(["git", "--git-dir=" + os.path.join(src_dir, ".git"),
"log", "-n", "1",
"--format=%%%s" % field, "HEAD"])
def local_rev_full_sha():
return local_rev_info("H").split()[0]
def local_rev_short_sha():
return local_rev_info("h").split()[0]
def local_rev_committer_date():
return local_rev_info("ci")
def get_url_to_file(u, f):
# no security issue, just to stop partial download leaving a stale file
tmpf = f + '.tmp'
returncode = -1
if distutils.spawn.find_executable("curl"):
returncode = subprocess.call(["curl", "-o", tmpf, u])
elif distutils.spawn.find_executable("wget"):
returncode = subprocess.call(["wget", "-O", tmpf, u])
if returncode != 0:
try:
os.unlink(tmpf)
except OSError:
pass
raise Exception("failed to fetch url")
os.rename(tmpf, f)
def snap_filename_hash_part(snap):
match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
if not match:
raise Exception("unable to find hash in filename: " + snap)
return match.group(1)
def hash_file(x):
h = sha_func()
h.update(open(x, "rb").read())
return scrub(h.hexdigest())
def get_winnt_runtime_deps(platform):
"""Returns a list of paths of Rust's system runtime dependencies"""
if platform == "winnt-x86_64":
deps = winnt_runtime_deps_64
else:
deps = winnt_runtime_deps_32
runtime_deps = []
path_dirs = os.environ["PATH"].split(os.pathsep)
for name in deps:
for dir in path_dirs:
filepath = os.path.join(dir, name)
if os.path.isfile(filepath):
runtime_deps.append(filepath)
break
else:
raise Exception("Could not find runtime dependency: %s" % name)
return runtime_deps
def make_snapshot(stage, triple):
kernel = get_kernel(triple)
platform = get_platform(triple)
rev = local_rev_short_sha()
date = local_rev_committer_date().split()[0]
file0 = partial_snapshot_name(date, rev, platform)
def in_tar_name(fn):
cs = re.split(r"[\\/]", fn)
if len(cs) >= 2:
return os.sep.join(cs[-2:])
tar = tarfile.open(file0, "w:bz2")
for name in snapshot_files[kernel]:
dir = stage
if stage == "stage1" and re.match(r"^lib/(lib)?std.*", name):
dir = "stage0"
fn_glob = os.path.join(triple, dir, name)
matches = glob.glob(fn_glob)
if not matches:
raise Exception("Not found file with name like " + fn_glob)
if len(matches) == 1:
tar.add(matches[0], "rust-stage0/" + in_tar_name(matches[0]))
else:
raise Exception("Found stale files: \n %s\n"
"Please make a clean build." % "\n ".join(matches))
if kernel == "winnt":
for path in get_winnt_runtime_deps(platform):
tar.add(path, "rust-stage0/bin/" + os.path.basename(path))
tar.add(os.path.join(os.path.dirname(__file__), "third-party"),
"rust-stage0/bin/third-party")
tar.close()
h = hash_file(file0)
file1 = full_snapshot_name(date, rev, platform, h)
shutil.move(file0, file1)
return file1
def curr_snapshot_rev():
i = 0
found_snap = False
date = None
rev = None
f = open(snapshotfile)
for line in f.readlines():
i += 1
parsed = parse_line(i, line)
if not parsed:
continue
if parsed["type"] == "snapshot":
date = parsed["date"]
rev = parsed["rev"]
found_snap = True
break
if not found_snap:
raise Exception("no snapshot entries in file")
return (date, rev)
def determine_curr_snapshot(triple):
i = 0
platform = get_platform(triple)
found_file = False
found_snap = False
hsh = None
date = None
rev = None
f = open(snapshotfile)
for line in f.readlines():
i += 1
parsed = parse_line(i, line)
if not parsed:
continue
if found_snap and parsed["type"] == "file":
if parsed["platform"] == platform:
hsh = parsed["hash"]
found_file = True
break
elif parsed["type"] == "snapshot":
date = parsed["date"]
rev = parsed["rev"]
found_snap = True
if not found_snap:
raise Exception("no snapshot entries in file")
if not found_file:
raise Exception("no snapshot file found for platform %s, rev %s" %
(platform, rev))
return full_snapshot_name(date, rev, platform, hsh)

View File

@ -53,34 +53,14 @@ extern "rust-intrinsic" {
// NB: These intrinsics take raw pointers because they mutate aliased
// memory, which is not valid for either `&` or `&mut`.
#[cfg(all(stage0, not(cargobuild)))]
pub fn atomic_cxchg<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(all(stage0, not(cargobuild)))]
pub fn atomic_cxchg_acq<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(all(stage0, not(cargobuild)))]
pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(all(stage0, not(cargobuild)))]
pub fn atomic_cxchg_acqrel<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(all(stage0, not(cargobuild)))]
pub fn atomic_cxchg_relaxed<T>(dst: *mut T, old: T, src: T) -> T;
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_acq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_rel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_acqrel<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_relaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_failacq<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_acq_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
#[cfg(any(not(stage0), cargobuild))]
pub fn atomic_cxchg_acqrel_failrelaxed<T>(dst: *mut T, old: T, src: T) -> (T, bool);
pub fn atomic_cxchgweak<T>(dst: *mut T, old: T, src: T) -> (T, bool);
@ -548,27 +528,22 @@ extern "rust-intrinsic" {
/// Float addition that allows optimizations based on algebraic rules.
/// May assume inputs are finite.
#[cfg(not(stage0))]
pub fn fadd_fast<T>(a: T, b: T) -> T;
/// Float subtraction that allows optimizations based on algebraic rules.
/// May assume inputs are finite.
#[cfg(not(stage0))]
pub fn fsub_fast<T>(a: T, b: T) -> T;
/// Float multiplication that allows optimizations based on algebraic rules.
/// May assume inputs are finite.
#[cfg(not(stage0))]
pub fn fmul_fast<T>(a: T, b: T) -> T;
/// Float division that allows optimizations based on algebraic rules.
/// May assume inputs are finite.
#[cfg(not(stage0))]
pub fn fdiv_fast<T>(a: T, b: T) -> T;
/// Float remainder that allows optimizations based on algebraic rules.
/// May assume inputs are finite.
#[cfg(not(stage0))]
pub fn frem_fast<T>(a: T, b: T) -> T;

View File

@ -1380,7 +1380,6 @@ unsafe fn atomic_sub<T>(dst: *mut T, val: T, order: Ordering) -> T {
}
#[inline]
#[cfg(any(not(stage0), cargobuild))]
unsafe fn atomic_compare_exchange<T>(dst: *mut T,
old: T,
new: T,
@ -1407,29 +1406,6 @@ unsafe fn atomic_compare_exchange<T>(dst: *mut T,
}
}
#[inline]
#[cfg(all(stage0, not(cargobuild)))]
unsafe fn atomic_compare_exchange<T>(dst: *mut T,
old: T,
new: T,
success: Ordering,
_: Ordering) -> Result<T, T>
where T: ::cmp::Eq + ::marker::Copy
{
let val = match success {
Acquire => intrinsics::atomic_cxchg_acq(dst, old, new),
Release => intrinsics::atomic_cxchg_rel(dst, old, new),
AcqRel => intrinsics::atomic_cxchg_acqrel(dst, old, new),
Relaxed => intrinsics::atomic_cxchg_relaxed(dst, old, new),
SeqCst => intrinsics::atomic_cxchg(dst, old, new),
};
if val == old {
Ok(val)
} else {
Err(val)
}
}
#[inline]
unsafe fn atomic_compare_exchange_weak<T>(dst: *mut T,
old: T,

View File

@ -1,2 +0,0 @@
rustc: 2016-03-20
cargo: 2016-03-11

File diff suppressed because it is too large Load Diff

17
src/stage0.txt Normal file
View File

@ -0,0 +1,17 @@
# This file describes the stage0 compiler that's used to then bootstrap the Rust
# compiler itself. For the rustbuild build system, this also describes the
# relevant Cargo revision that we're using.
#
# Currently Rust always bootstrap from the previous stable release, and in our
# train model this means that the master branch bootstraps from beta, beta
# bootstraps from current stable, and stable bootstraps from the previous stable
# release.
#
# If you're looking at this file on the master branch, you'll likely see that
# rustc bootstraps from `beta-$date`, whereas if you're looking at a source
# tarball for a stable release you'll likely see `1.x.0-$date` where `1.x.0` was
# released on `$date`
rustc: beta-2016-04-13
rustc_key: c2743eb4
cargo: nightly-2016-04-10