rustbuild: Tweak for vendored dependencies

A few changes are included here:

* The `winapi` and `url` dependencies were dropped. The source code for these
  projects is pretty weighty, and we're about to vendor them, so let's not
  commit to that intake just yet. If necessary we can vendor them later but for
  now it shouldn't be necessary.

* The `--frozen` flag is now always passed to Cargo, obviating the need for
  tidy's `cargo_lock` check.

* Tidy was updated to not check the vendor directory

Closes #34687
This commit is contained in:
Alex Crichton 2016-11-01 13:46:38 -07:00
parent 319f46fba3
commit 31a8638e5e
17 changed files with 145 additions and 168 deletions

1
.gitattributes vendored
View File

@ -7,3 +7,4 @@
src/etc/pkg/rust-logo.ico binary
src/etc/pkg/rust-logo.png binary
*.woff binary
src/vendor/* binary

1
.gitignore vendored
View File

@ -98,3 +98,4 @@ tmp.*.rs
version.md
version.ml
version.texi
.cargo

View File

@ -15,7 +15,7 @@ before_install:
script:
- docker run -v `pwd`:/build rust
sh -c "
./configure --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
./configure --enable-vendor --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
make tidy &&
make check -j4
"

1
configure vendored
View File

@ -634,6 +634,7 @@ opt rustbuild 0 "use the rust and cargo based build system"
opt codegen-tests 1 "run the src/test/codegen tests"
opt option-checking 1 "complain about unrecognized options in this configure script"
opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
opt vendor 0 "enable usage of vendored Rust crates"
# Optimization and debugging options. These may be overridden by the release channel, etc.
opt_nosave optimize 1 "build optimized rust code"

View File

@ -65,7 +65,8 @@ PKG_FILES := \
stage0.txt \
rust-installer \
tools \
test) \
test \
vendor) \
$(PKG_GITMODULES) \
$(filter-out config.stamp, \
$(MKFILES_FOR_TARBALL))

69
src/Cargo.lock generated
View File

@ -44,13 +44,11 @@ dependencies = [
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
"getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -149,25 +147,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
name = "graphviz"
version = "0.0.0"
[[package]]
name = "idna"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "kernel32-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libc"
version = "0.0.0"
@ -183,9 +162,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "linkchecker"
version = "0.1.0"
dependencies = [
"url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "log"
@ -196,11 +172,6 @@ name = "log"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "matches"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "md5"
version = "0.1.1"
@ -705,55 +676,15 @@ dependencies = [
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-bidi"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-normalization"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "url"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-build"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum cmake 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0e5bcf27e097a184c1df4437654ed98df3d7a516e8508a6ba45d8b092bbdf283"
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
"checksum gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)" = "553f11439bdefe755bf366b264820f1da70f3aaf3924e594b886beb9c831bcf5"
"checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "044d1360593a78f5c8e5e710beccdc24ab71d1f01bc19a29bcacdba22e8475d8"
"checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
"checksum matches 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc3ad8109fa4b522f9b0cd81440422781f564aaf8c195de6b9d6642177ad0dd"
"checksum md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5539a8dee9b4ae308c9c406a379838b435a8f2c84cf9fedc6d5a576be9888db"
"checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3"
"checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
"checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796"
"checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f"
"checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
"checksum url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "48ccf7bd87a81b769cf84ad556e034541fb90e1cd6d4bc375c822ed9500cd9d7"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"

View File

@ -27,10 +27,6 @@ num_cpus = "0.2"
toml = "0.1"
getopts = "0.2"
rustc-serialize = "0.3"
gcc = "0.3.36"
gcc = "0.3.38"
libc = "0.2"
md5 = "0.1"
[target.'cfg(windows)'.dependencies]
winapi = "0.2"
kernel32-sys = "0.2"

View File

@ -259,9 +259,11 @@ class RustBuild(object):
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
os.pathsep + env["PATH"]
self.run([self.cargo(), "build", "--manifest-path",
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
env)
args = [self.cargo(), "build", "--manifest-path",
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
if self.use_vendored_sources:
args.append("--frozen")
self.run(args, env)
def run(self, args, env):
proc = subprocess.Popen(args, env=env)
@ -384,6 +386,25 @@ def main():
except:
pass
rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
'CFG_ENABLE_VENDOR' in rb.config_mk
if rb.use_vendored_sources:
if not os.path.exists('.cargo'):
os.makedirs('.cargo')
f = open('.cargo/config','w')
f.write("""
[source.crates-io]
replace-with = 'vendored-sources'
registry = 'https://example.com'
[source.vendored-sources]
directory = '{}/src/vendor'
""".format(rb.rust_root))
f.close()
else:
if os.path.exists('.cargo'):
shutil.rmtree('.cargo')
data = stage0_data(rb.rust_root)
rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

View File

@ -44,6 +44,7 @@ pub struct Config {
pub submodules: bool,
pub compiler_docs: bool,
pub docs: bool,
pub vendor: bool,
pub target_config: HashMap<String, Target>,
// llvm codegen options
@ -126,6 +127,7 @@ struct Build {
docs: Option<bool>,
submodules: Option<bool>,
gdb: Option<String>,
vendor: Option<bool>,
}
/// TOML representation of how the LLVM build is configured.
@ -234,6 +236,7 @@ impl Config {
set(&mut config.compiler_docs, build.compiler_docs);
set(&mut config.docs, build.docs);
set(&mut config.submodules, build.submodules);
set(&mut config.vendor, build.vendor);
if let Some(ref llvm) = toml.llvm {
set(&mut config.ccache, llvm.ccache);
@ -347,6 +350,7 @@ impl Config {
("LOCAL_REBUILD", self.local_rebuild),
("NINJA", self.ninja),
("CODEGEN_TESTS", self.codegen_tests),
("VENDOR", self.vendor),
}
match key {

View File

@ -82,6 +82,9 @@
# The path to (or name of) the GDB executable to use
#gdb = "gdb"
# Indicate whether the vendored sources are used for Rust dependencies or not
#vendor = false
# =============================================================================
# Options for compiling Rust code itself
# =============================================================================

View File

@ -37,15 +37,82 @@
//! Note that this module has a #[cfg(windows)] above it as none of this logic
//! is required on Unix.
extern crate kernel32;
extern crate winapi;
#![allow(bad_style, dead_code)]
use std::env;
use std::io;
use std::mem;
use self::winapi::*;
use self::kernel32::*;
type HANDLE = *mut u8;
type BOOL = i32;
type DWORD = u32;
type LPHANDLE = *mut HANDLE;
type LPVOID = *mut u8;
type JOBOBJECTINFOCLASS = i32;
type SIZE_T = usize;
type LARGE_INTEGER = i64;
type ULONG_PTR = usize;
type ULONGLONG = u64;
const FALSE: BOOL = 0;
const DUPLICATE_SAME_ACCESS: DWORD = 0x2;
const PROCESS_DUP_HANDLE: DWORD = 0x40;
const JobObjectExtendedLimitInformation: JOBOBJECTINFOCLASS = 9;
const JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: DWORD = 0x2000;
extern "system" {
fn CreateJobObjectW(lpJobAttributes: *mut u8, lpName: *const u8) -> HANDLE;
fn CloseHandle(hObject: HANDLE) -> BOOL;
fn GetCurrentProcess() -> HANDLE;
fn OpenProcess(dwDesiredAccess: DWORD,
bInheritHandle: BOOL,
dwProcessId: DWORD) -> HANDLE;
fn DuplicateHandle(hSourceProcessHandle: HANDLE,
hSourceHandle: HANDLE,
hTargetProcessHandle: HANDLE,
lpTargetHandle: LPHANDLE,
dwDesiredAccess: DWORD,
bInheritHandle: BOOL,
dwOptions: DWORD) -> BOOL;
fn AssignProcessToJobObject(hJob: HANDLE, hProcess: HANDLE) -> BOOL;
fn SetInformationJobObject(hJob: HANDLE,
JobObjectInformationClass: JOBOBJECTINFOCLASS,
lpJobObjectInformation: LPVOID,
cbJobObjectInformationLength: DWORD) -> BOOL;
}
#[repr(C)]
struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION {
BasicLimitInformation: JOBOBJECT_BASIC_LIMIT_INFORMATION,
IoInfo: IO_COUNTERS,
ProcessMemoryLimit: SIZE_T,
JobMemoryLimit: SIZE_T,
PeakProcessMemoryUsed: SIZE_T,
PeakJobMemoryUsed: SIZE_T,
}
#[repr(C)]
struct IO_COUNTERS {
ReadOperationCount: ULONGLONG,
WriteOperationCount: ULONGLONG,
OtherOperationCount: ULONGLONG,
ReadTransferCount: ULONGLONG,
WriteTransferCount: ULONGLONG,
OtherTransferCount: ULONGLONG,
}
#[repr(C)]
struct JOBOBJECT_BASIC_LIMIT_INFORMATION {
PerProcessUserTimeLimit: LARGE_INTEGER,
PerJobUserTimeLimit: LARGE_INTEGER,
LimitFlags: DWORD,
MinimumWorkingsetSize: SIZE_T,
MaximumWorkingsetSize: SIZE_T,
ActiveProcessLimit: DWORD,
Affinity: ULONG_PTR,
PriorityClass: DWORD,
SchedulingClass: DWORD,
}
pub unsafe fn setup() {
// Create a new job object for us to use

View File

@ -460,6 +460,9 @@ impl Build {
if self.config.rust_optimize {
cargo.arg("--release");
}
if self.config.vendor {
cargo.arg("--frozen");
}
return cargo
}

View File

@ -3,9 +3,6 @@ name = "linkchecker"
version = "0.1.0"
authors = ["Alex Crichton <alex@alexcrichton.com>"]
[dependencies]
url = "1.2"
[[bin]]
name = "linkchecker"
path = "main.rs"

View File

@ -24,17 +24,13 @@
//! A few whitelisted exceptions are allowed as there's known bugs in rustdoc,
//! but this should catch the majority of "broken link" cases.
extern crate url;
use std::env;
use std::fs::File;
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::path::{Path, PathBuf, Component};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Entry;
use url::Url;
use Redirect::*;
macro_rules! t {
@ -47,9 +43,8 @@ macro_rules! t {
fn main() {
let docs = env::args().nth(1).unwrap();
let docs = env::current_dir().unwrap().join(docs);
let mut url = Url::from_file_path(&docs).unwrap();
let mut errors = false;
walk(&mut HashMap::new(), &docs, &docs, &mut url, &mut errors);
walk(&mut HashMap::new(), &docs, &docs, &mut errors);
if errors {
panic!("found some broken links");
}
@ -88,15 +83,14 @@ impl FileEntry {
}
}
fn walk(cache: &mut Cache, root: &Path, dir: &Path, url: &mut Url, errors: &mut bool) {
fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
for entry in t!(dir.read_dir()).map(|e| t!(e)) {
let path = entry.path();
let kind = t!(entry.file_type());
url.path_segments_mut().unwrap().push(entry.file_name().to_str().unwrap());
if kind.is_dir() {
walk(cache, root, &path, url, errors);
walk(cache, root, &path, errors);
} else {
let pretty_path = check(cache, root, &path, url, errors);
let pretty_path = check(cache, root, &path, errors);
if let Some(pretty_path) = pretty_path {
let entry = cache.get_mut(&pretty_path).unwrap();
// we don't need the source anymore,
@ -104,14 +98,12 @@ fn walk(cache: &mut Cache, root: &Path, dir: &Path, url: &mut Url, errors: &mut
entry.source = String::new();
}
}
url.path_segments_mut().unwrap().pop();
}
}
fn check(cache: &mut Cache,
root: &Path,
file: &Path,
base: &Url,
errors: &mut bool)
-> Option<PathBuf> {
// ignore js files as they are not prone to errors as the rest of the
@ -157,19 +149,28 @@ fn check(cache: &mut Cache,
url.starts_with("irc:") || url.starts_with("data:") {
return;
}
let mut parts = url.splitn(2, "#");
let url = parts.next().unwrap();
if url.is_empty() {
return
}
let fragment = parts.next();
let mut parts = url.splitn(2, "?");
let url = parts.next().unwrap();
// Once we've plucked out the URL, parse it using our base url and
// then try to extract a file path.
let (parsed_url, path) = match url_to_file_path(&base, url) {
Some((url, path)) => (url, PathBuf::from(path)),
None => {
*errors = true;
println!("{}:{}: invalid link - {}",
pretty_file.display(),
i + 1,
url);
return;
let mut path = file.to_path_buf();
path.pop();
for part in Path::new(url).components() {
match part {
Component::Prefix(_) |
Component::RootDir => panic!(),
Component::CurDir => {}
Component::ParentDir => { path.pop(); }
Component::Normal(s) => { path.push(s); }
}
};
}
// Alright, if we've found a file name then this file had better
// exist! If it doesn't then we register and print an error.
@ -200,7 +201,7 @@ fn check(cache: &mut Cache,
Err(LoadError::IsRedirect) => unreachable!(),
};
if let Some(ref fragment) = parsed_url.fragment() {
if let Some(ref fragment) = fragment {
// Fragments like `#1-6` are most likely line numbers to be
// interpreted by javascript, so we're ignoring these
if fragment.splitn(2, '-')
@ -231,7 +232,7 @@ fn check(cache: &mut Cache,
fn load_file(cache: &mut Cache,
root: &Path,
file: PathBuf,
mut file: PathBuf,
redirect: Redirect)
-> Result<(PathBuf, String), LoadError> {
let mut contents = String::new();
@ -266,10 +267,9 @@ fn load_file(cache: &mut Cache,
maybe
}
};
let base = Url::from_file_path(&file).unwrap();
match maybe_redirect.and_then(|url| url_to_file_path(&base, &url)) {
Some((_, redirect_file)) => {
file.pop();
match maybe_redirect.map(|url| file.join(url)) {
Some(redirect_file) => {
let path = PathBuf::from(redirect_file);
load_file(cache, root, path, FromRedirect(true))
}
@ -293,12 +293,6 @@ fn maybe_redirect(source: &str) -> Option<String> {
})
}
fn url_to_file_path(parser: &Url, url: &str) -> Option<(Url, PathBuf)> {
parser.join(url)
.ok()
.and_then(|parsed_url| parsed_url.to_file_path().ok().map(|f| (parsed_url, f)))
}
fn with_attrs_in_source<F: FnMut(&str, usize)>(contents: &str, attr: &str, mut f: F) {
for (i, mut line) in contents.lines().enumerate() {
while let Some(j) = line.find(attr) {

View File

@ -20,6 +20,9 @@ use std::fs::File;
use std::path::Path;
pub fn check(path: &Path, bad: &mut bool) {
if path.ends_with("vendor") {
return
}
for entry in t!(path.read_dir(), path).map(|e| t!(e)) {
// Look for `Cargo.toml` with a sibling `src/lib.rs` or `lib.rs`
if entry.file_name().to_str() == Some("Cargo.toml") {

View File

@ -1,45 +0,0 @@
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::path::Path;
use std::ffi::OsStr;
const CARGO_LOCK: &'static str = "Cargo.lock";
pub fn check(path: &Path, bad: &mut bool) {
use std::process::Command;
super::walk(path,
&mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
&mut |file| {
if let Some(CARGO_LOCK) = file.file_name().and_then(OsStr::to_str) {
let rel_path = file.strip_prefix(path).unwrap();
let git_friendly_path = rel_path.to_str().unwrap().replace("\\", "/");
let ret_code = Command::new("git")
.arg("diff")
.arg("--exit-code")
.arg("--patch")
.arg("HEAD")
.arg(&git_friendly_path)
.current_dir(path)
.status()
.unwrap_or_else(|e| {
panic!("could not run git diff-index: {}", e);
});
if !ret_code.success() {
let parent_path = file.parent().unwrap().join("Cargo.toml");
print!("dirty lock file found at {} ", rel_path.display());
println!("please commit your changes or update the lock file by running:");
println!("\n\tcargo update --manifest-path {}", parent_path.display());
*bad = true;
}
}
});
}

View File

@ -35,7 +35,6 @@ mod style;
mod errors;
mod features;
mod cargo;
mod cargo_lock;
mod pal;
fn main() {
@ -48,7 +47,6 @@ fn main() {
errors::check(&path, &mut bad);
cargo::check(&path, &mut bad);
features::check(&path, &mut bad);
cargo_lock::check(&path, &mut bad);
pal::check(&path, &mut bad);
if bad {
@ -66,6 +64,7 @@ fn filter_dirs(path: &Path) -> bool {
"src/rustllvm",
"src/rust-installer",
"src/liblibc",
"src/vendor",
];
skip.iter().any(|p| path.ends_with(p))
}