Autogenerate stubs and the summary of the unstable book

src/Cargo.lock

@@ -1892,6 +1892,13 @@ dependencies = [
  "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "unstable-book-gen"
+version = "0.1.0"
+dependencies = [
+ "tidy 0.1.0",
+]
+
 [[package]]
 name = "url"
 version = "1.4.0"

src/Cargo.toml

@@ -9,6 +9,7 @@ members = [
   "tools/error_index_generator",
   "tools/linkchecker",
   "tools/rustbook",
+  "tools/unstable-book-gen",
   "tools/tidy",
   "tools/build-manifest",
   "tools/remote-test-client",

src/bootstrap/doc.rs

@@ -27,18 +27,26 @@ use {Build, Compiler, Mode};
 use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
-/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
-/// `name` into the `out` path.
+/// Invoke `rustbook` for `target` for the doc book `name`.
 ///
 /// This will not actually generate any documentation if the documentation has
 /// already been generated.
 pub fn rustbook(build: &Build, target: &str, name: &str) {
+    let src = build.src.join("src/doc");
+    rustbook_src(build, target, name, &src);
+}
+
+/// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
+///
+/// This will not actually generate any documentation if the documentation has
+/// already been generated.
+pub fn rustbook_src(build: &Build, target: &str, name: &str, src: &Path) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
 
     let out = out.join(name);
     let compiler = Compiler::new(0, &build.config.build);
-    let src = build.src.join("src/doc").join(name);
+    let src = src.join(name);
     let index = out.join("index.html");
     let rustbook = build.tool(&compiler, "rustbook");
     if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
@@ -354,6 +362,19 @@ pub fn error_index(build: &Build, target: &str) {
     build.run(&mut index);
 }
 
+pub fn unstable_book_gen(build: &Build, target: &str) {
+    println!("Generating unstable book md files ({})", target);
+    let out = build.md_doc_out(target).join("unstable-book");
+    t!(fs::create_dir_all(&out));
+    t!(fs::remove_dir_all(&out));
+    let compiler = Compiler::new(0, &build.config.build);
+    let mut cmd = build.tool_cmd(&compiler, "unstable-book-gen");
+    cmd.arg(build.src.join("src"));
+    cmd.arg(out);
+
+    build.run(&mut cmd);
+}
+
 fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
     if let Ok(m) = fs::symlink_metadata(dst) {
         if m.file_type().is_dir() {

src/bootstrap/lib.rs

@@ -677,6 +677,11 @@ impl Build {
         self.out.join(target).join("doc")
     }
 
+    /// Output directory for some generated md crate documentation for a target (temporary)
+    fn md_doc_out(&self, target: &str) -> PathBuf {
+        self.out.join(target).join("md-doc")
+    }
+
     /// Output directory for all crate documentation for a target (temporary)
     ///
     /// The artifacts here are then copied into `doc_out` above.

src/bootstrap/step.rs

@@ -548,6 +548,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("librustc-tool"))
          .run(move |s| compile::tool(build, s.stage, s.target, "error_index_generator"));
+    rules.build("tool-unstable-book-gen", "src/tools/unstable-book-gen")
+         .dep(|s| s.name("maybe-clean-tools"))
+         .dep(|s| s.name("libstd-tool"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "unstable-book-gen"));
     rules.build("tool-tidy", "src/tools/tidy")
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("libstd-tool"))
@@ -662,8 +666,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
               .target(&build.config.build)
               .stage(0)
          })
+         .dep(move |s| {
+             s.name("doc-unstable-book-gen")
+              .host(&build.config.build)
+              .target(&build.config.build)
+              .stage(0)
+         })
          .default(build.config.docs)
-         .run(move |s| doc::rustbook(build, s.target, "unstable-book"));
+         .run(move |s| doc::rustbook_src(build,
+                                         s.target,
+                                         "unstable-book",
+                                         &build.md_doc_out(s.target)));
     rules.doc("doc-standalone", "src/doc")
          .dep(move |s| {
              s.name("rustc")
@@ -679,6 +692,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .default(build.config.docs)
          .host(true)
          .run(move |s| doc::error_index(build, s.target));
+    rules.doc("doc-unstable-book-gen", "src/tools/unstable-book-gen")
+         .dep(move |s| s.name("tool-unstable-book-gen").target(&build.config.build).stage(0))
+         .dep(move |s| s.name("librustc-link"))
+         .default(build.config.docs)
+         .host(true)
+         .run(move |s| doc::unstable_book_gen(build, s.target));
     for (krate, path, default) in krates("std") {
         rules.doc(&krate.doc_step, path)
              .dep(|s| s.name("libstd-link"))

src/doc/unstable-book/src/language-features/asm.md

@@ -190,4 +190,4 @@ constraints, etc.
 [llvm-docs]: http://llvm.org/docs/LangRef.html#inline-assembler-expressions
 
 If you need more power and don't mind losing some of the niceties of
-`asm!`, check out [global_asm](language-features/global_asm.html).
+`asm!`, check out [global_asm](language-features/global-asm.html).

src/tools/tidy/src/features.rs

@@ -24,7 +24,7 @@ use std::fs::File;
 use std::io::prelude::*;
 use std::path::Path;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum Status {
     Stable,
     Removed,
@@ -42,13 +42,16 @@ impl fmt::Display for Status {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Feature {
     pub level: Status,
     pub since: String,
     pub has_gate_test: bool,
+    pub tracking_issue: Option<u32>,
 }
 
+pub type Features = HashMap<String, Feature>;
+
 pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
     let mut features = collect_lang_features(path);
     assert!(!features.is_empty());
@@ -168,8 +171,7 @@ fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
         .map(|(i, j)| &line[i..j])
 }
 
-fn test_filen_gate(filen_underscore: &str,
-                   features: &mut HashMap<String, Feature>) -> bool {
+fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
     if filen_underscore.starts_with("feature_gate") {
         for (n, f) in features.iter_mut() {
             if filen_underscore == format!("feature_gate_{}", n) {
@@ -181,7 +183,7 @@ fn test_filen_gate(filen_underscore: &str,
     return false;
 }
 
-pub fn collect_lang_features(base_src_path: &Path) -> HashMap<String, Feature> {
+pub fn collect_lang_features(base_src_path: &Path) -> Features {
     let mut contents = String::new();
     let path = base_src_path.join("libsyntax/feature_gate.rs");
     t!(t!(File::open(path)).read_to_string(&mut contents));
@@ -197,11 +199,19 @@ pub fn collect_lang_features(base_src_path: &Path) -> HashMap<String, Feature> {
             };
             let name = parts.next().unwrap().trim();
             let since = parts.next().unwrap().trim().trim_matches('"');
+            let issue_str = parts.next().unwrap().trim();
+            let tracking_issue = if issue_str.starts_with("None") {
+                None
+            } else {
+                let s = issue_str.split("(").nth(1).unwrap().split(")").nth(0).unwrap();
+                Some(s.parse().unwrap())
+            };
             Some((name.to_owned(),
                 Feature {
-                    level: level,
+                    level,
                     since: since.to_owned(),
                     has_gate_test: false,
+                    tracking_issue,
                 }))
         })
         .collect()
@@ -209,8 +219,8 @@ pub fn collect_lang_features(base_src_path: &Path) -> HashMap<String, Feature> {
 
 pub fn collect_lib_features(base_src_path: &Path,
                             bad: &mut bool,
-                            features: &HashMap<String, Feature>) -> HashMap<String, Feature> {
-    let mut lib_features = HashMap::<String, Feature>::new();
+                            features: &Features) -> Features {
+    let mut lib_features = Features::new();
     let mut contents = String::new();
     super::walk(base_src_path,
                 &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
@@ -224,10 +234,32 @@ pub fn collect_lib_features(base_src_path: &Path,
         contents.truncate(0);
         t!(t!(File::open(&file), &file).read_to_string(&mut contents));
 
+        let mut becoming_feature: Option<(String, Feature)> = None;
         for (i, line) in contents.lines().enumerate() {
             let mut err = |msg: &str| {
                 tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
             };
+            if let Some((ref name, ref mut f)) = becoming_feature {
+                if f.tracking_issue.is_none() {
+                    f.tracking_issue = find_attr_val(line, "issue")
+                    .map(|s| s.parse().unwrap());
+                }
+                if line.ends_with("]") {
+                    lib_features.insert(name.to_owned(), f.clone());
+                } else if !line.ends_with(",") && !line.ends_with("\\") {
+                    // We need to bail here because we might have missed the
+                    // end of a stability attribute above because the "]"
+                    // might not have been at the end of the line.
+                    // We could then get into the very unfortunate situation that
+                    // we continue parsing the file assuming the current stability
+                    // attribute has not ended, and ignoring possible feature
+                    // attributes in the process.
+                    err("malformed stability attribute");
+                } else {
+                    continue;
+                }
+            }
+            becoming_feature = None;
             let level = if line.contains("[unstable(") {
                 Status::Unstable
             } else if line.contains("[stable(") {
@@ -250,6 +282,7 @@ pub fn collect_lib_features(base_src_path: &Path,
                 }
                 None => "None",
             };
+            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());
 
             if features.contains_key(feature_name) {
                 err("duplicating a lang feature");
@@ -263,12 +296,17 @@ pub fn collect_lib_features(base_src_path: &Path,
                 }
                 continue;
             }
-            lib_features.insert(feature_name.to_owned(),
-                                Feature {
-                                    level: level,
-                                    since: since.to_owned(),
-                                    has_gate_test: false,
-                                });
+            let feature = Feature {
+                level,
+                since: since.to_owned(),
+                has_gate_test: false,
+                tracking_issue,
+            };
+            if line.contains("]") {
+                lib_features.insert(feature_name.to_owned(), feature);
+            } else {
+                becoming_feature = Some((feature_name.to_owned(), feature));
+            }
         }
     });
     lib_features

src/tools/tidy/src/unstable_book.rs

@@ -11,26 +11,28 @@
 use std::collections::HashSet;
 use std::fs;
 use std::path;
-use features::{collect_lang_features, collect_lib_features, Status};
+use features::{collect_lang_features, collect_lib_features, Features, Status};
 
-const PATH_STR: &'static str = "doc/unstable-book/src";
+pub const PATH_STR: &str = "doc/unstable-book/src";
 
-const LANG_FEATURES_DIR: &'static str = "language-features";
+pub const COMPILER_FLAGS_DIR: &str = "compiler-flags";
 
-const LIB_FEATURES_DIR: &'static str = "library-features";
+pub const LANG_FEATURES_DIR: &str = "language-features";
+
+pub const LIB_FEATURES_DIR: &str = "library-features";
 
 /// Build the path to the Unstable Book source directory from the Rust 'src' directory
-fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf {
+pub fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf {
     base_src_path.join(PATH_STR)
 }
 
 /// Directory where the features are documented within the Unstable Book source directory
-fn unstable_book_lang_features_path(base_src_path: &path::Path) -> path::PathBuf {
+pub fn unstable_book_lang_features_path(base_src_path: &path::Path) -> path::PathBuf {
     unstable_book_path(base_src_path).join(LANG_FEATURES_DIR)
 }
 
 /// Directory where the features are documented within the Unstable Book source directory
-fn unstable_book_lib_features_path(base_src_path: &path::Path) -> path::PathBuf {
+pub fn unstable_book_lib_features_path(base_src_path: &path::Path) -> path::PathBuf {
     unstable_book_path(base_src_path).join(LIB_FEATURES_DIR)
 }
 
@@ -42,27 +44,16 @@ fn dir_entry_is_file(dir_entry: &fs::DirEntry) -> bool {
         .is_file()
 }
 
-/// Retrieve names of all lang-related unstable features
-fn collect_unstable_lang_feature_names(base_src_path: &path::Path) -> HashSet<String> {
-    collect_lang_features(base_src_path)
-        .into_iter()
+/// Retrieve names of all unstable features
+pub fn collect_unstable_feature_names(features: &Features) -> HashSet<String> {
+    features
+        .iter()
         .filter(|&(_, ref f)| f.level == Status::Unstable)
-        .map(|(ref name, _)| name.to_owned())
+        .map(|(name, _)| name.to_owned())
         .collect()
 }
 
-/// Retrieve names of all lib-related unstable features
-fn collect_unstable_lib_feature_names(base_src_path: &path::Path) -> HashSet<String> {
-    let mut bad = true;
-    let lang_features = collect_lang_features(base_src_path);
-    collect_lib_features(base_src_path, &mut bad, &lang_features)
-        .into_iter()
-        .filter(|&(_, ref f)| f.level == Status::Unstable)
-        .map(|(ref name, _)| name.to_owned())
-        .collect()
-}
-
-fn collect_unstable_book_section_file_names(dir: &path::Path) -> HashSet<String> {
+pub fn collect_unstable_book_section_file_names(dir: &path::Path) -> HashSet<String> {
     fs::read_dir(dir)
         .expect("could not read directory")
         .into_iter()
@@ -95,7 +86,10 @@ pub fn check(path: &path::Path, bad: &mut bool) {
 
     // Library features
 
-    let unstable_lib_feature_names = collect_unstable_lib_feature_names(path);
+    let lang_features = collect_lang_features(path);
+    let lib_features = collect_lib_features(path, bad, &lang_features);
+
+    let unstable_lib_feature_names = collect_unstable_feature_names(&lib_features);
     let unstable_book_lib_features_section_file_names =
         collect_unstable_book_lib_features_section_file_names(path);
 
@@ -119,7 +113,7 @@ pub fn check(path: &path::Path, bad: &mut bool) {
 
     // Language features
 
-    let unstable_lang_feature_names = collect_unstable_lang_feature_names(path);
+    let unstable_lang_feature_names = collect_unstable_feature_names(&lang_features);
     let unstable_book_lang_features_section_file_names =
         collect_unstable_book_lang_features_section_file_names(path);
 

src/tools/unstable-book-gen/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+authors = ["est31 <MTest31@outlook.com>",
+           "The Rust Project Developers"]
+name = "unstable-book-gen"
+version = "0.1.0"
+license = "MIT/Apache-2.0"
+
+[dependencies]
+tidy = { path = "../tidy" }

src/tools/unstable-book-gen/src/SUMMARY.md

@@ -0,0 +1,8 @@
+[The Unstable Book](the-unstable-book.md)
+
+- [Compiler flags](compiler-flags.md)
+{compiler_flags}
+- [Language features](language-features.md)
+{language_features}
+- [Library Features](library-features.md)
+{library_features}

src/tools/unstable-book-gen/src/main.rs

@@ -0,0 +1,149 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Auto-generate stub docs for the unstable book
+
+#![deny(warnings)]
+
+extern crate tidy;
+
+use tidy::features::{Feature, Features, collect_lib_features, collect_lang_features};
+use tidy::unstable_book::{collect_unstable_feature_names, collect_unstable_book_section_file_names,
+                          PATH_STR, LANG_FEATURES_DIR, LIB_FEATURES_DIR};
+use std::collections::HashSet;
+use std::io::Write;
+use std::fs::{self, File};
+use std::env;
+use std::path::Path;
+
+/// A helper macro to `unwrap` a result except also print out details like:
+///
+/// * The file/line of the panic
+/// * The expression that failed
+/// * The error itself
+macro_rules! t {
+    ($e:expr) => (match $e {
+        Ok(e) => e,
+        Err(e) => panic!("{} failed with {}", stringify!($e), e),
+    })
+}
+
+fn generate_stub_issue(path: &Path, name: &str, issue: u32) {
+    let mut file = t!(File::create(path));
+    t!(file.write_fmt(format_args!(include_str!("stub-issue.md"),
+                                   name = name,
+                                   issue = issue)));
+}
+
+fn generate_stub_no_issue(path: &Path, name: &str) {
+    let mut file = t!(File::create(path));
+    t!(file.write_fmt(format_args!(include_str!("stub-no-issue.md"),
+                                   name = name)));
+}
+
+fn hset_to_summary_str(hset: HashSet<String>, dir: &str
+) -> String {
+    hset
+        .iter()
+        .map(|ref n| format!("    - [{}]({}/{}.md)",
+                                      n,
+                                      dir,
+                                      n.replace('_', "-")))
+        .fold("".to_owned(), |s, a| s + &a + "\n")
+}
+
+fn generate_summary(path: &Path, lang_features: &Features, lib_features: &Features) {
+    let compiler_flags = collect_unstable_book_section_file_names(
+        &path.join("compiler-flags"));
+
+    let compiler_flags_str = hset_to_summary_str(compiler_flags,
+                                                 "compiler-flags");
+
+    let unstable_lang_features = collect_unstable_feature_names(&lang_features);
+    let unstable_lib_features = collect_unstable_feature_names(&lib_features);
+
+    let lang_features_str = hset_to_summary_str(unstable_lang_features,
+                                                LANG_FEATURES_DIR);
+    let lib_features_str = hset_to_summary_str(unstable_lib_features,
+                                               LIB_FEATURES_DIR);
+
+    let mut file = t!(File::create(&path.join("SUMMARY.md")));
+    t!(file.write_fmt(format_args!(include_str!("SUMMARY.md"),
+                                   compiler_flags = compiler_flags_str,
+                                   language_features = lang_features_str,
+                                   library_features = lib_features_str)));
+
+}
+
+fn has_valid_tracking_issue(f: &Feature) -> bool {
+    if let Some(n) = f.tracking_issue {
+        if n > 0 {
+            return true;
+        }
+    }
+    false
+}
+
+fn generate_unstable_book_files(src :&Path, out: &Path, features :&Features) {
+    let unstable_features = collect_unstable_feature_names(features);
+    let unstable_section_file_names = collect_unstable_book_section_file_names(src);
+    t!(fs::create_dir_all(&out));
+    for feature_name in &unstable_features - &unstable_section_file_names {
+        let file_name = format!("{}.md", feature_name.replace('_', "-"));
+        let out_file_path = out.join(&file_name);
+        let feature = &features[&feature_name];
+
+        if has_valid_tracking_issue(&feature) {
+            generate_stub_issue(&out_file_path, &feature_name, feature.tracking_issue.unwrap());
+        } else {
+            generate_stub_no_issue(&out_file_path, &feature_name);
+        }
+    }
+}
+
+fn copy_recursive(path: &Path, to: &Path) {
+    for entry in t!(fs::read_dir(path)) {
+        let e = t!(entry);
+        let t = t!(e.metadata());
+        let dest = &to.join(e.file_name());
+        if t.is_file() {
+            t!(fs::copy(&e.path(), dest));
+        } else if t.is_dir() {
+            t!(fs::create_dir_all(dest));
+            copy_recursive(&e.path(), dest);
+        }
+    }
+}
+
+fn main() {
+    let src_path_str = env::args_os().skip(1).next().expect("source path required");
+    let dest_path_str = env::args_os().skip(2).next().expect("destination path required");
+    let src_path = Path::new(&src_path_str);
+    let dest_path = Path::new(&dest_path_str).join("src");
+
+    let lang_features = collect_lang_features(src_path);
+    let mut bad = false;
+    let lib_features = collect_lib_features(src_path, &mut bad, &lang_features);
+
+    let doc_src_path = src_path.join(PATH_STR);
+
+    t!(fs::create_dir_all(&dest_path));
+
+    generate_unstable_book_files(&doc_src_path.join(LANG_FEATURES_DIR),
+                                 &dest_path.join(LANG_FEATURES_DIR),
+                                 &lang_features);
+    generate_unstable_book_files(&doc_src_path.join(LIB_FEATURES_DIR),
+                                 &dest_path.join(LIB_FEATURES_DIR),
+                                 &lib_features);
+
+    copy_recursive(&doc_src_path, &dest_path);
+
+    generate_summary(&dest_path, &lang_features, &lib_features);
+}

src/tools/unstable-book-gen/src/stub-issue.md

@@ -0,0 +1,7 @@
+# `{name}`
+
+The tracking issue for this feature is: [#{issue}]
+
+[#{issue}]: https://github.com/rust-lang/rust/issues/{issue}
+
+------------------------

src/tools/unstable-book-gen/src/stub-no-issue.md

@@ -0,0 +1,5 @@
+# `{name}`
+
+This feature has no tracking issue, and is therefore likely internal to the compiler, not being intended for general use.
+
+------------------------