Auto merge of #47761 - GuillaumeGomez:test-themes, r=Mark-Simulacrum

Test themes

r? @QuietMisdreavus

cc @Mark-Simulacrum

src/Cargo.lock

@@ -2217,6 +2217,10 @@ dependencies = [
  "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rustdoc-themes"
+version = "0.1.0"
+
 [[package]]
 name = "rustdoc-tool"
 version = "0.0.0"

src/Cargo.toml

@@ -22,6 +22,7 @@ members = [
   "tools/rls",
   "tools/rustfmt",
   "tools/miri",
+  "tools/rustdoc-themes",
   # FIXME(https://github.com/rust-lang/cargo/issues/4089): move these to exclude
   "tools/rls/test_data/bin_lib",
   "tools/rls/test_data/borrow_error",

src/bootstrap/builder.rs

@@ -258,7 +258,7 @@ impl<'a> Builder<'a> {
                 test::HostCompiletest, test::Crate, test::CrateLibrustc, test::Rustdoc,
                 test::Linkcheck, test::Cargotest, test::Cargo, test::Rls, test::Docs,
                 test::ErrorIndex, test::Distcheck, test::Rustfmt, test::Miri, test::Clippy,
-                test::RustdocJS),
+                test::RustdocJS, test::RustdocTheme),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
             Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook,
                 doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon,

src/bootstrap/check.rs

@@ -160,4 +160,3 @@ pub fn libtest_stamp(build: &Build, compiler: Compiler, target: Interned<String>
 pub fn librustc_stamp(build: &Build, compiler: Compiler, target: Interned<String>) -> PathBuf {
     build.cargo_out(compiler, Mode::Librustc, target).join(".librustc-check.stamp")
 }
-

src/bootstrap/test.rs

@@ -113,7 +113,7 @@ impl Step for Linkcheck {
 
         let _time = util::timeit();
         try_run(build, builder.tool_cmd(Tool::Linkchecker)
-                            .arg(build.out.join(host).join("doc")));
+                              .arg(build.out.join(host).join("doc")));
     }
 
     fn should_run(run: ShouldRun) -> ShouldRun {
@@ -424,6 +424,47 @@ fn path_for_cargo(builder: &Builder, compiler: Compiler) -> OsString {
     env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("")
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocTheme {
+    pub compiler: Compiler,
+}
+
+impl Step for RustdocTheme {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/tools/rustdoc-themes")
+    }
+
+    fn make_run(run: RunConfig) {
+        let compiler = run.builder.compiler(run.builder.top_stage, run.host);
+
+        run.builder.ensure(RustdocTheme {
+            compiler: compiler,
+        });
+    }
+
+    fn run(self, builder: &Builder) {
+        let rustdoc = builder.rustdoc(self.compiler.host);
+        let mut cmd = builder.tool_cmd(Tool::RustdocTheme);
+        cmd.arg(rustdoc.to_str().unwrap())
+           .arg(builder.src.join("src/librustdoc/html/static/themes").to_str().unwrap())
+           .env("RUSTC_STAGE", self.compiler.stage.to_string())
+           .env("RUSTC_SYSROOT", builder.sysroot(self.compiler))
+           .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host))
+           .env("CFG_RELEASE_CHANNEL", &builder.build.config.channel)
+           .env("RUSTDOC_REAL", builder.rustdoc(self.compiler.host))
+           .env("RUSTDOC_CRATE_VERSION", builder.build.rust_version())
+           .env("RUSTC_BOOTSTRAP", "1");
+        if let Some(linker) = builder.build.linker(self.compiler.host) {
+            cmd.env("RUSTC_TARGET_LINKER", linker);
+        }
+        try_run(builder.build, &mut cmd);
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct RustdocJS {
     pub host: Interned<String>,

src/bootstrap/tool.rs

@@ -260,6 +260,7 @@ tool!(
     BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::Libstd;
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::Libstd;
     RustInstaller, "src/tools/rust-installer", "fabricate", Mode::Libstd;
+    RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::Libstd;
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

src/librustdoc/html/static/themes/dark.css

@@ -112,10 +112,13 @@ pre {
 }
 .content .highlighted a, .content .highlighted span { color: #eee !important; }
 .content .highlighted.trait { background-color: #013191; }
+.content .highlighted.mod,
+.content .highlighted.externcrate { background-color: #afc6e4; }
 .content .highlighted.mod { background-color: #803a1b; }
 .content .highlighted.externcrate { background-color: #396bac; }
 .content .highlighted.enum { background-color: #5b4e68; }
 .content .highlighted.struct { background-color: #194e9f; }
+.content .highlighted.union { background-color: #b7bd49; }
 .content .highlighted.fn,
 .content .highlighted.method,
 .content .highlighted.tymethod { background-color: #4950ed; }

src/librustdoc/lib.rs

@@ -91,6 +91,7 @@ pub mod plugins;
 pub mod visit_ast;
 pub mod visit_lib;
 pub mod test;
+pub mod theme;
 
 use clean::AttributesExt;
 
@@ -267,6 +268,11 @@ pub fn opts() -> Vec<RustcOptGroup> {
                        "additional themes which will be added to the generated docs",
                        "FILES")
         }),
+        unstable("theme-checker", |o| {
+            o.optmulti("", "theme-checker",
+                       "check if given theme is valid",
+                       "FILES")
+        }),
     ]
 }
 
@@ -316,6 +322,31 @@ pub fn main_args(args: &[String]) -> isize {
         return 0;
     }
 
+    let to_check = matches.opt_strs("theme-checker");
+    if !to_check.is_empty() {
+        let paths = theme::load_css_paths(include_bytes!("html/static/themes/main.css"));
+        let mut errors = 0;
+
+        println!("rustdoc: [theme-checker] Starting tests!");
+        for theme_file in to_check.iter() {
+            print!(" - Checking \"{}\"...", theme_file);
+            let (success, differences) = theme::test_theme_against(theme_file, &paths);
+            if !differences.is_empty() || !success {
+                println!(" FAILED");
+                errors += 1;
+                if !differences.is_empty() {
+                    println!("{}", differences.join("\n"));
+                }
+            } else {
+                println!(" OK");
+            }
+        }
+        if errors != 0 {
+            return 1;
+        }
+        return 0;
+    }
+
     if matches.free.is_empty() {
         print_error("missing file operand");
         return 1;
@@ -369,12 +400,24 @@ pub fn main_args(args: &[String]) -> isize {
     }
 
     let mut themes = Vec::new();
-    for theme in matches.opt_strs("themes").iter().map(|s| PathBuf::from(&s)) {
-        if !theme.is_file() {
-            eprintln!("rustdoc: option --themes arguments must all be files");
-            return 1;
+    if matches.opt_present("themes") {
+        let paths = theme::load_css_paths(include_bytes!("html/static/themes/main.css"));
+
+        for (theme_file, theme_s) in matches.opt_strs("themes")
+                                            .iter()
+                                            .map(|s| (PathBuf::from(&s), s.to_owned())) {
+            if !theme_file.is_file() {
+                println!("rustdoc: option --themes arguments must all be files");
+                return 1;
+            }
+            let (success, ret) = theme::test_theme_against(&theme_file, &paths);
+            if !success || !ret.is_empty() {
+                println!("rustdoc: invalid theme: \"{}\"", theme_s);
+                println!("         Check what's wrong with the \"theme-checker\" option");
+                return 1;
+            }
+            themes.push(theme_file);
         }
-        themes.push(theme);
     }
 
     let external_html = match ExternalHtml::load(

src/librustdoc/theme.rs

@@ -0,0 +1,379 @@
+// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::collections::HashSet;
+use std::fs::File;
+use std::hash::{Hash, Hasher};
+use std::io::Read;
+use std::path::Path;
+
+macro_rules! try_something {
+    ($e:expr, $out:expr) => ({
+        match $e {
+            Ok(c) => c,
+            Err(e) => {
+                eprintln!("rustdoc: got an error: {}", e);
+                return $out;
+            }
+        }
+    })
+}
+
+#[derive(Debug, Clone, Eq)]
+pub struct CssPath {
+    pub name: String,
+    pub children: HashSet<CssPath>,
+}
+
+// This PartialEq implementation IS NOT COMMUTATIVE!!!
+//
+// The order is very important: the second object must have all first's rules.
+// However, the first doesn't require to have all second's rules.
+impl PartialEq for CssPath {
+    fn eq(&self, other: &CssPath) -> bool {
+        if self.name != other.name {
+            false
+        } else {
+            for child in &self.children {
+                if !other.children.iter().any(|c| child == c) {
+                    return false;
+                }
+            }
+            true
+        }
+    }
+}
+
+impl Hash for CssPath {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.name.hash(state);
+        for x in &self.children {
+            x.hash(state);
+        }
+    }
+}
+
+impl CssPath {
+    fn new(name: String) -> CssPath {
+        CssPath {
+            name,
+            children: HashSet::new(),
+        }
+    }
+}
+
+/// All variants contain the position they occur.
+#[derive(Debug, Clone, Copy)]
+enum Events {
+    StartLineComment(usize),
+    StartComment(usize),
+    EndComment(usize),
+    InBlock(usize),
+    OutBlock(usize),
+}
+
+impl Events {
+    fn get_pos(&self) -> usize {
+        match *self {
+            Events::StartLineComment(p) |
+            Events::StartComment(p) |
+            Events::EndComment(p) |
+            Events::InBlock(p) |
+            Events::OutBlock(p) => p,
+        }
+    }
+
+    fn is_comment(&self) -> bool {
+        match *self {
+            Events::StartLineComment(_) |
+            Events::StartComment(_) |
+            Events::EndComment(_) => true,
+            _ => false,
+        }
+    }
+}
+
+fn previous_is_line_comment(events: &[Events]) -> bool {
+    if let Some(&Events::StartLineComment(_)) = events.last() {
+        true
+    } else {
+        false
+    }
+}
+
+fn is_line_comment(pos: usize, v: &[u8], events: &[Events]) -> bool {
+    if let Some(&Events::StartComment(_)) = events.last() {
+        return false;
+    }
+    pos + 1 < v.len() && v[pos + 1] == b'/'
+}
+
+fn load_css_events(v: &[u8]) -> Vec<Events> {
+    let mut pos = 0;
+    let mut events = Vec::with_capacity(100);
+
+    while pos < v.len() - 1 {
+        match v[pos] {
+            b'/' if pos + 1 < v.len() && v[pos + 1] == b'*' => {
+                events.push(Events::StartComment(pos));
+                pos += 1;
+            }
+            b'/' if is_line_comment(pos, v, &events) => {
+                events.push(Events::StartLineComment(pos));
+                pos += 1;
+            }
+            b'\n' if previous_is_line_comment(&events) => {
+                events.push(Events::EndComment(pos));
+            }
+            b'*' if pos + 1 < v.len() && v[pos + 1] == b'/' => {
+                events.push(Events::EndComment(pos + 2));
+                pos += 1;
+            }
+            b'{' if !previous_is_line_comment(&events) => {
+                if let Some(&Events::StartComment(_)) = events.last() {
+                    pos += 1;
+                    continue
+                }
+                events.push(Events::InBlock(pos + 1));
+            }
+            b'}' if !previous_is_line_comment(&events) => {
+                if let Some(&Events::StartComment(_)) = events.last() {
+                    pos += 1;
+                    continue
+                }
+                events.push(Events::OutBlock(pos + 1));
+            }
+            _ => {}
+        }
+        pos += 1;
+    }
+    events
+}
+
+fn get_useful_next(events: &[Events], pos: &mut usize) -> Option<Events> {
+    while *pos < events.len() {
+        if !events[*pos].is_comment() {
+            return Some(events[*pos]);
+        }
+        *pos += 1;
+    }
+    None
+}
+
+fn get_previous_positions(events: &[Events], mut pos: usize) -> Vec<usize> {
+    let mut ret = Vec::with_capacity(3);
+
+    ret.push(events[pos].get_pos());
+    if pos > 0 {
+        pos -= 1;
+    }
+    loop {
+        if pos < 1 || !events[pos].is_comment() {
+            let x = events[pos].get_pos();
+            if *ret.last().unwrap() != x {
+                ret.push(x);
+            } else {
+                ret.push(0);
+            }
+            break
+        }
+        ret.push(events[pos].get_pos());
+        pos -= 1;
+    }
+    if ret.len() & 1 != 0 && events[pos].is_comment() {
+        ret.push(0);
+    }
+    ret.iter().rev().cloned().collect()
+}
+
+fn build_rule(v: &[u8], positions: &[usize]) -> String {
+    positions.chunks(2)
+             .map(|x| ::std::str::from_utf8(&v[x[0]..x[1]]).unwrap_or(""))
+             .collect::<String>()
+             .trim()
+             .replace("\n", " ")
+             .replace("/", "")
+             .replace("\t", " ")
+             .replace("{", "")
+             .replace("}", "")
+             .split(" ")
+             .filter(|s| s.len() > 0)
+             .collect::<Vec<&str>>()
+             .join(" ")
+}
+
+fn inner(v: &[u8], events: &[Events], pos: &mut usize) -> HashSet<CssPath> {
+    let mut paths = Vec::with_capacity(50);
+
+    while *pos < events.len() {
+        if let Some(Events::OutBlock(_)) = get_useful_next(events, pos) {
+            *pos += 1;
+            break
+        }
+        if let Some(Events::InBlock(_)) = get_useful_next(events, pos) {
+            paths.push(CssPath::new(build_rule(v, &get_previous_positions(events, *pos))));
+            *pos += 1;
+        }
+        while let Some(Events::InBlock(_)) = get_useful_next(events, pos) {
+            if let Some(ref mut path) = paths.last_mut() {
+                for entry in inner(v, events, pos).iter() {
+                    path.children.insert(entry.clone());
+                }
+            }
+        }
+        if let Some(Events::OutBlock(_)) = get_useful_next(events, pos) {
+            *pos += 1;
+        }
+    }
+    paths.iter().cloned().collect()
+}
+
+pub fn load_css_paths(v: &[u8]) -> CssPath {
+    let events = load_css_events(v);
+    let mut pos = 0;
+
+    let mut parent = CssPath::new("parent".to_owned());
+    parent.children = inner(v, &events, &mut pos);
+    parent
+}
+
+pub fn get_differences(against: &CssPath, other: &CssPath, v: &mut Vec<String>) {
+    if against.name != other.name {
+        return
+    } else {
+        for child in &against.children {
+            let mut found = false;
+            let mut found_working = false;
+            let mut tmp = Vec::new();
+
+            for other_child in &other.children {
+                if child.name == other_child.name {
+                    if child != other_child {
+                        get_differences(child, other_child, &mut tmp);
+                    } else {
+                        found_working = true;
+                    }
+                    found = true;
+                    break
+                }
+            }
+            if found == false {
+                v.push(format!("  Missing \"{}\" rule", child.name));
+            } else if found_working == false {
+                v.extend(tmp.iter().cloned());
+            }
+        }
+    }
+}
+
+pub fn test_theme_against<P: AsRef<Path>>(f: &P, against: &CssPath) -> (bool, Vec<String>) {
+    let mut file = try_something!(File::open(f), (false, Vec::new()));
+    let mut data = Vec::with_capacity(1000);
+
+    try_something!(file.read_to_end(&mut data), (false, Vec::new()));
+    let paths = load_css_paths(&data);
+    let mut ret = Vec::new();
+    get_differences(against, &paths, &mut ret);
+    (true, ret)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_comments_in_rules() {
+        let text = r#"
+rule a {}
+
+rule b, c
+// a line comment
+{}
+
+rule d
+// another line comment
+e {}
+
+rule f/* a multine
+
+comment*/{}
+
+rule g/* another multine
+
+comment*/h
+
+i {}
+
+rule j/*commeeeeent
+
+you like things like "{}" in there? :)
+*/
+end {}"#;
+
+        let against = r#"
+rule a {}
+
+rule b, c {}
+
+rule d e {}
+
+rule f {}
+
+rule gh i {}
+
+rule j end {}
+"#;
+
+        let mut ret = Vec::new();
+        get_differences(&load_css_paths(against.as_bytes()),
+                        &load_css_paths(text.as_bytes()),
+                        &mut ret);
+        assert!(ret.is_empty());
+    }
+
+    #[test]
+    fn test_text() {
+        let text = r#"
+a
+/* sdfs
+*/ b
+c // sdf
+d {}
+"#;
+        let paths = load_css_paths(text.as_bytes());
+        assert!(paths.children.contains(&CssPath::new("a b c d".to_owned())));
+    }
+
+    #[test]
+    fn test_comparison() {
+        let x = r#"
+a {
+    b {
+        c {}
+    }
+}
+"#;
+
+        let y = r#"
+a {
+    b {}
+}
+"#;
+
+        let against = load_css_paths(y.as_bytes());
+        let other = load_css_paths(x.as_bytes());
+
+        let mut ret = Vec::new();
+        get_differences(&against, &other, &mut ret);
+        assert!(ret.is_empty());
+        get_differences(&other, &against, &mut ret);
+        assert_eq!(ret, vec!["  Missing \"c\" rule".to_owned()]);
+    }
+}

src/tools/rustdoc-themes/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "rustdoc-themes"
+version = "0.1.0"
+authors = ["Guillaume Gomez <guillaume1.gomez@gmail.com>"]
+
+[[bin]]
+name = "rustdoc-themes"
+path = "main.rs"

src/tools/rustdoc-themes/main.rs

@@ -0,0 +1,59 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::env::args;
+use std::fs::read_dir;
+use std::path::Path;
+use std::process::{Command, exit};
+
+const FILES_TO_IGNORE: &[&str] = &["main.css"];
+
+fn get_folders<P: AsRef<Path>>(folder_path: P) -> Vec<String> {
+    let mut ret = Vec::with_capacity(10);
+
+    for entry in read_dir(folder_path.as_ref()).expect("read_dir failed") {
+        let entry = entry.expect("Couldn't unwrap entry");
+        let path = entry.path();
+
+        if !path.is_file() {
+            continue
+        }
+        let filename = path.file_name().expect("file_name failed");
+        if FILES_TO_IGNORE.iter().any(|x| x == &filename) {
+            continue
+        }
+        ret.push(format!("{}", path.display()));
+    }
+    ret
+}
+
+fn main() {
+    let argv: Vec<String> = args().collect();
+
+    if argv.len() < 3 {
+        eprintln!("Needs rustdoc binary path");
+        exit(1);
+    }
+    let rustdoc_bin = &argv[1];
+    let themes_folder = &argv[2];
+    let themes = get_folders(&themes_folder);
+    if themes.is_empty() {
+        eprintln!("No theme found in \"{}\"...", themes_folder);
+        exit(1);
+    }
+    let status = Command::new(rustdoc_bin)
+                        .args(&["-Z", "unstable-options", "--theme-checker"])
+                        .args(&themes)
+                        .status()
+                        .expect("failed to execute child");
+    if !status.success() {
+        exit(1);
+    }
+}