Auto merge of #30300 - sanxiyn:syntax-ext, r=nikomatsakis

This reduces iteration time (`make rustc-stage1`) for moved syntax extensions from 11 minutes to 3 minutes on my machine.

Because of the signature change, this is a [breaking-change] for people directly calling `expand_crate`. I think it is rare: from GitHub search, only case I found is [glassful](https://github.com/kmcallister/glassful).

mk/crates.mk

@@ -58,7 +58,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures rustc_front rustc_platform_intrinsics \
                 rustc_plugin rustc_metadata
-HOST_CRATES := syntax $(RUSTC_CRATES) rustdoc fmt_macros
+HOST_CRATES := syntax syntax_ext $(RUSTC_CRATES) rustdoc fmt_macros
 TOOLS := compiletest rustdoc rustc rustbook error-index-generator
 
 DEPS_core :=
@@ -86,9 +86,10 @@ DEPS_serialize := std log
 DEPS_term := std log
 DEPS_test := std getopts serialize rbml term native:rust_test_helpers
 
-DEPS_syntax := std term serialize log fmt_macros arena libc rustc_bitflags
+DEPS_syntax := std term serialize log arena libc rustc_bitflags
+DEPS_syntax_ext := syntax fmt_macros
 
-DEPS_rustc := syntax flate arena serialize getopts rbml rustc_front\
+DEPS_rustc := syntax fmt_macros flate arena serialize getopts rbml rustc_front\
               log graphviz rustc_llvm rustc_back rustc_data_structures
 DEPS_rustc_back := std syntax rustc_llvm rustc_front flate log libc
 DEPS_rustc_borrowck := rustc rustc_front log graphviz syntax
@@ -96,7 +97,7 @@ DEPS_rustc_data_structures := std log serialize
 DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
                      rustc_typeck rustc_mir rustc_resolve log syntax serialize rustc_llvm \
 	             rustc_trans rustc_privacy rustc_lint rustc_front rustc_plugin \
-                     rustc_metadata
+                     rustc_metadata syntax_ext
 DEPS_rustc_front := std syntax log serialize
 DEPS_rustc_lint := rustc log syntax
 DEPS_rustc_llvm := native:rustllvm libc std rustc_bitflags

src/librustc_driver/driver.rs

@@ -54,6 +54,7 @@ use syntax::parse::token;
 use syntax::util::node_count::NodeCounter;
 use syntax::visit;
 use syntax;
+use syntax_ext;
 
 pub fn compile_input(sess: Session,
                      cstore: &CStore,
@@ -563,12 +564,15 @@ pub fn phase_2_configure_and_expand(sess: &Session,
             recursion_limit: sess.recursion_limit.get(),
             trace_mac: sess.opts.debugging_opts.trace_macros,
         };
-        let (ret, macro_names) = syntax::ext::expand::expand_crate(&sess.parse_sess,
-                                                                    cfg,
-                                                                    macros,
-                                                                    syntax_exts,
-                                                                    &mut feature_gated_cfgs,
-                                                                    krate);
+        let mut ecx = syntax::ext::base::ExtCtxt::new(&sess.parse_sess,
+                                                      krate.config.clone(),
+                                                      cfg,
+                                                      &mut feature_gated_cfgs);
+        syntax_ext::register_builtins(&mut ecx.syntax_env);
+        let (ret, macro_names) = syntax::ext::expand::expand_crate(ecx,
+                                                                   macros,
+                                                                   syntax_exts,
+                                                                   krate);
         if cfg!(windows) {
             env::set_var("PATH", &_old_path);
         }

src/librustc_driver/lib.rs

@@ -57,6 +57,7 @@ extern crate rustc_llvm as llvm;
 extern crate log;
 #[macro_use]
 extern crate syntax;
+extern crate syntax_ext;
 
 pub use syntax::diagnostic;
 

src/libsyntax/ext/base.rs

@@ -464,26 +464,6 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
 
     let mut syntax_expanders = SyntaxEnv::new();
     syntax_expanders.insert(intern("macro_rules"), MacroRulesTT);
-    syntax_expanders.insert(intern("format_args"),
-                            // format_args uses `unstable` things internally.
-                            NormalTT(Box::new(ext::format::expand_format_args), None, true));
-    syntax_expanders.insert(intern("env"),
-                            builtin_normal_expander(
-                                    ext::env::expand_env));
-    syntax_expanders.insert(intern("option_env"),
-                            builtin_normal_expander(
-                                    ext::env::expand_option_env));
-    syntax_expanders.insert(intern("concat_idents"),
-                            builtin_normal_expander(
-                                    ext::concat_idents::expand_syntax_ext));
-    syntax_expanders.insert(intern("concat"),
-                            builtin_normal_expander(
-                                    ext::concat::expand_syntax_ext));
-    syntax_expanders.insert(intern("log_syntax"),
-                            builtin_normal_expander(
-                                    ext::log_syntax::expand_syntax_ext));
-
-    ext::deriving::register_all(&mut syntax_expanders);
 
     if ecfg.enable_quotes() {
         // Quasi-quoting expanders
@@ -552,15 +532,6 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
     syntax_expanders.insert(intern("module_path"),
                             builtin_normal_expander(
                                     ext::source_util::expand_mod));
-    syntax_expanders.insert(intern("asm"),
-                            builtin_normal_expander(
-                                    ext::asm::expand_asm));
-    syntax_expanders.insert(intern("cfg"),
-                            builtin_normal_expander(
-                                    ext::cfg::expand_cfg));
-    syntax_expanders.insert(intern("trace_macros"),
-                            builtin_normal_expander(
-                                    ext::trace_macros::expand_trace_macros));
     syntax_expanders
 }
 

src/libsyntax/ext/expand.rs

@@ -21,7 +21,7 @@ use attr::{AttrMetaMethods, WithAttrs};
 use codemap;
 use codemap::{Span, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
 use ext::base::*;
-use feature_gate::{self, Features, GatedCfgAttr};
+use feature_gate::{self, Features};
 use fold;
 use fold::*;
 use util::move_map::MoveMap;
@@ -1276,15 +1276,11 @@ impl<'feat> ExpansionConfig<'feat> {
     }
 }
 
-pub fn expand_crate<'feat>(parse_sess: &parse::ParseSess,
-                           cfg: ExpansionConfig<'feat>,
-                           // these are the macros being imported to this crate:
-                           imported_macros: Vec<ast::MacroDef>,
-                           user_exts: Vec<NamedSyntaxExtension>,
-                           feature_gated_cfgs: &mut Vec<GatedCfgAttr>,
-                           c: Crate) -> (Crate, HashSet<Name>) {
-    let mut cx = ExtCtxt::new(parse_sess, c.config.clone(), cfg,
-                              feature_gated_cfgs);
+pub fn expand_crate(mut cx: ExtCtxt,
+                    // these are the macros being imported to this crate:
+                    imported_macros: Vec<ast::MacroDef>,
+                    user_exts: Vec<NamedSyntaxExtension>,
+                    c: Crate) -> (Crate, HashSet<Name>) {
     if std_inject::no_core(&c) {
         cx.crate_root = None;
     } else if std_inject::no_std(&c) {
@@ -1305,7 +1301,7 @@ pub fn expand_crate<'feat>(parse_sess: &parse::ParseSess,
 
         let mut ret = expander.fold_crate(c);
         ret.exported_macros = expander.cx.exported_macros.clone();
-        parse_sess.span_diagnostic.handler().abort_if_errors();
+        cx.parse_sess.span_diagnostic.handler().abort_if_errors();
         ret
     };
     return (ret, cx.syntax_env.names);
@@ -1401,6 +1397,7 @@ mod tests {
     use ast;
     use ast::Name;
     use codemap;
+    use ext::base::ExtCtxt;
     use ext::mtwt;
     use fold::Folder;
     use parse;
@@ -1471,7 +1468,9 @@ mod tests {
             src,
             Vec::new(), &sess);
         // should fail:
-        expand_crate(&sess,test_ecfg(),vec!(),vec!(), &mut vec![], crate_ast);
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast);
     }
 
     // make sure that macros can't escape modules
@@ -1484,7 +1483,9 @@ mod tests {
             "<test>".to_string(),
             src,
             Vec::new(), &sess);
-        expand_crate(&sess,test_ecfg(),vec!(),vec!(), &mut vec![], crate_ast);
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast);
     }
 
     // macro_use modules should allow macros to escape
@@ -1496,14 +1497,18 @@ mod tests {
             "<test>".to_string(),
             src,
             Vec::new(), &sess);
-        expand_crate(&sess, test_ecfg(), vec!(), vec!(), &mut vec![], crate_ast);
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast);
     }
 
     fn expand_crate_str(crate_str: String) -> ast::Crate {
         let ps = parse::ParseSess::new();
         let crate_ast = panictry!(string_to_parser(&ps, crate_str).parse_crate_mod());
         // the cfg argument actually does matter, here...
-        expand_crate(&ps,test_ecfg(),vec!(),vec!(), &mut vec![], crate_ast).0
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&ps, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast).0
     }
 
     // find the pat_ident paths in a crate

src/libsyntax/lib.rs

@@ -37,7 +37,6 @@
 #![feature(str_escape)]
 #![feature(unicode)]
 
-extern crate fmt_macros;
 extern crate serialize;
 extern crate term;
 extern crate libc;
@@ -110,21 +109,12 @@ pub mod print {
 }
 
 pub mod ext {
-    pub mod asm;
     pub mod base;
     pub mod build;
-    pub mod cfg;
-    pub mod concat;
-    pub mod concat_idents;
-    pub mod deriving;
-    pub mod env;
     pub mod expand;
-    pub mod format;
-    pub mod log_syntax;
     pub mod mtwt;
     pub mod quote;
     pub mod source_util;
-    pub mod trace_macros;
 
     pub mod tt {
         pub mod transcribe;

src/libsyntax_ext/asm.rs

@@ -13,15 +13,15 @@
  */
 use self::State::*;
 
-use ast;
-use codemap;
-use codemap::Span;
-use ext::base;
-use ext::base::*;
-use feature_gate;
-use parse::token::{intern, InternedString};
-use parse::token;
-use ptr::P;
+use syntax::ast;
+use syntax::codemap;
+use syntax::codemap::Span;
+use syntax::ext::base;
+use syntax::ext::base::*;
+use syntax::feature_gate;
+use syntax::parse::token::{intern, InternedString};
+use syntax::parse::token;
+use syntax::ptr::P;
 use syntax::ast::AsmDialect;
 
 enum State {

src/libsyntax_ext/cfg.rs

@@ -12,15 +12,15 @@
 /// a literal `true` or `false` based on whether the given cfg matches the
 /// current compilation environment.
 
-use ast;
-use codemap::Span;
-use ext::base::*;
-use ext::base;
-use ext::build::AstBuilder;
-use attr;
-use attr::*;
-use parse::token;
-use config::CfgDiagReal;
+use syntax::ast;
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ext::base;
+use syntax::ext::build::AstBuilder;
+use syntax::attr;
+use syntax::attr::*;
+use syntax::parse::token;
+use syntax::config::CfgDiagReal;
 
 pub fn expand_cfg<'cx>(cx: &mut ExtCtxt,
                        sp: Span,

src/libsyntax_ext/concat.rs

@@ -8,11 +8,11 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast;
-use codemap;
-use ext::base;
-use ext::build::AstBuilder;
-use parse::token;
+use syntax::ast;
+use syntax::codemap;
+use syntax::ext::base;
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token;
 
 use std::string::String;
 

src/libsyntax_ext/concat_idents.rs

@@ -8,14 +8,14 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{self, TokenTree};
-use codemap::Span;
-use ext::base::*;
-use ext::base;
-use feature_gate;
-use parse::token;
-use parse::token::str_to_ident;
-use ptr::P;
+use syntax::ast::{self, TokenTree};
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ext::base;
+use syntax::feature_gate;
+use syntax::parse::token;
+use syntax::parse::token::str_to_ident;
+use syntax::ptr::P;
 
 pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                               -> Box<base::MacResult+'cx> {

src/libsyntax_ext/deriving/bounds.rs

@@ -8,11 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::MetaItem;
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::MetaItem;
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
 
 pub fn expand_deriving_unsafe_bound(cx: &mut ExtCtxt,
                                     span: Span,

src/libsyntax_ext/deriving/clone.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_clone(cx: &mut ExtCtxt,
                              span: Span,

src/libsyntax_ext/deriving/cmp/eq.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_eq(cx: &mut ExtCtxt,
                           span: Span,

src/libsyntax_ext/deriving/cmp/ord.rs

@@ -8,15 +8,16 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast;
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast;
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_ord(cx: &mut ExtCtxt,
                            span: Span,

src/libsyntax_ext/deriving/cmp/partial_eq.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr, self};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr, self};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_partial_eq(cx: &mut ExtCtxt,
                                   span: Span,

src/libsyntax_ext/deriving/cmp/partial_ord.rs

@@ -10,15 +10,16 @@
 
 pub use self::OrderingOp::*;
 
-use ast;
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast;
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_partial_ord(cx: &mut ExtCtxt,
                                    span: Span,

src/libsyntax_ext/deriving/debug.rs

@@ -8,15 +8,16 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast;
-use ast::{MetaItem, Expr};
-use codemap::{Span, respan};
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast;
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::{Span, respan};
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token;
+use syntax::ptr::P;
 
 pub fn expand_deriving_debug(cx: &mut ExtCtxt,
                             span: Span,

src/libsyntax_ext/deriving/decodable.rs

@@ -10,16 +10,17 @@
 
 //! The compiler code necessary for `#[derive(Decodable)]`. See encodable.rs for more.
 
-use ast;
-use ast::{MetaItem, Expr, MutMutable};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use parse::token;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast;
+use syntax::ast::{MetaItem, Expr, MutMutable};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::parse::token;
+use syntax::ptr::P;
 
 pub fn expand_deriving_rustc_decodable(cx: &mut ExtCtxt,
                                        span: Span,

src/libsyntax_ext/deriving/default.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_default(cx: &mut ExtCtxt,
                                span: Span,

src/libsyntax_ext/deriving/encodable.rs

@@ -88,14 +88,15 @@
 //! }
 //! ```
 
-use ast::{MetaItem, Expr, ExprRet, MutMutable};
-use codemap::Span;
-use ext::base::{ExtCtxt,Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr, ExprRet, MutMutable};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt,Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token;
+use syntax::ptr::P;
 
 pub fn expand_deriving_rustc_encodable(cx: &mut ExtCtxt,
                                        span: Span,

src/libsyntax_ext/deriving/generic/mod.rs

@@ -192,23 +192,23 @@ use std::cell::RefCell;
 use std::collections::HashSet;
 use std::vec;
 
-use abi::Abi;
-use abi;
-use ast;
-use ast::{EnumDef, Expr, Ident, Generics, VariantData};
-use ast_util;
-use attr;
-use attr::AttrMetaMethods;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use codemap::{self, DUMMY_SP};
-use codemap::Span;
-use diagnostic::SpanHandler;
-use util::move_map::MoveMap;
-use owned_slice::OwnedSlice;
-use parse::token::{intern, InternedString};
-use parse::token::special_idents;
-use ptr::P;
+use syntax::abi::Abi;
+use syntax::abi;
+use syntax::ast;
+use syntax::ast::{EnumDef, Expr, Ident, Generics, VariantData};
+use syntax::ast_util;
+use syntax::attr;
+use syntax::attr::AttrMetaMethods;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::codemap::{self, DUMMY_SP};
+use syntax::codemap::Span;
+use syntax::diagnostic::SpanHandler;
+use syntax::util::move_map::MoveMap;
+use syntax::owned_slice::OwnedSlice;
+use syntax::parse::token::{intern, InternedString};
+use syntax::parse::token::special_idents;
+use syntax::ptr::P;
 
 use self::ty::{LifetimeBounds, Path, Ptr, PtrTy, Self_, Ty};
 
@@ -347,7 +347,7 @@ pub fn combine_substructure<'a>(f: CombineSubstructureFunc<'a>)
 /// type. For a type parameter `<T>`, it looks for either a `TyPath` that
 /// is not global and starts with `T`, or a `TyQPath`.
 fn find_type_parameters(ty: &ast::Ty, ty_param_names: &[ast::Name]) -> Vec<P<ast::Ty>> {
-    use visit;
+    use syntax::visit;
 
     struct Visitor<'a> {
         ty_param_names: &'a [ast::Name],
@@ -1632,73 +1632,3 @@ pub fn cs_same_method<F>(f: F,
         }
     }
 }
-
-/// Fold together the results of calling the derived method on all the
-/// fields. `use_foldl` controls whether this is done left-to-right
-/// (`true`) or right-to-left (`false`).
-#[inline]
-pub fn cs_same_method_fold<F>(use_foldl: bool,
-                              mut f: F,
-                              base: P<Expr>,
-                              enum_nonmatch_f: EnumNonMatchCollapsedFunc,
-                              cx: &mut ExtCtxt,
-                              trait_span: Span,
-                              substructure: &Substructure)
-                              -> P<Expr> where
-    F: FnMut(&mut ExtCtxt, Span, P<Expr>, P<Expr>) -> P<Expr>,
-{
-    cs_same_method(
-        |cx, span, vals| {
-            if use_foldl {
-                vals.into_iter().fold(base.clone(), |old, new| {
-                    f(cx, span, old, new)
-                })
-            } else {
-                vals.into_iter().rev().fold(base.clone(), |old, new| {
-                    f(cx, span, old, new)
-                })
-            }
-        },
-        enum_nonmatch_f,
-        cx, trait_span, substructure)
-}
-
-/// Use a given binop to combine the result of calling the derived method
-/// on all the fields.
-#[inline]
-pub fn cs_binop(binop: ast::BinOp_, base: P<Expr>,
-                enum_nonmatch_f: EnumNonMatchCollapsedFunc,
-                cx: &mut ExtCtxt, trait_span: Span,
-                substructure: &Substructure) -> P<Expr> {
-    cs_same_method_fold(
-        true, // foldl is good enough
-        |cx, span, old, new| {
-            cx.expr_binary(span,
-                           binop,
-                           old, new)
-
-        },
-        base,
-        enum_nonmatch_f,
-        cx, trait_span, substructure)
-}
-
-/// cs_binop with binop == or
-#[inline]
-pub fn cs_or(enum_nonmatch_f: EnumNonMatchCollapsedFunc,
-             cx: &mut ExtCtxt, span: Span,
-             substructure: &Substructure) -> P<Expr> {
-    cs_binop(ast::BiOr, cx.expr_bool(span, false),
-             enum_nonmatch_f,
-             cx, span, substructure)
-}
-
-/// cs_binop with binop == and
-#[inline]
-pub fn cs_and(enum_nonmatch_f: EnumNonMatchCollapsedFunc,
-              cx: &mut ExtCtxt, span: Span,
-              substructure: &Substructure) -> P<Expr> {
-    cs_binop(ast::BiAnd, cx.expr_bool(span, true),
-             enum_nonmatch_f,
-             cx, span, substructure)
-}

src/libsyntax_ext/deriving/generic/ty.rs

@@ -14,17 +14,18 @@
 pub use self::PtrTy::*;
 pub use self::Ty::*;
 
-use ast;
-use ast::{Expr,Generics,Ident};
-use ext::base::ExtCtxt;
-use ext::build::AstBuilder;
-use codemap::{Span,respan};
-use owned_slice::OwnedSlice;
-use parse::token::special_idents;
-use ptr::P;
+use syntax::ast;
+use syntax::ast::{Expr,Generics,Ident};
+use syntax::ext::base::ExtCtxt;
+use syntax::ext::build::AstBuilder;
+use syntax::codemap::{Span,respan};
+use syntax::owned_slice::OwnedSlice;
+use syntax::parse::token::special_idents;
+use syntax::ptr::P;
 
 /// The types of pointers
 #[derive(Clone, Eq, PartialEq)]
+#[allow(dead_code)]
 pub enum PtrTy<'a> {
     /// &'lifetime mut
     Borrowed(Option<&'a str>, ast::Mutability),

src/libsyntax_ext/deriving/hash.rs

@@ -8,13 +8,14 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr, MutMutable};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr, MutMutable};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::ptr::P;
 
 pub fn expand_deriving_hash(cx: &mut ExtCtxt,
                             span: Span,

src/libsyntax_ext/deriving/mod.rs

@@ -13,13 +13,14 @@
 //! FIXME (#2810): hygiene. Search for "__" strings (in other files too). We also assume "extra" is
 //! the standard library, and "std" is the core library.
 
-use ast::{MetaItem, MetaWord};
-use attr::AttrMetaMethods;
-use ext::base::{ExtCtxt, SyntaxEnv, MultiDecorator, MultiItemDecorator, MultiModifier, Annotatable};
-use ext::build::AstBuilder;
-use feature_gate;
-use codemap::Span;
-use parse::token::{intern, intern_and_get_ident};
+use syntax::ast::{MetaItem, MetaWord};
+use syntax::attr::AttrMetaMethods;
+use syntax::ext::base::{ExtCtxt, SyntaxEnv, Annotatable};
+use syntax::ext::base::{MultiDecorator, MultiItemDecorator, MultiModifier};
+use syntax::ext::build::AstBuilder;
+use syntax::feature_gate;
+use syntax::codemap::Span;
+use syntax::parse::token::{intern, intern_and_get_ident};
 
 macro_rules! pathvec {
     ($($x:ident)::+) => (
@@ -35,7 +36,7 @@ macro_rules! path {
 
 macro_rules! path_local {
     ($x:ident) => (
-        ::ext::deriving::generic::ty::Path::new_local(stringify!($x))
+        ::deriving::generic::ty::Path::new_local(stringify!($x))
     )
 }
 
@@ -51,7 +52,7 @@ macro_rules! pathvec_std {
 
 macro_rules! path_std {
     ($($x:tt)*) => (
-        ::ext::deriving::generic::ty::Path::new( pathvec_std!( $($x)* ) )
+        ::deriving::generic::ty::Path::new( pathvec_std!( $($x)* ) )
     )
 }
 

src/libsyntax_ext/deriving/primitive.rs

@@ -8,15 +8,16 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr};
-use ast;
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr};
+use syntax::ast;
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_from_primitive(cx: &mut ExtCtxt,
                                       span: Span,

src/libsyntax_ext/env.rs

@@ -14,12 +14,12 @@
  * interface.
  */
 
-use ast;
-use codemap::Span;
-use ext::base::*;
-use ext::base;
-use ext::build::AstBuilder;
-use parse::token;
+use syntax::ast;
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ext::base;
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token;
 
 use std::env;
 

src/libsyntax_ext/format.rs

@@ -11,16 +11,17 @@
 use self::ArgumentType::*;
 use self::Position::*;
 
-use ast;
-use codemap::{Span, respan};
-use ext::base::*;
-use ext::base;
-use ext::build::AstBuilder;
 use fmt_macros as parse;
-use fold::Folder;
-use parse::token::special_idents;
-use parse::token;
-use ptr::P;
+
+use syntax::ast;
+use syntax::codemap::{Span, respan};
+use syntax::ext::base::*;
+use syntax::ext::base;
+use syntax::ext::build::AstBuilder;
+use syntax::fold::Folder;
+use syntax::parse::token::special_idents;
+use syntax::parse::token;
+use syntax::ptr::P;
 
 use std::collections::HashMap;
 

src/libsyntax_ext/lib.rs

@@ -0,0 +1,89 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Syntax extensions in the Rust compiler.
+
+#![crate_name = "syntax_ext"]
+#![crate_type = "dylib"]
+#![crate_type = "rlib"]
+#![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
+       html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
+       html_root_url = "https://doc.rust-lang.org/nightly/")]
+
+#![cfg_attr(stage0, feature(custom_attribute))]
+#![cfg_attr(stage0, staged_api)]
+#![unstable(feature = "rustc_private", issue = "27812")]
+
+#![feature(rustc_private)]
+#![feature(staged_api)]
+#![feature(str_char)]
+
+extern crate fmt_macros;
+extern crate syntax;
+
+use syntax::ext::base::{MacroExpanderFn, NormalTT};
+use syntax::ext::base::{SyntaxEnv, SyntaxExtension};
+use syntax::parse::token::intern;
+
+// A variant of 'try!' that panics on Err(FatalError). This is used as a
+// crutch on the way towards a non-panic!-prone parser. It should be used
+// for fatal parsing errors; eventually we plan to convert all code using
+// panictry to just use normal try
+macro_rules! panictry {
+    ($e:expr) => ({
+        use std::result::Result::{Ok, Err};
+        use syntax::diagnostic::FatalError;
+        match $e {
+            Ok(e) => e,
+            Err(FatalError) => panic!(FatalError)
+        }
+    })
+}
+
+mod asm;
+mod cfg;
+mod concat;
+mod concat_idents;
+mod env;
+mod format;
+mod log_syntax;
+mod trace_macros;
+
+// for custom_derive
+pub mod deriving;
+
+pub fn register_builtins(env: &mut SyntaxEnv) {
+    // utility function to simplify creating NormalTT syntax extensions
+    fn builtin_normal_expander(f: MacroExpanderFn) -> SyntaxExtension {
+        NormalTT(Box::new(f), None, false)
+    }
+
+    env.insert(intern("asm"),
+               builtin_normal_expander(asm::expand_asm));
+    env.insert(intern("cfg"),
+               builtin_normal_expander(cfg::expand_cfg));
+    env.insert(intern("concat"),
+               builtin_normal_expander(concat::expand_syntax_ext));
+    env.insert(intern("concat_idents"),
+               builtin_normal_expander(concat_idents::expand_syntax_ext));
+    env.insert(intern("env"),
+               builtin_normal_expander(env::expand_env));
+    env.insert(intern("option_env"),
+               builtin_normal_expander(env::expand_option_env));
+    env.insert(intern("format_args"),
+               // format_args uses `unstable` things internally.
+               NormalTT(Box::new(format::expand_format_args), None, true));
+    env.insert(intern("log_syntax"),
+               builtin_normal_expander(log_syntax::expand_syntax_ext));
+    env.insert(intern("trace_macros"),
+               builtin_normal_expander(trace_macros::expand_trace_macros));
+
+    deriving::register_all(env);
+}

src/libsyntax_ext/log_syntax.rs

@@ -8,11 +8,11 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast;
-use codemap;
-use ext::base;
-use feature_gate;
-use print;
+use syntax::ast;
+use syntax::codemap;
+use syntax::ext::base;
+use syntax::feature_gate;
+use syntax::print;
 
 pub fn expand_syntax_ext<'cx>(cx: &'cx mut base::ExtCtxt,
                               sp: codemap::Span,

src/libsyntax_ext/trace_macros.rs

@@ -8,12 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::TokenTree;
-use codemap::Span;
-use ext::base::ExtCtxt;
-use ext::base;
-use feature_gate;
-use parse::token::keywords;
+use syntax::ast::TokenTree;
+use syntax::codemap::Span;
+use syntax::ext::base::ExtCtxt;
+use syntax::ext::base;
+use syntax::feature_gate;
+use syntax::parse::token::keywords;
 
 
 pub fn expand_trace_macros(cx: &mut ExtCtxt,

src/test/auxiliary/custom_derive_plugin.rs

@@ -15,6 +15,7 @@
 #![feature(rustc_private)]
 
 extern crate syntax;
+extern crate syntax_ext;
 extern crate rustc;
 extern crate rustc_plugin;
 
@@ -22,10 +23,10 @@ use syntax::ast;
 use syntax::codemap::Span;
 use syntax::ext::base::{MultiDecorator, ExtCtxt, Annotatable};
 use syntax::ext::build::AstBuilder;
-use syntax::ext::deriving::generic::{cs_fold, TraitDef, MethodDef, combine_substructure};
-use syntax::ext::deriving::generic::ty::{Literal, LifetimeBounds, Path, borrowed_explicit_self};
 use syntax::parse::token;
 use syntax::ptr::P;
+use syntax_ext::deriving::generic::{cs_fold, TraitDef, MethodDef, combine_substructure};
+use syntax_ext::deriving::generic::ty::{Literal, LifetimeBounds, Path, borrowed_explicit_self};
 use rustc_plugin::Registry;
 
 #[plugin_registrar]

src/test/auxiliary/custom_derive_plugin_attr.rs

@@ -15,6 +15,7 @@
 #![feature(rustc_private)]
 
 extern crate syntax;
+extern crate syntax_ext;
 extern crate rustc;
 extern crate rustc_plugin;
 
@@ -23,11 +24,11 @@ use syntax::attr::AttrMetaMethods;
 use syntax::codemap::Span;
 use syntax::ext::base::{MultiDecorator, ExtCtxt, Annotatable};
 use syntax::ext::build::AstBuilder;
-use syntax::ext::deriving::generic::{cs_fold, TraitDef, MethodDef, combine_substructure};
-use syntax::ext::deriving::generic::{Substructure, Struct, EnumMatching};
-use syntax::ext::deriving::generic::ty::{Literal, LifetimeBounds, Path, borrowed_explicit_self};
 use syntax::parse::token;
 use syntax::ptr::P;
+use syntax_ext::deriving::generic::{cs_fold, TraitDef, MethodDef, combine_substructure};
+use syntax_ext::deriving::generic::{Substructure, Struct, EnumMatching};
+use syntax_ext::deriving::generic::ty::{Literal, LifetimeBounds, Path, borrowed_explicit_self};
 use rustc_plugin::Registry;
 
 #[plugin_registrar]