Auto merge of #36214 - jseyfried:stackless_expansion, r=nrc

macros: stackless expansion

After this PR, macro expansion cannot overflow the stack unless the expanded crate is too deep to fold.
Everything but the stackless placeholder expansion commit is also groundwork for macro modularization.

r? @nrc or @eddyb

src/librustc_driver/driver.rs

@@ -555,7 +555,7 @@ pub struct ExpansionResult<'a> {
 /// Returns `None` if we're aborting after handling -W help.
 pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
                                            cstore: &CStore,
-                                           mut krate: ast::Crate,
+                                           krate: ast::Crate,
                                            registry: Option<Registry>,
                                            crate_name: &'a str,
                                            addl_plugins: Option<Vec<String>>,
@@ -566,21 +566,9 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session,
 {
     let time_passes = sess.time_passes();
 
-    // strip before anything else because crate metadata may use #[cfg_attr]
-    // and so macros can depend on configuration variables, such as
-    //
-    //   #[macro_use] #[cfg(foo)]
-    //   mod bar { macro_rules! baz!(() => {{}}) }
-    //
-    // baz! should not use this definition unless foo is enabled.
-
-    krate = time(time_passes, "configuration", || {
-        let (krate, features) =
-            syntax::config::strip_unconfigured_items(krate, &sess.parse_sess, sess.opts.test);
-        // these need to be set "early" so that expansion sees `quote` if enabled.
-        *sess.features.borrow_mut() = features;
-        krate
-    });
+    let (mut krate, features) = syntax::config::features(krate, &sess.parse_sess, sess.opts.test);
+    // these need to be set "early" so that expansion sees `quote` if enabled.
+    *sess.features.borrow_mut() = features;
 
     *sess.crate_types.borrow_mut() = collect_crate_types(sess, &krate.attrs);
     *sess.crate_disambiguator.borrow_mut() =

src/libsyntax/config.rs

@@ -10,11 +10,10 @@
 
 use attr::HasAttrs;
 use feature_gate::{emit_feature_err, EXPLAIN_STMT_ATTR_SYNTAX, Features, get_features, GateIssue};
-use fold::Folder;
 use {fold, attr};
 use ast;
 use codemap::{Spanned, respan};
-use parse::{ParseSess, token};
+use parse::ParseSess;
 use ptr::P;
 
 use util::small_vector::SmallVector;
@@ -27,8 +26,51 @@ pub struct StripUnconfigured<'a> {
     pub features: Option<&'a Features>,
 }
 
+// `cfg_attr`-process the crate's attributes and compute the crate's features.
+pub fn features(mut krate: ast::Crate, sess: &ParseSess, should_test: bool)
+                -> (ast::Crate, Features) {
+    let features;
+    {
+        let mut strip_unconfigured = StripUnconfigured {
+            config: &krate.config.clone(),
+            should_test: should_test,
+            sess: sess,
+            features: None,
+        };
+
+        let unconfigured_attrs = krate.attrs.clone();
+        let err_count = sess.span_diagnostic.err_count();
+        if let Some(attrs) = strip_unconfigured.configure(krate.attrs) {
+            krate.attrs = attrs;
+        } else { // the entire crate is unconfigured
+            krate.attrs = Vec::new();
+            krate.module.items = Vec::new();
+            return (krate, Features::new());
+        }
+
+        features = get_features(&sess.span_diagnostic, &krate.attrs);
+
+        // Avoid reconfiguring malformed `cfg_attr`s
+        if err_count == sess.span_diagnostic.err_count() {
+            strip_unconfigured.features = Some(&features);
+            strip_unconfigured.configure(unconfigured_attrs);
+        }
+    }
+
+    (krate, features)
+}
+
+macro_rules! configure {
+    ($this:ident, $node:ident) => {
+        match $this.configure($node) {
+            Some(node) => node,
+            None => return Default::default(),
+        }
+    }
+}
+
 impl<'a> StripUnconfigured<'a> {
-    fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
+    pub fn configure<T: HasAttrs>(&mut self, node: T) -> Option<T> {
         let node = self.process_cfg_attrs(node);
         if self.in_cfg(node.attrs()) { Some(node) } else { None }
     }
@@ -123,65 +165,35 @@ impl<'a> StripUnconfigured<'a> {
             }
         }
     }
-}
 
-// Support conditional compilation by transforming the AST, stripping out
-// any items that do not belong in the current configuration
-pub fn strip_unconfigured_items(mut krate: ast::Crate, sess: &ParseSess, should_test: bool)
-                                -> (ast::Crate, Features) {
-    let features;
-    {
-        let mut strip_unconfigured = StripUnconfigured {
-            config: &krate.config.clone(),
-            should_test: should_test,
-            sess: sess,
-            features: None,
-        };
-
-        let err_count = sess.span_diagnostic.err_count();
-        let krate_attrs = strip_unconfigured.configure(krate.attrs.clone()).unwrap_or_default();
-        features = get_features(&sess.span_diagnostic, &krate_attrs);
-        if err_count < sess.span_diagnostic.err_count() {
-            krate.attrs = krate_attrs.clone(); // Avoid reconfiguring malformed `cfg_attr`s
-        }
-
-        strip_unconfigured.features = Some(&features);
-        krate = strip_unconfigured.fold_crate(krate);
-        krate.attrs = krate_attrs;
-    }
-
-    (krate, features)
-}
-
-impl<'a> fold::Folder for StripUnconfigured<'a> {
-    fn fold_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
+    pub fn configure_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
         ast::ForeignMod {
             abi: foreign_mod.abi,
-            items: foreign_mod.items.into_iter().filter_map(|item| {
-                self.configure(item).map(|item| fold::noop_fold_foreign_item(item, self))
-            }).collect(),
+            items: foreign_mod.items.into_iter().filter_map(|item| self.configure(item)).collect(),
         }
     }
 
-    fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
-        let fold_struct = |this: &mut Self, vdata| match vdata {
+    fn configure_variant_data(&mut self, vdata: ast::VariantData) -> ast::VariantData {
+        match vdata {
             ast::VariantData::Struct(fields, id) => {
-                let fields = fields.into_iter().filter_map(|field| this.configure(field));
+                let fields = fields.into_iter().filter_map(|field| self.configure(field));
                 ast::VariantData::Struct(fields.collect(), id)
             }
             ast::VariantData::Tuple(fields, id) => {
-                let fields = fields.into_iter().filter_map(|field| this.configure(field));
+                let fields = fields.into_iter().filter_map(|field| self.configure(field));
                 ast::VariantData::Tuple(fields.collect(), id)
             }
             ast::VariantData::Unit(id) => ast::VariantData::Unit(id)
-        };
+        }
+    }
 
-        let item = match item {
+    pub fn configure_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
+        match item {
             ast::ItemKind::Struct(def, generics) => {
-                ast::ItemKind::Struct(fold_struct(self, def), generics)
+                ast::ItemKind::Struct(self.configure_variant_data(def), generics)
             }
             ast::ItemKind::Union(def, generics) => {
-                ast::ItemKind::Union(fold_struct(self, def), generics)
+                ast::ItemKind::Union(self.configure_variant_data(def), generics)
             }
             ast::ItemKind::Enum(def, generics) => {
                 let variants = def.variants.into_iter().filter_map(|v| {
@@ -190,7 +202,7 @@ impl<'a> fold::Folder for StripUnconfigured<'a> {
                             node: ast::Variant_ {
                                 name: v.node.name,
                                 attrs: v.node.attrs,
-                                data: fold_struct(self, v.node.data),
+                                data: self.configure_variant_data(v.node.data),
                                 disr_expr: v.node.disr_expr,
                             },
                             span: v.span
@@ -202,12 +214,19 @@ impl<'a> fold::Folder for StripUnconfigured<'a> {
                 }, generics)
             }
             item => item,
-        };
-
-        fold::noop_fold_item_kind(item, self)
+        }
     }
 
-    fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
+    pub fn configure_expr_kind(&mut self, expr_kind: ast::ExprKind) -> ast::ExprKind {
+        if let ast::ExprKind::Match(m, arms) = expr_kind {
+            let arms = arms.into_iter().filter_map(|a| self.configure(a)).collect();
+            ast::ExprKind::Match(m, arms)
+        } else {
+            expr_kind
+        }
+    }
+
+    pub fn configure_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
         self.visit_stmt_or_expr_attrs(expr.attrs());
 
         // If an expr is valid to cfg away it will have been removed by the
@@ -222,62 +241,62 @@ impl<'a> fold::Folder for StripUnconfigured<'a> {
             self.sess.span_diagnostic.span_err(attr.span, msg);
         }
 
-        let expr = self.process_cfg_attrs(expr);
-        fold_expr(self, expr)
+        self.process_cfg_attrs(expr)
     }
 
-    fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
-        self.configure(expr).map(|expr| fold_expr(self, expr))
-    }
-
-    fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
+    pub fn configure_stmt(&mut self, stmt: ast::Stmt) -> Option<ast::Stmt> {
         self.visit_stmt_or_expr_attrs(stmt.attrs());
-        self.configure(stmt).map(|stmt| fold::noop_fold_stmt(stmt, self))
-                            .unwrap_or(SmallVector::zero())
-    }
-
-    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
-        fold::noop_fold_mac(mac, self)
-    }
-
-    fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
-        self.configure(item).map(|item| fold::noop_fold_item(item, self))
-                            .unwrap_or(SmallVector::zero())
-    }
-
-    fn fold_impl_item(&mut self, item: ast::ImplItem) -> SmallVector<ast::ImplItem> {
-        self.configure(item).map(|item| fold::noop_fold_impl_item(item, self))
-                            .unwrap_or(SmallVector::zero())
-    }
-
-    fn fold_trait_item(&mut self, item: ast::TraitItem) -> SmallVector<ast::TraitItem> {
-        self.configure(item).map(|item| fold::noop_fold_trait_item(item, self))
-                            .unwrap_or(SmallVector::zero())
-    }
-
-    fn fold_interpolated(&mut self, nt: token::Nonterminal) -> token::Nonterminal {
-        // Don't configure interpolated AST (c.f. #34171).
-        // Interpolated AST will get configured once the surrounding tokens are parsed.
-        nt
+        self.configure(stmt)
     }
 }
 
-fn fold_expr(folder: &mut StripUnconfigured, expr: P<ast::Expr>) -> P<ast::Expr> {
-    expr.map(|ast::Expr {id, span, node, attrs}| {
-        fold::noop_fold_expr(ast::Expr {
-            id: id,
-            node: match node {
-                ast::ExprKind::Match(m, arms) => {
-                    ast::ExprKind::Match(m, arms.into_iter()
-                                        .filter_map(|a| folder.configure(a))
-                                        .collect())
-                }
-                _ => node
-            },
-            span: span,
-            attrs: attrs,
-        }, folder)
-    })
+impl<'a> fold::Folder for StripUnconfigured<'a> {
+    fn fold_foreign_mod(&mut self, foreign_mod: ast::ForeignMod) -> ast::ForeignMod {
+        let foreign_mod = self.configure_foreign_mod(foreign_mod);
+        fold::noop_fold_foreign_mod(foreign_mod, self)
+    }
+
+    fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
+        let item = self.configure_item_kind(item);
+        fold::noop_fold_item_kind(item, self)
+    }
+
+    fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
+        let mut expr = self.configure_expr(expr).unwrap();
+        expr.node = self.configure_expr_kind(expr.node);
+        P(fold::noop_fold_expr(expr, self))
+    }
+
+    fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
+        let mut expr = configure!(self, expr).unwrap();
+        expr.node = self.configure_expr_kind(expr.node);
+        Some(P(fold::noop_fold_expr(expr, self)))
+    }
+
+    fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
+        match self.configure_stmt(stmt) {
+            Some(stmt) => fold::noop_fold_stmt(stmt, self),
+            None => return SmallVector::zero(),
+        }
+    }
+
+    fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
+        fold::noop_fold_item(configure!(self, item), self)
+    }
+
+    fn fold_impl_item(&mut self, item: ast::ImplItem) -> SmallVector<ast::ImplItem> {
+        fold::noop_fold_impl_item(configure!(self, item), self)
+    }
+
+    fn fold_trait_item(&mut self, item: ast::TraitItem) -> SmallVector<ast::TraitItem> {
+        fold::noop_fold_trait_item(configure!(self, item), self)
+    }
+
+    fn fold_mac(&mut self, mac: ast::Mac) -> ast::Mac {
+        // Don't configure interpolated AST (c.f. #34171).
+        // Interpolated AST will get configured once the surrounding tokens are parsed.
+        mac
+    }
 }
 
 fn is_cfg(attr: &ast::Attribute) -> bool {

src/libsyntax/ext/base.rs

@@ -24,12 +24,14 @@ use parse::parser;
 use parse::token;
 use parse::token::{InternedString, intern, str_to_ident};
 use ptr::P;
+use std_inject;
 use util::small_vector::SmallVector;
 use util::lev_distance::find_best_match_for_name;
 use fold::Folder;
 use feature_gate;
 
 use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
 use std::rc::Rc;
 use tokenstream;
 
@@ -90,16 +92,6 @@ impl Annotatable {
             _ => panic!("expected Item")
         }
     }
-
-    pub fn fold_with<F: Folder>(self, folder: &mut F) -> SmallVector<Self> {
-        match self {
-            Annotatable::Item(item) => folder.fold_item(item).map(Annotatable::Item),
-            Annotatable::ImplItem(item) =>
-                folder.fold_impl_item(item.unwrap()).map(|item| Annotatable::ImplItem(P(item))),
-            Annotatable::TraitItem(item) =>
-                folder.fold_trait_item(item.unwrap()).map(|item| Annotatable::TraitItem(P(item))),
-        }
-    }
 }
 
 // A more flexible ItemDecorator.
@@ -472,19 +464,6 @@ pub enum SyntaxExtension {
 
 pub type NamedSyntaxExtension = (Name, SyntaxExtension);
 
-pub struct BlockInfo {
-    /// Should macros escape from this scope?
-    pub macros_escape: bool,
-}
-
-impl BlockInfo {
-    pub fn new() -> BlockInfo {
-        BlockInfo {
-            macros_escape: false,
-        }
-    }
-}
-
 /// The base map of methods for expanding syntax extension
 /// AST nodes into full ASTs
 fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
@@ -595,16 +574,11 @@ pub struct ExtCtxt<'a> {
     pub crate_root: Option<&'static str>,
     pub loader: &'a mut MacroLoader,
 
-    pub mod_path: Vec<ast::Ident> ,
     pub exported_macros: Vec<ast::MacroDef>,
 
     pub syntax_env: SyntaxEnv,
     pub derive_modes: HashMap<InternedString, Box<MultiItemModifier>>,
     pub recursion_count: usize,
-
-    pub filename: Option<String>,
-    pub mod_path_stack: Vec<InternedString>,
-    pub in_block: bool,
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -612,23 +586,17 @@ impl<'a> ExtCtxt<'a> {
                ecfg: expand::ExpansionConfig<'a>,
                loader: &'a mut MacroLoader)
                -> ExtCtxt<'a> {
-        let env = initial_syntax_expander_table(&ecfg);
         ExtCtxt {
+            syntax_env: initial_syntax_expander_table(&ecfg),
             parse_sess: parse_sess,
             cfg: cfg,
             backtrace: NO_EXPANSION,
-            mod_path: Vec::new(),
             ecfg: ecfg,
             crate_root: None,
             exported_macros: Vec::new(),
             loader: loader,
-            syntax_env: env,
             derive_modes: HashMap::new(),
             recursion_count: 0,
-
-            filename: None,
-            mod_path_stack: Vec::new(),
-            in_block: false,
         }
     }
 
@@ -677,16 +645,7 @@ impl<'a> ExtCtxt<'a> {
         last_macro.expect("missing expansion backtrace")
     }
 
-    pub fn mod_push(&mut self, i: ast::Ident) { self.mod_path.push(i); }
-    pub fn mod_pop(&mut self) { self.mod_path.pop().unwrap(); }
-    pub fn mod_path(&self) -> Vec<ast::Ident> {
-        let mut v = Vec::new();
-        v.push(token::str_to_ident(&self.ecfg.crate_name));
-        v.extend(self.mod_path.iter().cloned());
-        return v;
-    }
     pub fn bt_push(&mut self, ei: ExpnInfo) {
-        self.recursion_count += 1;
         if self.recursion_count > self.ecfg.recursion_limit {
             self.span_fatal(ei.call_site,
                             &format!("recursion limit reached while expanding the macro `{}`",
@@ -700,17 +659,7 @@ impl<'a> ExtCtxt<'a> {
             callee: ei.callee
         });
     }
-    pub fn bt_pop(&mut self) {
-        match self.backtrace {
-            NO_EXPANSION => self.bug("tried to pop without a push"),
-            expn_id => {
-                self.recursion_count -= 1;
-                self.backtrace = self.codemap().with_expn_info(expn_id, |expn_info| {
-                    expn_info.map_or(NO_EXPANSION, |ei| ei.call_site.expn_id)
-                });
-            }
-        }
-    }
+    pub fn bt_pop(&mut self) {}
 
     pub fn insert_macro(&mut self, def: ast::MacroDef) {
         if def.export {
@@ -829,6 +778,28 @@ impl<'a> ExtCtxt<'a> {
             }
         }
     }
+
+    pub fn initialize(&mut self, user_exts: Vec<NamedSyntaxExtension>, krate: &ast::Crate) {
+        if std_inject::no_core(&krate) {
+            self.crate_root = None;
+        } else if std_inject::no_std(&krate) {
+            self.crate_root = Some("core");
+        } else {
+            self.crate_root = Some("std");
+        }
+
+        for (name, extension) in user_exts {
+            self.syntax_env.insert(name, extension);
+        }
+
+        self.syntax_env.current_module = Module(0);
+        let mut paths = ModulePaths {
+            mod_path: vec![token::str_to_ident(&self.ecfg.crate_name)],
+            directory: PathBuf::from(self.parse_sess.codemap().span_to_filename(krate.span)),
+        };
+        paths.directory.pop();
+        self.syntax_env.module_data[0].paths = Rc::new(paths);
+    }
 }
 
 /// Extract a string literal from the macro expanded version of `expr`,
@@ -915,79 +886,97 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt,
 ///
 /// This environment maps Names to SyntaxExtensions.
 pub struct SyntaxEnv {
-    chain: Vec<MapChainFrame>,
+    module_data: Vec<ModuleData>,
+    pub current_module: Module,
+
     /// All bang-style macro/extension names
     /// encountered so far; to be used for diagnostics in resolve
     pub names: HashSet<Name>,
 }
 
-// impl question: how to implement it? Initially, the
-// env will contain only macros, so it might be painful
-// to add an empty frame for every context. Let's just
-// get it working, first....
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub struct Module(u32);
 
-// NB! the mutability of the underlying maps means that
-// if expansion is out-of-order, a deeper scope may be
-// able to refer to a macro that was added to an enclosing
-// scope lexically later than the deeper scope.
+struct ModuleData {
+    parent: Module,
+    paths: Rc<ModulePaths>,
+    macros: HashMap<Name, Rc<SyntaxExtension>>,
+    macros_escape: bool,
+    in_block: bool,
+}
 
-struct MapChainFrame {
-    info: BlockInfo,
-    map: HashMap<Name, Rc<SyntaxExtension>>,
+#[derive(Clone)]
+pub struct ModulePaths {
+    pub mod_path: Vec<ast::Ident>,
+    pub directory: PathBuf,
 }
 
 impl SyntaxEnv {
     fn new() -> SyntaxEnv {
-        let mut map = SyntaxEnv { chain: Vec::new() , names: HashSet::new()};
-        map.push_frame();
-        map
+        let mut env = SyntaxEnv {
+            current_module: Module(0),
+            module_data: Vec::new(),
+            names: HashSet::new(),
+        };
+        let paths = Rc::new(ModulePaths { mod_path: Vec::new(), directory: PathBuf::new() });
+        env.add_module(false, false, paths);
+        env
     }
 
-    pub fn push_frame(&mut self) {
-        self.chain.push(MapChainFrame {
-            info: BlockInfo::new(),
-            map: HashMap::new(),
-        });
+    fn data(&self, module: Module) -> &ModuleData {
+        &self.module_data[module.0 as usize]
     }
 
-    pub fn pop_frame(&mut self) {
-        assert!(self.chain.len() > 1, "too many pops on MapChain!");
-        self.chain.pop();
+    pub fn paths(&self) -> Rc<ModulePaths> {
+        self.data(self.current_module).paths.clone()
     }
 
-    fn find_escape_frame(&mut self) -> &mut MapChainFrame {
-        for (i, frame) in self.chain.iter_mut().enumerate().rev() {
-            if !frame.info.macros_escape || i == 0 {
-                return frame
+    pub fn in_block(&self) -> bool {
+        self.data(self.current_module).in_block
+    }
+
+    pub fn add_module(&mut self, macros_escape: bool, in_block: bool, paths: Rc<ModulePaths>)
+                      -> Module {
+        let data = ModuleData {
+            parent: self.current_module,
+            paths: paths,
+            macros: HashMap::new(),
+            macros_escape: macros_escape,
+            in_block: in_block,
+        };
+
+        self.module_data.push(data);
+        Module(self.module_data.len() as u32 - 1)
+    }
+
+    pub fn find(&self, name: Name) -> Option<Rc<SyntaxExtension>> {
+        let mut module = self.current_module;
+        let mut module_data;
+        loop {
+            module_data = self.data(module);
+            if let Some(ext) = module_data.macros.get(&name) {
+                return Some(ext.clone());
             }
-        }
-        unreachable!()
-    }
-
-    pub fn find(&self, k: Name) -> Option<Rc<SyntaxExtension>> {
-        for frame in self.chain.iter().rev() {
-            if let Some(v) = frame.map.get(&k) {
-                return Some(v.clone());
+            if module == module_data.parent {
+                return None;
             }
+            module = module_data.parent;
         }
-        None
     }
 
-    pub fn insert(&mut self, k: Name, v: SyntaxExtension) {
-        if let NormalTT(..) = v {
-            self.names.insert(k);
+    pub fn insert(&mut self, name: Name, ext: SyntaxExtension) {
+        if let NormalTT(..) = ext {
+            self.names.insert(name);
         }
-        self.find_escape_frame().map.insert(k, Rc::new(v));
-    }
 
-    pub fn info(&mut self) -> &mut BlockInfo {
-        let last_chain_index = self.chain.len() - 1;
-        &mut self.chain[last_chain_index].info
+        let mut module = self.current_module;
+        while self.data(module).macros_escape {
+            module = self.data(module).parent;
+        }
+        self.module_data[module.0 as usize].macros.insert(name, Rc::new(ext));
     }
 
     pub fn is_crate_root(&mut self) -> bool {
-        // The first frame is pushed in `SyntaxEnv::new()` and the second frame is
-        // pushed when folding the crate root pseudo-module (c.f. noop_fold_crate).
-        self.chain.len() <= 2
+        self.current_module == Module(0)
     }
 }

src/libsyntax/ext/hygiene.rs

@@ -40,6 +40,10 @@ impl Mark {
             ::std::mem::replace(&mut data.next_mark, next_mark)
         })
     }
+
+    pub fn as_u32(&self) -> u32 {
+        self.0
+    }
 }
 
 struct HygieneData {

src/libsyntax/ext/placeholders.rs

@@ -0,0 +1,175 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use ast;
+use codemap::{DUMMY_SP, dummy_spanned};
+use ext::expand::{Expansion, ExpansionKind};
+use fold::*;
+use parse::token::keywords;
+use ptr::P;
+use util::small_vector::SmallVector;
+
+use std::collections::HashMap;
+
+pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion {
+    fn mac_placeholder() -> ast::Mac {
+        dummy_spanned(ast::Mac_ {
+            path: ast::Path { span: DUMMY_SP, global: false, segments: Vec::new() },
+            tts: Vec::new(),
+        })
+    }
+
+    let ident = keywords::Invalid.ident();
+    let attrs = Vec::new();
+    let vis = ast::Visibility::Inherited;
+    let span = DUMMY_SP;
+    let expr_placeholder = || P(ast::Expr {
+        id: id, span: span,
+        attrs: ast::ThinVec::new(),
+        node: ast::ExprKind::Mac(mac_placeholder()),
+    });
+
+    match kind {
+        ExpansionKind::Expr => Expansion::Expr(expr_placeholder()),
+        ExpansionKind::OptExpr => Expansion::OptExpr(Some(expr_placeholder())),
+        ExpansionKind::Items => Expansion::Items(SmallVector::one(P(ast::Item {
+            id: id, span: span, ident: ident, vis: vis, attrs: attrs,
+            node: ast::ItemKind::Mac(mac_placeholder()),
+        }))),
+        ExpansionKind::TraitItems => Expansion::TraitItems(SmallVector::one(ast::TraitItem {
+            id: id, span: span, ident: ident, attrs: attrs,
+            node: ast::TraitItemKind::Macro(mac_placeholder()),
+        })),
+        ExpansionKind::ImplItems => Expansion::ImplItems(SmallVector::one(ast::ImplItem {
+            id: id, span: span, ident: ident, vis: vis, attrs: attrs,
+            node: ast::ImplItemKind::Macro(mac_placeholder()),
+            defaultness: ast::Defaultness::Final,
+        })),
+        ExpansionKind::Pat => Expansion::Pat(P(ast::Pat {
+            id: id, span: span, node: ast::PatKind::Mac(mac_placeholder()),
+        })),
+        ExpansionKind::Ty => Expansion::Ty(P(ast::Ty {
+            id: id, span: span, node: ast::TyKind::Mac(mac_placeholder()),
+        })),
+        ExpansionKind::Stmts => Expansion::Stmts(SmallVector::one({
+            let mac = P((mac_placeholder(), ast::MacStmtStyle::Braces, ast::ThinVec::new()));
+            ast::Stmt { id: id, span: span, node: ast::StmtKind::Mac(mac) }
+        })),
+    }
+}
+
+pub fn macro_scope_placeholder() -> Expansion {
+    placeholder(ExpansionKind::Items, ast::DUMMY_NODE_ID)
+}
+
+pub struct PlaceholderExpander {
+    expansions: HashMap<ast::NodeId, Expansion>,
+}
+
+impl PlaceholderExpander {
+    pub fn new() -> Self {
+        PlaceholderExpander {
+            expansions: HashMap::new(),
+        }
+    }
+
+    pub fn add(&mut self, id: ast::NodeId, expansion: Expansion) {
+        self.expansions.insert(id, expansion);
+    }
+
+    pub fn remove(&mut self, id: ast::NodeId) -> Expansion {
+        self.expansions.remove(&id).unwrap()
+    }
+}
+
+impl Folder for PlaceholderExpander {
+    fn fold_item(&mut self, item: P<ast::Item>) -> SmallVector<P<ast::Item>> {
+        match item.node {
+            // Scope placeholder
+            ast::ItemKind::Mac(_) if item.id == ast::DUMMY_NODE_ID => SmallVector::one(item),
+            ast::ItemKind::Mac(_) => self.remove(item.id).make_items(),
+            _ => noop_fold_item(item, self),
+        }
+    }
+
+    fn fold_trait_item(&mut self, item: ast::TraitItem) -> SmallVector<ast::TraitItem> {
+        match item.node {
+            ast::TraitItemKind::Macro(_) => self.remove(item.id).make_trait_items(),
+            _ => noop_fold_trait_item(item, self),
+        }
+    }
+
+    fn fold_impl_item(&mut self, item: ast::ImplItem) -> SmallVector<ast::ImplItem> {
+        match item.node {
+            ast::ImplItemKind::Macro(_) => self.remove(item.id).make_impl_items(),
+            _ => noop_fold_impl_item(item, self),
+        }
+    }
+
+    fn fold_expr(&mut self, expr: P<ast::Expr>) -> P<ast::Expr> {
+        match expr.node {
+            ast::ExprKind::Mac(_) => self.remove(expr.id).make_expr(),
+            _ => expr.map(|expr| noop_fold_expr(expr, self)),
+        }
+    }
+
+    fn fold_opt_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
+        match expr.node {
+            ast::ExprKind::Mac(_) => self.remove(expr.id).make_opt_expr(),
+            _ => noop_fold_opt_expr(expr, self),
+        }
+    }
+
+    fn fold_stmt(&mut self, stmt: ast::Stmt) -> SmallVector<ast::Stmt> {
+        let (style, mut expansion) = match stmt.node {
+            ast::StmtKind::Mac(mac) => (mac.1, self.remove(stmt.id).make_stmts()),
+            _ => return noop_fold_stmt(stmt, self),
+        };
+
+        if style == ast::MacStmtStyle::Semicolon {
+            if let Some(stmt) = expansion.pop() {
+                expansion.push(stmt.add_trailing_semicolon());
+            }
+        }
+
+        expansion
+    }
+
+    fn fold_pat(&mut self, pat: P<ast::Pat>) -> P<ast::Pat> {
+        match pat.node {
+            ast::PatKind::Mac(_) => self.remove(pat.id).make_pat(),
+            _ => noop_fold_pat(pat, self),
+        }
+    }
+
+    fn fold_ty(&mut self, ty: P<ast::Ty>) -> P<ast::Ty> {
+        match ty.node {
+            ast::TyKind::Mac(_) => self.remove(ty.id).make_ty(),
+            _ => noop_fold_ty(ty, self),
+        }
+    }
+}
+
+pub fn reconstructed_macro_rules(def: &ast::MacroDef, path: &ast::Path) -> Expansion {
+    Expansion::Items(SmallVector::one(P(ast::Item {
+        ident: def.ident,
+        attrs: def.attrs.clone(),
+        id: ast::DUMMY_NODE_ID,
+        node: ast::ItemKind::Mac(ast::Mac {
+            span: def.span,
+            node: ast::Mac_ {
+                path: path.clone(),
+                tts: def.body.clone(),
+            }
+        }),
+        vis: ast::Visibility::Inherited,
+        span: def.span,
+    })))
+}

src/libsyntax/ext/source_util.rs

@@ -74,11 +74,9 @@ pub fn expand_stringify(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTre
 pub fn expand_mod(cx: &mut ExtCtxt, sp: Span, tts: &[tokenstream::TokenTree])
                   -> Box<base::MacResult+'static> {
     base::check_zero_tts(cx, sp, tts, "module_path!");
-    let string = cx.mod_path()
-                   .iter()
-                   .map(|x| x.to_string())
-                   .collect::<Vec<String>>()
-                   .join("::");
+    let paths = cx.syntax_env.paths();
+    let string = paths.mod_path.iter().map(|x| x.to_string()).collect::<Vec<String>>().join("::");
+
     base::MacEager::expr(cx.expr_str(
             sp,
             token::intern_and_get_ident(&string[..])))

src/libsyntax/ext/tt/macro_rules.rs

@@ -211,9 +211,8 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                                            imported_from,
                                            rhs);
                 let mut p = Parser::new(cx.parse_sess(), cx.cfg(), Box::new(trncbr));
-                p.filename = cx.filename.clone();
-                p.mod_path_stack = cx.mod_path_stack.clone();
-                p.restrictions = match cx.in_block {
+                p.directory = cx.syntax_env.paths().directory.clone();
+                p.restrictions = match cx.syntax_env.in_block() {
                     true => Restrictions::NO_NONINLINE_MOD,
                     false => Restrictions::empty(),
                 };

src/libsyntax/lib.rs

@@ -104,6 +104,7 @@ pub mod abi;
 pub mod ast;
 pub mod attr;
 pub mod codemap;
+#[macro_use]
 pub mod config;
 pub mod entry;
 pub mod feature_gate;
@@ -126,6 +127,7 @@ pub mod ext {
     pub mod base;
     pub mod build;
     pub mod expand;
+    pub mod placeholders;
     pub mod hygiene;
     pub mod proc_macro_shim;
     pub mod quote;

src/libsyntax/parse/parser.rs

@@ -264,8 +264,7 @@ pub struct Parser<'a> {
     /// extra detail when the same error is seen twice
     pub obsolete_set: HashSet<ObsoleteSyntax>,
     /// Used to determine the path to externally loaded source files
-    pub filename: Option<String>,
-    pub mod_path_stack: Vec<InternedString>,
+    pub directory: PathBuf,
     /// Stack of open delimiters and their spans. Used for error message.
     pub open_braces: Vec<(token::DelimToken, Span)>,
     /// Flag if this parser "owns" the directory that it is currently parsing
@@ -346,9 +345,11 @@ impl<'a> Parser<'a> {
     {
         let tok0 = rdr.real_token();
         let span = tok0.sp;
-        let filename = if span != syntax_pos::DUMMY_SP {
-            Some(sess.codemap().span_to_filename(span))
-        } else { None };
+        let mut directory = match span {
+            syntax_pos::DUMMY_SP => PathBuf::new(),
+            _ => PathBuf::from(sess.codemap().span_to_filename(span)),
+        };
+        directory.pop();
         let placeholder = TokenAndSpan {
             tok: token::Underscore,
             sp: span,
@@ -377,8 +378,7 @@ impl<'a> Parser<'a> {
             quote_depth: 0,
             parsing_token_tree: false,
             obsolete_set: HashSet::new(),
-            mod_path_stack: Vec::new(),
-            filename: filename,
+            directory: directory,
             open_braces: Vec::new(),
             owns_directory: true,
             root_module_name: None,
@@ -5306,27 +5306,24 @@ impl<'a> Parser<'a> {
             let (m, attrs) = self.eval_src_mod(id, &outer_attrs, id_span)?;
             Ok((id, m, Some(attrs)))
         } else {
-            self.push_mod_path(id, &outer_attrs);
+            let directory = self.directory.clone();
+            self.push_directory(id, &outer_attrs);
             self.expect(&token::OpenDelim(token::Brace))?;
             let mod_inner_lo = self.span.lo;
             let attrs = self.parse_inner_attributes()?;
             let m = self.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo)?;
-            self.pop_mod_path();
+            self.directory = directory;
             Ok((id, ItemKind::Mod(m), Some(attrs)))
         }
     }
 
-    fn push_mod_path(&mut self, id: Ident, attrs: &[Attribute]) {
+    fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) {
         let default_path = self.id_to_interned_str(id);
         let file_path = match ::attr::first_attr_value_str_by_name(attrs, "path") {
             Some(d) => d,
             None => default_path,
         };
-        self.mod_path_stack.push(file_path)
-    }
-
-    fn pop_mod_path(&mut self) {
-        self.mod_path_stack.pop().unwrap();
+        self.directory.push(&*file_path)
     }
 
     pub fn submod_path_from_attr(attrs: &[ast::Attribute], dir_path: &Path) -> Option<PathBuf> {
@@ -5374,18 +5371,11 @@ impl<'a> Parser<'a> {
                    id: ast::Ident,
                    outer_attrs: &[ast::Attribute],
                    id_sp: Span) -> PResult<'a, ModulePathSuccess> {
-        let mut prefix = PathBuf::from(self.filename.as_ref().unwrap());
-        prefix.pop();
-        let mut dir_path = prefix;
-        for part in &self.mod_path_stack {
-            dir_path.push(&**part);
-        }
-
-        if let Some(p) = Parser::submod_path_from_attr(outer_attrs, &dir_path) {
+        if let Some(p) = Parser::submod_path_from_attr(outer_attrs, &self.directory) {
             return Ok(ModulePathSuccess { path: p, owns_directory: true });
         }
 
-        let paths = Parser::default_submod_path(id, &dir_path, self.sess.codemap());
+        let paths = Parser::default_submod_path(id, &self.directory, self.sess.codemap());
 
         if self.restrictions.contains(Restrictions::NO_NONINLINE_MOD) {
             let msg =
@@ -5400,8 +5390,8 @@ impl<'a> Parser<'a> {
         } else if !self.owns_directory {
             let mut err = self.diagnostic().struct_span_err(id_sp,
                 "cannot declare a new module at this location");
-            let this_module = match self.mod_path_stack.last() {
-                Some(name) => name.to_string(),
+            let this_module = match self.directory.file_name() {
+                Some(file_name) => file_name.to_str().unwrap().to_owned(),
                 None => self.root_module_name.as_ref().unwrap().clone(),
             };
             err.span_note(id_sp,

src/libsyntax/test.rs

@@ -300,14 +300,11 @@ fn generate_test_harness(sess: &ParseSess,
         }
     });
 
-    let mut fold = TestHarnessGenerator {
+    TestHarnessGenerator {
         cx: cx,
         tests: Vec::new(),
         tested_submods: Vec::new(),
-    };
-    let res = fold.fold_crate(krate);
-    fold.cx.ext_cx.bt_pop();
-    return res;
+    }.fold_crate(krate)
 }
 
 /// Craft a span that will be ignored by the stability lint's

src/libsyntax/util/small_vector.rs

@@ -29,6 +29,12 @@ enum SmallVectorRepr<T> {
     Many(Vec<T>),
 }
 
+impl<T> Default for SmallVector<T> {
+    fn default() -> Self {
+        SmallVector { repr: Zero }
+    }
+}
+
 impl<T> Into<Vec<T>> for SmallVector<T> {
     fn into(self) -> Vec<T> {
         match self.repr {