Auto merge of #65495 - Centril:rollup-tguwjt5, r=Centril

Rollup of 8 pull requests

Successful merges:

 - #65237 (Move debug_map assertions after check for err)
 - #65316 (make File::try_clone produce non-inheritable handles on Windows)
 - #65319 (InterpCx: make memory field public)
 - #65461 (Don't recommend ONCE_INIT in std::sync::Once)
 - #65465 (Move syntax::ext to a syntax_expand and refactor some attribute logic)
 - #65475 (add example for type_name)
 - #65478 (fmt::Write is about string slices, not byte slices)
 - #65486 (doc: fix typo in OsStrExt and OsStringExt)

Failed merges:

r? @ghost

Cargo.lock

@@ -3112,6 +3112,7 @@ dependencies = [
  "serialize",
  "smallvec",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
 ]
 
@@ -3427,6 +3428,7 @@ dependencies = [
  "rustc_target",
  "serialize",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
  "tempfile",
 ]
@@ -3559,6 +3561,7 @@ dependencies = [
  "serialize",
  "smallvec",
  "syntax",
+ "syntax_expand",
  "syntax_ext",
  "syntax_pos",
  "tempfile",
@@ -3630,6 +3633,7 @@ dependencies = [
  "smallvec",
  "stable_deref_trait",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
 ]
 
@@ -3678,6 +3682,7 @@ dependencies = [
  "rustc_index",
  "rustc_target",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
 ]
 
@@ -3695,6 +3700,7 @@ dependencies = [
  "rustc",
  "rustc_metadata",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
 ]
 
@@ -3723,6 +3729,7 @@ dependencies = [
  "rustc_metadata",
  "smallvec",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
 ]
 
@@ -4336,6 +4343,25 @@ dependencies = [
  "syntax_pos",
 ]
 
+[[package]]
+name = "syntax_expand"
+version = "0.0.0"
+dependencies = [
+ "bitflags",
+ "lazy_static 1.3.0",
+ "log",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_index",
+ "rustc_lexer",
+ "rustc_target",
+ "scoped-tls",
+ "serialize",
+ "smallvec",
+ "syntax",
+ "syntax_pos",
+]
+
 [[package]]
 name = "syntax_ext"
 version = "0.0.0"
@@ -4347,6 +4373,7 @@ dependencies = [
  "rustc_target",
  "smallvec",
  "syntax",
+ "syntax_expand",
  "syntax_pos",
 ]
 

src/libcore/any.rs

@@ -445,6 +445,15 @@ impl TypeId {
 ///
 /// The current implementation uses the same infrastructure as compiler
 /// diagnostics and debuginfo, but this is not guaranteed.
+///
+/// # Example
+///
+/// ```rust
+/// assert_eq!(
+///     std::any::type_name::<Option<String>>(),
+///     "core::option::Option<alloc::string::String>",
+/// );
+/// ```
 #[stable(feature = "type_name", since = "1.38.0")]
 #[rustc_const_unstable(feature = "const_type_name")]
 pub const fn type_name<T: ?Sized>() -> &'static str {

src/libcore/fmt/builders.rs

@@ -775,10 +775,10 @@ impl<'a, 'b: 'a> DebugMap<'a, 'b> {
                reason = "recently added",
                issue = "62482")]
     pub fn key(&mut self, key: &dyn fmt::Debug) -> &mut DebugMap<'a, 'b> {
-        assert!(!self.has_key, "attempted to begin a new map entry \
-                                without completing the previous one");
-
         self.result = self.result.and_then(|_| {
+            assert!(!self.has_key, "attempted to begin a new map entry \
+                                    without completing the previous one");
+
             if self.is_pretty() {
                 if !self.has_fields {
                     self.fmt.write_str("\n")?;
@@ -839,9 +839,9 @@ impl<'a, 'b: 'a> DebugMap<'a, 'b> {
                reason = "recently added",
                issue = "62482")]
     pub fn value(&mut self, value: &dyn fmt::Debug) -> &mut DebugMap<'a, 'b> {
-        assert!(self.has_key, "attempted to format a map value before its key");
-
         self.result = self.result.and_then(|_| {
+            assert!(self.has_key, "attempted to format a map value before its key");
+
             if self.is_pretty() {
                 let mut slot = None;
                 let mut writer = PadAdapter::wrap(&mut self.fmt, &mut slot, &mut self.state);
@@ -924,9 +924,11 @@ impl<'a, 'b: 'a> DebugMap<'a, 'b> {
     /// ```
     #[stable(feature = "debug_builders", since = "1.2.0")]
     pub fn finish(&mut self) -> fmt::Result {
-        assert!(!self.has_key, "attempted to finish a map with a partial entry");
+        self.result.and_then(|_| {
+            assert!(!self.has_key, "attempted to finish a map with a partial entry");
 
-        self.result.and_then(|_| self.fmt.write_str("}"))
+            self.fmt.write_str("}")
+        })
     }
 
     fn is_pretty(&self) -> bool {

src/libcore/fmt/mod.rs

@@ -108,10 +108,10 @@ pub struct Error;
 /// [`io::Write`]: ../../std/io/trait.Write.html
 #[stable(feature = "rust1", since = "1.0.0")]
 pub trait Write {
-    /// Writes a slice of bytes into this writer, returning whether the write
+    /// Writes a string slice into this writer, returning whether the write
     /// succeeded.
     ///
-    /// This method can only succeed if the entire byte slice was successfully
+    /// This method can only succeed if the entire string slice was successfully
     /// written, and this method will not return until all data has been
     /// written or an error occurs.
     ///

src/libcore/tests/fmt/builders.rs

@@ -319,6 +319,46 @@ mod debug_map {
                    format!("{:#?}", Bar));
     }
 
+    #[test]
+    fn test_entry_err() {
+        // Ensure errors in a map entry don't trigger panics (#65231)
+        use std::fmt::Write;
+
+        struct ErrorFmt;
+
+        impl fmt::Debug for ErrorFmt {
+            fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
+                Err(fmt::Error)
+            }
+        }
+
+        struct KeyValue<K, V>(usize, K, V);
+
+        impl<K, V> fmt::Debug for KeyValue<K, V>
+        where
+            K: fmt::Debug,
+            V: fmt::Debug,
+        {
+            fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+                let mut map = fmt.debug_map();
+
+                for _ in 0..self.0 {
+                    map.entry(&self.1, &self.2);
+                }
+
+                map.finish()
+            }
+        }
+
+        let mut buf = String::new();
+
+        assert!(write!(&mut buf, "{:?}", KeyValue(1, ErrorFmt, "bar")).is_err());
+        assert!(write!(&mut buf, "{:?}", KeyValue(1, "foo", ErrorFmt)).is_err());
+
+        assert!(write!(&mut buf, "{:?}", KeyValue(2, ErrorFmt, "bar")).is_err());
+        assert!(write!(&mut buf, "{:?}", KeyValue(2, "foo", ErrorFmt)).is_err());
+    }
+
     #[test]
     #[should_panic]
     fn test_invalid_key_when_entry_is_incomplete() {

src/librustc/Cargo.toml

@@ -29,6 +29,7 @@ rustc_index = { path = "../librustc_index" }
 errors = { path = "../librustc_errors", package = "rustc_errors" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }
 backtrace = "0.3.3"
 parking_lot = "0.9"

src/librustc/hir/def.rs

@@ -6,7 +6,7 @@ use crate::ty;
 use crate::util::nodemap::DefIdMap;
 
 use syntax::ast;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::ast::NodeId;
 use syntax_pos::Span;
 use rustc_macros::HashStable;

src/librustc/hir/lowering.rs

@@ -64,15 +64,15 @@ use syntax::ast;
 use syntax::ptr::P as AstP;
 use syntax::ast::*;
 use syntax::errors;
-use syntax::ext::base::SpecialDerives;
+use syntax_expand::base::SpecialDerives;
-use syntax::ext::hygiene::ExpnId;
 use syntax::print::pprust;
-use syntax::tokenstream::{TokenStream, TokenTree};
 use syntax::parse::token::{self, Nonterminal, Token};
+use syntax::tokenstream::{TokenStream, TokenTree};
 use syntax::sess::ParseSess;
 use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned};
 use syntax::symbol::{kw, sym, Symbol};
 use syntax::visit::{self, Visitor};
+use syntax_pos::hygiene::ExpnId;
 use syntax_pos::Span;
 
 const HIR_ID_COUNTER_LOCKED: u32 = 0xFFFFFFFF;

src/librustc/hir/lowering/item.rs

@@ -18,7 +18,7 @@ use smallvec::SmallVec;
 use syntax::attr;
 use syntax::ast::*;
 use syntax::visit::{self, Visitor};
-use syntax::ext::base::SpecialDerives;
+use syntax_expand::base::SpecialDerives;
 use syntax::source_map::{respan, DesugaringKind, Spanned};
 use syntax::symbol::{kw, sym};
 use syntax_pos::Span;

src/librustc/hir/map/def_collector.rs

@@ -2,7 +2,7 @@ use crate::hir::map::definitions::*;
 use crate::hir::def_id::DefIndex;
 
 use syntax::ast::*;
-use syntax::ext::hygiene::ExpnId;
+use syntax_expand::hygiene::ExpnId;
 use syntax::visit;
 use syntax::symbol::{kw, sym};
 use syntax::parse::token::{self, Token};

src/librustc/hir/map/definitions.rs

@@ -17,7 +17,7 @@ use std::borrow::Borrow;
 use std::fmt::Write;
 use std::hash::Hash;
 use syntax::ast;
-use syntax::ext::hygiene::ExpnId;
+use syntax_expand::hygiene::ExpnId;
 use syntax::symbol::{Symbol, sym, InternedString};
 use syntax_pos::{Span, DUMMY_SP};
 

src/librustc/hir/map/mod.rs

@@ -20,7 +20,7 @@ use rustc_data_structures::svh::Svh;
 use rustc_index::vec::IndexVec;
 use syntax::ast::{self, Name, NodeId};
 use syntax::source_map::Spanned;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax_pos::{Span, DUMMY_SP};
 
 pub mod blocks;

src/librustc/ich/hcx.rs

@@ -13,7 +13,7 @@ use std::cell::RefCell;
 
 use syntax::ast;
 use syntax::source_map::SourceMap;
-use syntax::ext::hygiene::SyntaxContext;
+use syntax_expand::hygiene::SyntaxContext;
 use syntax::symbol::Symbol;
 use syntax::tokenstream::DelimSpan;
 use syntax_pos::{Span, DUMMY_SP};

src/librustc/ich/impls_syntax.rs

@@ -59,7 +59,7 @@ impl_stable_hash_for!(enum ::syntax::ast::AsmDialect {
     Intel
 });
 
-impl_stable_hash_for!(enum ::syntax::ext::base::MacroKind {
+impl_stable_hash_for!(enum ::syntax_expand::base::MacroKind {
     Bang,
     Attr,
     Derive,

src/librustc/lint/mod.rs

@@ -39,7 +39,7 @@ use syntax::ast;
 use syntax::source_map::{MultiSpan, ExpnKind, DesugaringKind};
 use syntax::early_buffered_lints::BufferedEarlyLintId;
 use syntax::edition::Edition;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::symbol::{Symbol, sym};
 use syntax_pos::Span;
 

src/librustc/session/mod.rs

@@ -24,7 +24,7 @@ use errors::emitter::HumanReadableErrorType;
 use errors::annotate_snippet_emitter_writer::{AnnotateSnippetEmitterWriter};
 use syntax::ast::{self, NodeId};
 use syntax::edition::Edition;
-use syntax::ext::allocator::AllocatorKind;
+use syntax_expand::allocator::AllocatorKind;
 use syntax::feature_gate::{self, AttributeType};
 use syntax::json::JsonEmitter;
 use syntax::source_map;

src/librustc/ty/mod.rs

@@ -45,7 +45,7 @@ use std::{mem, ptr};
 use std::ops::Range;
 use syntax::ast::{self, Name, Ident, NodeId};
 use syntax::attr;
-use syntax::ext::hygiene::ExpnId;
+use syntax_expand::hygiene::ExpnId;
 use syntax::symbol::{kw, sym, Symbol, InternedString};
 use syntax_pos::Span;
 

src/librustc_codegen_llvm/allocator.rs

@@ -3,7 +3,7 @@ use std::ffi::CString;
 use crate::attributes;
 use libc::c_uint;
 use rustc::ty::TyCtxt;
-use syntax::ext::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
+use syntax_expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 
 use crate::ModuleLlvm;
 use crate::llvm::{self, False, True};

src/librustc_codegen_llvm/lib.rs

@@ -39,6 +39,7 @@ extern crate rustc_driver as _;
 
 #[macro_use] extern crate log;
 extern crate syntax;
+extern crate syntax_expand;
 extern crate syntax_pos;
 extern crate rustc_errors as errors;
 
@@ -48,7 +49,7 @@ use rustc_codegen_ssa::back::lto::{SerializedModule, LtoModuleCodegen, ThinModul
 use rustc_codegen_ssa::CompiledModule;
 use errors::{FatalError, Handler};
 use rustc::dep_graph::WorkProduct;
-use syntax::ext::allocator::AllocatorKind;
+use syntax_expand::allocator::AllocatorKind;
 use syntax_pos::symbol::InternedString;
 pub use llvm_util::target_features;
 use std::any::Any;

src/librustc_codegen_ssa/Cargo.toml

@@ -21,6 +21,7 @@ tempfile = "3.1"
 
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc = { path = "../librustc" }
 rustc_apfloat = { path = "../librustc_apfloat" }

src/librustc_codegen_ssa/back/symbol_export.rs

@@ -14,7 +14,7 @@ use rustc::ty::query::Providers;
 use rustc::ty::subst::SubstsRef;
 use rustc::util::nodemap::{FxHashMap, DefIdMap};
 use rustc_index::vec::IndexVec;
-use syntax::ext::allocator::ALLOCATOR_METHODS;
+use syntax_expand::allocator::ALLOCATOR_METHODS;
 
 pub type ExportedSymbols = FxHashMap<
     CrateNum,

src/librustc_codegen_ssa/back/write.rs

@@ -27,7 +27,7 @@ use rustc_errors::{Handler, Level, FatalError, DiagnosticId, SourceMapperDyn};
 use rustc_errors::emitter::{Emitter};
 use rustc_target::spec::MergeFunctions;
 use syntax::attr;
-use syntax::ext::hygiene::ExpnId;
+use syntax_expand::hygiene::ExpnId;
 use syntax_pos::symbol::{Symbol, sym};
 use jobserver::{Client, Acquired};
 

src/librustc_codegen_ssa/traits/backend.rs

@@ -9,7 +9,7 @@ use rustc::ty::TyCtxt;
 use rustc_codegen_utils::codegen_backend::CodegenBackend;
 use std::sync::Arc;
 use std::sync::mpsc;
-use syntax::ext::allocator::AllocatorKind;
+use syntax_expand::allocator::AllocatorKind;
 use syntax_pos::symbol::InternedString;
 
 pub trait BackendTypes {

src/librustc_interface/Cargo.toml

@@ -15,6 +15,7 @@ rayon = { version = "0.3.0", package = "rustc-rayon" }
 smallvec = { version = "0.6.7", features = ["union", "may_dangle"] }
 syntax = { path = "../libsyntax" }
 syntax_ext = { path = "../libsyntax_ext" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 rustc = { path = "../librustc" }

src/librustc_interface/passes.rs

@@ -35,7 +35,7 @@ use rustc_traits;
 use rustc_typeck as typeck;
 use syntax::{self, ast, visit};
 use syntax::early_buffered_lints::BufferedEarlyLint;
-use syntax::ext::base::{NamedSyntaxExtension, ExtCtxt};
+use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt};
 use syntax::mut_visit::MutVisitor;
 use syntax::parse::{self, PResult};
 use syntax::util::node_count::NodeCounter;
@@ -397,12 +397,12 @@ fn configure_and_expand_inner<'a>(
 
         // Create the config for macro expansion
         let features = sess.features_untracked();
-        let cfg = syntax::ext::expand::ExpansionConfig {
+        let cfg = syntax_expand::expand::ExpansionConfig {
             features: Some(&features),
             recursion_limit: *sess.recursion_limit.get(),
             trace_mac: sess.opts.debugging_opts.trace_macros,
             should_test: sess.opts.test,
-            ..syntax::ext::expand::ExpansionConfig::default(crate_name.to_string())
+            ..syntax_expand::expand::ExpansionConfig::default(crate_name.to_string())
         };
 
         let mut ecx = ExtCtxt::new(&sess.parse_sess, cfg, &mut resolver);
@@ -559,7 +559,7 @@ pub fn lower_to_hir(
 
     // Discard hygiene data, which isn't required after lowering to HIR.
     if !sess.opts.debugging_opts.keep_hygiene_data {
-        syntax::ext::hygiene::clear_syntax_context_map();
+        syntax_expand::hygiene::clear_syntax_context_map();
     }
 
     Ok(hir_forest)

src/librustc_metadata/Cargo.toml

@@ -22,4 +22,5 @@ rustc_index = { path = "../librustc_index" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
 stable_deref_trait = "1.0.0"
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }

src/librustc_metadata/creader.rs

@@ -26,7 +26,7 @@ use std::{cmp, fs};
 
 use syntax::ast;
 use syntax::attr;
-use syntax::ext::allocator::{global_allocator_spans, AllocatorKind};
+use syntax_expand::allocator::{global_allocator_spans, AllocatorKind};
 use syntax::symbol::{Symbol, sym};
 use syntax::{span_err, span_fatal};
 use syntax_pos::{Span, DUMMY_SP};

src/librustc_metadata/cstore.rs

@@ -11,7 +11,7 @@ use rustc_index::vec::IndexVec;
 use rustc::util::nodemap::FxHashMap;
 use rustc_data_structures::sync::{Lrc, RwLock, Lock, MetadataRef, AtomicCell};
 use syntax::ast;
-use syntax::ext::base::SyntaxExtension;
+use syntax_expand::base::SyntaxExtension;
 use syntax_pos;
 use proc_macro::bridge::client::ProcMacro;
 

src/librustc_metadata/decoder.rs

@@ -34,11 +34,11 @@ use syntax::attr;
 use syntax::ast::{self, Ident};
 use syntax::source_map::{self, respan, Spanned};
 use syntax::symbol::{Symbol, sym};
-use syntax::ext::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension};
+use syntax_expand::base::{MacroKind, SyntaxExtensionKind, SyntaxExtension};
 use syntax_pos::{self, Span, BytePos, Pos, DUMMY_SP, symbol::{InternedString}};
 use log::debug;
 use proc_macro::bridge::client::ProcMacro;
-use syntax::ext::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro};
+use syntax_expand::proc_macro::{AttrProcMacro, ProcMacroDerive, BangProcMacro};
 
 crate struct DecodeContext<'a, 'tcx> {
     opaque: opaque::Decoder<'a>,

src/librustc_metadata/encoder.rs

@@ -32,7 +32,7 @@ use std::path::Path;
 use std::u32;
 use syntax::ast;
 use syntax::attr;
-use syntax::ext::proc_macro::is_proc_macro_attr;
+use syntax_expand::proc_macro::is_proc_macro_attr;
 use syntax::source_map::Spanned;
 use syntax::symbol::{kw, sym, Ident, Symbol};
 use syntax_pos::{self, FileName, SourceFile, Span};

src/librustc_mir/interpret/eval_context.rs

@@ -35,7 +35,7 @@ pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
     pub(crate) param_env: ty::ParamEnv<'tcx>,
 
     /// The virtual memory system.
-    pub(crate) memory: Memory<'mir, 'tcx, M>,
+    pub memory: Memory<'mir, 'tcx, M>,
 
     /// The virtual call stack.
     pub(crate) stack: Vec<Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>>,
@@ -211,16 +211,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
     }
 
-    #[inline(always)]
-    pub fn memory(&self) -> &Memory<'mir, 'tcx, M> {
-        &self.memory
-    }
-
-    #[inline(always)]
-    pub fn memory_mut(&mut self) -> &mut Memory<'mir, 'tcx, M> {
-        &mut self.memory
-    }
-
     #[inline(always)]
     pub fn force_ptr(
         &self,

src/librustc_mir/interpret/intern.rs

@@ -73,8 +73,7 @@ fn intern_shallow<'rt, 'mir, 'tcx>(
     );
     // remove allocation
     let tcx = ecx.tcx;
-    let memory = ecx.memory_mut();
+    let (kind, mut alloc) = match ecx.memory.alloc_map.remove(&alloc_id) {
-    let (kind, mut alloc) = match memory.alloc_map.remove(&alloc_id) {
         Some(entry) => entry,
         None => {
             // Pointer not found in local memory map. It is either a pointer to the global
@@ -332,7 +331,7 @@ pub fn intern_const_alloc_recursive(
 
     let mut todo: Vec<_> = leftover_allocations.iter().cloned().collect();
     while let Some(alloc_id) = todo.pop() {
-        if let Some((_, mut alloc)) = ecx.memory_mut().alloc_map.remove(&alloc_id) {
+        if let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) {
             // We can't call the `intern_shallow` method here, as its logic is tailored to safe
             // references and a `leftover_allocations` set (where we only have a todo-list here).
             // So we hand-roll the interning logic here again.
@@ -350,7 +349,7 @@ pub fn intern_const_alloc_recursive(
                     todo.push(reloc);
                 }
             }
-        } else if ecx.memory().dead_alloc_map.contains_key(&alloc_id) {
+        } else if ecx.memory.dead_alloc_map.contains_key(&alloc_id) {
             // dangling pointer
             throw_unsup!(ValidationFailure("encountered dangling pointer in final constant".into()))
         }

src/librustc_mir/interpret/terminator.rs

@@ -140,12 +140,12 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                                 .read_immediate(self.eval_operand(len, None)?)
                                 .expect("can't eval len")
                                 .to_scalar()?
-                                .to_bits(self.memory().pointer_size())? as u64;
+                                .to_bits(self.memory.pointer_size())? as u64;
                             let index = self
                                 .read_immediate(self.eval_operand(index, None)?)
                                 .expect("can't eval index")
                                 .to_scalar()?
-                                .to_bits(self.memory().pointer_size())? as u64;
+                                .to_bits(self.memory.pointer_size())? as u64;
                             err_panic!(BoundsCheck { len, index })
                         }
                         Overflow(op) => err_panic!(Overflow(*op)),

src/librustc_passes/Cargo.toml

@@ -13,6 +13,7 @@ log = "0.4"
 rustc = { path = "../librustc" }
 rustc_data_structures = { path = "../librustc_data_structures" }
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }
 errors = { path = "../librustc_errors", package = "rustc_errors" }
 rustc_target = { path = "../librustc_target" }

src/librustc_passes/ast_validation.rs

@@ -14,7 +14,7 @@ use rustc::session::Session;
 use rustc_data_structures::fx::FxHashMap;
 use syntax::ast::*;
 use syntax::attr;
-use syntax::ext::proc_macro::is_proc_macro_attr;
+use syntax_expand::proc_macro::is_proc_macro_attr;
 use syntax::feature_gate::is_builtin_attr;
 use syntax::source_map::Spanned;
 use syntax::symbol::{kw, sym};

src/librustc_plugin/Cargo.toml

@@ -14,4 +14,5 @@ doctest = false
 rustc = { path = "../librustc" }
 rustc_metadata = { path = "../librustc_metadata" }
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }

src/librustc_plugin/lib.rs

@@ -21,7 +21,7 @@
 //! extern crate syntax_pos;
 //!
 //! use rustc_driver::plugin::Registry;
-//! use syntax::ext::base::{ExtCtxt, MacResult};
+//! use syntax_expand::base::{ExtCtxt, MacResult};
 //! use syntax_pos::Span;
 //! use syntax::tokenstream::TokenTree;
 //!

src/librustc_plugin/registry.rs

@@ -4,8 +4,8 @@ use rustc::lint::{EarlyLintPassObject, LateLintPassObject, LintId, Lint};
 use rustc::session::Session;
 use rustc::util::nodemap::FxHashMap;
 
-use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind, NamedSyntaxExtension};
+use syntax_expand::base::{SyntaxExtension, SyntaxExtensionKind, NamedSyntaxExtension};
-use syntax::ext::base::MacroExpanderFn;
+use syntax_expand::base::MacroExpanderFn;
 use syntax::symbol::Symbol;
 use syntax::ast;
 use syntax::feature_gate::AttributeType;

src/librustc_resolve/Cargo.toml

@@ -14,6 +14,7 @@ doctest = false
 bitflags = "1.0"
 log = "0.4"
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 rustc = { path = "../librustc" }
 arena = { path = "../libarena" }
 errors = { path = "../librustc_errors", package = "rustc_errors" }

src/librustc_resolve/build_reduced_graph.rs

@@ -32,9 +32,9 @@ use syntax::attr;
 
 use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, NodeId};
 use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind};
-use syntax::ext::base::{MacroKind, SyntaxExtension};
+use syntax_expand::base::{MacroKind, SyntaxExtension};
-use syntax::ext::expand::AstFragment;
+use syntax_expand::expand::AstFragment;
-use syntax::ext::hygiene::ExpnId;
+use syntax_expand::hygiene::ExpnId;
 use syntax::feature_gate::is_builtin_attr;
 use syntax::parse::token::{self, Token};
 use syntax::print::pprust;

src/librustc_resolve/diagnostics.rs

@@ -10,7 +10,7 @@ use rustc::session::Session;
 use rustc::ty::{self, DefIdTree};
 use rustc::util::nodemap::FxHashSet;
 use syntax::ast::{self, Ident, Path};
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::feature_gate::BUILTIN_ATTRIBUTES;
 use syntax::source_map::SourceMap;
 use syntax::struct_span_err;

src/librustc_resolve/late/diagnostics.rs

@@ -13,7 +13,7 @@ use rustc::hir::PrimTy;
 use rustc::session::config::nightly_options;
 use rustc::util::nodemap::FxHashSet;
 use syntax::ast::{self, Expr, ExprKind, Ident, NodeId, Path, Ty, TyKind};
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::symbol::kw;
 use syntax::util::lev_distance::find_best_match_for_name;
 use syntax_pos::Span;

src/librustc_resolve/lib.rs

@@ -35,18 +35,16 @@ use rustc::span_bug;
 use rustc_metadata::creader::CrateLoader;
 use rustc_metadata::cstore::CStore;
 
-use syntax::ext::hygiene::{ExpnId, Transparency, SyntaxContext};
+use syntax_expand::hygiene::{ExpnId, Transparency, SyntaxContext};
+use syntax_expand::base::{SyntaxExtension, MacroKind, SpecialDerives};
+use syntax::{struct_span_err, unwrap_or};
+use syntax::attr;
 use syntax::ast::{self, Name, NodeId, Ident, FloatTy, IntTy, UintTy};
-use syntax::ext::base::{SyntaxExtension, MacroKind, SpecialDerives};
+use syntax::ast::{ItemKind, Path, CRATE_NODE_ID, Crate};
 use syntax::print::pprust;
 use syntax::symbol::{kw, sym};
-
-use syntax::visit::{self, Visitor};
-use syntax::attr;
-use syntax::ast::{CRATE_NODE_ID, Crate};
-use syntax::ast::{ItemKind, Path};
-use syntax::{struct_span_err, unwrap_or};
 use syntax::source_map::Spanned;
+use syntax::visit::{self, Visitor};
 
 use syntax_pos::{Span, DUMMY_SP};
 use errors::{Applicability, DiagnosticBuilder};

src/librustc_resolve/macros.rs

@@ -14,11 +14,11 @@ use rustc::{ty, lint, span_bug};
 use syntax::ast::{self, NodeId, Ident};
 use syntax::attr::StabilityLevel;
 use syntax::edition::Edition;
-use syntax::ext::base::{self, InvocationRes, Indeterminate, SpecialDerives};
+use syntax_expand::base::{self, InvocationRes, Indeterminate, SpecialDerives};
-use syntax::ext::base::{MacroKind, SyntaxExtension};
+use syntax_expand::base::{MacroKind, SyntaxExtension};
-use syntax::ext::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind};
+use syntax_expand::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind};
-use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind};
+use syntax_expand::hygiene::{self, ExpnId, ExpnData, ExpnKind};
-use syntax::ext::compile_declarative_macro;
+use syntax_expand::compile_declarative_macro;
 use syntax::feature_gate::{emit_feature_err, is_builtin_attr_name};
 use syntax::feature_gate::GateIssue;
 use syntax::print::pprust;

src/librustc_resolve/resolve_imports.rs

@@ -28,7 +28,7 @@ use rustc::util::nodemap::FxHashSet;
 use rustc::{bug, span_bug};
 
 use syntax::ast::{Ident, Name, NodeId, CRATE_NODE_ID};
-use syntax::ext::hygiene::ExpnId;
+use syntax_expand::hygiene::ExpnId;
 use syntax::symbol::kw;
 use syntax::util::lev_distance::find_best_match_for_name;
 use syntax::{struct_span_err, unwrap_or};

src/librustdoc/clean/inline.rs

@@ -3,7 +3,7 @@
 use std::iter::once;
 
 use syntax::ast;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::symbol::sym;
 use syntax_pos::Span;
 

src/librustdoc/clean/mod.rs

@@ -28,7 +28,7 @@ use rustc::ty::layout::VariantIdx;
 use rustc::util::nodemap::{FxHashMap, FxHashSet};
 use syntax::ast::{self, AttrStyle, Ident};
 use syntax::attr;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::source_map::DUMMY_SP;
 use syntax::symbol::{Symbol, kw, sym};
 use syntax::symbol::InternedString;

src/librustdoc/doctree.rs

@@ -4,7 +4,7 @@ pub use self::StructType::*;
 
 use syntax::ast;
 use syntax::ast::Name;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax_pos::{self, Span};
 
 use rustc::hir;

src/librustdoc/html/item_type.rs

@@ -1,7 +1,7 @@
 //! Item types.
 
 use std::fmt;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use crate::clean;
 
 /// Item type. Corresponds to `clean::ItemEnum` variants.

src/librustdoc/html/render.rs

@@ -45,11 +45,11 @@ use errors;
 use serialize::json::{ToJson, Json, as_json};
 use syntax::ast;
 use syntax::edition::Edition;
-use syntax::ext::base::MacroKind;
+use syntax::feature_gate::UnstableFeatures;
 use syntax::print::pprust;
 use syntax::source_map::FileName;
-use syntax::feature_gate::UnstableFeatures;
 use syntax::symbol::{Symbol, sym};
+use syntax_expand::base::MacroKind;
 use rustc::hir::def_id::DefId;
 use rustc::middle::privacy::AccessLevels;
 use rustc::middle::stability;

src/librustdoc/lib.rs

@@ -34,6 +34,7 @@ extern crate rustc_typeck;
 extern crate rustc_lexer;
 extern crate serialize;
 extern crate syntax;
+extern crate syntax_expand;
 extern crate syntax_pos;
 extern crate test as testing;
 #[macro_use] extern crate log;

src/librustdoc/passes/collect_intra_doc_links.rs

@@ -7,7 +7,7 @@ use rustc::ty;
 use rustc_resolve::ParentScope;
 use syntax;
 use syntax::ast::{self, Ident};
-use syntax::ext::base::SyntaxExtensionKind;
+use syntax_expand::base::SyntaxExtensionKind;
 use syntax::feature_gate::UnstableFeatures;
 use syntax::symbol::Symbol;
 use syntax_pos::DUMMY_SP;

src/librustdoc/visit_ast.rs

@@ -8,7 +8,7 @@ use rustc::middle::privacy::AccessLevel;
 use rustc::util::nodemap::{FxHashSet, FxHashMap};
 use rustc::ty::TyCtxt;
 use syntax::ast;
-use syntax::ext::base::MacroKind;
+use syntax_expand::base::MacroKind;
 use syntax::source_map::Spanned;
 use syntax::symbol::sym;
 use syntax_pos::{self, Span};

src/libstd/sync/once.rs

@@ -60,10 +60,9 @@ use crate::thread::{self, Thread};
 
 /// A synchronization primitive which can be used to run a one-time global
 /// initialization. Useful for one-time initialization for FFI or related
-/// functionality. This type can only be constructed with the [`ONCE_INIT`]
+/// functionality. This type can only be constructed with the [`Once::new`]
-/// value or the equivalent [`Once::new`] constructor.
+/// constructor.
 ///
-/// [`ONCE_INIT`]: constant.ONCE_INIT.html
 /// [`Once::new`]: struct.Once.html#method.new
 ///
 /// # Examples

src/libstd/sys/windows/fs.rs

@@ -412,7 +412,7 @@ impl File {
 
     pub fn duplicate(&self) -> io::Result<File> {
         Ok(File {
-            handle: self.handle.duplicate(0, true, c::DUPLICATE_SAME_ACCESS)?,
+            handle: self.handle.duplicate(0, false, c::DUPLICATE_SAME_ACCESS)?,
         })
     }
 

src/libstd/sys_common/os_str_bytes.rs

@@ -193,7 +193,7 @@ impl Slice {
 pub trait OsStringExt {
     /// Creates an [`OsString`] from a byte vector.
     ///
-    /// See the module docmentation for an example.
+    /// See the module documentation for an example.
     ///
     /// [`OsString`]: ../../../ffi/struct.OsString.html
     #[stable(feature = "rust1", since = "1.0.0")]
@@ -201,7 +201,7 @@ pub trait OsStringExt {
 
     /// Yields the underlying byte vector of this [`OsString`].
     ///
-    /// See the module docmentation for an example.
+    /// See the module documentation for an example.
     ///
     /// [`OsString`]: ../../../ffi/struct.OsString.html
     #[stable(feature = "rust1", since = "1.0.0")]
@@ -226,14 +226,14 @@ pub trait OsStrExt {
     #[stable(feature = "rust1", since = "1.0.0")]
     /// Creates an [`OsStr`] from a byte slice.
     ///
-    /// See the module docmentation for an example.
+    /// See the module documentation for an example.
     ///
     /// [`OsStr`]: ../../../ffi/struct.OsStr.html
     fn from_bytes(slice: &[u8]) -> &Self;
 
     /// Gets the underlying byte view of the [`OsStr`] slice.
     ///
-    /// See the module docmentation for an example.
+    /// See the module documentation for an example.
     ///
     /// [`OsStr`]: ../../../ffi/struct.OsStr.html
     #[stable(feature = "rust1", since = "1.0.0")]

src/libsyntax/ast.rs

@@ -2,30 +2,29 @@
 
 pub use GenericArgs::*;
 pub use UnsafeSource::*;
-pub use crate::symbol::{Ident, Symbol as Name};
 pub use crate::util::parser::ExprPrecedence;
 
-use crate::ext::hygiene::ExpnId;
 use crate::parse::token::{self, DelimToken};
 use crate::ptr::P;
 use crate::source_map::{dummy_spanned, respan, Spanned};
-use crate::symbol::{kw, sym, Symbol};
 use crate::tokenstream::TokenStream;
-use crate::ThinVec;
+
+use rustc_target::spec::abi::Abi;
+pub use rustc_target::abi::FloatTy;
+
+use syntax_pos::{Span, DUMMY_SP, ExpnId};
+use syntax_pos::symbol::{kw, sym, Symbol};
+pub use syntax_pos::symbol::{Ident, Symbol as Name};
 
 use rustc_index::vec::Idx;
 #[cfg(target_arch = "x86_64")]
 use rustc_data_structures::static_assert_size;
-use rustc_target::spec::abi::Abi;
-use syntax_pos::{Span, DUMMY_SP};
-
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::thin_vec::ThinVec;
 use rustc_serialize::{self, Decoder, Encoder};
 use std::fmt;
 
-pub use rustc_target::abi::FloatTy;
-
 #[cfg(test)]
 mod tests;
 

src/libsyntax/attr/builtin.rs

@@ -2,7 +2,6 @@
 
 use crate::ast::{self, Attribute, MetaItem, NestedMetaItem};
 use crate::early_buffered_lints::BufferedEarlyLintId;
-use crate::ext::base::ExtCtxt;
 use crate::feature_gate::{Features, GatedCfg};
 use crate::print::pprust;
 use crate::sess::ParseSess;
@@ -32,6 +31,10 @@ pub struct AttributeTemplate {
 }
 
 impl AttributeTemplate {
+    pub fn only_word() -> Self {
+        Self { word: true, list: None, name_value_str: None }
+    }
+
     /// Checks that the given meta-item is compatible with this template.
     fn compatible(&self, meta_item_kind: &ast::MetaItemKind) -> bool {
         match meta_item_kind {
@@ -937,14 +940,7 @@ pub fn find_transparency(
     (transparency.map_or(fallback, |t| t.0), error)
 }
 
-pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) {
+pub fn check_builtin_attribute(
-    // All the built-in macro attributes are "words" at the moment.
-    let template = AttributeTemplate { word: true, list: None, name_value_str: None };
-    let attr = ecx.attribute(meta_item.clone());
-    check_builtin_attribute(ecx.parse_sess, &attr, name, template);
-}
-
-crate fn check_builtin_attribute(
     sess: &ParseSess, attr: &ast::Attribute, name: Symbol, template: AttributeTemplate
 ) {
     // Some special attributes like `cfg` must be checked

src/libsyntax/attr/mod.rs

@@ -280,7 +280,7 @@ impl Attribute {
         self.item.meta(self.span)
     }
 
-    pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T>
+    crate fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T>
         where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>,
     {
         let mut parser = Parser::new(
@@ -298,24 +298,11 @@ impl Attribute {
         Ok(result)
     }
 
-    pub fn parse_list<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, Vec<T>>
+    pub fn parse_derive_paths<'a>(&self, sess: &'a ParseSess) -> PResult<'a, Vec<Path>> {
-        where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    {
         if self.tokens.is_empty() {
             return Ok(Vec::new());
         }
-        self.parse(sess, |parser| {
+        self.parse(sess, |p| p.parse_derive_paths())
-            parser.expect(&token::OpenDelim(token::Paren))?;
-            let mut list = Vec::new();
-            while !parser.eat(&token::CloseDelim(token::Paren)) {
-                list.push(f(parser)?);
-                if !parser.eat(&token::Comma) {
-                   parser.expect(&token::CloseDelim(token::Paren))?;
-                    break
-                }
-            }
-            Ok(list)
-        })
     }
 
     pub fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> {

src/libsyntax/config.rs

@@ -10,7 +10,6 @@ use crate::attr;
 use crate::ast;
 use crate::edition::Edition;
 use crate::mut_visit::*;
-use crate::parse::token;
 use crate::ptr::P;
 use crate::sess::ParseSess;
 use crate::symbol::sym;
@@ -57,6 +56,7 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition,
     (krate, features)
 }
 
+#[macro_export]
 macro_rules! configure {
     ($this:ident, $node:ident) => {
         match $this.configure($node) {
@@ -112,25 +112,7 @@ impl<'a> StripUnconfigured<'a> {
             return vec![];
         }
 
-        let (cfg_predicate, expanded_attrs) = match attr.parse(self.sess, |parser| {
+        let (cfg_predicate, expanded_attrs) = match attr.parse(self.sess, |p| p.parse_cfg_attr()) {
-            parser.expect(&token::OpenDelim(token::Paren))?;
-
-            let cfg_predicate = parser.parse_meta_item()?;
-            parser.expect(&token::Comma)?;
-
-            // Presumably, the majority of the time there will only be one attr.
-            let mut expanded_attrs = Vec::with_capacity(1);
-
-            while !parser.check(&token::CloseDelim(token::Paren)) {
-                let lo = parser.token.span.lo();
-                let item = parser.parse_attr_item()?;
-                expanded_attrs.push((item, parser.prev_span.with_lo(lo)));
-                parser.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Paren)])?;
-            }
-
-            parser.expect(&token::CloseDelim(token::Paren))?;
-            Ok((cfg_predicate, expanded_attrs))
-        }) {
             Ok(result) => result,
             Err(mut e) => {
                 e.emit();

src/libsyntax/feature_gate/check.rs

@@ -56,7 +56,7 @@ macro_rules! gate_feature {
     };
 }
 
-crate fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) {
+pub fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) {
     PostExpansionVisitor { parse_sess, features }.visit_attribute(attr)
 }
 

src/libsyntax/feature_gate/mod.rs

@@ -58,8 +58,7 @@ pub use builtin_attrs::{
     deprecated_attributes, is_builtin_attr,  is_builtin_attr_name,
 };
 pub use check::{
-    check_crate, get_features, feature_err, emit_feature_err,
+    check_crate, check_attribute, get_features, feature_err, emit_feature_err,
     Stability, GateIssue, UnstableFeatures,
     EXPLAIN_STMT_ATTR_SYNTAX, EXPLAIN_UNSIZED_TUPLE_COERCION,
 };
-crate use check::check_attribute;

src/libsyntax/lib.rs

@@ -13,17 +13,12 @@
 #![feature(crate_visibility_modifier)]
 #![feature(label_break_value)]
 #![feature(nll)]
-#![feature(proc_macro_diagnostic)]
-#![feature(proc_macro_internals)]
-#![feature(proc_macro_span)]
 #![feature(try_trait)]
 #![feature(slice_patterns)]
 #![feature(unicode_internals)]
 
 #![recursion_limit="256"]
 
-extern crate proc_macro;
-
 pub use errors;
 use rustc_data_structures::sync::Lock;
 use rustc_index::bit_set::GrowableBitSet;
@@ -34,26 +29,7 @@ use syntax_pos::edition::Edition;
 #[cfg(test)]
 mod tests;
 
-const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments");
+pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments");
-
-// A variant of 'try!' that panics on an Err. This is used as a crutch on the
-// way towards a non-panic!-prone parser. It should be used for fatal parsing
-// errors; eventually we plan to convert all code using panictry to just use
-// normal try.
-#[macro_export]
-macro_rules! panictry {
-    ($e:expr) => ({
-        use std::result::Result::{Ok, Err};
-        use errors::FatalError;
-        match $e {
-            Ok(e) => e,
-            Err(mut e) => {
-                e.emit();
-                FatalError.raise()
-            }
-        }
-    })
-}
 
 // A variant of 'panictry!' that works on a Vec<Diagnostic> instead of a single DiagnosticBuilder.
 macro_rules! panictry_buffer {
@@ -157,19 +133,4 @@ pub mod print {
     mod helpers;
 }
 
-pub mod ext {
-    mod placeholders;
-    mod proc_macro_server;
-
-    pub use syntax_pos::hygiene;
-    pub use mbe::macro_rules::compile_declarative_macro;
-    pub mod allocator;
-    pub mod base;
-    pub mod build;
-    pub mod expand;
-    pub mod proc_macro;
-
-    crate mod mbe;
-}
-
 pub mod early_buffered_lints;

src/libsyntax/parse/mod.rs

@@ -2,7 +2,7 @@
 
 use crate::ast;
 use crate::parse::parser::{Parser, emit_unclosed_delims};
-use crate::parse::token::{Nonterminal, TokenKind};
+use crate::parse::token::Nonterminal;
 use crate::tokenstream::{self, TokenStream, TokenTree};
 use crate::print::pprust;
 use crate::sess::ParseSess;
@@ -24,12 +24,10 @@ mod tests;
 
 #[macro_use]
 pub mod parser;
-pub mod attr;
 pub mod lexer;
 pub mod token;
 
 crate mod classify;
-crate mod diagnostics;
 crate mod literal;
 crate mod unescape_error_reporting;
 
@@ -273,30 +271,6 @@ pub fn stream_to_parser_with_base_dir<'a>(
     Parser::new(sess, stream, Some(base_dir), true, false, None)
 }
 
-/// A sequence separator.
-pub struct SeqSep {
-    /// The separator token.
-    pub sep: Option<TokenKind>,
-    /// `true` if a trailing separator is allowed.
-    pub trailing_sep_allowed: bool,
-}
-
-impl SeqSep {
-    pub fn trailing_allowed(t: TokenKind) -> SeqSep {
-        SeqSep {
-            sep: Some(t),
-            trailing_sep_allowed: true,
-        }
-    }
-
-    pub fn none() -> SeqSep {
-        SeqSep {
-            sep: None,
-            trailing_sep_allowed: false,
-        }
-    }
-}
-
 // NOTE(Centril): The following probably shouldn't be here but it acknowledges the
 // fact that architecturally, we are using parsing (read on below to understand why).
 

src/libsyntax/parse/parser.rs

@@ -1,21 +1,21 @@
+pub mod attr;
 mod expr;
 mod pat;
 mod item;
-pub use item::AliasKind;
 mod module;
-pub use module::{ModulePath, ModulePathSuccess};
 mod ty;
 mod path;
 pub use path::PathStyle;
 mod stmt;
 mod generics;
-use super::diagnostics::Error;
+mod diagnostics;
+use diagnostics::Error;
 
 use crate::ast::{
     self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Ident,
     IsAsync, MacDelimiter, Mutability, StrStyle, Visibility, VisibilityKind, Unsafety,
 };
-use crate::parse::{PResult, Directory, DirectoryOwnership, SeqSep};
+use crate::parse::{PResult, Directory, DirectoryOwnership};
 use crate::parse::lexer::UnmatchedBrace;
 use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
 use crate::parse::token::{self, Token, TokenKind, DelimToken};
@@ -44,14 +44,14 @@ bitflags::bitflags! {
 }
 
 #[derive(Clone, Copy, PartialEq, Debug)]
-crate enum SemiColonMode {
+enum SemiColonMode {
     Break,
     Ignore,
     Comma,
 }
 
 #[derive(Clone, Copy, PartialEq, Debug)]
-crate enum BlockMode {
+enum BlockMode {
     Break,
     Ignore,
 }
@@ -124,33 +124,33 @@ pub struct Parser<'a> {
     prev_token_kind: PrevTokenKind,
     restrictions: Restrictions,
     /// Used to determine the path to externally loaded source files.
-    crate directory: Directory<'a>,
+    pub(super) directory: Directory<'a>,
     /// `true` to parse sub-modules in other files.
-    pub recurse_into_file_modules: bool,
+    pub(super) recurse_into_file_modules: bool,
     /// Name of the root module this parser originated from. If `None`, then the
     /// name is not known. This does not change while the parser is descending
     /// into modules, and sub-parsers have new values for this name.
     pub root_module_name: Option<String>,
-    crate expected_tokens: Vec<TokenType>,
+    expected_tokens: Vec<TokenType>,
     token_cursor: TokenCursor,
     desugar_doc_comments: bool,
     /// `true` we should configure out of line modules as we parse.
-    pub cfg_mods: bool,
+    cfg_mods: bool,
     /// This field is used to keep track of how many left angle brackets we have seen. This is
     /// required in order to detect extra leading left angle brackets (`<` characters) and error
     /// appropriately.
     ///
     /// See the comments in the `parse_path_segment` function for more details.
-    crate unmatched_angle_bracket_count: u32,
+    unmatched_angle_bracket_count: u32,
-    crate max_angle_bracket_count: u32,
+    max_angle_bracket_count: u32,
     /// A list of all unclosed delimiters found by the lexer. If an entry is used for error recovery
     /// it gets removed from here. Every entry left at the end gets emitted as an independent
     /// error.
-    crate unclosed_delims: Vec<UnmatchedBrace>,
+    pub(super) unclosed_delims: Vec<UnmatchedBrace>,
-    crate last_unexpected_token_span: Option<Span>,
+    last_unexpected_token_span: Option<Span>,
-    crate last_type_ascription: Option<(Span, bool /* likely path typo */)>,
+    pub last_type_ascription: Option<(Span, bool /* likely path typo */)>,
     /// If present, this `Parser` is not parsing Rust code but rather a macro call.
-    crate subparser_name: Option<&'static str>,
+    subparser_name: Option<&'static str>,
 }
 
 impl<'a> Drop for Parser<'a> {
@@ -194,7 +194,7 @@ struct TokenCursorFrame {
 /// You can find some more example usage of this in the `collect_tokens` method
 /// on the parser.
 #[derive(Clone)]
-crate enum LastToken {
+enum LastToken {
     Collecting(Vec<TreeAndJoint>),
     Was(Option<TreeAndJoint>),
 }
@@ -297,7 +297,7 @@ impl TokenCursor {
 }
 
 #[derive(Clone, PartialEq)]
-crate enum TokenType {
+enum TokenType {
     Token(TokenKind),
     Keyword(Symbol),
     Operator,
@@ -309,7 +309,7 @@ crate enum TokenType {
 }
 
 impl TokenType {
-    crate fn to_string(&self) -> String {
+    fn to_string(&self) -> String {
         match *self {
             TokenType::Token(ref t) => format!("`{}`", pprust::token_kind_to_string(t)),
             TokenType::Keyword(kw) => format!("`{}`", kw),
@@ -324,11 +324,35 @@ impl TokenType {
 }
 
 #[derive(Copy, Clone, Debug)]
-crate enum TokenExpectType {
+enum TokenExpectType {
     Expect,
     NoExpect,
 }
 
+/// A sequence separator.
+struct SeqSep {
+    /// The separator token.
+    sep: Option<TokenKind>,
+    /// `true` if a trailing separator is allowed.
+    trailing_sep_allowed: bool,
+}
+
+impl SeqSep {
+    fn trailing_allowed(t: TokenKind) -> SeqSep {
+        SeqSep {
+            sep: Some(t),
+            trailing_sep_allowed: true,
+        }
+    }
+
+    fn none() -> SeqSep {
+        SeqSep {
+            sep: None,
+            trailing_sep_allowed: false,
+        }
+    }
+}
+
 impl<'a> Parser<'a> {
     pub fn new(
         sess: &'a ParseSess,
@@ -405,7 +429,7 @@ impl<'a> Parser<'a> {
         pprust::token_to_string(&self.token)
     }
 
-    crate fn token_descr(&self) -> Option<&'static str> {
+    fn token_descr(&self) -> Option<&'static str> {
         Some(match &self.token.kind {
             _ if self.token.is_special_ident() => "reserved identifier",
             _ if self.token.is_used_keyword() => "keyword",
@@ -415,7 +439,7 @@ impl<'a> Parser<'a> {
         })
     }
 
-    crate fn this_token_descr(&self) -> String {
+    pub(super) fn this_token_descr(&self) -> String {
         if let Some(prefix) = self.token_descr() {
             format!("{} `{}`", prefix, self.this_token_to_string())
         } else {
@@ -465,7 +489,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub fn parse_ident(&mut self) -> PResult<'a, ast::Ident> {
+    fn parse_ident(&mut self) -> PResult<'a, ast::Ident> {
         self.parse_ident_common(true)
     }
 
@@ -498,7 +522,7 @@ impl<'a> Parser<'a> {
     ///
     /// This method will automatically add `tok` to `expected_tokens` if `tok` is not
     /// encountered.
-    crate fn check(&mut self, tok: &TokenKind) -> bool {
+    fn check(&mut self, tok: &TokenKind) -> bool {
         let is_present = self.token == *tok;
         if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); }
         is_present
@@ -520,7 +544,7 @@ impl<'a> Parser<'a> {
 
     /// If the next token is the given keyword, eats it and returns `true`.
     /// Otherwise, returns `false`. An expectation is also added for diagnostics purposes.
-    pub fn eat_keyword(&mut self, kw: Symbol) -> bool {
+    fn eat_keyword(&mut self, kw: Symbol) -> bool {
         if self.check_keyword(kw) {
             self.bump();
             true
@@ -558,7 +582,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn check_ident(&mut self) -> bool {
+    fn check_ident(&mut self) -> bool {
         self.check_or_expected(self.token.is_ident(), TokenType::Ident)
     }
 
@@ -723,7 +747,7 @@ impl<'a> Parser<'a> {
     /// Parses a sequence, including the closing delimiter. The function
     /// `f` must consume tokens until reaching the next separator or
     /// closing bracket.
-    pub fn parse_seq_to_end<T>(
+    fn parse_seq_to_end<T>(
         &mut self,
         ket: &TokenKind,
         sep: SeqSep,
@@ -739,7 +763,7 @@ impl<'a> Parser<'a> {
     /// Parses a sequence, not including the closing delimiter. The function
     /// `f` must consume tokens until reaching the next separator or
     /// closing bracket.
-    pub fn parse_seq_to_before_end<T>(
+    fn parse_seq_to_before_end<T>(
         &mut self,
         ket: &TokenKind,
         sep: SeqSep,
@@ -757,7 +781,7 @@ impl<'a> Parser<'a> {
         })
     }
 
-    crate fn parse_seq_to_before_tokens<T>(
+    fn parse_seq_to_before_tokens<T>(
         &mut self,
         kets: &[&TokenKind],
         sep: SeqSep,
@@ -1003,7 +1027,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn process_potential_macro_variable(&mut self) {
+    pub fn process_potential_macro_variable(&mut self) {
         self.token = match self.token.kind {
             token::Dollar if self.token.span.from_expansion() &&
                              self.look_ahead(1, |t| t.is_ident()) => {
@@ -1037,7 +1061,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a single token tree from the input.
-    crate fn parse_token_tree(&mut self) -> TokenTree {
+    pub fn parse_token_tree(&mut self) -> TokenTree {
         match self.token.kind {
             token::OpenDelim(..) => {
                 let frame = mem::replace(&mut self.token_cursor.frame,
@@ -1323,7 +1347,7 @@ impl<'a> Parser<'a> {
                                    *t == token::BinOp(token::Star))
     }
 
-    pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
+    fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
         let ret = match self.token.kind {
             token::Literal(token::Lit { kind: token::Str, symbol, suffix }) =>
                 (symbol, ast::StrStyle::Cooked, suffix),

src/libsyntax/parse/parser/attr.rs

@@ -1,8 +1,7 @@
+use super::{SeqSep, PResult, Parser, TokenType, PathStyle};
 use crate::attr;
 use crate::ast;
-use crate::parse::{SeqSep, PResult};
 use crate::parse::token::{self, Nonterminal, DelimToken};
-use crate::parse::parser::{Parser, TokenType, PathStyle};
 use crate::tokenstream::{TokenStream, TokenTree};
 use crate::source_map::Span;
 
@@ -20,7 +19,7 @@ const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &str = "an inner attribute is not \
 
 impl<'a> Parser<'a> {
     /// Parses attributes that appear before an item.
-    crate fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
+    pub(super) fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> {
         let mut attrs: Vec<ast::Attribute> = Vec::new();
         let mut just_parsed_doc_comment = false;
         loop {
@@ -84,9 +83,10 @@ impl<'a> Parser<'a> {
 
     /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy`
     /// that prescribes how to handle inner attributes.
-    fn parse_attribute_with_inner_parse_policy(&mut self,
+    fn parse_attribute_with_inner_parse_policy(
-                                               inner_parse_policy: InnerAttributeParsePolicy<'_>)
+        &mut self,
-                                               -> PResult<'a, ast::Attribute> {
+        inner_parse_policy: InnerAttributeParsePolicy<'_>
+    ) -> PResult<'a, ast::Attribute> {
         debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}",
                inner_parse_policy,
                self.token);
@@ -260,6 +260,27 @@ impl<'a> Parser<'a> {
         Ok(lit)
     }
 
+    /// Parses `cfg_attr(pred, attr_item_list)` where `attr_item_list` is comma-delimited.
+    crate fn parse_cfg_attr(&mut self) -> PResult<'a, (ast::MetaItem, Vec<(ast::AttrItem, Span)>)> {
+        self.expect(&token::OpenDelim(token::Paren))?;
+
+        let cfg_predicate = self.parse_meta_item()?;
+        self.expect(&token::Comma)?;
+
+        // Presumably, the majority of the time there will only be one attr.
+        let mut expanded_attrs = Vec::with_capacity(1);
+
+        while !self.check(&token::CloseDelim(token::Paren)) {
+            let lo = self.token.span.lo();
+            let item = self.parse_attr_item()?;
+            expanded_attrs.push((item, self.prev_span.with_lo(lo)));
+            self.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Paren)])?;
+        }
+
+        self.expect(&token::CloseDelim(token::Paren))?;
+        Ok((cfg_predicate, expanded_attrs))
+    }
+
     /// Matches the following grammar (per RFC 1559).
     ///
     ///     meta_item : PATH ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ;

src/libsyntax/parse/parser/diagnostics.rs

@@ -1,9 +1,11 @@
+use super::{
+    BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType,
+    SeqSep, PResult, Parser
+};
 use crate::ast::{
     self, Param, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind,
     Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind,
 };
-use crate::parse::{SeqSep, PResult, Parser};
-use crate::parse::parser::{BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType};
 use crate::parse::token::{self, TokenKind};
 use crate::print::pprust;
 use crate::ptr::P;
@@ -17,8 +19,9 @@ use log::{debug, trace};
 use std::mem;
 
 const TURBOFISH: &'static str = "use `::<...>` instead of `<...>` to specify type arguments";
+
 /// Creates a placeholder argument.
-crate fn dummy_arg(ident: Ident) -> Param {
+pub(super) fn dummy_arg(ident: Ident) -> Param {
     let pat = P(Pat {
         id: ast::DUMMY_NODE_ID,
         kind: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None),
@@ -121,7 +124,7 @@ impl Error {
     }
 }
 
-pub trait RecoverQPath: Sized + 'static {
+pub(super) trait RecoverQPath: Sized + 'static {
     const PATH_STYLE: PathStyle = PathStyle::Expr;
     fn to_ty(&self) -> Option<P<Ty>>;
     fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self;
@@ -173,39 +176,43 @@ impl<'a> Parser<'a> {
         self.span_fatal(self.token.span, m)
     }
 
-    pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> {
+    crate fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> {
         self.sess.span_diagnostic.struct_span_fatal(sp, m)
     }
 
-    pub fn span_fatal_err<S: Into<MultiSpan>>(&self, sp: S, err: Error) -> DiagnosticBuilder<'a> {
+    pub(super) fn span_fatal_err<S: Into<MultiSpan>>(
+        &self,
+        sp: S,
+        err: Error,
+    ) -> DiagnosticBuilder<'a> {
         err.span_err(sp, self.diagnostic())
     }
 
-    pub fn bug(&self, m: &str) -> ! {
+    pub(super) fn bug(&self, m: &str) -> ! {
         self.sess.span_diagnostic.span_bug(self.token.span, m)
     }
 
-    pub fn span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) {
+    pub(super) fn span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) {
         self.sess.span_diagnostic.span_err(sp, m)
     }
 
-    crate fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> {
+    pub fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> {
         self.sess.span_diagnostic.struct_span_err(sp, m)
     }
 
-    crate fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! {
+    pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! {
         self.sess.span_diagnostic.span_bug(sp, m)
     }
 
-    crate fn diagnostic(&self) -> &'a errors::Handler {
+    pub(super) fn diagnostic(&self) -> &'a errors::Handler {
         &self.sess.span_diagnostic
     }
 
-    crate fn span_to_snippet(&self, span: Span) -> Result<String, SpanSnippetError> {
+    pub(super) fn span_to_snippet(&self, span: Span) -> Result<String, SpanSnippetError> {
         self.sess.source_map().span_to_snippet(span)
     }
 
-    crate fn expected_ident_found(&self) -> DiagnosticBuilder<'a> {
+    pub(super) fn expected_ident_found(&self) -> DiagnosticBuilder<'a> {
         let mut err = self.struct_span_err(
             self.token.span,
             &format!("expected identifier, found {}", self.this_token_descr()),
@@ -236,7 +243,7 @@ impl<'a> Parser<'a> {
         err
     }
 
-    pub fn expected_one_of_not_found(
+    pub(super) fn expected_one_of_not_found(
         &mut self,
         edible: &[TokenKind],
         inedible: &[TokenKind],
@@ -423,7 +430,7 @@ impl<'a> Parser<'a> {
 
     /// Eats and discards tokens until one of `kets` is encountered. Respects token trees,
     /// passes through any errors encountered. Used for error recovery.
-    crate fn eat_to_tokens(&mut self, kets: &[&TokenKind]) {
+    pub(super) fn eat_to_tokens(&mut self, kets: &[&TokenKind]) {
         if let Err(ref mut err) = self.parse_seq_to_before_tokens(
             kets,
             SeqSep::none(),
@@ -441,7 +448,7 @@ impl<'a> Parser<'a> {
     /// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>();
     ///                                                        ^^ help: remove extra angle brackets
     /// ```
-    crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: TokenKind) {
+    pub(super) fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: TokenKind) {
         // This function is intended to be invoked after parsing a path segment where there are two
         // cases:
         //
@@ -560,7 +567,7 @@ impl<'a> Parser<'a> {
     ///     inner_op   r2
     ///        /  \
     ///     l1    r1
-    crate fn check_no_chained_comparison(
+    pub(super) fn check_no_chained_comparison(
         &mut self,
         lhs: &Expr,
         outer_op: &AssocOp,
@@ -695,7 +702,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn maybe_report_ambiguous_plus(
+    pub(super) fn maybe_report_ambiguous_plus(
         &mut self,
         allow_plus: bool,
         impl_dyn_multi: bool,
@@ -768,7 +775,7 @@ impl<'a> Parser<'a> {
     /// Tries to recover from associated item paths like `[T]::AssocItem` / `(T, U)::AssocItem`.
     /// Attempts to convert the base expression/pattern/type into a type, parses the `::AssocItem`
     /// tail, and combines them into a `<Ty>::AssocItem` expression/pattern/type.
-    crate fn maybe_recover_from_bad_qpath<T: RecoverQPath>(
+    pub(super) fn maybe_recover_from_bad_qpath<T: RecoverQPath>(
         &mut self,
         base: P<T>,
         allow_recovery: bool,
@@ -784,7 +791,7 @@ impl<'a> Parser<'a> {
 
     /// Given an already parsed `Ty`, parses the `::AssocItem` tail and
     /// combines them into a `<Ty>::AssocItem` expression/pattern/type.
-    crate fn maybe_recover_from_bad_qpath_stage_2<T: RecoverQPath>(
+    pub(super) fn maybe_recover_from_bad_qpath_stage_2<T: RecoverQPath>(
         &mut self,
         ty_span: Span,
         ty: P<Ty>,
@@ -823,7 +830,7 @@ impl<'a> Parser<'a> {
         )))
     }
 
-    crate fn maybe_consume_incorrect_semicolon(&mut self, items: &[P<Item>]) -> bool {
+    pub(super) fn maybe_consume_incorrect_semicolon(&mut self, items: &[P<Item>]) -> bool {
         if self.eat(&token::Semi) {
             let mut err = self.struct_span_err(self.prev_span, "expected item, found `;`");
             err.span_suggestion_short(
@@ -859,7 +866,7 @@ impl<'a> Parser<'a> {
 
     /// Creates a `DiagnosticBuilder` for an unexpected token `t` and tries to recover if it is a
     /// closing delimiter.
-    pub fn unexpected_try_recover(
+    pub(super) fn unexpected_try_recover(
         &mut self,
         t: &TokenKind,
     ) -> PResult<'a, bool /* recovered */> {
@@ -909,7 +916,7 @@ impl<'a> Parser<'a> {
         Err(err)
     }
 
-    crate fn parse_semi_or_incorrect_foreign_fn_body(
+    pub(super) fn parse_semi_or_incorrect_foreign_fn_body(
         &mut self,
         ident: &Ident,
         extern_sp: Span,
@@ -947,7 +954,7 @@ impl<'a> Parser<'a> {
 
     /// Consumes alternative await syntaxes like `await!(<expr>)`, `await <expr>`,
     /// `await? <expr>`, `await(<expr>)`, and `await { <expr> }`.
-    crate fn parse_incorrect_await_syntax(
+    pub(super) fn parse_incorrect_await_syntax(
         &mut self,
         lo: Span,
         await_sp: Span,
@@ -999,7 +1006,7 @@ impl<'a> Parser<'a> {
     }
 
     /// If encountering `future.await()`, consumes and emits an error.
-    crate fn recover_from_await_method_call(&mut self) {
+    pub(super) fn recover_from_await_method_call(&mut self) {
         if self.token == token::OpenDelim(token::Paren) &&
             self.look_ahead(1, |t| t == &token::CloseDelim(token::Paren))
         {
@@ -1022,7 +1029,7 @@ impl<'a> Parser<'a> {
     /// and suggest writing `for $pat in $expr` instead.
     ///
     /// This should be called before parsing the `$block`.
-    crate fn recover_parens_around_for_head(
+    pub(super) fn recover_parens_around_for_head(
         &mut self,
         pat: P<Pat>,
         expr: &Expr,
@@ -1060,7 +1067,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn could_ascription_be_path(&self, node: &ast::ExprKind) -> bool {
+    pub(super) fn could_ascription_be_path(&self, node: &ast::ExprKind) -> bool {
         self.token.is_ident() &&
             if let ast::ExprKind::Path(..) = node { true } else { false } &&
             !self.token.is_reserved_ident() &&           // v `foo:bar(baz)`
@@ -1074,7 +1081,7 @@ impl<'a> Parser<'a> {
              self.look_ahead(2, |t| t == &token::Lt))  // `foo:bar::<baz>`
     }
 
-    crate fn recover_seq_parse_error(
+    pub(super) fn recover_seq_parse_error(
         &mut self,
         delim: token::DelimToken,
         lo: Span,
@@ -1091,7 +1098,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn recover_closing_delimiter(
+    pub(super) fn recover_closing_delimiter(
         &mut self,
         tokens: &[TokenKind],
         mut err: DiagnosticBuilder<'a>,
@@ -1142,7 +1149,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Recovers from `pub` keyword in places where it seems _reasonable_ but isn't valid.
-    crate fn eat_bad_pub(&mut self) {
+    pub(super) fn eat_bad_pub(&mut self) {
         if self.token.is_keyword(kw::Pub) {
             match self.parse_visibility(false) {
                 Ok(vis) => {
@@ -1160,7 +1167,7 @@ impl<'a> Parser<'a> {
     /// statement. This is something of a best-effort heuristic.
     ///
     /// We terminate when we find an unmatched `}` (without consuming it).
-    crate fn recover_stmt(&mut self) {
+    pub(super) fn recover_stmt(&mut self) {
         self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore)
     }
 
@@ -1171,7 +1178,11 @@ impl<'a> Parser<'a> {
     ///
     /// If `break_on_block` is `Break`, then we will stop consuming tokens
     /// after finding (and consuming) a brace-delimited block.
-    crate fn recover_stmt_(&mut self, break_on_semi: SemiColonMode, break_on_block: BlockMode) {
+    pub(super) fn recover_stmt_(
+        &mut self,
+        break_on_semi: SemiColonMode,
+        break_on_block: BlockMode,
+    ) {
         let mut brace_depth = 0;
         let mut bracket_depth = 0;
         let mut in_block = false;
@@ -1239,7 +1250,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn check_for_for_in_in_typo(&mut self, in_span: Span) {
+    pub(super) fn check_for_for_in_in_typo(&mut self, in_span: Span) {
         if self.eat_keyword(kw::In) {
             // a common typo: `for _ in in bar {}`
             self.struct_span_err(self.prev_span, "expected iterable, found keyword `in`")
@@ -1253,14 +1264,14 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn expected_semi_or_open_brace<T>(&mut self) -> PResult<'a, T> {
+    pub(super) fn expected_semi_or_open_brace<T>(&mut self) -> PResult<'a, T> {
         let token_str = self.this_token_descr();
         let mut err = self.fatal(&format!("expected `;` or `{{`, found {}", token_str));
         err.span_label(self.token.span, "expected `;` or `{`");
         Err(err)
     }
 
-    crate fn eat_incorrect_doc_comment_for_param_type(&mut self) {
+    pub(super) fn eat_incorrect_doc_comment_for_param_type(&mut self) {
         if let token::DocComment(_) = self.token.kind {
             self.struct_span_err(
                 self.token.span,
@@ -1288,7 +1299,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn parameter_without_type(
+    pub(super) fn parameter_without_type(
         &mut self,
         err: &mut DiagnosticBuilder<'_>,
         pat: P<ast::Pat>,
@@ -1351,7 +1362,7 @@ impl<'a> Parser<'a> {
         None
     }
 
-    crate fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::Ty>)> {
+    pub(super) fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::Ty>)> {
         let pat = self.parse_pat(Some("argument name"))?;
         self.expect(&token::Colon)?;
         let ty = self.parse_ty()?;
@@ -1379,7 +1390,7 @@ impl<'a> Parser<'a> {
         Ok((pat, ty))
     }
 
-    crate fn recover_bad_self_param(
+    pub(super) fn recover_bad_self_param(
         &mut self,
         mut param: ast::Param,
         is_trait_item: bool,
@@ -1397,7 +1408,7 @@ impl<'a> Parser<'a> {
         Ok(param)
     }
 
-    crate fn consume_block(&mut self, delim: token::DelimToken) {
+    pub(super) fn consume_block(&mut self, delim: token::DelimToken) {
         let mut brace_depth = 0;
         loop {
             if self.eat(&token::OpenDelim(delim)) {
@@ -1417,7 +1428,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> {
+    pub(super) fn expected_expression_found(&self) -> DiagnosticBuilder<'a> {
         let (span, msg) = match (&self.token.kind, self.subparser_name) {
             (&token::Eof, Some(origin)) => {
                 let sp = self.sess.source_map().next_point(self.token.span);
@@ -1462,7 +1473,7 @@ impl<'a> Parser<'a> {
     /// the parameters are *names* (so we don't emit errors about not being able to find `b` in
     /// the local scope), but if we find the same name multiple times, like in `fn foo(i8, i8)`,
     /// we deduplicate them to not complain about duplicated parameter names.
-    crate fn deduplicate_recovered_params_names(&self, fn_inputs: &mut Vec<Param>) {
+    pub(super) fn deduplicate_recovered_params_names(&self, fn_inputs: &mut Vec<Param>) {
         let mut seen_inputs = FxHashSet::default();
         for input in fn_inputs.iter_mut() {
             let opt_ident = if let (PatKind::Ident(_, ident, _), TyKind::Err) = (

src/libsyntax/parse/parser/expr.rs

@@ -1,6 +1,7 @@
 use super::{Parser, PResult, Restrictions, PrevTokenKind, TokenType, PathStyle, BlockMode};
 use super::{SemiColonMode, SeqSep, TokenExpectType};
 use super::pat::{GateOr, PARAM_EXPECTED};
+use super::diagnostics::Error;
 
 use crate::parse::literal::LitError;
 
@@ -12,7 +13,6 @@ use crate::ast::{
 use crate::maybe_recover_from_interpolated_ty_qpath;
 use crate::parse::classify;
 use crate::parse::token::{self, Token, TokenKind};
-use crate::parse::diagnostics::Error;
 use crate::print::pprust;
 use crate::ptr::P;
 use crate::source_map::{self, Span};
@@ -1074,7 +1074,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Matches `lit = true | false | token_lit`.
-    crate fn parse_lit(&mut self) -> PResult<'a, Lit> {
+    pub(super) fn parse_lit(&mut self) -> PResult<'a, Lit> {
         let mut recovered = None;
         if self.token == token::Dot {
             // Attempt to recover `.4` as `0.4`.
@@ -1233,7 +1233,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
-    crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
+    pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
         maybe_whole_expr!(self);
 
         let minus_lo = self.token.span;
@@ -1253,7 +1253,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a block or unsafe block.
-    crate fn parse_block_expr(
+    pub(super) fn parse_block_expr(
         &mut self,
         opt_label: Option<Label>,
         lo: Span,
@@ -1558,7 +1558,7 @@ impl<'a> Parser<'a> {
         return Ok(self.mk_expr(lo.to(hi), ExprKind::Match(discriminant, arms), attrs));
     }
 
-    crate fn parse_arm(&mut self) -> PResult<'a, Arm> {
+    pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
         let attrs = self.parse_outer_attributes()?;
         let lo = self.token.span;
         let pat = self.parse_top_pat(GateOr::No)?;
@@ -1666,7 +1666,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses an `async move? {...}` expression.
-    pub fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
+    fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> {
         let span_lo = self.token.span;
         self.expect_keyword(kw::Async)?;
         let capture_clause = self.parse_capture_clause();
@@ -1946,4 +1946,8 @@ impl<'a> Parser<'a> {
     crate fn mk_expr(&self, span: Span, kind: ExprKind, attrs: ThinVec<Attribute>) -> P<Expr> {
         P(Expr { kind, span, attrs, id: DUMMY_NODE_ID })
     }
+
+    pub(super) fn mk_expr_err(&self, span: Span) -> P<Expr> {
+        self.mk_expr(span, ExprKind::Err, ThinVec::new())
+    }
 }

src/libsyntax/parse/parser/generics.rs

@@ -74,7 +74,7 @@ impl<'a> Parser<'a> {
 
     /// Parses a (possibly empty) list of lifetime and type parameters, possibly including
     /// a trailing comma and erroneous trailing attributes.
-    crate fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> {
+    pub(super) fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> {
         let mut params = Vec::new();
         loop {
             let attrs = self.parse_outer_attributes()?;

src/libsyntax/parse/parser/item.rs

@@ -1,4 +1,6 @@
 use super::{Parser, PResult, PathStyle, SemiColonMode, BlockMode};
+use super::diagnostics::{Error, dummy_arg};
+
 use crate::maybe_whole;
 use crate::ptr::P;
 use crate::ast::{self, DUMMY_NODE_ID, Ident, Attribute, AttrStyle, AnonConst, Item, ItemKind};
@@ -7,10 +9,8 @@ use crate::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness}
 use crate::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind};
 use crate::ast::{Ty, TyKind, Generics, GenericBounds, TraitRef, EnumDef, VariantData, StructField};
 use crate::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, MethodSig, SelfKind, Param};
-use crate::ext::base::DummyResult;
 use crate::parse::token;
 use crate::parse::parser::maybe_append;
-use crate::parse::diagnostics::{Error, dummy_arg};
 use crate::tokenstream::{TokenTree, TokenStream};
 use crate::symbol::{kw, sym};
 use crate::source_map::{self, respan, Span};
@@ -23,7 +23,7 @@ use errors::{Applicability, DiagnosticBuilder, DiagnosticId, StashKey};
 
 /// Whether the type alias or associated type is a concrete type or an opaque type.
 #[derive(Debug)]
-pub enum AliasKind {
+pub(super) enum AliasKind {
     /// Just a new name for the same type.
     Weak(P<Ty>),
     /// Only trait impls of the type will be usable, not the actual type itself.
@@ -605,7 +605,7 @@ impl<'a> Parser<'a> {
         let ty_second = if self.token == token::DotDot {
             // We need to report this error after `cfg` expansion for compatibility reasons
             self.bump(); // `..`, do not add it to expected tokens
-            Some(DummyResult::raw_ty(self.prev_span, true))
+            Some(self.mk_ty(self.prev_span, TyKind::Err))
         } else if has_for || self.token.can_begin_type() {
             Some(self.parse_ty()?)
         } else {
@@ -1116,7 +1116,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a foreign item.
-    crate fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> {
+    pub fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> {
         maybe_whole!(self, NtForeignItem, |ni| ni);
 
         let attrs = self.parse_outer_attributes()?;

src/libsyntax/parse/parser/module.rs

@@ -1,24 +1,24 @@
 use super::{Parser, PResult};
 use super::item::ItemInfo;
+use super::diagnostics::Error;
 
 use crate::attr;
 use crate::ast::{self, Ident, Attribute, ItemKind, Mod, Crate};
 use crate::parse::{new_sub_parser_from_file, DirectoryOwnership};
 use crate::parse::token::{self, TokenKind};
-use crate::parse::diagnostics::{Error};
 use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName};
 use crate::symbol::sym;
 
 use std::path::{self, Path, PathBuf};
 
 /// Information about the path to a module.
-pub struct ModulePath {
+pub(super) struct ModulePath {
     name: String,
     path_exists: bool,
     pub result: Result<ModulePathSuccess, Error>,
 }
 
-pub struct ModulePathSuccess {
+pub(super) struct ModulePathSuccess {
     pub path: PathBuf,
     pub directory_ownership: DirectoryOwnership,
     warn: bool,
@@ -39,6 +39,8 @@ impl<'a> Parser<'a> {
     /// Parses a `mod <foo> { ... }` or `mod <foo>;` item.
     pub(super) fn parse_item_mod(&mut self, outer_attrs: &[Attribute]) -> PResult<'a, ItemInfo> {
         let (in_cfg, outer_attrs) = {
+            // FIXME(Centril): This results in a cycle between config and parsing.
+            // Consider using dynamic dispatch via `self.sess` to disentangle the knot.
             let mut strip_unconfigured = crate::config::StripUnconfigured {
                 sess: self.sess,
                 features: None, // Don't perform gated feature checking.
@@ -198,7 +200,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> {
+    pub(super) fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> {
         if let Some(s) = attr::first_attr_value_str_by_name(attrs, sym::path) {
             let s = s.as_str();
 
@@ -215,7 +217,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Returns a path to a module.
-    pub fn default_submod_path(
+    pub(super) fn default_submod_path(
         id: ast::Ident,
         relative: Option<ast::Ident>,
         dir_path: &Path,

src/libsyntax/parse/parser/pat.rs

@@ -22,7 +22,7 @@ const WHILE_PARSING_OR_MSG: &str = "while parsing this or-pattern starting here"
 
 /// Whether or not an or-pattern should be gated when occurring in the current context.
 #[derive(PartialEq)]
-pub enum GateOr { Yes, No }
+pub(super) enum GateOr { Yes, No }
 
 /// Whether or not to recover a `,` when parsing or-patterns.
 #[derive(PartialEq, Copy, Clone)]

src/libsyntax/parse/parser/path.rs

@@ -111,7 +111,7 @@ impl<'a> Parser<'a> {
     /// Like `parse_path`, but also supports parsing `Word` meta items into paths for
     /// backwards-compatibility. This is used when parsing derive macro paths in `#[derive]`
     /// attributes.
-    pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> {
+    fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, Path> {
         let meta_ident = match self.token.kind {
             token::Interpolated(ref nt) => match **nt {
                 token::NtMeta(ref item) => match item.tokens.is_empty() {
@@ -129,7 +129,22 @@ impl<'a> Parser<'a> {
         self.parse_path(style)
     }
 
-    crate fn parse_path_segments(
+    /// Parse a list of paths inside `#[derive(path_0, ..., path_n)]`.
+    crate fn parse_derive_paths(&mut self) -> PResult<'a, Vec<Path>> {
+        self.expect(&token::OpenDelim(token::Paren))?;
+        let mut list = Vec::new();
+        while !self.eat(&token::CloseDelim(token::Paren)) {
+            let path = self.parse_path_allowing_meta(PathStyle::Mod)?;
+            list.push(path);
+            if !self.eat(&token::Comma) {
+                self.expect(&token::CloseDelim(token::Paren))?;
+                break
+            }
+        }
+        Ok(list)
+    }
+
+    pub(super) fn parse_path_segments(
         &mut self,
         segments: &mut Vec<PathSegment>,
         style: PathStyle,

src/libsyntax/parse/parser/stmt.rs

@@ -2,14 +2,13 @@ use super::{Parser, PResult, Restrictions, PrevTokenKind, SemiColonMode, BlockMo
 use super::expr::LhsExpr;
 use super::path::PathStyle;
 use super::pat::GateOr;
+use super::diagnostics::Error;
 
 use crate::ptr::P;
 use crate::{maybe_whole, ThinVec};
 use crate::ast::{self, DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind};
 use crate::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter};
-use crate::ext::base::DummyResult;
 use crate::parse::{classify, DirectoryOwnership};
-use crate::parse::diagnostics::Error;
 use crate::parse::token;
 use crate::source_map::{respan, Span};
 use crate::symbol::{kw, sym};
@@ -373,7 +372,9 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a block. Inner attributes are allowed.
-    crate fn parse_inner_attrs_and_block(&mut self) -> PResult<'a, (Vec<Attribute>, P<Block>)> {
+    pub(super) fn parse_inner_attrs_and_block(
+        &mut self
+    ) -> PResult<'a, (Vec<Attribute>, P<Block>)> {
         maybe_whole!(self, NtBlock, |x| (Vec::new(), x));
 
         let lo = self.token.span;
@@ -400,7 +401,7 @@ impl<'a> Parser<'a> {
                     self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore);
                     Some(Stmt {
                         id: DUMMY_NODE_ID,
-                        kind: StmtKind::Expr(DummyResult::raw_expr(self.token.span, true)),
+                        kind: StmtKind::Expr(self.mk_expr_err(self.token.span)),
                         span: self.token.span,
                     })
                 }
@@ -422,7 +423,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a statement, including the trailing semicolon.
-    crate fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> {
+    pub fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> {
         // Skip looking for a trailing semicolon when we have an interpolated statement.
         maybe_whole!(self, NtStmt, |x| Some(x));
 
@@ -443,7 +444,7 @@ impl<'a> Parser<'a> {
                         self.recover_stmt();
                         // Don't complain about type errors in body tail after parse error (#57383).
                         let sp = expr.span.to(self.prev_span);
-                        stmt.kind = StmtKind::Expr(DummyResult::raw_expr(sp, true));
+                        stmt.kind = StmtKind::Expr(self.mk_expr_err(sp));
                     }
                 }
             }

src/libsyntax/parse/parser/ty.rs

@@ -210,7 +210,7 @@ impl<'a> Parser<'a> {
         };
 
         let span = lo.to(self.prev_span);
-        let ty = P(Ty { kind, span, id: ast::DUMMY_NODE_ID });
+        let ty = self.mk_ty(span, kind);
 
         // Try to recover from use of `+` with incorrect priority.
         self.maybe_report_ambiguous_plus(allow_plus, impl_dyn_multi, &ty);
@@ -296,7 +296,7 @@ impl<'a> Parser<'a> {
         })))
     }
 
-    crate fn parse_generic_bounds(&mut self,
+    pub(super) fn parse_generic_bounds(&mut self,
                                   colon_span: Option<Span>) -> PResult<'a, GenericBounds> {
         self.parse_generic_bounds_common(true, colon_span)
     }
@@ -433,13 +433,13 @@ impl<'a> Parser<'a> {
         }
     }
 
-    crate fn check_lifetime(&mut self) -> bool {
+    pub fn check_lifetime(&mut self) -> bool {
         self.expected_tokens.push(TokenType::Lifetime);
         self.token.is_lifetime()
     }
 
     /// Parses a single lifetime `'a` or panics.
-    crate fn expect_lifetime(&mut self) -> Lifetime {
+    pub fn expect_lifetime(&mut self) -> Lifetime {
         if let Some(ident) = self.token.lifetime() {
             let span = self.token.span;
             self.bump();
@@ -448,4 +448,8 @@ impl<'a> Parser<'a> {
             self.span_bug(self.token.span, "not a lifetime")
         }
     }
+
+    pub(super) fn mk_ty(&self, span: Span, kind: TyKind) -> P<Ty> {
+        P(Ty { kind, span, id: ast::DUMMY_NODE_ID })
+    }
 }

src/libsyntax/parse/token.rs

@@ -285,7 +285,7 @@ impl TokenKind {
 }
 
 impl Token {
-    crate fn new(kind: TokenKind, span: Span) -> Self {
+    pub fn new(kind: TokenKind, span: Span) -> Self {
         Token { kind, span }
     }
 
@@ -295,12 +295,12 @@ impl Token {
     }
 
     /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
-    crate fn from_ast_ident(ident: ast::Ident) -> Self {
+    pub fn from_ast_ident(ident: ast::Ident) -> Self {
         Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
     }
 
     /// Return this token by value and leave a dummy token in its place.
-    crate fn take(&mut self) -> Self {
+    pub fn take(&mut self) -> Self {
         mem::replace(self, Token::dummy())
     }
 
@@ -321,7 +321,7 @@ impl Token {
     }
 
     /// Returns `true` if the token can appear at the start of an expression.
-    crate fn can_begin_expr(&self) -> bool {
+    pub fn can_begin_expr(&self) -> bool {
         match self.kind {
             Ident(name, is_raw)              =>
                 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
@@ -353,7 +353,7 @@ impl Token {
     }
 
     /// Returns `true` if the token can appear at the start of a type.
-    crate fn can_begin_type(&self) -> bool {
+    pub fn can_begin_type(&self) -> bool {
         match self.kind {
             Ident(name, is_raw)        =>
                 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
@@ -396,7 +396,7 @@ impl Token {
     }
 
     /// Returns `true` if the token is any literal
-    crate fn is_lit(&self) -> bool {
+    pub fn is_lit(&self) -> bool {
         match self.kind {
             Literal(..) => true,
             _           => false,
@@ -412,7 +412,7 @@ impl Token {
 
     /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
     /// for example a '-42', or one of the boolean idents).
-    crate fn can_begin_literal_or_bool(&self) -> bool {
+    pub fn can_begin_literal_or_bool(&self) -> bool {
         match self.kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,

src/libsyntax/tests.rs

@@ -1,4 +1,4 @@
-use crate::{ast, panictry};
+use crate::ast;
 use crate::parse::{PResult, source_file_to_stream};
 use crate::parse::new_parser_from_source_str;
 use crate::parse::parser::Parser;
@@ -28,7 +28,7 @@ crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F)
     F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
 {
     let mut p = string_to_parser(&ps, s);
-    let x = panictry!(f(&mut p));
+    let x = f(&mut p).unwrap();
     p.sess.span_diagnostic.abort_if_errors();
     x
 }

src/libsyntax/tokenstream.rs

@@ -156,7 +156,7 @@ use IsJoint::*;
 impl TokenStream {
     /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
     /// separating the two arguments with a comma for diagnostic suggestions.
-    pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> {
+    pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
         // Used to suggest if a user writes `foo!(a b);`
         let mut suggestion = None;
         let mut iter = self.0.iter().enumerate().peekable();

src/libsyntax_expand/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "syntax_expand"
+version = "0.0.0"
+edition = "2018"
+build = false
+
+[lib]
+name = "syntax_expand"
+path = "lib.rs"
+doctest = false
+
+[dependencies]
+bitflags = "1.0"
+rustc_serialize = { path = "../libserialize", package = "serialize" }
+log = "0.4"
+scoped-tls = "1.0"
+lazy_static = "1.0.0"
+syntax_pos = { path = "../libsyntax_pos" }
+errors = { path = "../librustc_errors", package = "rustc_errors" }
+rustc_data_structures = { path = "../librustc_data_structures" }
+rustc_index = { path = "../librustc_index" }
+rustc_lexer = { path = "../librustc_lexer" }
+rustc_target = { path = "../librustc_target" }
+smallvec = { version = "0.6.7", features = ["union", "may_dangle"] }
+syntax = { path = "../libsyntax" }

src/libsyntax_expand/allocator.rs

@@ -1,5 +1,5 @@
-use crate::{ast, attr, visit};
+use syntax::{ast, attr, visit};
-use crate::symbol::{sym, Symbol};
+use syntax::symbol::{sym, Symbol};
 use syntax_pos::Span;
 
 #[derive(Clone, Copy)]

src/libsyntax_expand/base.rs

@@ -1,18 +1,19 @@
-use crate::ast::{self, NodeId, Attribute, Name, PatKind};
+use crate::expand::{self, AstFragment, Invocation};
-use crate::attr::{self, HasAttrs, Stability, Deprecation};
+use crate::hygiene::ExpnId;
-use crate::source_map::SourceMap;
+
-use crate::edition::Edition;
+use syntax::ast::{self, NodeId, Attribute, Name, PatKind};
-use crate::ext::expand::{self, AstFragment, Invocation};
+use syntax::attr::{self, HasAttrs, Stability, Deprecation};
-use crate::ext::hygiene::ExpnId;
+use syntax::source_map::SourceMap;
-use crate::mut_visit::{self, MutVisitor};
+use syntax::edition::Edition;
-use crate::parse::{self, parser, DirectoryOwnership};
+use syntax::mut_visit::{self, MutVisitor};
-use crate::parse::token;
+use syntax::parse::{self, parser, DirectoryOwnership};
-use crate::ptr::P;
+use syntax::parse::token;
-use crate::sess::ParseSess;
+use syntax::ptr::P;
-use crate::symbol::{kw, sym, Ident, Symbol};
+use syntax::sess::ParseSess;
-use crate::{ThinVec, MACRO_ARGUMENTS};
+use syntax::symbol::{kw, sym, Ident, Symbol};
-use crate::tokenstream::{self, TokenStream};
+use syntax::{ThinVec, MACRO_ARGUMENTS};
-use crate::visit::Visitor;
+use syntax::tokenstream::{self, TokenStream};
+use syntax::visit::Visitor;
 
 use errors::{DiagnosticBuilder, DiagnosticId};
 use smallvec::{smallvec, SmallVec};

src/libsyntax_expand/build.rs

@@ -1,10 +1,11 @@
-use crate::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind};
+use crate::base::ExtCtxt;
-use crate::attr;
+
-use crate::source_map::{respan, Spanned};
+use syntax::ast::{self, Ident, Expr, BlockCheckMode, UnOp, PatKind};
-use crate::ext::base::ExtCtxt;
+use syntax::attr;
-use crate::ptr::P;
+use syntax::source_map::{respan, Spanned};
-use crate::symbol::{kw, sym, Symbol};
+use syntax::ptr::P;
-use crate::ThinVec;
+use syntax::symbol::{kw, sym, Symbol};
+use syntax::ThinVec;
 
 use syntax_pos::{Pos, Span};
 

src/libsyntax_expand/expand.rs

@@ -1,24 +1,26 @@
-use crate::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
+use crate::base::*;
-use crate::ast::{MacStmtStyle, StmtKind, ItemKind};
+use crate::proc_macro::{collect_derives, MarkAttrs};
-use crate::attr::{self, HasAttrs};
+use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
-use crate::source_map::respan;
+use crate::mbe::macro_rules::annotate_err_with_kind;
-use crate::config::StripUnconfigured;
+use crate::placeholders::{placeholder, PlaceholderExpander};
-use crate::ext::base::*;
+
-use crate::ext::proc_macro::{collect_derives, MarkAttrs};
+use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path};
-use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
+use syntax::ast::{MacStmtStyle, StmtKind, ItemKind};
-use crate::ext::mbe::macro_rules::annotate_err_with_kind;
+use syntax::attr::{self, HasAttrs};
-use crate::ext::placeholders::{placeholder, PlaceholderExpander};
+use syntax::source_map::respan;
-use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
+use syntax::configure;
-use crate::mut_visit::*;
+use syntax::config::StripUnconfigured;
-use crate::parse::{DirectoryOwnership, PResult};
+use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
-use crate::parse::token;
+use syntax::mut_visit::*;
-use crate::parse::parser::Parser;
+use syntax::parse::{DirectoryOwnership, PResult};
-use crate::print::pprust;
+use syntax::parse::token;
-use crate::ptr::P;
+use syntax::parse::parser::Parser;
-use crate::symbol::{sym, Symbol};
+use syntax::print::pprust;
-use crate::tokenstream::{TokenStream, TokenTree};
+use syntax::ptr::P;
-use crate::visit::Visitor;
+use syntax::symbol::{sym, Symbol};
-use crate::util::map_in_place::MapInPlace;
+use syntax::tokenstream::{TokenStream, TokenTree};
+use syntax::visit::Visitor;
+use syntax::util::map_in_place::MapInPlace;
 
 use errors::{Applicability, FatalError};
 use smallvec::{smallvec, SmallVec};
@@ -116,8 +118,8 @@ macro_rules! ast_fragments {
             }
         }
 
-        impl<'a> MacResult for crate::ext::mbe::macro_rules::ParserAnyMacro<'a> {
+        impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> {
-            $(fn $make_ast(self: Box<crate::ext::mbe::macro_rules::ParserAnyMacro<'a>>)
+            $(fn $make_ast(self: Box<crate::mbe::macro_rules::ParserAnyMacro<'a>>)
                            -> Option<$AstTy> {
                 Some(self.make(AstFragmentKind::$Kind).$make_ast())
             })*
@@ -752,9 +754,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         span: Span,
     ) -> AstFragment {
         let mut parser = self.cx.new_parser_from_tts(toks);
-        match parser.parse_ast_fragment(kind, false) {
+        match parse_ast_fragment(&mut parser, kind, false) {
             Ok(fragment) => {
-                parser.ensure_complete_parse(path, kind.name(), span);
+                ensure_complete_parse(&mut parser, path, kind.name(), span);
                 fragment
             }
             Err(mut err) => {
@@ -768,100 +770,106 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     }
 }
 
-impl<'a> Parser<'a> {
+pub fn parse_ast_fragment<'a>(
-    pub fn parse_ast_fragment(&mut self, kind: AstFragmentKind, macro_legacy_warnings: bool)
+    this: &mut Parser<'a>,
-                              -> PResult<'a, AstFragment> {
+    kind: AstFragmentKind,
-        Ok(match kind {
+    macro_legacy_warnings: bool,
-            AstFragmentKind::Items => {
+) -> PResult<'a, AstFragment> {
-                let mut items = SmallVec::new();
+    Ok(match kind {
-                while let Some(item) = self.parse_item()? {
+        AstFragmentKind::Items => {
-                    items.push(item);
+            let mut items = SmallVec::new();
-                }
+            while let Some(item) = this.parse_item()? {
-                AstFragment::Items(items)
+                items.push(item);
             }
-            AstFragmentKind::TraitItems => {
+            AstFragment::Items(items)
-                let mut items = SmallVec::new();
-                while self.token != token::Eof {
-                    items.push(self.parse_trait_item(&mut false)?);
-                }
-                AstFragment::TraitItems(items)
-            }
-            AstFragmentKind::ImplItems => {
-                let mut items = SmallVec::new();
-                while self.token != token::Eof {
-                    items.push(self.parse_impl_item(&mut false)?);
-                }
-                AstFragment::ImplItems(items)
-            }
-            AstFragmentKind::ForeignItems => {
-                let mut items = SmallVec::new();
-                while self.token != token::Eof {
-                    items.push(self.parse_foreign_item(DUMMY_SP)?);
-                }
-                AstFragment::ForeignItems(items)
-            }
-            AstFragmentKind::Stmts => {
-                let mut stmts = SmallVec::new();
-                while self.token != token::Eof &&
-                      // won't make progress on a `}`
-                      self.token != token::CloseDelim(token::Brace) {
-                    if let Some(stmt) = self.parse_full_stmt(macro_legacy_warnings)? {
-                        stmts.push(stmt);
-                    }
-                }
-                AstFragment::Stmts(stmts)
-            }
-            AstFragmentKind::Expr => AstFragment::Expr(self.parse_expr()?),
-            AstFragmentKind::OptExpr => {
-                if self.token != token::Eof {
-                    AstFragment::OptExpr(Some(self.parse_expr()?))
-                } else {
-                    AstFragment::OptExpr(None)
-                }
-            },
-            AstFragmentKind::Ty => AstFragment::Ty(self.parse_ty()?),
-            AstFragmentKind::Pat => AstFragment::Pat(self.parse_pat(None)?),
-            AstFragmentKind::Arms
-            | AstFragmentKind::Fields
-            | AstFragmentKind::FieldPats
-            | AstFragmentKind::GenericParams
-            | AstFragmentKind::Params
-            | AstFragmentKind::StructFields
-            | AstFragmentKind::Variants
-                => panic!("unexpected AST fragment kind"),
-        })
-    }
-
-    pub fn ensure_complete_parse(&mut self, macro_path: &Path, kind_name: &str, span: Span) {
-        if self.token != token::Eof {
-            let msg = format!("macro expansion ignores token `{}` and any following",
-                              self.this_token_to_string());
-            // Avoid emitting backtrace info twice.
-            let def_site_span = self.token.span.with_ctxt(SyntaxContext::root());
-            let mut err = self.diagnostic().struct_span_err(def_site_span, &msg);
-            err.span_label(span, "caused by the macro expansion here");
-            let msg = format!(
-                "the usage of `{}!` is likely invalid in {} context",
-                pprust::path_to_string(&macro_path),
-                kind_name,
-            );
-            err.note(&msg);
-            let semi_span = self.sess.source_map().next_point(span);
-
-            let semi_full_span = semi_span.to(self.sess.source_map().next_point(semi_span));
-            match self.sess.source_map().span_to_snippet(semi_full_span) {
-                Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => {
-                    err.span_suggestion(
-                        semi_span,
-                        "you might be missing a semicolon here",
-                        ";".to_owned(),
-                        Applicability::MaybeIncorrect,
-                    );
-                }
-                _ => {}
-            }
-            err.emit();
         }
+        AstFragmentKind::TraitItems => {
+            let mut items = SmallVec::new();
+            while this.token != token::Eof {
+                items.push(this.parse_trait_item(&mut false)?);
+            }
+            AstFragment::TraitItems(items)
+        }
+        AstFragmentKind::ImplItems => {
+            let mut items = SmallVec::new();
+            while this.token != token::Eof {
+                items.push(this.parse_impl_item(&mut false)?);
+            }
+            AstFragment::ImplItems(items)
+        }
+        AstFragmentKind::ForeignItems => {
+            let mut items = SmallVec::new();
+            while this.token != token::Eof {
+                items.push(this.parse_foreign_item(DUMMY_SP)?);
+            }
+            AstFragment::ForeignItems(items)
+        }
+        AstFragmentKind::Stmts => {
+            let mut stmts = SmallVec::new();
+            while this.token != token::Eof &&
+                    // won't make progress on a `}`
+                    this.token != token::CloseDelim(token::Brace) {
+                if let Some(stmt) = this.parse_full_stmt(macro_legacy_warnings)? {
+                    stmts.push(stmt);
+                }
+            }
+            AstFragment::Stmts(stmts)
+        }
+        AstFragmentKind::Expr => AstFragment::Expr(this.parse_expr()?),
+        AstFragmentKind::OptExpr => {
+            if this.token != token::Eof {
+                AstFragment::OptExpr(Some(this.parse_expr()?))
+            } else {
+                AstFragment::OptExpr(None)
+            }
+        },
+        AstFragmentKind::Ty => AstFragment::Ty(this.parse_ty()?),
+        AstFragmentKind::Pat => AstFragment::Pat(this.parse_pat(None)?),
+        AstFragmentKind::Arms
+        | AstFragmentKind::Fields
+        | AstFragmentKind::FieldPats
+        | AstFragmentKind::GenericParams
+        | AstFragmentKind::Params
+        | AstFragmentKind::StructFields
+        | AstFragmentKind::Variants
+            => panic!("unexpected AST fragment kind"),
+    })
+}
+
+pub fn ensure_complete_parse<'a>(
+    this: &mut Parser<'a>,
+    macro_path: &Path,
+    kind_name: &str,
+    span: Span,
+) {
+    if this.token != token::Eof {
+        let msg = format!("macro expansion ignores token `{}` and any following",
+                            this.this_token_to_string());
+        // Avoid emitting backtrace info twice.
+        let def_site_span = this.token.span.with_ctxt(SyntaxContext::root());
+        let mut err = this.struct_span_err(def_site_span, &msg);
+        err.span_label(span, "caused by the macro expansion here");
+        let msg = format!(
+            "the usage of `{}!` is likely invalid in {} context",
+            pprust::path_to_string(macro_path),
+            kind_name,
+        );
+        err.note(&msg);
+        let semi_span = this.sess.source_map().next_point(span);
+
+        let semi_full_span = semi_span.to(this.sess.source_map().next_point(semi_span));
+        match this.sess.source_map().span_to_snippet(semi_full_span) {
+            Ok(ref snippet) if &snippet[..] != ";" && kind_name == "expression" => {
+                err.span_suggestion(
+                    semi_span,
+                    "you might be missing a semicolon here",
+                    ";".to_owned(),
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            _ => {}
+        }
+        err.emit();
     }
 }
 

src/libsyntax_expand/lib.rs

@@ -0,0 +1,38 @@
+#![feature(crate_visibility_modifier)]
+#![feature(proc_macro_diagnostic)]
+#![feature(proc_macro_internals)]
+#![feature(proc_macro_span)]
+
+extern crate proc_macro as pm;
+
+// A variant of 'try!' that panics on an Err. This is used as a crutch on the
+// way towards a non-panic!-prone parser. It should be used for fatal parsing
+// errors; eventually we plan to convert all code using panictry to just use
+// normal try.
+#[macro_export]
+macro_rules! panictry {
+    ($e:expr) => ({
+        use std::result::Result::{Ok, Err};
+        use errors::FatalError;
+        match $e {
+            Ok(e) => e,
+            Err(mut e) => {
+                e.emit();
+                FatalError.raise()
+            }
+        }
+    })
+}
+
+mod placeholders;
+mod proc_macro_server;
+
+pub use syntax_pos::hygiene;
+pub use mbe::macro_rules::compile_declarative_macro;
+pub mod allocator;
+pub mod base;
+pub mod build;
+pub mod expand;
+pub mod proc_macro;
+
+crate mod mbe;

src/libsyntax_expand/mbe.rs

@@ -9,9 +9,9 @@ crate mod macro_parser;
 crate mod macro_rules;
 crate mod quoted;
 
-use crate::ast;
+use syntax::ast;
-use crate::parse::token::{self, Token, TokenKind};
+use syntax::parse::token::{self, Token, TokenKind};
-use crate::tokenstream::{DelimSpan};
+use syntax::tokenstream::{DelimSpan};
 
 use syntax_pos::{BytePos, Span};
 

src/libsyntax_expand/mbe/macro_check.rs

@@ -104,13 +104,13 @@
 //! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks
 //! stored when entering a macro definition starting from the state in which the meta-variable is
 //! bound.
-use crate::ast::NodeId;
+use crate::mbe::{KleeneToken, TokenTree};
-use crate::early_buffered_lints::BufferedEarlyLintId;
+
-use crate::ext::mbe::{KleeneToken, TokenTree};
+use syntax::ast::NodeId;
-use crate::parse::token::TokenKind;
+use syntax::early_buffered_lints::BufferedEarlyLintId;
-use crate::parse::token::{DelimToken, Token};
+use syntax::parse::token::{DelimToken, Token, TokenKind};
-use crate::sess::ParseSess;
+use syntax::sess::ParseSess;
-use crate::symbol::{kw, sym};
+use syntax::symbol::{kw, sym};
 
 use rustc_data_structures::fx::FxHashMap;
 use smallvec::SmallVec;

src/libsyntax_expand/mbe/macro_parser.rs

@@ -74,15 +74,16 @@ crate use NamedMatch::*;
 crate use ParseResult::*;
 use TokenTreeOrTokenTreeSlice::*;
 
-use crate::ast::{Ident, Name};
+use crate::mbe::{self, TokenTree};
-use crate::ext::mbe::{self, TokenTree};
+
-use crate::parse::{Directory, PResult};
+use syntax::ast::{Ident, Name};
-use crate::parse::parser::{Parser, PathStyle};
+use syntax::parse::{Directory, PResult};
-use crate::parse::token::{self, DocComment, Nonterminal, Token};
+use syntax::parse::parser::{Parser, PathStyle};
-use crate::print::pprust;
+use syntax::parse::token::{self, DocComment, Nonterminal, Token};
-use crate::sess::ParseSess;
+use syntax::print::pprust;
-use crate::symbol::{kw, sym, Symbol};
+use syntax::sess::ParseSess;
-use crate::tokenstream::{DelimSpan, TokenStream};
+use syntax::symbol::{kw, sym, Symbol};
+use syntax::tokenstream::{DelimSpan, TokenStream};
 
 use errors::FatalError;
 use smallvec::{smallvec, SmallVec};
@@ -651,7 +652,7 @@ pub(super) fn parse(
         directory,
         recurse_into_modules,
         true,
-        crate::MACRO_ARGUMENTS,
+        syntax::MACRO_ARGUMENTS,
     );
 
     // A queue of possible matcher positions. We initialize it with the matcher position in which
@@ -889,6 +890,9 @@ fn may_begin_with(token: &Token, name: Name) -> bool {
 ///
 /// The parsed non-terminal.
 fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal {
+    // FIXME(Centril): Consider moving this to `parser.rs` to make
+    // the visibilities of the methods used below `pub(super)` at most.
+
     if name == sym::tt {
         return token::NtTT(p.parse_token_tree());
     }

src/libsyntax_expand/mbe/macro_rules.rs

@@ -1,24 +1,25 @@
-use crate::ast;
+use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander};
-use crate::attr::{self, TransparencyError};
+use crate::base::{SyntaxExtension, SyntaxExtensionKind};
-use crate::edition::Edition;
+use crate::expand::{AstFragment, AstFragmentKind, ensure_complete_parse, parse_ast_fragment};
-use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander};
+use crate::mbe;
-use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind};
+use crate::mbe::macro_check;
-use crate::ext::expand::{AstFragment, AstFragmentKind};
+use crate::mbe::macro_parser::parse;
-use crate::ext::mbe;
+use crate::mbe::macro_parser::{Error, Failure, Success};
-use crate::ext::mbe::macro_check;
+use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult};
-use crate::ext::mbe::macro_parser::parse;
+use crate::mbe::transcribe::transcribe;
-use crate::ext::mbe::macro_parser::{Error, Failure, Success};
+
-use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult};
+use syntax::ast;
-use crate::ext::mbe::transcribe::transcribe;
+use syntax::attr::{self, TransparencyError};
-use crate::feature_gate::Features;
+use syntax::edition::Edition;
-use crate::parse::parser::Parser;
+use syntax::feature_gate::Features;
-use crate::parse::token::TokenKind::*;
+use syntax::parse::parser::Parser;
-use crate::parse::token::{self, NtTT, Token};
+use syntax::parse::token::TokenKind::*;
-use crate::parse::Directory;
+use syntax::parse::token::{self, NtTT, Token};
-use crate::print::pprust;
+use syntax::parse::Directory;
-use crate::sess::ParseSess;
+use syntax::print::pprust;
-use crate::symbol::{kw, sym, Symbol};
+use syntax::sess::ParseSess;
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
+use syntax::symbol::{kw, sym, Symbol};
+use syntax::tokenstream::{DelimSpan, TokenStream};
 
 use errors::{DiagnosticBuilder, FatalError};
 use log::debug;
@@ -66,7 +67,7 @@ crate fn annotate_err_with_kind(
 impl<'a> ParserAnyMacro<'a> {
     crate fn make(mut self: Box<ParserAnyMacro<'a>>, kind: AstFragmentKind) -> AstFragment {
         let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self;
-        let fragment = panictry!(parser.parse_ast_fragment(kind, true).map_err(|mut e| {
+        let fragment = panictry!(parse_ast_fragment(parser, kind, true).map_err(|mut e| {
             if parser.token == token::Eof && e.message().ends_with(", found `<eof>`") {
                 if !e.span.is_dummy() {
                     // early end of macro arm (#52866)
@@ -128,7 +129,7 @@ impl<'a> ParserAnyMacro<'a> {
 
         // Make sure we don't have any tokens left to parse so we don't silently drop anything.
         let path = ast::Path::from_ident(macro_ident.with_span_pos(site_span));
-        parser.ensure_complete_parse(&path, kind.name(), site_span);
+        ensure_complete_parse(parser, &path, kind.name(), site_span);
         fragment
     }
 }
@@ -189,7 +190,7 @@ fn generic_extension<'cx>(
             _ => cx.span_bug(sp, "malformed macro lhs"),
         };
 
-        match TokenTree::parse(cx, lhs_tt, arg.clone()) {
+        match parse_tt(cx, lhs_tt, arg.clone()) {
             Success(named_matches) => {
                 let rhs = match rhses[i] {
                     // ignore delimiters
@@ -265,7 +266,7 @@ fn generic_extension<'cx>(
                 mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..],
                 _ => continue,
             };
-            match TokenTree::parse(cx, lhs_tt, arg.clone()) {
+            match parse_tt(cx, lhs_tt, arg.clone()) {
                 Success(_) => {
                     if comma_span.is_dummy() {
                         err.note("you might be missing a comma");
@@ -1158,7 +1159,7 @@ fn is_legal_fragment_specifier(
 
 fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     match *tt {
-        mbe::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token),
+        mbe::TokenTree::Token(ref token) => pprust::token_to_string(&token),
         mbe::TokenTree::MetaVar(_, name) => format!("${}", name),
         mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind),
         _ => panic!(
@@ -1168,17 +1169,14 @@ fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     }
 }
 
-impl TokenTree {
+/// Use this token tree as a matcher to parse given tts.
-    /// Use this token tree as a matcher to parse given tts.
+fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult {
-    fn parse(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream)
+    // `None` is because we're not interpolating
-             -> NamedParseResult {
+    let directory = Directory {
-        // `None` is because we're not interpolating
+        path: Cow::from(cx.current_expansion.module.directory.as_path()),
-        let directory = Directory {
+        ownership: cx.current_expansion.directory_ownership,
-            path: Cow::from(cx.current_expansion.module.directory.as_path()),
+    };
-            ownership: cx.current_expansion.directory_ownership,
+    parse(cx.parse_sess(), tts, mtch, Some(directory), true)
-        };
-        parse(cx.parse_sess(), tts, mtch, Some(directory), true)
-    }
 }
 
 /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For

src/libsyntax_expand/mbe/quoted.rs

@@ -1,11 +1,12 @@
-use crate::ast;
+use crate::mbe::macro_parser;
-use crate::ext::mbe::macro_parser;
+use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited};
-use crate::ext::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited};
+
-use crate::parse::token::{self, Token};
+use syntax::ast;
-use crate::print::pprust;
+use syntax::parse::token::{self, Token};
-use crate::sess::ParseSess;
+use syntax::print::pprust;
-use crate::symbol::kw;
+use syntax::sess::ParseSess;
-use crate::tokenstream;
+use syntax::symbol::kw;
+use syntax::tokenstream;
 
 use syntax_pos::Span;
 

src/libsyntax_expand/mbe/transcribe.rs

@@ -1,10 +1,11 @@
-use crate::ast::{Ident, Mac};
+use crate::base::ExtCtxt;
-use crate::ext::base::ExtCtxt;
+use crate::mbe;
-use crate::ext::mbe;
+use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
-use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
+
-use crate::mut_visit::{self, MutVisitor};
+use syntax::ast::{Ident, Mac};
-use crate::parse::token::{self, NtTT, Token};
+use syntax::mut_visit::{self, MutVisitor};
-use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
+use syntax::parse::token::{self, NtTT, Token};
+use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
 
 use smallvec::{smallvec, SmallVec};
 

src/libsyntax_expand/placeholders.rs

@@ -1,11 +1,12 @@
-use crate::ast::{self, NodeId};
+use crate::base::ExtCtxt;
-use crate::source_map::{DUMMY_SP, dummy_spanned};
+use crate::expand::{AstFragment, AstFragmentKind};
-use crate::ext::base::ExtCtxt;
+
-use crate::ext::expand::{AstFragment, AstFragmentKind};
+use syntax::ast::{self, NodeId};
-use crate::tokenstream::TokenStream;
+use syntax::source_map::{DUMMY_SP, dummy_spanned};
-use crate::mut_visit::*;
+use syntax::tokenstream::TokenStream;
-use crate::ptr::P;
+use syntax::mut_visit::*;
-use crate::ThinVec;
+use syntax::ptr::P;
+use syntax::ThinVec;
 
 use smallvec::{smallvec, SmallVec};
 

src/libsyntax_expand/proc_macro.rs

@@ -1,23 +1,22 @@
-use crate::ast::{self, ItemKind, Attribute, Mac};
+use crate::base::{self, *};
-use crate::attr::{mark_used, mark_known};
+use crate::proc_macro_server;
-use crate::errors::{Applicability, FatalError};
+
-use crate::ext::base::{self, *};
+use syntax::ast::{self, ItemKind, Attribute, Mac};
-use crate::ext::proc_macro_server;
+use syntax::attr::{mark_used, mark_known};
-use crate::parse::{self, token};
+use syntax::errors::{Applicability, FatalError};
-use crate::parse::parser::PathStyle;
+use syntax::parse::{self, token};
-use crate::symbol::sym;
+use syntax::symbol::sym;
-use crate::tokenstream::{self, TokenStream};
+use syntax::tokenstream::{self, TokenStream};
-use crate::visit::Visitor;
+use syntax::visit::Visitor;
 
 use rustc_data_structures::sync::Lrc;
 use syntax_pos::{Span, DUMMY_SP};
 
-const EXEC_STRATEGY: proc_macro::bridge::server::SameThread =
+const EXEC_STRATEGY: pm::bridge::server::SameThread = pm::bridge::server::SameThread;
-    proc_macro::bridge::server::SameThread;
 
 pub struct BangProcMacro {
-    pub client: proc_macro::bridge::client::Client<
+    pub client: pm::bridge::client::Client<
-        fn(proc_macro::TokenStream) -> proc_macro::TokenStream,
+        fn(pm::TokenStream) -> pm::TokenStream,
     >,
 }
 
@@ -45,9 +44,7 @@ impl base::ProcMacro for BangProcMacro {
 }
 
 pub struct AttrProcMacro {
-    pub client: proc_macro::bridge::client::Client<
+    pub client: pm::bridge::client::Client<fn(pm::TokenStream, pm::TokenStream) -> pm::TokenStream>,
-        fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream,
-    >,
 }
 
 impl base::AttrProcMacro for AttrProcMacro {
@@ -75,9 +72,7 @@ impl base::AttrProcMacro for AttrProcMacro {
 }
 
 pub struct ProcMacroDerive {
-    pub client: proc_macro::bridge::client::Client<
+    pub client: pm::bridge::client::Client<fn(pm::TokenStream) -> pm::TokenStream>,
-        fn(proc_macro::TokenStream) -> proc_macro::TokenStream,
-    >,
 }
 
 impl MultiItemModifier for ProcMacroDerive {
@@ -205,8 +200,7 @@ crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>)
             return false;
         }
 
-        match attr.parse_list(cx.parse_sess,
+        match attr.parse_derive_paths(cx.parse_sess) {
-                              |parser| parser.parse_path_allowing_meta(PathStyle::Mod)) {
             Ok(traits) => {
                 result.extend(traits);
                 true

src/libsyntax_expand/proc_macro_server.rs

@@ -1,18 +1,19 @@
-use crate::ast;
+use crate::base::ExtCtxt;
-use crate::ext::base::ExtCtxt;
+
-use crate::parse::{self, token};
+use syntax::ast;
-use crate::parse::lexer::comments;
+use syntax::parse::{self, token};
-use crate::print::pprust;
+use syntax::parse::lexer::comments;
-use crate::sess::ParseSess;
+use syntax::print::pprust;
-use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
+use syntax::sess::ParseSess;
+use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
 
 use errors::Diagnostic;
 use rustc_data_structures::sync::Lrc;
 use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
 use syntax_pos::symbol::{kw, sym, Symbol};
 
-use proc_macro::{Delimiter, Level, LineColumn, Spacing};
+use pm::{Delimiter, Level, LineColumn, Spacing};
-use proc_macro::bridge::{server, TokenTree};
+use pm::bridge::{server, TokenTree};
 use std::{ascii, panic};
 use std::ops::Bound;
 
@@ -51,7 +52,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
 {
     fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>))
                     -> Self {
-        use crate::parse::token::*;
+        use syntax::parse::token::*;
 
         let joint = is_joint == Joint;
         let Token { kind, span } = match tree {
@@ -192,7 +193,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
 
 impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
     fn to_internal(self) -> TokenStream {
-        use crate::parse::token::*;
+        use syntax::parse::token::*;
 
         let (ch, joint, span) = match self {
             TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),

src/libsyntax_ext/Cargo.toml

@@ -17,4 +17,5 @@ rustc_data_structures = { path = "../librustc_data_structures" }
 rustc_target = { path = "../librustc_target" }
 smallvec = { version = "0.6.7", features = ["union", "may_dangle"] }
 syntax = { path = "../libsyntax" }
+syntax_expand = { path = "../libsyntax_expand" }
 syntax_pos = { path = "../libsyntax_pos" }

src/libsyntax_ext/asm.rs

@@ -7,7 +7,7 @@ use rustc_data_structures::thin_vec::ThinVec;
 use errors::DiagnosticBuilder;
 
 use syntax::ast;
-use syntax::ext::base::{self, *};
+use syntax_expand::base::{self, *};
 use syntax::parse::token::{self, Token};
 use syntax::ptr::P;
 use syntax::symbol::{kw, sym, Symbol};

src/libsyntax_ext/assert.rs

@@ -1,7 +1,7 @@
 use errors::{Applicability, DiagnosticBuilder};
 
 use syntax::ast::{self, *};
-use syntax::ext::base::*;
+use syntax_expand::base::*;
 use syntax::parse::token::{self, TokenKind};
 use syntax::parse::parser::Parser;
 use syntax::print::pprust;

src/libsyntax_ext/cfg.rs

@@ -5,7 +5,7 @@
 use errors::DiagnosticBuilder;
 
 use syntax::ast;
-use syntax::ext::base::{self, *};
+use syntax_expand::base::{self, *};
 use syntax::attr;
 use syntax::tokenstream::TokenStream;
 use syntax::parse::token;

src/libsyntax_ext/cmdline_attrs.rs

@@ -2,9 +2,9 @@
 
 use syntax::ast::{self, AttrItem, AttrStyle};
 use syntax::attr::mk_attr;
-use syntax::panictry;
 use syntax::parse::{self, token};
 use syntax::sess::ParseSess;
+use syntax_expand::panictry;
 use syntax_pos::FileName;
 
 pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate {

src/libsyntax_ext/compile_error.rs

@@ -1,6 +1,6 @@
 // The compiler code necessary to support the compile_error! extension.
 
-use syntax::ext::base::{self, *};
+use syntax_expand::base::{self, *};
 use syntax_pos::Span;
 use syntax::tokenstream::TokenStream;
 

src/libsyntax_ext/concat.rs

@@ -1,5 +1,5 @@
 use syntax::ast;
-use syntax::ext::base::{self, DummyResult};
+use syntax_expand::base::{self, DummyResult};
 use syntax::symbol::Symbol;
 use syntax::tokenstream::TokenStream;