explain the magic

crates/ra_mbe/src/lib.rs

@@ -109,6 +109,13 @@ mod tests {
 
     use super::*;
 
+    // Good first issue (although a slightly chellegning one):
+    //
+    // * Pick a random test from here
+    //   https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
+    // * Port the test to rust and add it to this module
+    // * Make it pass :-)
+
     #[test]
     fn test_convert_tt() {
         let macro_definition = r#"

crates/ra_mbe/src/mbe_expander.rs

@@ -1,9 +1,12 @@
+/// This module takes a (parsed) defenition of `macro_rules` invocation, a
+/// `tt::TokenTree` representing an argument of macro invocation, and produces a
+/// `tt::TokenTree` for the result of the expansion.
 use rustc_hash::FxHashMap;
 use ra_syntax::SmolStr;
 
 use crate::tt_cursor::TtCursor;
 
-pub fn exapnd(rules: &crate::MacroRules, input: &tt::Subtree) -> Option<tt::Subtree> {
+pub(crate) fn exapnd(rules: &crate::MacroRules, input: &tt::Subtree) -> Option<tt::Subtree> {
     rules.rules.iter().find_map(|it| expand_rule(it, input))
 }
 
@@ -13,6 +16,51 @@ fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Option<tt::Subtree> {
     expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
 }
 
+/// The actual algorithm for expansion is not too hard, but is pretty tricky.
+/// `Bindings` structure is the key to understanding what we are doing here.
+///
+/// On the high level, it stores mapping from meta variables to the bits of
+/// syntax it should be substituted with. For example, if `$e:expr` is matched
+/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
+///
+/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
+///
+/// ```ignore
+/// macro_rules! foo {
+///     ($($ i:ident $($ e:expr),*);*) => {
+///         $(fn $ i() { $($ e);*; })*
+///     }
+/// }
+/// foo! { foo 1,2,3; bar 4,5,6 }
+/// ```
+///
+/// Here, the `$i` meta variable is matched first with `foo` and then with
+/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
+///
+/// To represent such "multi-mappings", we use a recursive structures: we map
+/// variables not to values, but to *lists* of values or other lists (that is,
+/// to the trees).
+///
+/// For the above example, the bindings would store
+///
+/// ```ignore
+/// i -> [foo, bar]
+/// e -> [[1, 2, 3], [4, 5, 6]]
+/// ```
+///
+/// We construct `Bindings` in the `match_lhs`. The interesting case is
+/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
+/// nesting structure.
+///
+/// The other side of the puzzle is `expand_subtree`, where we use the bindings
+/// to substitute meta variables in the output template. When expanding, we
+/// maintain a `nesteing` stack of indicies whihc tells us which occurence from
+/// the `Bindings` we should take. We push to the stack when we enter a
+/// repetition.
+///
+/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
+/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
+/// many is not a plain `usize`, but an `&[usize]`.
 #[derive(Debug, Default)]
 struct Bindings {
     inner: FxHashMap<SmolStr, Binding>,
@@ -95,24 +143,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Option<Bindings>
     Some(res)
 }
 
-/*
-
-macro_rules! impl_froms {
-    ($e:ident: $($v:ident),*) => {
-        $(
-            impl From<$v> for $e {
-                fn from(it: $v) -> $e {
-                    $e::$v(it)
-                }
-            }
-        )*
-    }
-}
-
-impl_froms! (Foo: Bar, Baz)
-
-*/
-
 fn expand_subtree(
     template: &crate::Subtree,
     bindings: &Bindings,