Merge #3392

3392: Implement concat eager macro  r=matklad a=edwin0cheng

This PR implements the following things:

1. Add basic eager macro infrastructure by introducing `EagerCallId` such that the new `MacroCallId` is defined as :

```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum MacroCallId {
    LazyMacro(LazyMacroId),
    EagerMacro(EagerMacroId),
}
```

2. Add `concat!` builtin macro.




Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>

crates/ra_hir_def/src/lib.rs

@@ -47,8 +47,8 @@ mod marks;
 use std::hash::Hash;
 
 use hir_expand::{
-    ast_id_map::FileAstId, db::AstDatabase, hygiene::Hygiene, AstId, HirFileId, InFile,
-    MacroCallId, MacroCallKind, MacroDefId,
+    ast_id_map::FileAstId, db::AstDatabase, eager::expand_eager_macro, hygiene::Hygiene, AstId,
+    HirFileId, InFile, MacroCallId, MacroCallKind, MacroDefId, MacroDefKind,
 };
 use ra_arena::{impl_arena_id, RawId};
 use ra_db::{impl_intern_key, salsa, CrateId};
@@ -459,8 +459,21 @@ impl AsMacroCall for AstIdWithPath<ast::MacroCall> {
         db: &impl AstDatabase,
         resolver: impl Fn(path::ModPath) -> Option<MacroDefId>,
     ) -> Option<MacroCallId> {
-        let def = resolver(self.path.clone())?;
-        Some(def.as_call_id(db, MacroCallKind::FnLike(self.ast_id)))
+        let def: MacroDefId = resolver(self.path.clone())?;
+
+        if let MacroDefKind::BuiltInEager(_) = def.kind {
+            let macro_call = InFile::new(self.ast_id.file_id, self.ast_id.to_node(db));
+            let hygiene = Hygiene::new(db, self.ast_id.file_id);
+
+            Some(
+                expand_eager_macro(db, macro_call, def, &|path: ast::Path| {
+                    resolver(path::ModPath::from_src(path, &hygiene)?)
+                })?
+                .into(),
+            )
+        } else {
+            Some(def.as_lazy_macro(db, MacroCallKind::FnLike(self.ast_id)).into())
+        }
     }
 }
 
@@ -471,6 +484,6 @@ impl AsMacroCall for AstIdWithPath<ast::ModuleItem> {
         resolver: impl Fn(path::ModPath) -> Option<MacroDefId>,
     ) -> Option<MacroCallId> {
         let def = resolver(self.path.clone())?;
-        Some(def.as_call_id(db, MacroCallKind::Attr(self.ast_id)))
+        Some(def.as_lazy_macro(db, MacroCallKind::Attr(self.ast_id)).into())
     }
 }

crates/ra_hir_expand/src/builtin_derive.rs

@@ -9,7 +9,7 @@ use ra_syntax::{
 };
 
 use crate::db::AstDatabase;
-use crate::{name, quote, MacroCallId, MacroDefId, MacroDefKind};
+use crate::{name, quote, LazyMacroId, MacroDefId, MacroDefKind};
 
 macro_rules! register_builtin {
     ( $($trait:ident => $expand:ident),* ) => {
@@ -22,7 +22,7 @@ macro_rules! register_builtin {
             pub fn expand(
                 &self,
                 db: &dyn AstDatabase,
-                id: MacroCallId,
+                id: LazyMacroId,
                 tt: &tt::Subtree,
             ) -> Result<tt::Subtree, mbe::ExpandError> {
                 let expander = match *self {
@@ -155,7 +155,7 @@ fn expand_simple_derive(
 
 fn copy_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::marker::Copy })
@@ -163,7 +163,7 @@ fn copy_expand(
 
 fn clone_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::clone::Clone })
@@ -171,7 +171,7 @@ fn clone_expand(
 
 fn default_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::default::Default })
@@ -179,7 +179,7 @@ fn default_expand(
 
 fn debug_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::fmt::Debug })
@@ -187,7 +187,7 @@ fn debug_expand(
 
 fn hash_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::hash::Hash })
@@ -195,7 +195,7 @@ fn hash_expand(
 
 fn eq_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::cmp::Eq })
@@ -203,7 +203,7 @@ fn eq_expand(
 
 fn partial_eq_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::cmp::PartialEq })
@@ -211,7 +211,7 @@ fn partial_eq_expand(
 
 fn ord_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::cmp::Ord })
@@ -219,7 +219,7 @@ fn ord_expand(
 
 fn partial_ord_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     expand_simple_derive(tt, quote! { std::cmp::PartialOrd })
@@ -228,7 +228,7 @@ fn partial_ord_expand(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{test_db::TestDB, AstId, MacroCallKind, MacroCallLoc};
+    use crate::{test_db::TestDB, AstId, MacroCallId, MacroCallKind, MacroCallLoc};
     use ra_db::{fixture::WithFixture, SourceDatabase};
 
     fn expand_builtin_derive(s: &str, expander: BuiltinDeriveExpander) -> String {
@@ -248,7 +248,7 @@ mod tests {
             kind: MacroCallKind::Attr(AstId::new(file_id.into(), ast_id_map.ast_id(&items[0]))),
         };
 
-        let id = db.intern_macro(loc);
+        let id: MacroCallId = db.intern_macro(loc).into();
         let parsed = db.parse_or_expand(id.as_file()).unwrap();
 
         // FIXME text() for syntax nodes parsed from token tree looks weird

crates/ra_hir_expand/src/builtin_macro.rs

@@ -1,24 +1,31 @@
 //! Builtin macro
 use crate::db::AstDatabase;
 use crate::{
-    ast::{self},
-    name, AstId, CrateId, MacroCallId, MacroDefId, MacroDefKind, TextUnit,
+    ast::{self, AstToken, HasStringValue},
+    name, AstId, CrateId, MacroDefId, MacroDefKind, TextUnit,
 };
 
-use crate::quote;
+use crate::{quote, LazyMacroId};
+use either::Either;
+use ra_parser::FragmentKind;
 
 macro_rules! register_builtin {
-    ( $(($name:ident, $kind: ident) => $expand:ident),* ) => {
+    ( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),*  ) => {
         #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
         pub enum BuiltinFnLikeExpander {
             $($kind),*
         }
 
+        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+        pub enum EagerExpander {
+            $($e_kind),*
+        }
+
         impl BuiltinFnLikeExpander {
             pub fn expand(
                 &self,
                 db: &dyn AstDatabase,
-                id: MacroCallId,
+                id: LazyMacroId,
                 tt: &tt::Subtree,
             ) -> Result<tt::Subtree, mbe::ExpandError> {
                 let expander = match *self {
@@ -26,28 +33,54 @@ macro_rules! register_builtin {
                 };
                 expander(db, id, tt)
             }
+        }
 
-            fn by_name(ident: &name::Name) -> Option<BuiltinFnLikeExpander> {
-                match ident {
-                    $( id if id == &name::name![$name] => Some(BuiltinFnLikeExpander::$kind), )*
-                    _ => return None,
-                }
+        impl EagerExpander {
+            pub fn expand(
+                &self,
+                tt: &tt::Subtree,
+            ) -> Result<(tt::Subtree, FragmentKind), mbe::ExpandError> {
+                let expander = match *self {
+                    $( EagerExpander::$e_kind => $e_expand, )*
+                };
+                expander(tt)
             }
         }
 
-        pub fn find_builtin_macro(
-            ident: &name::Name,
-            krate: CrateId,
-            ast_id: AstId<ast::MacroCall>,
-        ) -> Option<MacroDefId> {
-            let kind = BuiltinFnLikeExpander::by_name(ident)?;
-
-            Some(MacroDefId { krate: Some(krate), ast_id: Some(ast_id), kind: MacroDefKind::BuiltIn(kind) })
+        fn find_by_name(ident: &name::Name) -> Option<Either<BuiltinFnLikeExpander, EagerExpander>> {
+            match ident {
+                $( id if id == &name::name![$name] => Some(Either::Left(BuiltinFnLikeExpander::$kind)), )*
+                $( id if id == &name::name![$e_name] => Some(Either::Right(EagerExpander::$e_kind)), )*
+                _ => return None,
+            }
         }
     };
 }
 
+pub fn find_builtin_macro(
+    ident: &name::Name,
+    krate: CrateId,
+    ast_id: AstId<ast::MacroCall>,
+) -> Option<MacroDefId> {
+    let kind = find_by_name(ident)?;
+
+    match kind {
+        Either::Left(kind) => Some(MacroDefId {
+            krate: Some(krate),
+            ast_id: Some(ast_id),
+            kind: MacroDefKind::BuiltIn(kind),
+        }),
+        Either::Right(kind) => Some(MacroDefId {
+            krate: Some(krate),
+            ast_id: Some(ast_id),
+            kind: MacroDefKind::BuiltInEager(kind),
+        }),
+    }
+}
+
 register_builtin! {
+    LAZY:
+
     (column, Column) => column_expand,
     (compile_error, CompileError) => compile_error_expand,
     (file, File) => file_expand,
@@ -58,12 +91,16 @@ register_builtin! {
     (option_env, OptionEnv) => option_env_expand,
     // format_args_nl only differs in that it adds a newline in the end,
     // so we use the same stub expansion for now
-    (format_args_nl, FormatArgsNl) => format_args_expand
+    (format_args_nl, FormatArgsNl) => format_args_expand,
+
+    EAGER:
+    // eagers
+    (concat, Concat) => concat_expand
 }
 
 fn line_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     _tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     // dummy implementation for type-checking purposes
@@ -77,7 +114,7 @@ fn line_expand(
 
 fn stringify_expand(
     db: &dyn AstDatabase,
-    id: MacroCallId,
+    id: LazyMacroId,
     _tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     let loc = db.lookup_intern_macro(id);
@@ -99,7 +136,7 @@ fn stringify_expand(
 
 fn env_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     _tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     // dummy implementation for type-checking purposes
@@ -110,7 +147,7 @@ fn env_expand(
 
 fn option_env_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     _tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     // dummy implementation for type-checking purposes
@@ -121,7 +158,7 @@ fn option_env_expand(
 
 fn column_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     _tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     // dummy implementation for type-checking purposes
@@ -135,7 +172,7 @@ fn column_expand(
 
 fn file_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     _tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     // FIXME: RA purposefully lacks knowledge of absolute file names
@@ -151,7 +188,7 @@ fn file_expand(
 
 fn compile_error_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     if tt.count() == 1 {
@@ -168,7 +205,7 @@ fn compile_error_expand(
 
 fn format_args_expand(
     _db: &dyn AstDatabase,
-    _id: MacroCallId,
+    _id: LazyMacroId,
     tt: &tt::Subtree,
 ) -> Result<tt::Subtree, mbe::ExpandError> {
     // We expand `format_args!("", a1, a2)` to
@@ -208,23 +245,44 @@ fn format_args_expand(
     Ok(expanded)
 }
 
+fn unquote_str(lit: &tt::Literal) -> Option<String> {
+    let lit = ast::make::tokens::literal(&lit.to_string());
+    let token = ast::String::cast(lit)?;
+    token.value()
+}
+
+fn concat_expand(tt: &tt::Subtree) -> Result<(tt::Subtree, FragmentKind), mbe::ExpandError> {
+    let mut text = String::new();
+    for (i, t) in tt.token_trees.iter().enumerate() {
+        match t {
+            tt::TokenTree::Leaf(tt::Leaf::Literal(it)) if i % 2 == 0 => {
+                text += &unquote_str(&it).ok_or_else(|| mbe::ExpandError::ConversionError)?;
+            }
+            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if i % 2 == 1 && punct.char == ',' => (),
+            _ => return Err(mbe::ExpandError::UnexpectedToken),
+        }
+    }
+
+    Ok((quote!(#text), FragmentKind::Expr))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{name::AsName, test_db::TestDB, AstNode, MacroCallKind, MacroCallLoc};
+    use crate::{name::AsName, test_db::TestDB, AstNode, MacroCallId, MacroCallKind, MacroCallLoc};
     use ra_db::{fixture::WithFixture, SourceDatabase};
     use ra_syntax::ast::NameOwner;
 
-    fn expand_builtin_macro(s: &str) -> String {
-        let (db, file_id) = TestDB::with_single_file(&s);
+    fn expand_builtin_macro(ra_fixture: &str) -> String {
+        let (db, file_id) = TestDB::with_single_file(&ra_fixture);
         let parsed = db.parse(file_id);
         let macro_calls: Vec<_> =
             parsed.syntax_node().descendants().filter_map(ast::MacroCall::cast).collect();
 
         let ast_id_map = db.ast_id_map(file_id.into());
 
-        let expander =
-            BuiltinFnLikeExpander::by_name(&macro_calls[0].name().unwrap().as_name()).unwrap();
+        let expander = find_by_name(&macro_calls[0].name().unwrap().as_name()).unwrap();
+        let expander = expander.left().unwrap();
 
         // the first one should be a macro_rules
         let def = MacroDefId {
@@ -241,7 +299,7 @@ mod tests {
             )),
         };
 
-        let id = db.intern_macro(loc);
+        let id: MacroCallId = db.intern_macro(loc).into();
         let parsed = db.parse_or_expand(id.as_file()).unwrap();
 
         parsed.text().to_string()

crates/ra_hir_expand/src/db.rs

@@ -9,8 +9,9 @@ use ra_prof::profile;
 use ra_syntax::{AstNode, Parse, SyntaxKind::*, SyntaxNode};
 
 use crate::{
-    ast_id_map::AstIdMap, BuiltinDeriveExpander, BuiltinFnLikeExpander, HirFileId, HirFileIdRepr,
-    MacroCallId, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile,
+    ast_id_map::AstIdMap, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallLoc, EagerMacroId,
+    HirFileId, HirFileIdRepr, LazyMacroId, MacroCallId, MacroCallLoc, MacroDefId, MacroDefKind,
+    MacroFile,
 };
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -24,7 +25,7 @@ impl TokenExpander {
     pub fn expand(
         &self,
         db: &dyn AstDatabase,
-        id: MacroCallId,
+        id: LazyMacroId,
         tt: &tt::Subtree,
     ) -> Result<tt::Subtree, mbe::ExpandError> {
         match self {
@@ -60,12 +61,15 @@ pub trait AstDatabase: SourceDatabase {
     fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode>;
 
     #[salsa::interned]
-    fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId;
+    fn intern_macro(&self, macro_call: MacroCallLoc) -> LazyMacroId;
     fn macro_arg(&self, id: MacroCallId) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>>;
     fn macro_def(&self, id: MacroDefId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>>;
     fn parse_macro(&self, macro_file: MacroFile)
         -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)>;
     fn macro_expand(&self, macro_call: MacroCallId) -> Result<Arc<tt::Subtree>, String>;
+
+    #[salsa::interned]
+    fn intern_eager_expansion(&self, eager: EagerCallLoc) -> EagerMacroId;
 }
 
 pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc<AstIdMap> {
@@ -101,6 +105,7 @@ pub(crate) fn macro_def(
         MacroDefKind::BuiltInDerive(expander) => {
             Some(Arc::new((TokenExpander::BuiltinDerive(expander), mbe::TokenMap::default())))
         }
+        MacroDefKind::BuiltInEager(_expander) => None,
     }
 }
 
@@ -108,6 +113,13 @@ pub(crate) fn macro_arg(
     db: &dyn AstDatabase,
     id: MacroCallId,
 ) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>> {
+    let id = match id {
+        MacroCallId::LazyMacro(id) => id,
+        MacroCallId::EagerMacro(_id) => {
+            // FIXME: support macro_arg for eager macro
+            return None;
+        }
+    };
     let loc = db.lookup_intern_macro(id);
     let arg = loc.kind.arg(db)?;
     let (tt, tmap) = mbe::syntax_node_to_token_tree(&arg)?;
@@ -118,11 +130,18 @@ pub(crate) fn macro_expand(
     db: &dyn AstDatabase,
     id: MacroCallId,
 ) -> Result<Arc<tt::Subtree>, String> {
-    let loc = db.lookup_intern_macro(id);
+    let lazy_id = match id {
+        MacroCallId::LazyMacro(id) => id,
+        MacroCallId::EagerMacro(id) => {
+            return Ok(db.lookup_intern_eager_expansion(id).subtree);
+        }
+    };
+
+    let loc = db.lookup_intern_macro(lazy_id);
     let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?;
 
     let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?;
-    let tt = macro_rules.0.expand(db, id, &macro_arg.0).map_err(|err| format!("{:?}", err))?;
+    let tt = macro_rules.0.expand(db, lazy_id, &macro_arg.0).map_err(|err| format!("{:?}", err))?;
     // Set a hard limit for the expanded tt
     let count = tt.count();
     if count > 65536 {
@@ -153,9 +172,20 @@ pub(crate) fn parse_macro(
             // Note:
             // The final goal we would like to make all parse_macro success,
             // such that the following log will not call anyway.
-            let loc: MacroCallLoc = db.lookup_intern_macro(macro_call_id);
-            let node = loc.kind.node(db);
-            log::warn!("fail on macro_parse: (reason: {} macro_call: {:#})", err, node.value);
+            match macro_call_id {
+                MacroCallId::LazyMacro(id) => {
+                    let loc: MacroCallLoc = db.lookup_intern_macro(id);
+                    let node = loc.kind.node(db);
+                    log::warn!(
+                        "fail on macro_parse: (reason: {} macro_call: {:#})",
+                        err,
+                        node.value
+                    );
+                }
+                _ => {
+                    log::warn!("fail on macro_parse: (reason: {})", err);
+                }
+            }
         })
         .ok()?;
 
@@ -167,8 +197,14 @@ pub(crate) fn parse_macro(
 
 /// Given a `MacroCallId`, return what `FragmentKind` it belongs to.
 /// FIXME: Not completed
-fn to_fragment_kind(db: &dyn AstDatabase, macro_call_id: MacroCallId) -> FragmentKind {
-    let syn = db.lookup_intern_macro(macro_call_id).kind.node(db).value;
+fn to_fragment_kind(db: &dyn AstDatabase, id: MacroCallId) -> FragmentKind {
+    let lazy_id = match id {
+        MacroCallId::LazyMacro(id) => id,
+        MacroCallId::EagerMacro(id) => {
+            return db.lookup_intern_eager_expansion(id).fragment;
+        }
+    };
+    let syn = db.lookup_intern_macro(lazy_id).kind.node(db).value;
 
     let parent = match syn.parent() {
         Some(it) => it,

crates/ra_hir_expand/src/eager.rs

@@ -0,0 +1,112 @@
+//! Eager expansion related utils
+//!
+//! Here is a dump of a discussion from Vadim Petrochenkov about Eager Expansion and
+//! Its name resolution :
+//!
+//! > Eagerly expanded macros (and also macros eagerly expanded by eagerly expanded macros,
+//! > which actually happens in practice too!) are resolved at the location of the "root" macro
+//! > that performs the eager expansion on its arguments.
+//! > If some name cannot be resolved at the eager expansion time it's considered unresolved,
+//! > even if becomes available later (e.g. from a glob import or other macro).
+//!
+//! > Eagerly expanded macros don't add anything to the module structure of the crate and
+//! > don't build any speculative module structures, i.e. they are expanded in a "flat"
+//! > way even if tokens in them look like modules.
+//!
+//! > In other words, it kinda works for simple cases for which it was originally intended,
+//! > and we need to live with it because it's available on stable and widely relied upon.
+//!
+//!
+//! See the full discussion : https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Eager.20expansion.20of.20built-in.20macros
+
+use crate::{
+    ast::{self, AstNode},
+    db::AstDatabase,
+    EagerCallLoc, EagerMacroId, InFile, MacroCallId, MacroCallKind, MacroDefId, MacroDefKind,
+};
+
+use ra_parser::FragmentKind;
+use ra_syntax::{algo::replace_descendants, SyntaxElement, SyntaxNode};
+use std::{collections::HashMap, sync::Arc};
+
+pub fn expand_eager_macro(
+    db: &impl AstDatabase,
+    macro_call: InFile<ast::MacroCall>,
+    def: MacroDefId,
+    resolver: &dyn Fn(ast::Path) -> Option<MacroDefId>,
+) -> Option<EagerMacroId> {
+    let args = macro_call.value.token_tree()?;
+    let parsed_args = mbe::ast_to_token_tree(&args)?.0;
+    let parsed_args = mbe::token_tree_to_syntax_node(&parsed_args, FragmentKind::Expr).ok()?.0;
+    let result = eager_macro_recur(db, macro_call.with_value(parsed_args.syntax_node()), resolver)?;
+
+    let subtree = to_subtree(&result)?;
+
+    if let MacroDefKind::BuiltInEager(eager) = def.kind {
+        let (subtree, fragment) = eager.expand(&subtree).ok()?;
+        let eager =
+            EagerCallLoc { def, fragment, subtree: Arc::new(subtree), file_id: macro_call.file_id };
+
+        Some(db.intern_eager_expansion(eager))
+    } else {
+        None
+    }
+}
+
+fn to_subtree(node: &SyntaxNode) -> Option<tt::Subtree> {
+    let mut subtree = mbe::syntax_node_to_token_tree(node)?.0;
+    subtree.delimiter = None;
+    Some(subtree)
+}
+
+fn lazy_expand(
+    db: &impl AstDatabase,
+    def: &MacroDefId,
+    macro_call: InFile<ast::MacroCall>,
+) -> Option<InFile<SyntaxNode>> {
+    let ast_id = db.ast_id_map(macro_call.file_id).ast_id(&macro_call.value);
+
+    let id: MacroCallId =
+        def.as_lazy_macro(db, MacroCallKind::FnLike(macro_call.with_value(ast_id))).into();
+
+    db.parse_or_expand(id.as_file()).map(|node| InFile::new(id.as_file(), node))
+}
+
+fn eager_macro_recur(
+    db: &impl AstDatabase,
+    curr: InFile<SyntaxNode>,
+    macro_resolver: &dyn Fn(ast::Path) -> Option<MacroDefId>,
+) -> Option<SyntaxNode> {
+    let mut original = curr.value.clone();
+
+    let children = curr.value.descendants().filter_map(ast::MacroCall::cast);
+    let mut replaces: HashMap<SyntaxElement, SyntaxElement> = HashMap::default();
+
+    // Collect replacement
+    for child in children {
+        let def: MacroDefId = macro_resolver(child.path()?)?;
+        let insert = match def.kind {
+            MacroDefKind::BuiltInEager(_) => {
+                let id: MacroCallId =
+                    expand_eager_macro(db, curr.with_value(child.clone()), def, macro_resolver)?
+                        .into();
+                db.parse_or_expand(id.as_file())?
+            }
+            MacroDefKind::Declarative
+            | MacroDefKind::BuiltIn(_)
+            | MacroDefKind::BuiltInDerive(_) => {
+                let expanded = lazy_expand(db, &def, curr.with_value(child.clone()))?;
+                // replace macro inside
+                eager_macro_recur(db, expanded, macro_resolver)?
+            }
+        };
+
+        replaces.insert(child.syntax().clone().into(), insert.into());
+    }
+
+    if !replaces.is_empty() {
+        original = replace_descendants(&original, |n| replaces.get(n).cloned());
+    }
+
+    Some(original)
+}

crates/ra_hir_expand/src/hygiene.rs

@@ -9,7 +9,7 @@ use ra_syntax::ast;
 use crate::{
     db::AstDatabase,
     name::{AsName, Name},
-    HirFileId, HirFileIdRepr, MacroDefKind,
+    HirFileId, HirFileIdRepr, MacroCallId, MacroDefKind,
 };
 
 #[derive(Debug)]
@@ -22,14 +22,18 @@ impl Hygiene {
     pub fn new(db: &impl AstDatabase, file_id: HirFileId) -> Hygiene {
         let def_crate = match file_id.0 {
             HirFileIdRepr::FileId(_) => None,
-            HirFileIdRepr::MacroFile(macro_file) => {
-                let loc = db.lookup_intern_macro(macro_file.macro_call_id);
-                match loc.def.kind {
-                    MacroDefKind::Declarative => loc.def.krate,
-                    MacroDefKind::BuiltIn(_) => None,
-                    MacroDefKind::BuiltInDerive(_) => None,
+            HirFileIdRepr::MacroFile(macro_file) => match macro_file.macro_call_id {
+                MacroCallId::LazyMacro(id) => {
+                    let loc = db.lookup_intern_macro(id);
+                    match loc.def.kind {
+                        MacroDefKind::Declarative => loc.def.krate,
+                        MacroDefKind::BuiltIn(_) => None,
+                        MacroDefKind::BuiltInDerive(_) => None,
+                        MacroDefKind::BuiltInEager(_) => None,
+                    }
                 }
-            }
+                MacroCallId::EagerMacro(_id) => None,
+            },
         };
         Hygiene { def_crate }
     }

crates/ra_hir_expand/src/lib.rs

@@ -12,6 +12,7 @@ pub mod diagnostics;
 pub mod builtin_derive;
 pub mod builtin_macro;
 pub mod quote;
+pub mod eager;
 
 use std::hash::Hash;
 use std::sync::Arc;
@@ -25,7 +26,7 @@ use ra_syntax::{
 
 use crate::ast_id_map::FileAstId;
 use crate::builtin_derive::BuiltinDeriveExpander;
-use crate::builtin_macro::BuiltinFnLikeExpander;
+use crate::builtin_macro::{BuiltinFnLikeExpander, EagerExpander};
 
 #[cfg(test)]
 mod test_db;
@@ -70,8 +71,17 @@ impl HirFileId {
         match self.0 {
             HirFileIdRepr::FileId(file_id) => file_id,
             HirFileIdRepr::MacroFile(macro_file) => {
-                let loc = db.lookup_intern_macro(macro_file.macro_call_id);
-                loc.kind.file_id().original_file(db)
+                let file_id = match macro_file.macro_call_id {
+                    MacroCallId::LazyMacro(id) => {
+                        let loc = db.lookup_intern_macro(id);
+                        loc.kind.file_id()
+                    }
+                    MacroCallId::EagerMacro(id) => {
+                        let loc = db.lookup_intern_eager_expansion(id);
+                        loc.file_id
+                    }
+                };
+                file_id.original_file(db)
             }
         }
     }
@@ -81,7 +91,14 @@ impl HirFileId {
         match self.0 {
             HirFileIdRepr::FileId(_) => None,
             HirFileIdRepr::MacroFile(macro_file) => {
-                let loc = db.lookup_intern_macro(macro_file.macro_call_id);
+                let lazy_id = match macro_file.macro_call_id {
+                    MacroCallId::LazyMacro(id) => id,
+                    MacroCallId::EagerMacro(_id) => {
+                        // FIXME: handle call node for eager macro
+                        return None;
+                    }
+                };
+                let loc = db.lookup_intern_macro(lazy_id);
                 Some(loc.kind.node(db))
             }
         }
@@ -92,7 +109,14 @@ impl HirFileId {
         match self.0 {
             HirFileIdRepr::FileId(_) => None,
             HirFileIdRepr::MacroFile(macro_file) => {
-                let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
+                let lazy_id = match macro_file.macro_call_id {
+                    MacroCallId::LazyMacro(id) => id,
+                    MacroCallId::EagerMacro(_id) => {
+                        // FIXME: handle expansion_info for eager macro
+                        return None;
+                    }
+                };
+                let loc: MacroCallLoc = db.lookup_intern_macro(lazy_id);
 
                 let arg_tt = loc.kind.arg(db)?;
                 let def_tt = loc.def.ast_id?.to_node(db).token_tree()?;
@@ -118,7 +142,13 @@ impl HirFileId {
         match self.0 {
             HirFileIdRepr::FileId(_) => None,
             HirFileIdRepr::MacroFile(macro_file) => {
-                let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id);
+                let lazy_id = match macro_file.macro_call_id {
+                    MacroCallId::LazyMacro(id) => id,
+                    MacroCallId::EagerMacro(_id) => {
+                        return None;
+                    }
+                };
+                let loc: MacroCallLoc = db.lookup_intern_macro(lazy_id);
                 let item = match loc.def.kind {
                     MacroDefKind::BuiltInDerive(_) => loc.kind.node(db),
                     _ => return None,
@@ -137,16 +167,44 @@ pub struct MacroFile {
 /// `MacroCallId` identifies a particular macro invocation, like
 /// `println!("Hello, {}", world)`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct MacroCallId(salsa::InternId);
-impl salsa::InternKey for MacroCallId {
+pub enum MacroCallId {
+    LazyMacro(LazyMacroId),
+    EagerMacro(EagerMacroId),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct LazyMacroId(salsa::InternId);
+impl salsa::InternKey for LazyMacroId {
     fn from_intern_id(v: salsa::InternId) -> Self {
-        MacroCallId(v)
+        LazyMacroId(v)
     }
     fn as_intern_id(&self) -> salsa::InternId {
         self.0
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct EagerMacroId(salsa::InternId);
+impl salsa::InternKey for EagerMacroId {
+    fn from_intern_id(v: salsa::InternId) -> Self {
+        EagerMacroId(v)
+    }
+    fn as_intern_id(&self) -> salsa::InternId {
+        self.0
+    }
+}
+
+impl From<LazyMacroId> for MacroCallId {
+    fn from(it: LazyMacroId) -> Self {
+        MacroCallId::LazyMacro(it)
+    }
+}
+impl From<EagerMacroId> for MacroCallId {
+    fn from(it: EagerMacroId) -> Self {
+        MacroCallId::EagerMacro(it)
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct MacroDefId {
     // FIXME: krate and ast_id are currently optional because we don't have a
@@ -161,7 +219,7 @@ pub struct MacroDefId {
 }
 
 impl MacroDefId {
-    pub fn as_call_id(self, db: &dyn db::AstDatabase, kind: MacroCallKind) -> MacroCallId {
+    pub fn as_lazy_macro(self, db: &dyn db::AstDatabase, kind: MacroCallKind) -> LazyMacroId {
         db.intern_macro(MacroCallLoc { def: self, kind })
     }
 }
@@ -172,6 +230,7 @@ pub enum MacroDefKind {
     BuiltIn(BuiltinFnLikeExpander),
     // FIXME: maybe just Builtin and rename BuiltinFnLikeExpander to BuiltinExpander
     BuiltInDerive(BuiltinDeriveExpander),
+    BuiltInEager(EagerExpander),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -217,6 +276,14 @@ impl MacroCallId {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct EagerCallLoc {
+    pub(crate) def: MacroDefId,
+    pub(crate) fragment: FragmentKind,
+    pub(crate) subtree: Arc<tt::Subtree>,
+    pub(crate) file_id: HirFileId,
+}
+
 /// ExpansionInfo mainly describes how to map text range between src and expanded macro
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct ExpansionInfo {
@@ -230,6 +297,7 @@ pub struct ExpansionInfo {
 }
 
 pub use mbe::Origin;
+use ra_parser::FragmentKind;
 
 impl ExpansionInfo {
     pub fn call_node(&self) -> Option<InFile<SyntaxNode>> {

crates/ra_hir_expand/src/name.rs

@@ -173,6 +173,7 @@ pub mod known {
         compile_error,
         line,
         stringify,
+        concat,
         format_args,
         format_args_nl,
         env,

crates/ra_hir_ty/src/tests/macros.rs

@@ -419,6 +419,25 @@ fn main() {
     );
 }
 
+#[test]
+fn infer_builtin_macros_concat() {
+    assert_snapshot!(
+        infer(r#"
+#[rustc_builtin_macro]
+macro_rules! concat {() => {}}
+
+fn main() {
+    let x = concat!("hello", concat!("world", "!"));
+}
+"#),
+        @r###"
+    ![0; 13) '"helloworld!"': &str
+    [66; 122) '{     ...")); }': ()
+    [76; 77) 'x': &str
+    "###
+    );
+}
+
 #[test]
 fn infer_derive_clone_simple() {
     let (db, pos) = TestDB::with_position(

crates/ra_parser/src/lib.rs

@@ -83,7 +83,7 @@ pub fn parse(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::root);
 }
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub enum FragmentKind {
     Path,
     Expr,

crates/ra_syntax/src/ast/make.rs

@@ -219,7 +219,7 @@ fn unroot(n: SyntaxNode) -> SyntaxNode {
 }
 
 pub mod tokens {
-    use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken, T};
+    use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken, T};
     use once_cell::sync::Lazy;
 
     pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> =
@@ -251,6 +251,12 @@ pub mod tokens {
         sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
     }
 
+    pub fn literal(text: &str) -> SyntaxToken {
+        assert_eq!(text.trim(), text);
+        let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
+        lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
+    }
+
     pub fn single_newline() -> SyntaxToken {
         SOURCE_FILE
             .tree()