Rust: fix macro expansion in library code

rust/extractor/src/config.rs

@@ -71,6 +71,7 @@ pub struct Config {
     pub proc_macro_server: Option<PathBuf>,
     pub skip_path_resolution: bool,
     pub extract_dependencies_as_source: bool,
+    pub force_library_mode: bool, // for testing purposes
 }
 
 impl Config {

rust/extractor/src/main.rs

@@ -293,6 +293,11 @@ fn main() -> anyhow::Result<()> {
     } else {
         (SourceKind::Library, ResolvePaths::No)
     };
+    let (source_mode, source_resolve_paths) = if cfg.force_library_mode {
+        (library_mode, library_resolve_paths)
+    } else {
+        (SourceKind::Source, resolve_paths)
+    };
     let mut processed_files: HashSet<PathBuf, RandomState> =
         HashSet::from_iter(files.iter().cloned());
     for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
@@ -311,12 +316,10 @@ fn main() -> anyhow::Result<()> {
                         file,
                         &semantics,
                         vfs,
-                        resolve_paths,
-                        SourceKind::Source,
+                        source_resolve_paths,
+                        source_mode,
                     ),
-                    Err(reason) => {
-                        extractor.extract_without_semantics(file, SourceKind::Source, &reason)
-                    }
+                    Err(reason) => extractor.extract_without_semantics(file, source_mode, &reason),
                 };
             }
             for (file_id, file) in vfs.iter() {

rust/extractor/src/qltest.rs

@@ -19,7 +19,7 @@ fn dump_lib() -> anyhow::Result<()> {
         .iter()
         .map(|p| p.file_stem().expect("results of glob must have a name"))
         .filter(|&p| !["main", "lib", "proc_macro"].map(OsStr::new).contains(&p))
-        .map(|p| format!("mod {};", p.to_string_lossy()))
+        .map(|p| format!("pub mod {};", p.to_string_lossy()))
         .join("\n");
     fs::write("lib.rs", lib).context("writing lib.rs")
 }

rust/extractor/src/translate/base.rs

@@ -24,33 +24,31 @@ use ra_ap_syntax::{
 
 impl Emission<ast::Item> for Translator<'_> {
     fn pre_emit(&mut self, node: &ast::Item) -> Option<Label<generated::Item>> {
-        self.prepare_item_expansion(node).map(Into::into)
+        self.item_pre_emit(node).map(Into::into)
     }
 
     fn post_emit(&mut self, node: &ast::Item, label: Label<generated::Item>) {
-        self.emit_item_expansion(node, label);
+        self.item_post_emit(node, label);
     }
 }
 
 impl Emission<ast::AssocItem> for Translator<'_> {
     fn pre_emit(&mut self, node: &ast::AssocItem) -> Option<Label<generated::AssocItem>> {
-        self.prepare_item_expansion(&node.clone().into())
-            .map(Into::into)
+        self.item_pre_emit(&node.clone().into()).map(Into::into)
     }
 
     fn post_emit(&mut self, node: &ast::AssocItem, label: Label<generated::AssocItem>) {
-        self.emit_item_expansion(&node.clone().into(), label.into());
+        self.item_post_emit(&node.clone().into(), label.into());
     }
 }
 
 impl Emission<ast::ExternItem> for Translator<'_> {
     fn pre_emit(&mut self, node: &ast::ExternItem) -> Option<Label<generated::ExternItem>> {
-        self.prepare_item_expansion(&node.clone().into())
-            .map(Into::into)
+        self.item_pre_emit(&node.clone().into()).map(Into::into)
     }
 
     fn post_emit(&mut self, node: &ast::ExternItem, label: Label<generated::ExternItem>) {
-        self.emit_item_expansion(&node.clone().into(), label.into());
+        self.item_post_emit(&node.clone().into(), label.into());
     }
 }
 
@@ -849,35 +847,6 @@ impl<'a> Translator<'a> {
             })
     }
 
-    pub(crate) fn prepare_item_expansion(
-        &mut self,
-        node: &ast::Item,
-    ) -> Option<Label<generated::MacroCall>> {
-        if self.source_kind == SourceKind::Library {
-            // if the item expands via an attribute macro, we want to only emit the expansion
-            if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
-                // we wrap it in a dummy MacroCall to get a single Item label that can replace
-                // the original Item
-                let label = self.trap.emit(generated::MacroCall {
-                    id: TrapId::Star,
-                    attrs: vec![],
-                    path: None,
-                    token_tree: None,
-                });
-                generated::MacroCall::emit_macro_call_expansion(
-                    label,
-                    expanded.into(),
-                    &mut self.trap.writer,
-                );
-                return Some(label);
-            }
-        }
-        if self.is_attribute_macro_target(node) {
-            self.macro_context_depth += 1;
-        }
-        None
-    }
-
     fn process_item_macro_expansion(
         &mut self,
         node: &impl ast::AstNode,
@@ -915,10 +884,6 @@ impl<'a> Translator<'a> {
         &mut self,
         node: &ast::Item,
     ) -> Option<Label<generated::MacroItems>> {
-        if !self.is_attribute_macro_target(node) {
-            return None;
-        }
-        self.macro_context_depth -= 1;
         if self.macro_context_depth > 0 {
             // only expand the outermost attribute macro
             return None;
@@ -927,7 +892,48 @@ impl<'a> Translator<'a> {
         self.process_item_macro_expansion(node, expansion.map(|x| x.value))
     }
 
-    pub(crate) fn emit_item_expansion(&mut self, node: &ast::Item, label: Label<generated::Item>) {
+    pub(crate) fn item_pre_emit(
+        &mut self,
+        node: &ast::Item,
+    ) -> Option<Label<generated::MacroCall>> {
+        if !self.is_attribute_macro_target(node) {
+            return None;
+        }
+        if self.source_kind == SourceKind::Library {
+            // if the item expands via an attribute macro, we want to only emit the expansion
+            if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
+                // we wrap it in a dummy MacroCall to get a single Item label that can replace
+                // the original Item
+                let label = self.trap.emit(generated::MacroCall {
+                    id: TrapId::Star,
+                    attrs: vec![],
+                    path: None,
+                    token_tree: None,
+                });
+                generated::MacroCall::emit_macro_call_expansion(
+                    label,
+                    expanded.into(),
+                    &mut self.trap.writer,
+                );
+                return Some(label);
+            }
+        }
+        self.macro_context_depth += 1;
+        None
+    }
+
+    pub(crate) fn item_post_emit(&mut self, node: &ast::Item, label: Label<generated::Item>) {
+        if !self.is_attribute_macro_target(node) {
+            return;
+        }
+        // see `item_pre_emit`:
+        // if self.is_attribute_macro_target(node), then we either exited early with `Some(label)`
+        // and are not here, or we did self.macro_context_depth += 1
+        assert!(
+            self.macro_context_depth > 0,
+            "macro_context_depth should be > 0 for an attribute macro target"
+        );
+        self.macro_context_depth -= 1;
         if let Some(expanded) = self.emit_attribute_macro_expansion(node) {
             generated::Item::emit_attribute_macro_expansion(label, expanded, &mut self.trap.writer);
         }

rust/ql/test/TestUtils.qll

@@ -6,7 +6,7 @@ predicate toBeTested(Element e) {
   (
     not e instanceof Locatable
     or
-    e.(Locatable).fromSource()
+    exists(e.(Locatable).getFile().getRelativePath())
   )
 }
 

rust/ql/test/extractor-tests/macro-in-library/PrintAst.expected

@@ -0,0 +1,116 @@
+lib.rs:
+#    1| [SourceFile] SourceFile
+#    1|   getItem(0): [Module] mod macro_in_library
+#    1|     getName(): [Name] macro_in_library
+#    1|     getVisibility(): [Visibility] Visibility
+macro_in_library.rs:
+#    1| [SourceFile] SourceFile
+#    4|   getItem(1): [Function] fn bar
+#    4|     getParamList(): [ParamList] ParamList
+#    4|     getName(): [Name] bar
+#    4|     getVisibility(): [Visibility] Visibility
+#    2| [MacroItems] MacroItems
+#    2|   getItem(0): [Function] fn foo
+#    2|     getParamList(): [ParamList] ParamList
+#    2|     getName(): [Name] foo
+#    2|     getVisibility(): [Visibility] Visibility
+#    2|   getItem(1): [Function] fn foo_new
+#    2|     getParamList(): [ParamList] ParamList
+#    2|     getName(): [Name] foo_new
+#    2|     getVisibility(): [Visibility] Visibility
+proc_macro.rs:
+#    1| [SourceFile] SourceFile
+#    1|   getItem(0): [Use] use ...::TokenStream
+#    1|     getUseTree(): [UseTree] ...::TokenStream
+#    1|       getPath(): [Path] ...::TokenStream
+#    1|         getQualifier(): [Path] proc_macro
+#    1|           getSegment(): [PathSegment] proc_macro
+#    1|             getIdentifier(): [NameRef] proc_macro
+#    1|         getSegment(): [PathSegment] TokenStream
+#    1|           getIdentifier(): [NameRef] TokenStream
+#    2|   getItem(1): [Use] use ...::quote
+#    2|     getUseTree(): [UseTree] ...::quote
+#    2|       getPath(): [Path] ...::quote
+#    2|         getQualifier(): [Path] quote
+#    2|           getSegment(): [PathSegment] quote
+#    2|             getIdentifier(): [NameRef] quote
+#    2|         getSegment(): [PathSegment] quote
+#    2|           getIdentifier(): [NameRef] quote
+#    4|   getItem(2): [Function] fn add_one
+#    5|     getParamList(): [ParamList] ParamList
+#    5|       getParam(0): [Param] : TokenStream
+#    5|         getTypeRepr(): [PathTypeRepr] TokenStream
+#    5|           getPath(): [Path] TokenStream
+#    5|             getSegment(): [PathSegment] TokenStream
+#    5|               getIdentifier(): [NameRef] TokenStream
+#    5|       getParam(1): [Param] : TokenStream
+#    5|         getTypeRepr(): [PathTypeRepr] TokenStream
+#    5|           getPath(): [Path] TokenStream
+#    5|             getSegment(): [PathSegment] TokenStream
+#    5|               getIdentifier(): [NameRef] TokenStream
+#    4|     getAttr(0): [Attr] Attr
+#    4|       getMeta(): [Meta] Meta
+#    4|         getPath(): [Path] proc_macro_attribute
+#    4|           getSegment(): [PathSegment] proc_macro_attribute
+#    4|             getIdentifier(): [NameRef] proc_macro_attribute
+#    5|     getName(): [Name] add_one
+#    5|     getRetType(): [RetTypeRepr] RetTypeRepr
+#    5|       getTypeRepr(): [PathTypeRepr] TokenStream
+#    5|         getPath(): [Path] TokenStream
+#    5|           getSegment(): [PathSegment] TokenStream
+#    5|             getIdentifier(): [NameRef] TokenStream
+#    5|     getVisibility(): [Visibility] Visibility
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/allocator-api2-0.2.21/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/cfg-if-1.0.0/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/compiler_builtins-0.1.146/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/getopts-0.2.21/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/hashbrown-0.15.2/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/libc-0.2.169/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/proc-macro2-1.0.95/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/quote-1.0.40/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand-0.9.0/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand_core-0.9.0/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rand_xorshift-0.4.0/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/rustc-demangle-0.1.24/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/syn-2.0.104/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/unicode-ident-1.0.18/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/unicode-width-0.1.14/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/zerocopy-0.8.17/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/panic_abort/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/panic_unwind/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/proc_macro/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/stdarch/crates/std_detect/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/test/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crates/home/redsun82/.rustup/toolchains/1.86-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/unwind/src/lib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/crateslib.rs:
+#    1| [SourceFile] SourceFile
+resolved/macro-in-library.testproj/trap/rust/cratesproc_macro.rs:
+#    1| [SourceFile] SourceFile

rust/ql/test/extractor-tests/macro-in-library/PrintAst.qlref

@@ -0,0 +1 @@
+utils/PrintAst.ql

rust/ql/test/extractor-tests/macro-in-library/macro_in_library.rs

@@ -0,0 +1,6 @@
+#[proc_macro::add_one]
+pub fn foo() {}
+
+pub fn bar() {
+    foo_new();
+}

rust/ql/test/extractor-tests/macro-in-library/options.yml

@@ -0,0 +1 @@
+force_library_mode: true

rust/ql/test/extractor-tests/macro-in-library/proc_macro.rs

@@ -0,0 +1,14 @@
+use proc_macro::TokenStream;
+use quote::quote;
+
+#[proc_macro_attribute]
+pub fn add_one(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let ast = syn::parse_macro_input!(item as syn::ItemFn);
+    let mut new_ast = ast.clone();
+    new_ast.sig.ident = syn::Ident::new(&format!("{}_new", ast.sig.ident), ast.sig.ident.span());
+    quote! {
+        #ast
+        #new_ast
+    }.into()
+}
+

rust/ql/test/extractor-tests/macro-in-library/test.expected

@@ -0,0 +1,4 @@
+macro_items
+| macro_in_library.rs:2:1:2:14 | MacroItems | 0 | macro_in_library.rs:2:1:2:14 | fn foo |
+| macro_in_library.rs:2:1:2:14 | MacroItems | 1 | macro_in_library.rs:2:1:2:14 | fn foo_new |
+warnings