Discover NIFs at startup

This is a proof-of-concept for a mechanism that does away with the need
to list all NIF functions in the `rustler::init!` invocation. A similar
mechanism will be required and implemented for resource type
registrations.

- Convert `Nif` to a `struct`
- Use `inventory` to register the `Nif` instances
- In `rustler::init!`, ignore the passed functions and instead use the
  registered `Nif` instances

Next steps:

- See whether `linkme` is not a better option (creates the array
  directly at build-time, no need to leak)
- Implement resource registration using this mechanism
- See if we can move the macros to `rustler`

Author: filmor

rustler/Cargo.toml

@@ -22,6 +22,7 @@ nif_version_2_17 = ["nif_version_2_16", "rustler_sys/nif_version_2_17"]
 serde = ["dep:serde"]
 
 [dependencies]
+inventory = "0.3"
 rustler_codegen = { path = "../rustler_codegen", version = "0.32.1", optional = true}
 rustler_sys = { path = "../rustler_sys", version = "~2.4.0" }
 num-bigint = { version = "0.4", optional = true }

rustler/src/codegen_runtime.rs

@@ -5,6 +5,9 @@ use std::fmt;
 
 use crate::{Encoder, Env, OwnedBinary, Term};
 
+// Re-export of inventory
+pub use inventory;
+
 // Names used by the `rustler::init!` macro or other generated code.
 pub use crate::wrapper::exception::raise_exception;
 pub use crate::wrapper::{

rustler/src/nif.rs

@@ -1,13 +1,25 @@
 use crate::codegen_runtime::{c_char, c_int, c_uint, DEF_NIF_FUNC, NIF_ENV, NIF_TERM};
 
-pub trait Nif {
-    const NAME: *const c_char;
-    const ARITY: c_uint;
-    const FLAGS: c_uint;
-    const FUNC: DEF_NIF_FUNC;
-    const RAW_FUNC: unsafe extern "C" fn(
-        nif_env: NIF_ENV,
-        argc: c_int,
-        argv: *const NIF_TERM,
-    ) -> NIF_TERM;
+pub struct Nif {
+    pub name: *const c_char,
+    pub arity: c_uint,
+    pub flags: c_uint,
+    // pub func: DEF_NIF_FUNC,
+    pub raw_func:
+        unsafe extern "C" fn(nif_env: NIF_ENV, argc: c_int, argv: *const NIF_TERM) -> NIF_TERM,
 }
+
+impl Nif {
+    pub fn get_def(&self) -> DEF_NIF_FUNC {
+        DEF_NIF_FUNC {
+            arity: self.arity,
+            flags: self.flags,
+            function: self.raw_func,
+            name: self.name,
+        }
+    }
+}
+
+unsafe impl Sync for Nif {}
+
+inventory::collect!(Nif);

rustler_codegen/Cargo.toml

@@ -17,6 +17,7 @@ syn = { version = "2.0", features = ["full", "extra-traits"] }
 quote = "1.0"
 heck = "0.5"
 proc-macro2 = "1.0"
+inventory = "0.3"
 
 [dev-dependencies]
 trybuild = "1.0"

rustler_codegen/src/init.rs

@@ -1,26 +1,23 @@
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
 use syn::parse::{Parse, ParseStream};
-use syn::punctuated::Punctuated;
-use syn::token::Comma;
-use syn::{Expr, Ident, Result, Token};
+use syn::{Ident, Result, Token};
 
 #[derive(Debug)]
 pub struct InitMacroInput {
     name: syn::Lit,
-    funcs: syn::ExprArray,
     load: TokenStream,
 }
 
 impl Parse for InitMacroInput {
     fn parse(input: ParseStream) -> Result<Self> {
         let name = syn::Lit::parse(input)?;
         let _comma = <syn::Token![,]>::parse(input)?;
-        let funcs = syn::ExprArray::parse(input)?;
+        let _funcs = syn::ExprArray::parse(input)?;
         let options = parse_expr_assigns(input);
         let load = extract_option(options, "load");
 
-        Ok(InitMacroInput { name, funcs, load })
+        Ok(InitMacroInput { name, load })
     }
 }
 
@@ -55,20 +52,21 @@ fn extract_option(args: Vec<syn::ExprAssign>, name: &str) -> TokenStream {
 impl From<InitMacroInput> for proc_macro2::TokenStream {
     fn from(input: InitMacroInput) -> Self {
         let name = input.name;
-        let num_of_funcs = input.funcs.elems.len();
-        let funcs = nif_funcs(input.funcs.elems);
         let load = input.load;
 
         let inner = quote! {
             static mut NIF_ENTRY: Option<rustler::codegen_runtime::DEF_NIF_ENTRY> = None;
-            use rustler::Nif;
+            let nif_funcs: Box<[_]> =
+                rustler::codegen_runtime::inventory::iter::<rustler::Nif>()
+                .map(rustler::Nif::get_def)
+                .collect();
 
             let entry = rustler::codegen_runtime::DEF_NIF_ENTRY {
                 major: rustler::codegen_runtime::NIF_MAJOR_VERSION,
                 minor: rustler::codegen_runtime::NIF_MINOR_VERSION,
                 name: concat!(#name, "\0").as_ptr() as *const rustler::codegen_runtime::c_char,
-                num_of_funcs: #num_of_funcs as rustler::codegen_runtime::c_int,
-                funcs: [#funcs].as_ptr(),
+                num_of_funcs: nif_funcs.len() as rustler::codegen_runtime::c_int,
+                funcs: nif_funcs.as_ptr(),
                 load: {
                     extern "C" fn nif_load(
                         env: rustler::codegen_runtime::NIF_ENV,
@@ -91,6 +89,9 @@ impl From<InitMacroInput> for proc_macro2::TokenStream {
             };
 
             unsafe {
+                // Leak nif_funcs
+                std::mem::forget(nif_funcs);
+
                 NIF_ENTRY = Some(entry);
                 NIF_ENTRY.as_ref().unwrap()
             }
@@ -115,17 +116,3 @@ impl From<InitMacroInput> for proc_macro2::TokenStream {
         }
     }
 }
-
-fn nif_funcs(funcs: Punctuated<Expr, Comma>) -> TokenStream {
-    let mut tokens = TokenStream::new();
-
-    for func in funcs.iter() {
-        if let Expr::Path(_) = *func {
-            tokens.extend(quote!(#func::FUNC,));
-        } else {
-            panic!("Expected an expression, found: {}", stringify!(func));
-        }
-    }
-
-    tokens
-}

rustler_codegen/src/nif.rs

@@ -55,54 +55,47 @@ pub fn transcoder_decorator(nif_attributes: NifAttributes, fun: syn::ItemFn) ->
     }
 
     quote! {
-        #[allow(non_camel_case_types)]
-        pub struct #name;
-
-        impl rustler::Nif for #name {
-            const NAME: *const rustler::codegen_runtime::c_char = concat!(#erl_func_name, "\0").as_ptr() as *const rustler::codegen_runtime::c_char;
-            const ARITY: rustler::codegen_runtime::c_uint = #arity;
-            const FLAGS: rustler::codegen_runtime::c_uint = #flags as rustler::codegen_runtime::c_uint;
-            const RAW_FUNC: unsafe extern "C" fn(
-                nif_env: rustler::codegen_runtime::NIF_ENV,
-                argc: rustler::codegen_runtime::c_int,
-                argv: *const rustler::codegen_runtime::NIF_TERM
-            ) -> rustler::codegen_runtime::NIF_TERM = {
-                unsafe extern "C" fn nif_func(
-                    nif_env: rustler::codegen_runtime::NIF_ENV,
-                    argc: rustler::codegen_runtime::c_int,
-                    argv: *const rustler::codegen_runtime::NIF_TERM
-                ) -> rustler::codegen_runtime::NIF_TERM {
-                    let lifetime = ();
-                    let env = rustler::Env::new(&lifetime, nif_env);
-
-                    let terms = std::slice::from_raw_parts(argv, argc as usize)
-                        .iter()
-                        .map(|term| rustler::Term::new(env, *term))
-                        .collect::<Vec<rustler::Term>>();
-
-                    fn wrapper<'a>(
-                        env: rustler::Env<'a>,
-                        args: &[rustler::Term<'a>]
-                    ) -> rustler::codegen_runtime::NifReturned {
-                        let result: std::thread::Result<_> = std::panic::catch_unwind(move || {
-                            #decoded_terms
-                            #function
-                            Ok(#name(#argument_names))
-                        });
-
-                        rustler::codegen_runtime::handle_nif_result(result, env)
+        rustler::codegen_runtime::inventory::submit!(
+            rustler::Nif {
+                name: concat!(#erl_func_name, "\0").as_ptr()
+                    as *const rustler::codegen_runtime::c_char,
+                arity: #arity,
+                flags: #flags as rustler::codegen_runtime::c_uint,
+                raw_func: {
+                    unsafe extern "C" fn nif_func(
+                        nif_env: rustler::codegen_runtime::NIF_ENV,
+                        argc: rustler::codegen_runtime::c_int,
+                        argv: *const rustler::codegen_runtime::NIF_TERM
+                    ) -> rustler::codegen_runtime::NIF_TERM {
+                        let lifetime = ();
+                        let env = rustler::Env::new(&lifetime, nif_env);
+
+                        let terms = std::slice::from_raw_parts(argv, argc as usize)
+                            .iter()
+                            .map(|term| rustler::Term::new(env, *term))
+                            .collect::<Vec<rustler::Term>>();
+
+                        fn wrapper<'a>(
+                            env: rustler::Env<'a>,
+                            args: &[rustler::Term<'a>]
+                        ) -> rustler::codegen_runtime::NifReturned {
+                            let result: std::thread::Result<_> =
+                                std::panic::catch_unwind(move || {
+                                    #decoded_terms
+                                    #function
+                                    Ok(#name(#argument_names))
+                                });
+
+                            rustler::codegen_runtime::handle_nif_result(
+                                result, env
+                            )
+                        }
+                        wrapper(env, &terms).apply(env)
                     }
-                    wrapper(env, &terms).apply(env)
+                    nif_func
                 }
-                nif_func
-            };
-            const FUNC: rustler::codegen_runtime::DEF_NIF_FUNC = rustler::codegen_runtime::DEF_NIF_FUNC {
-                arity: Self::ARITY,
-                flags: Self::FLAGS,
-                function: Self::RAW_FUNC,
-                name: Self::NAME
-            };
-        }
+            }
+        );
     }
 }
 

rustler_sys/src/rustler_sys_api.rs

@@ -39,6 +39,7 @@ unsafe impl Send for ErlNifEnv {}
 
 /// See [ErlNifFunc](http://www.erlang.org/doc/man/erl_nif.html#ErlNifFunc) in the Erlang docs.
 // #[allow(missing_copy_implementations)]
+#[derive(Debug)]
 #[repr(C)]
 pub struct ErlNifFunc {
     pub name: *const c_char,

rustler_tests/native/rustler_serde_test/src/lib.rs

@@ -16,19 +16,19 @@ use rustler::{types::tuple, Encoder, Env, NifResult, SerdeTerm, Term};
 init! {
     "Elixir.SerdeRustlerTests",
     [
-        // json
-        json::decode_json,
-        json::decode_json_dirty,
-        json::encode_json_compact,
-        json::encode_json_compact_dirty,
-        json::encode_json_pretty,
-        json::encode_json_pretty_dirty,
-
-        // tests
-        readme,
-        test::test,
-        transcode,
-        transcode_dirty,
+        // // json
+        // json::decode_json,
+        // json::decode_json_dirty,
+        // json::encode_json_compact,
+        // json::encode_json_compact_dirty,
+        // json::encode_json_pretty,
+        // json::encode_json_pretty_dirty,
+        //
+        // // tests
+        // readme,
+        // test::test,
+        // transcode,
+        // transcode_dirty,
     ]
 }
 

rustler_tests/native/rustler_test/src/test_resource.rs

@@ -109,7 +109,7 @@ pub fn resource_make_binaries(
     )
 }
 
-#[rustler::nif]
+// #[rustler::nif]
 pub fn resource_make_binary_from_vec(env: Env, resource: ResourceArc<WithBinaries>) -> Binary {
     resource.make_binary(env, |w| &w.b)
 }