Merge pull request #98 from xch-dev/extern-import

Extern function import from hex files

Author: Rigidity

crates/rue-ast/src/lib.rs

@@ -232,6 +232,13 @@ impl AstFunctionItem {
         self.syntax().children().find_map(AstType::cast)
     }
 
+    pub fn source_path(&self) -> Option<SyntaxToken> {
+        self.syntax()
+            .children_with_tokens()
+            .filter_map(SyntaxElement::into_token)
+            .find(|token| token.kind() == SyntaxKind::String)
+    }
+
     pub fn body(&self) -> Option<AstBlock> {
         self.syntax().children().find_map(AstBlock::cast)
     }

crates/rue-compiler/src/compile/item/function.rs

@@ -1,13 +1,19 @@
+use std::{fs, path::Path};
+
+use clvmr::{Allocator, serde::node_from_bytes};
 use indexmap::IndexMap;
 use log::debug;
 use rue_ast::{AstFunctionItem, AstNode};
-use rue_diagnostic::DiagnosticKind;
-use rue_hir::{Declaration, FunctionKind, FunctionSymbol, ParameterSymbol, Symbol, SymbolId, Test};
+use rue_diagnostic::{DiagnosticKind, SourceKind};
+use rue_hir::{
+    Declaration, FunctionKind, FunctionSymbol, HirId, ParameterSymbol, Symbol, SymbolId, Test,
+};
+use rue_parser::SyntaxToken;
 use rue_types::{FunctionType, Type};
 
 use crate::{
     Compiler, CompletionContext, SyntaxItemKind, compile_block, compile_generic_parameters,
-    compile_type, create_binding,
+    compile_type, const_eval::decode_node, create_binding,
 };
 
 pub fn declare_function(ctx: &mut Compiler, function: &AstFunctionItem) -> SymbolId {
@@ -102,6 +108,12 @@ pub fn declare_function(ctx: &mut Compiler, function: &AstFunctionItem) -> Symbo
         ret: return_type,
     }));
 
+    if function.source_path().is_some() {
+        for &parameter in parameters.values() {
+            ctx.reference(Declaration::Symbol(parameter), None);
+        }
+    }
+
     let name = function.name().map(|name| ctx.local_name(&name));
 
     *ctx.symbol_mut(symbol) = Symbol::Function(FunctionSymbol {
@@ -115,6 +127,8 @@ pub fn declare_function(ctx: &mut Compiler, function: &AstFunctionItem) -> Symbo
         body,
         kind: if function.inline().is_some() {
             FunctionKind::Inline
+        } else if function.source_path().is_some() {
+            FunctionKind::External
         } else if function.extern_kw().is_some() {
             FunctionKind::Sequential
         } else {
@@ -172,7 +186,9 @@ pub fn compile_function(ctx: &mut Compiler, function: &AstFunctionItem, symbol:
         ctx.pop_declaration();
     }
 
-    let resolved_body = if let Some(body) = function.body() {
+    let resolved_body = if let Some(source_path) = function.source_path() {
+        compile_external_function(ctx, &source_path)
+    } else if let Some(body) = function.body() {
         let value = compile_block(
             ctx,
             &body,
@@ -181,10 +197,10 @@ pub fn compile_function(ctx: &mut Compiler, function: &AstFunctionItem, symbol:
             function.return_type().is_some(),
         );
         ctx.assign_type(body.syntax(), value.ty, return_type);
-        value
+        value.hir
     } else {
         debug!("Unresolved function body");
-        ctx.builtins().unresolved.clone()
+        ctx.builtins().unresolved.hir
     };
 
     ctx.pop_scope(range.end());
@@ -193,7 +209,97 @@ pub fn compile_function(ctx: &mut Compiler, function: &AstFunctionItem, symbol:
         unreachable!();
     };
 
-    *body = resolved_body.hir;
+    *body = resolved_body;
 
     ctx.pop_declaration();
 }
+
+fn compile_external_function(ctx: &mut Compiler, source_path: &SyntaxToken) -> HirId {
+    let unresolved = ctx.builtins().unresolved.hir;
+    let path_text = source_path
+        .text()
+        .strip_prefix('"')
+        .and_then(|path| path.strip_suffix('"'))
+        .unwrap_or(source_path.text());
+    let relative_path = Path::new(path_text);
+
+    if relative_path.is_absolute() {
+        ctx.diagnostic(source_path, DiagnosticKind::AbsoluteExternalPath);
+        return unresolved;
+    }
+
+    if relative_path
+        .extension()
+        .is_none_or(|extension| extension != "hex")
+    {
+        ctx.diagnostic(source_path, DiagnosticKind::InvalidExternalExtension);
+        return unresolved;
+    }
+
+    let SourceKind::File(source_file) = &ctx.source().kind else {
+        ctx.diagnostic(source_path, DiagnosticKind::ExternalFromNonFileSource);
+        return unresolved;
+    };
+    let Some(parent) = Path::new(source_file).parent() else {
+        ctx.diagnostic(source_path, DiagnosticKind::ExternalFromNonFileSource);
+        return unresolved;
+    };
+    let unresolved_path = parent.join(relative_path);
+    let resolved_path = match unresolved_path.canonicalize() {
+        Ok(path) => path,
+        Err(error) => {
+            ctx.diagnostic(
+                source_path,
+                DiagnosticKind::ExternalFileRead(format!("{path_text}: {error}")),
+            );
+            return unresolved;
+        }
+    };
+
+    let (bytes, should_cache) = if let Some(bytes) = ctx.external_program(&resolved_path) {
+        (bytes.to_vec(), false)
+    } else {
+        let contents = match fs::read_to_string(&resolved_path) {
+            Ok(contents) => contents,
+            Err(error) => {
+                ctx.diagnostic(
+                    source_path,
+                    DiagnosticKind::ExternalFileRead(format!("{path_text}: {error}")),
+                );
+                return unresolved;
+            }
+        };
+        let hex = contents
+            .chars()
+            .filter(|character| !character.is_ascii_whitespace())
+            .collect::<String>();
+        let bytes = match hex::decode(hex) {
+            Ok(bytes) => bytes,
+            Err(error) => {
+                ctx.diagnostic(
+                    source_path,
+                    DiagnosticKind::InvalidExternalHex(error.to_string()),
+                );
+                return unresolved;
+            }
+        };
+        (bytes, true)
+    };
+
+    let mut allocator = Allocator::new();
+    let program = match node_from_bytes(&mut allocator, &bytes) {
+        Ok(program) => program,
+        Err(error) => {
+            ctx.diagnostic(
+                source_path,
+                DiagnosticKind::InvalidExternalClvm(error.to_string()),
+            );
+            return unresolved;
+        }
+    };
+    if should_cache {
+        ctx.cache_external_program(resolved_path, bytes);
+    }
+    let hir = decode_node(ctx, &allocator, program);
+    ctx.alloc_hir(hir)
+}

crates/rue-compiler/src/compiler.rs

@@ -3,6 +3,7 @@ use std::{
     collections::{HashMap, HashSet},
     mem,
     ops::{Deref, DerefMut, Range},
+    path::{Path, PathBuf},
     sync::Arc,
 };
 
@@ -34,6 +35,7 @@ pub struct Compiler {
     defaults: HashMap<TypeId, HashMap<String, Value>>,
     declaration_stack: Vec<Declaration>,
     registered_scopes: HashSet<ScopeId>,
+    external_programs: HashMap<PathBuf, Vec<u8>>,
 }
 
 impl Deref for Compiler {
@@ -68,6 +70,7 @@ impl Compiler {
             defaults: HashMap::new(),
             declaration_stack: Vec::new(),
             registered_scopes: HashSet::new(),
+            external_programs: HashMap::new(),
         };
 
         if options.std {
@@ -112,6 +115,14 @@ impl Compiler {
         &self.options
     }
 
+    pub(crate) fn external_program(&self, path: &Path) -> Option<&[u8]> {
+        self.external_programs.get(path).map(Vec::as_slice)
+    }
+
+    pub(crate) fn cache_external_program(&mut self, path: PathBuf, program: Vec<u8>) {
+        self.external_programs.insert(path, program);
+    }
+
     pub fn set_source(&mut self, source: Source) {
         self.source = source;
     }

crates/rue-compiler/src/const_eval.rs

@@ -139,7 +139,7 @@ fn evaluate_const_expr(ctx: &mut Compiler, value: HirId) -> Result<Hir, ConstEva
     Ok(decode_node(ctx, &allocator, output.1))
 }
 
-fn decode_node(ctx: &mut Compiler, allocator: &Allocator, node: NodePtr) -> Hir {
+pub(crate) fn decode_node(ctx: &mut Compiler, allocator: &Allocator, node: NodePtr) -> Hir {
     match allocator.sexp(node) {
         SExp::Atom => Hir::Bytes(allocator.atom(node).to_vec()),
         SExp::Pair(first, rest) => {

crates/rue-diagnostic/src/kind.rs

@@ -28,6 +28,24 @@ pub enum DiagnosticKind {
     #[error("Missing function body")]
     MissingFunctionBody,
 
+    #[error("External function paths must be relative")]
+    AbsoluteExternalPath,
+
+    #[error("External function path must have a `.hex` extension")]
+    InvalidExternalExtension,
+
+    #[error("External functions cannot be loaded from this source")]
+    ExternalFromNonFileSource,
+
+    #[error("Failed to read external function: {0}")]
+    ExternalFileRead(String),
+
+    #[error("External function contains invalid hex: {0}")]
+    InvalidExternalHex(String),
+
+    #[error("External function contains invalid CLVM: {0}")]
+    InvalidExternalClvm(String),
+
     #[error("Duplicate symbol `{0}` found in scope")]
     DuplicateSymbol(String),
 
@@ -257,6 +275,12 @@ impl DiagnosticKind {
             | Self::UnterminatedBinary
             | Self::UnterminatedOctal
             | Self::MissingFunctionBody
+            | Self::AbsoluteExternalPath
+            | Self::InvalidExternalExtension
+            | Self::ExternalFromNonFileSource
+            | Self::ExternalFileRead(..)
+            | Self::InvalidExternalHex(..)
+            | Self::InvalidExternalClvm(..)
             | Self::DuplicateSymbol(..)
             | Self::DuplicateType(..)
             | Self::UndeclaredSymbol(..)

crates/rue-hir/src/lower.rs

@@ -135,6 +135,11 @@ impl<'d, 'a, 'g> Lowerer<'d, 'a, 'g> {
 
         let mut expr = self.lower_hir(&function_env, function.body);
 
+        if symbol == self.main && function.kind == FunctionKind::External {
+            let entire_env = self.arena.alloc(Lir::Path(1));
+            return self.arena.alloc(Lir::Run(expr, entire_env));
+        }
+
         if symbol == self.main {
             let mut map = HashMap::new();
 
@@ -171,6 +176,8 @@ impl<'d, 'a, 'g> Lowerer<'d, 'a, 'g> {
                 expr = self.arena.alloc(Lir::Run(expr, group_env));
             }
 
+            expr
+        } else if function.kind == FunctionKind::External {
             expr
         } else {
             self.arena.alloc(Lir::Quote(expr))
@@ -485,7 +492,9 @@ impl<'d, 'a, 'g> Lowerer<'d, 'a, 'g> {
             self.lower_symbol_reference(env, symbol)
         };
 
-        if let Symbol::Function(function) = self.db.symbol(symbol).clone() {
+        if let Symbol::Function(function) = self.db.symbol(symbol).clone()
+            && function.kind != FunctionKind::External
+        {
             let captures: Vec<SymbolId> = self
                 .graph
                 .dependencies(symbol, true)
@@ -902,6 +911,10 @@ impl<'d, 'a, 'g> Lowerer<'d, 'a, 'g> {
                 false
             }
             Symbol::Function(function) => {
+                if function.kind == FunctionKind::External {
+                    return false;
+                }
+
                 if self.graph.dependencies(symbol, false).contains(&symbol) {
                     return false;
                 }

crates/rue-hir/src/symbol.rs

@@ -65,6 +65,7 @@ pub struct FunctionSymbol {
 pub enum FunctionKind {
     BinaryTree,
     Sequential,
+    External,
     Inline,
 }
 

crates/rue-lexer/src/lexer.rs

@@ -55,6 +55,7 @@ impl<'a> Lexer<'a> {
             "import" => TokenKind::Import,
             "export" => TokenKind::Export,
             "extern" => TokenKind::Extern,
+            "from" => TokenKind::From,
             "inline" => TokenKind::Inline,
             "test" => TokenKind::Test,
             "mod" => TokenKind::Mod,
@@ -553,6 +554,13 @@ mod tests {
             "#]],
         );
 
+        check(
+            "from",
+            expect![[r#"
+                From
+            "#]],
+        );
+
         check(
             "inline",
             expect![[r#"

crates/rue-lexer/src/token_kind.rs

@@ -19,6 +19,7 @@ pub enum TokenKind {
     Import,
     Export,
     Extern,
+    From,
     Inline,
     Test,
     Mod,

crates/rue-lir/src/codegen.rs

@@ -48,6 +48,13 @@ fn codegen_impl(
             Ok(allocator.new_atom(&atom)?)
         }
         Lir::Run(callee, env) => {
+            if options.optimize_static_pairs
+                && matches!(arena[*env], Lir::Path(1))
+                && is_static_value(arena, *callee)
+            {
+                return codegen_static_value(arena, allocator, *callee);
+            }
+
             let callee = codegen(arena, allocator, *callee)?;
             let env = codegen(arena, allocator, *env)?;
             Ok(clvm_list!(ClvmOp::Apply, callee, env).to_clvm(allocator)?)
@@ -526,6 +533,28 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_run_static_program_with_entire_env() {
+        let mut arena = Arena::new();
+        let nil = arena.alloc(Lir::Atom(Vec::new()));
+        let five = arena.alloc(Lir::Atom(vec![5]));
+        let args = arena.alloc(Lir::Cons(five, nil));
+        let two = arena.alloc(Lir::Atom(vec![2]));
+        let args = arena.alloc(Lir::Cons(two, args));
+        let add = arena.alloc(Lir::Atom(vec![16]));
+        let program = arena.alloc(Lir::Cons(add, args));
+        let env = arena.alloc(Lir::Path(1));
+        let lir = arena.alloc(Lir::Run(program, env));
+
+        check_with_options(
+            &arena,
+            lir,
+            false,
+            expect!["(a (c (q . 16) (c (q . 2) (c (q . 5) ()))) 1)"],
+        );
+        check_with_options(&arena, lir, true, expect!["(+ 2 5)"]);
+    }
+
     #[test]
     fn test_closure() {
         let mut arena = Arena::new();