Cache macro imports between compiler runs.

This makes things a lot faster!

build.sh

@@ -64,7 +64,7 @@ cp -R src build/
 echo "Building stage 2..."
 FLAGS="$FLAGS -version=LLVMBackend"
 # see generation.md
-$NEAT $FLAGS -backend=llvm -macro-backend=c -macro-version=oldabi -Pcompiler:src -j src/main.nt \
+$NEAT $FLAGS -backend=llvm -macro-backend=c -Pcompiler:src -j src/main.nt \
     -o build/neat_stage2
 NEAT=build/neat_stage2
 

src/c/fcntl.nt

@@ -0,0 +1,16 @@
+module c.fcntl;
+
+extern(C) int open(char*, int flags, int mode);
+
+alias O_RDONLY = 0;
+alias O_WRONLY = 1;
+alias O_RDWR = 2;
+
+alias O_CREAT = 64;
+
+alias S_IWOTH = 2;
+alias S_IROTH = 4;
+alias S_IWGRP = 16;
+alias S_IRGRP = 32;
+alias S_IWUSR = 128;
+alias S_IRUSR = 256;

src/c/unistd.nt

@@ -1,4 +1,5 @@
 module c.unistd;
 
 extern(C) size_t read(int fd, void* buf, size_t count);
+extern(C) size_t write(int fd, void* buf, size_t count);
 extern(C) int close(int fd);

src/main.nt

@@ -252,7 +252,7 @@ class CompilationTask : ITask
 
     override string id() => id_;
     override bool fresh() => true;
-    override void run() {
+    override void run(WorkPoolBase workPool) {
         this.result = emitModule(
             this.module_, this.visitors, this.settings, this.fileIdTable, this.visitorLock, this.prehash);
     }
@@ -441,7 +441,7 @@ class ParserHelperImpl : ParserHelper
 
     this(this.settings, this.fileIdTable) { }
 
-    override (void | Error) proxyCall(ASTModuleBase astModule, string function_, void* ptr,
+    override (LibraryCallRecord | Error) proxyCall(ASTModuleBase astModule, string function_, void* ptr,
         LocRange locRange)
     {
         auto astModule = astModule.instanceOf(ASTModule)? else die;
@@ -463,6 +463,7 @@ class ParserHelperImpl : ParserHelper
             neat_runtime_system("mv $tmpname $soname");
         }
         neat_runtime_dlcall(soname, name, ptr);
+        return (soname=soname, fnname=name);
     }
 }
 

src/neat/base.nt

@@ -892,9 +892,11 @@ struct Parameter
         ~ "type=$type, defaultValue=..., locRange=$locRange)";
 }
 
+alias LibraryCallRecord = (string soname, string fnname);
+
 abstract class ParserHelper
 {
-    abstract (void | Error) proxyCall(ASTModuleBase astModule, string function_, void* ptr,
+    abstract (LibraryCallRecord | Error) proxyCall(ASTModuleBase astModule, string function_, void* ptr,
         LocRange locRange);
 }
 
@@ -1372,18 +1374,18 @@ class MacroState
 
     mut MacroState[] imports;
 
+    // A record of every dynamic library call done to
+    // get the macro state to where it currently is.
+    // This is used for macro caching.
+    mut LibraryCallRecord[] records;
+
     this() { }
 
     MacroState dup() {
         auto newState = new MacroState;
-        auto newMacros = new Macro mut[](this.macros.length);
-        for (i, macro_ in this.macros)
-            newMacros[i] = macro_;
-        newState.macros = newMacros.freeze;
-        auto newImports = new MacroState mut[](this.imports.length);
-        for (i, import_ in this.imports)
-            newImports[i] = import_;
-        newState.imports = newImports.freeze;
+        newState.macros = macros.dup.freeze;
+        newState.imports = imports.dup.freeze;
+        newState.records = records.dup.freeze;
         return newState;
     }
 
@@ -1779,11 +1781,23 @@ abstract class WorkPoolBase
     abstract ITask wait(string id);
     // if `provider` is not fresh, `consumer` is not fresh.
     abstract void dependency((string | :current) provider, (string | :current) consumer);
+    /**
+     * The bill-of-materials system allows validating that an artifact is still current
+     * without actually parsing anything. We attach bill items, which are key-value pairs,
+     * to the current task.
+     * Then we can query all bill items that are transitively part of a task.
+     * The point is that, for instance for a macro object, we can store the dynamic filename
+     * and bill of materials; then when we need to reload the macro, we just check that the
+     * BOM is still current.
+     * Though the API is kept generic, in practice `id` will be a filename and `value` a hash.
+     */
+    abstract void addBomItem(string id, string value);
+    abstract string[string] bom(string id);
 }
 
 interface ITask
 {
-    void run();
+    void run(WorkPoolBase workPool);
     string id();
     // Return false if environment conditions have changed from when the task was added.
     // For instance, if a file changed checksum or last-modified date.

src/neat/macrocache.nt

@@ -0,0 +1,94 @@
+/**
+ * Allow saving and loading macro state;
+ * or rather, the required information to reconstruct a macro state.
+ */
+module neat.macrocache;
+
+macro import package(compiler).std.macro.listcomprehension;
+
+import neat.base;
+import package(compiler).std.file;
+import package(compiler).std.json;
+import package(compiler).std.json.stream;
+import package(compiler).std.sha256;
+import package(compiler).std.stream;
+import package(compiler).std.string;
+import polyhash;
+
+private alias CompilerError = Error;
+
+(void | CompilerError) saveMacroCache(string importDesc, string[string] bom, LibraryCallRecord[] records) {
+    auto target = importDesc.targetFile;
+    auto tmp = importDesc.tmpFile;
+    auto file = fileSink(tmp).toCompilerError?;
+    mut (string key, JSONValue value)[] transformBom;
+    for (key in bom.keys) transformBom ~= (key, JSONValue(bom[key]));
+    auto dto = MacroCacheDto(
+        fileHashes=JSONValue(transformBom),
+        records=[RecordDto(soname=a.soname, fnname=a.fnname) for a in records],
+        compilerHash=compilerHashStr,
+    );
+    dto.encode(new JsonPrinter(file)).toCompilerError?;
+    file.close;
+    tmp.rename(target);
+}
+
+(string[string] bom, LibraryCallRecord[] records | :missing | CompilerError) loadMacroCache(string importDesc) {
+    auto target = importDesc.targetFile;
+    if (!target.exists) return :missing;
+    auto file = new FileSource(target);
+    auto source = new JsonLexer(file);
+    auto dto = source.decode!MacroCacheDto.toCompilerError?;
+    // cached from older compiler version
+    if (dto.compilerHash != compilerHashStr) return :missing;
+    mut string[string] bom;
+    for (value in dto.fileHashes.object) {
+        auto file = value.key;
+        // file removed
+        if (!file.exists) return :missing;
+        auto currentFile = file.readText;
+        auto digest = new Sha256;
+        digest.update(cast(ubyte[]) currentFile);
+        auto hexstr = digest.finalize.toHexString;
+        // hash changed
+        if (value.value.str != hexstr) return :missing;
+        // unchanged from cache.
+        bom[file] = value.value.str;
+    }
+    return (bom=bom, records=[(soname=a.soname, fnname=a.fnname) for a in dto.records]);
+}
+
+private auto toCompilerError(T)(T value) {
+    import package(compiler).std.error : Error;
+
+    return value.case(Error err: new CompilerError([err.range], err.message));
+}
+
+private string compilerHashStr() {
+    auto hash = new Hash;
+    hash.apply(compiler_hash_add, compiler_hash_mult);
+    return hash.text;
+}
+
+private string targetFile(string importDesc) => ".obj/macrocache_$(importDesc.filter)";
+
+private string tmpFile(string importDesc) => ".obj/.macrocache_$(importDesc.filter).tmp";
+
+private string filter(string desc) => desc.replace(" ", "_").replace("/", "_").replace("(", "_").replace(")", "_");
+
+struct MacroCacheDto {
+    // TODO
+    // string[string] fileHashes;
+    JSONValue fileHashes;
+    RecordDto[] records;
+    string compilerHash;
+
+}
+
+struct RecordDto {
+    string soname;
+    string fnname;
+}
+
+private extern(C) long compiler_hash_add();
+private extern(C) long compiler_hash_mult();

src/neat/stuff.nt

@@ -4408,7 +4408,7 @@ class AstCompileTask : ITask
         }
     }
 
-    override void run() {
+    override void run(WorkPoolBase workPool) {
         this.coldContext.modulePreProcessor.process(astModule);
 
         this.result = compileModule(
@@ -4483,15 +4483,51 @@ class ImportTask : ImportModuleBaseTask
         }
     }
 
-    override void run() {
-        this.lexicalContext.compiler.workPool
-            .dependency(provider=id_, consumer=moduleParseTaskId);
+    override void run(WorkPoolBase workPool) {
+        workPool.dependency(provider=id_, consumer=moduleParseTaskId);
+
+        void seterror(Error error) { this.result = error; }
+        version (firstpass) {}
+        else {
+            import neat.macrocache : loadMacroCache;
+            auto cached = loadMacroCache(id_).case(Error e: return seterror(e));
+            if let (auto cached = cached.case(:missing: breakelse)) {
+                // add "fake" bom from cache
+                auto bom = cached.bom;
+                for (key in bom.keys) {
+                    lexicalContext.compiler.workPool.addBomItem(key, bom[key]);
+                }
+
+                auto macroState = new MacroState;
+                for (entry in cached.records) {
+                    neat_runtime_dlcall(entry.soname, entry.fnname, macroState);
+                    macroState.records ~= entry;
+                }
+
+                // fake macro import module, will only be used for its macro state
+                // TODO: (ASTModule | MacroState | ...) result;
+                this.result = new ASTModule(name="", new Package(name="cachestub", "", []),
+                    path="", moduleParseTaskId="", macroState=macroState, locRange=__RANGE__,
+                    parent=null);
+                return;
+            }
+        }
         this.result = this.lexicalContext.resolveImport(import_).case(
             ASTModuleBase base: base.instanceOf(ASTModule)? else die,
             Error err: err);
+        version (firstpass) {}
+        else if (import_.isMacroImport) {
+            import neat.macrocache : saveMacroCache;
+            auto bom = lexicalContext.compiler.workPool.bom(id_);
+            if let (auto records = module_.case(Error: breakelse).macroState.records) {
+                saveMacroCache(id, bom, records).case(Error e: return seterror(e));
+            }
+        }
     }
 }
 
+extern(C) void neat_runtime_dlcall(string soname, string name, void* arg);
+
 (ASTImportStatement | :none | Error) parseImportStatement(Parser parser, LexicalContext lexicalContext)
 {
     parser.begin;
@@ -4571,8 +4607,7 @@ class ImportTask : ImportModuleBaseTask
     parser.expectToken(TokenType.semicolon)?;
 
     auto newMacroState = astModule.macroState.dup;
-
-    helper.proxyCall(astModule, identifier, newMacroState, parser.to(from))?;
+    newMacroState.records ~= helper.proxyCall(astModule, identifier, newMacroState, parser.to(from))?;
 
     auto subModule = new ASTModule(
         astModule.name, astModule.pak, astModule.path, astModule.moduleParseTaskId,
@@ -4653,8 +4688,7 @@ class LexicalContextImpl : LexicalContext
             return (:none);
         ASTImportStatement importStmt:
             parser.commit;
-            if (!toplevel && importStmt.isMacroImport)
-            {
+            if (!toplevel && importStmt.isMacroImport) {
                 return parser.fail("macro import not allowed here");
             }
 
@@ -4663,6 +4697,7 @@ class LexicalContextImpl : LexicalContext
             auto task = new ImportTask(id, lexicalContext, importStmt, lexicalContext.moduleParseTaskId);
             auto import_ = new ASTImport(importStmt, (pool, task));
 
+            // If not toplevel, the import may be conditional.
             if (toplevel) pool.queue(task);
 
             return (protection, import_);
@@ -4757,16 +4792,15 @@ class LexicalContextImpl : LexicalContext
                 continue;
             }
             auto id = import_.statement.repr(lexicalContext.pak.name);
+            auto task = lexicalContext.compiler.workPool.wait(id).instanceOf(ImportTask)? else die;
 
             module_.macroState = module_.macroState.dup;
             lexicalContext = new LexicalContextImpl(
                 compiler, module_.pak, moduleParseTaskId=module_.moduleParseTaskId,
                 module_.macroState, helper);
 
             lexicalContext.compiler.workPool.dependency(provider=id, consumer=:current);
-            lexicalContext.macroState.addImport(
-                lexicalContext.compiler.workPool.wait(id)
-                    .instanceOf(ImportTask).module_?.macroState);
+            lexicalContext.macroState.addImport(task.module_?.macroState);
         }
         importGroup = null;
     }
@@ -4989,7 +5023,7 @@ class ParseAstModuleTask : ITask
     }
     override string id() => "parse ast $path";
     override bool fresh() => true;
-    override void run() {
+    override void run(WorkPoolBase workPool) {
         this.result = parse;
     }
     (ASTModule | Error) parse() {