reimplement external intrinsic idea to make it embeded into final transpiled source code

Author: Viktor Yastrebov

lib/core/intrinsics/external_intrinsics.cpp

@@ -3,6 +3,7 @@
 #include <clang/Frontend/CompilerInstance.h>
 #include "core/transpiler_session/session_stage.h"
 #include "core/transpiler_session/transpiler_session.h"
+#include "oklt/util/io_helper.h"
 #include "util/string_utils.hpp"
 
 #include <algorithm>
@@ -73,10 +74,9 @@ std::optional<fs::path> getExternalInstrincisInclude(TranspilerSession& session,
     return std::nullopt;
 }
 
-tl::expected<std::unique_ptr<llvm::MemoryBuffer>, std::string> getExternalIntrinsicSource(
-    TargetBackend backend,
-    const fs::path& intrinsicPath,
-    clang::SourceManager& sm) {
+tl::expected<std::string, std::string> getExternalIntrinsicSource(TargetBackend backend,
+                                                                  const fs::path& intrinsicPath,
+                                                                  clang::SourceManager& sm) {
     auto implPathResult = getIntrincisImplSourcePath(backend, intrinsicPath);
     if (!implPathResult) {
         return tl::make_unexpected(implPathResult.error());
@@ -93,119 +93,85 @@ tl::expected<std::unique_ptr<llvm::MemoryBuffer>, std::string> getExternalIntrin
     }
 
     auto it = std::find_if(files.cbegin(), files.cend(), [](const fs::path& p) -> bool {
-        return p.extension().string() == std::string(".h");
+        return p.extension().string() == std::string(".cpp");
     });
 
     if (it == files.cend()) {
         std::string error = "Can't' find implementation file with path: " + sourceFolder.string();
         return tl::make_unexpected(error);
     }
 
-    auto& fm = sm.getFileManager();
-    auto backendFilePath = it->string();
-    auto maybeReplacedFile = fm.getFile(backendFilePath);
-    if (!maybeReplacedFile) {
-        std::string error = "Can't open file: " + backendFilePath;
+    auto contentResult = util::readFileAsStr(*it);
+    if (!contentResult) {
+        std::string error = "Can't get memory buffer for: " + it->string();
         return tl::make_unexpected(error);
     }
-    auto maybeBuffer = fm.getBufferForFile(maybeReplacedFile.get());
-    if (!maybeBuffer) {
-        std::string error = "Can't get memory buffer for: " + backendFilePath;
-        return tl::make_unexpected(error);
-    }
-    return std::move(maybeBuffer.get());
+    return contentResult.value();
 }
 
-bool overrideExternalIntrinsic(TranspilerSession& session,
+bool overrideExternalIntrinsic(SessionStage& stage,
+                               HeaderDepsInfo& deps,
                                const std::string& includedFileName,
-                               clang::OptionalFileEntryRef includedFile,
-                               clang::SourceManager& sourceManager) {
+                               clang::OptionalFileEntryRef includedFile) {
+    auto& session = stage.getSession();
+    auto& sourceManager = stage.getCompiler().getSourceManager();
     const auto& userIntrinsics = session.getInput().userIntrinsics;
     if (!userIntrinsics.empty()) {
         auto maybeIntrinsicPath = getExternalInstrincisInclude(session, includedFileName);
         if (!maybeIntrinsicPath) {
             return false;
         }
         auto intrinsicPath = maybeIntrinsicPath.value();
-        auto infoResult =
-            getExternalIntrinsicSource(session.getInput().backend, intrinsicPath, sourceManager);
-        if (!infoResult) {
-            session.pushError(std::error_code(), infoResult.error());
+        auto intrinsicResult =
+            getExternalIntrinsicSource(stage.getBackend(), intrinsicPath, sourceManager);
+        if (!intrinsicResult) {
+            session.pushError(std::error_code(), intrinsicResult.error());
             return false;
         }
-        auto buffer = std::move(infoResult.value());
-        auto& fm = sourceManager.getFileManager();
+        deps.externalIntrinsicsSources[includedFileName] = std::move(intrinsicResult.value());
+
+        auto emptyExternalIntrinsic = MemoryBuffer::getMemBuffer("");
         if (includedFile) {
             auto fileRef = includedFile;
             const auto& fileEntry = fileRef->getFileEntry();
-            sourceManager.overrideFileContents(&fileEntry, std::move(buffer));
+            sourceManager.overrideFileContents(&fileEntry, std::move(emptyExternalIntrinsic));
         } else {
             // INFO: case when the file can be found by relative path
             //       it happens when the include path is relative to WORKING DIR path
+            auto& fm = sourceManager.getFileManager();
             auto maybeFileRef = fm.getFileRef(includedFileName);
             if (maybeFileRef) {
                 auto foundFileRef = maybeFileRef.get();
-                sourceManager.overrideFileContents(foundFileRef, std::move(buffer));
+                sourceManager.overrideFileContents(foundFileRef, std::move(emptyExternalIntrinsic));
             }
         }
         return true;
     }
     return false;
 }
 
-void nullyLauncherExternalIntrinsics(TransformedFiles& inputs, SessionStage& stage) {
-    if (stage.getBackend() != TargetBackend::_LAUNCHER) {
-        return;
-    }
-
-    auto& session = stage.getSession();
-    const auto& intrinsics = session.getInput().userIntrinsics;
-    if (intrinsics.empty()) {
+void updateExternalIntrinsicMap(SessionStage& stage, HeaderDepsInfo& deps) {
+    if (deps.externalIntrinsicsSources.empty()) {
         return;
     }
 
-    for (auto& mappedFile : inputs.fileMap) {
-        auto fileName = normalizedFileName(mappedFile.first);
-        for (const auto& includePath : intrinsics) {
-            auto folderPrefix = includePath.filename().string();
-            if (util::startsWith(fileName, folderPrefix)) {
-                mappedFile.second.clear();
-            }
-        }
-    }
-}
-
-void embedLauncherExternalIntrinsics(std::string& input,
-                                     const HeaderDepsInfo& info,
-                                     SessionStage& stage) {
     auto backend = stage.getBackend();
-    if (backend != TargetBackend::_LAUNCHER) {
-        return;
-    }
-
-    const auto& usedIntrinsics = info.externalIntrinsics;
-    if (usedIntrinsics.empty()) {
-        return;
-    }
-
-    auto session = stage.getSession();
+    auto& session = stage.getSession();
     auto& sm = stage.getCompiler().getSourceManager();
-    for (const auto& includeIntrinsic : usedIntrinsics) {
-        auto maybeIntrinsicPath = getExternalInstrincisInclude(session, includeIntrinsic);
+    for (auto& mappedIntrinsic : deps.externalIntrinsicsSources) {
+        auto maybeIntrinsicPath = getExternalInstrincisInclude(session, mappedIntrinsic.first);
         if (!maybeIntrinsicPath) {
-            std::string error = "Count not find implementation for " + includeIntrinsic;
+            std::string error = "Count not find implementation for " + mappedIntrinsic.first;
             session.pushError(std::error_code(), error);
             return;
         }
         auto intrinsicPath = maybeIntrinsicPath.value();
-        auto infoResult = getExternalIntrinsicSource(backend, intrinsicPath, sm);
-        if (!infoResult) {
-            session.pushError(std::error_code(), infoResult.error());
+        auto intrinsicResult = getExternalIntrinsicSource(backend, intrinsicPath, sm);
+        if (!intrinsicResult) {
+            session.pushError(std::error_code(), intrinsicResult.error());
             return;
         }
-        auto buffer = std::move(infoResult.value());
-        // INFO: make temporary buffer because pointer could be not null terminated
-        input.insert(0, buffer->getBuffer().str());
+        mappedIntrinsic.second = std::move(intrinsicResult.value());
     }
 }
 }  // namespace oklt

lib/core/intrinsics/external_intrinsics.h

@@ -2,27 +2,15 @@
 #include <clang/Basic/FileEntry.h>
 #include <string>
 
-namespace clang {
-class CompilerInstance;
-class SourceManager;
-}  // namespace clang
-
 namespace oklt {
 
-class TranspilerSession;
-class TransformedFiles;
 class SessionStage;
 class HeaderDepsInfo;
 
-bool overrideExternalIntrinsic(TranspilerSession& session,
+bool overrideExternalIntrinsic(SessionStage& stage,
+                               HeaderDepsInfo& deps,
                                const std::string& includedFileName,
-                               clang::OptionalFileEntryRef includedFile,
-                               clang::SourceManager& sourceManager);
-
-void nullyLauncherExternalIntrinsics(TransformedFiles& inputs, SessionStage& stage);
-
-void embedLauncherExternalIntrinsics(std::string& input,
-                                     const HeaderDepsInfo& info,
-                                     SessionStage& stage);
+                               clang::OptionalFileEntryRef includedFile);
 
+void updateExternalIntrinsicMap(SessionStage& stage, HeaderDepsInfo& deps);
 }  // namespace oklt

lib/core/transpiler_session/code_generator.cpp

@@ -7,11 +7,11 @@
 #include "core/transpiler_session/transpiler_session.h"
 
 #include "core/intrinsics/builtin_intrinsics.h"
-#include "core/intrinsics/external_intrinsics.h"
+// #include "core/intrinsics/external_intrinsics.h"
 
 #include "core/handler_manager/handler_manager.h"
 
-#include "core/utils/attributes.h"
+// #include "core/utils/attributes.h"
 #include "core/vfs/overlay_fs.h"
 
 #include <clang/AST/Attr.h>
@@ -86,10 +86,8 @@ void removeSystemHeaders(SessionStage& stage, const HeaderDepsInfo& deps) {
         rewriter.RemoveText({dep.hashLoc, dep.filenameRange.getEnd()});
     }
 
-    for (const auto& intrinsicDep : deps.externalIntrinsicDeps) {
-        SPDLOG_TRACE(
-            "remove system include {} {}", intrinsicDep.relativePath, intrinsicDep.fileName);
-        rewriter.RemoveText({intrinsicDep.hashLoc, intrinsicDep.filenameRange.getEnd()});
+    for (const auto& intrinsic : deps.externalIntrinsicHeaders) {
+        rewriter.RemoveText({intrinsic.hashLoc, intrinsic.filenameRange.getEnd()});
     }
 }
 
@@ -169,18 +167,14 @@ std::string restoreSystemAndBackendHeaders(SessionStage& stage,
         }
     }
 
-    embedLauncherExternalIntrinsics(input, deps, stage);
+    for (const auto& externalIntrinsicSource : deps.externalIntrinsicsSources) {
+        input.insert(0, externalIntrinsicSource.second);
+    }
 
     for (auto it = deps.backendHeaders.rbegin(); it < deps.backendHeaders.rend(); ++it) {
         input.insert(0, *it);
     }
 
-    if (backend != TargetBackend::_LAUNCHER) {
-        for (const auto& intrinsicDep : deps.externalIntrinsicDeps) {
-            input.insert(0, "#include <" + intrinsicDep.fileName + ">\n");
-        }
-    }
-
     // restore system headers
     for (const auto& dep : deps.topLevelDeps) {
         if (!clang::SrcMgr::isSystem(dep.fileType)) {
@@ -197,8 +191,6 @@ tl::expected<std::string, Error> fuseIncludeDeps(SessionStage& stage, const Head
 
     auto inputs = gatherTransformedFiles(stage);
 
-    nullyLauncherExternalIntrinsics(inputs, stage);
-
     auto preprocessedResult = preprocesseInputs(stage, inputs);
     if (!preprocessedResult) {
         return preprocessedResult;

lib/core/transpiler_session/header_info.cpp

@@ -1,20 +1,19 @@
 #include "core/transpiler_session/header_info.h"
 #include "core/intrinsics/builtin_intrinsics.h"
 #include "core/intrinsics/external_intrinsics.h"
-#include "core/transpiler_session/transpiler_session.h"
+#include "core/transpiler_session/session_stage.h"
 
 namespace {}
 namespace oklt {
 
 using namespace llvm;
 
-InclusionDirectiveCallback::InclusionDirectiveCallback(TranspilerSession& session,
+InclusionDirectiveCallback::InclusionDirectiveCallback(SessionStage& stage,
                                                        HeaderDepsInfo& deps_,
                                                        clang::SourceManager& sm_)
     : deps(deps_),
       sm(sm_),
-      _session(session),
-      _isInExternalIntrinsic(false) {}
+      _stage(stage) {}
 
 void InclusionDirectiveCallback::InclusionDirective(clang::SourceLocation hashLoc,
                                                     const clang::Token& includeTok,
@@ -36,67 +35,25 @@ void InclusionDirectiveCallback::InclusionDirective(clang::SourceLocation hashLo
     }
 
     auto fileNameStr = fileName.str();
-    auto isIntrinsic = overrideExternalIntrinsic(_session, fileNameStr, file, sm);
-    if (isIntrinsic) {
-        deps.externalIntrinsics.push_back(fileNameStr);
-        _extIntrinsicFID = sm.getFileID(hashLoc);
-    }
+    bool isIntrinsic = overrideExternalIntrinsic(_stage, deps, fileNameStr, file);
+
+    auto dep = HeaderDep{
+        .hashLoc = hashLoc,
+        .includeTok = includeTok,
+        .fileName = fileNameStr,
+        .isAngled = isAngled,
+        .filenameRange = filenameRange,
+        .file = file,
+        .searchPath = searchPath.str(),
+        .relativePath = relativePath.str(),
+        .imported = imported,
+        .fileType = fileType,
+    };
 
-    // INFO: force instrinsic includes to be system includes
-    //       they will be removed & attached to final transpiled source
-    if (_isInExternalIntrinsic) {
-        fileType = clang::SrcMgr::CharacteristicKind::C_System;
-        deps.externalIntrinsicDeps.push_back(HeaderDep{
-            .hashLoc = hashLoc,
-            .includeTok = includeTok,
-            .fileName = fileNameStr,
-            .isAngled = isAngled,
-            .filenameRange = filenameRange,
-            .file = file,
-            .searchPath = searchPath.str(),
-            .relativePath = relativePath.str(),
-            .imported = imported,
-            .fileType = fileType,
-        });
+    if (isIntrinsic) {
+        deps.externalIntrinsicHeaders.push_back(std::move(dep));
     } else {
-        deps.topLevelDeps.push_back(HeaderDep{
-            .hashLoc = hashLoc,
-            .includeTok = includeTok,
-            .fileName = fileNameStr,
-            .isAngled = isAngled,
-            .filenameRange = filenameRange,
-            .file = file,
-            .searchPath = searchPath.str(),
-            .relativePath = relativePath.str(),
-            .imported = imported,
-            .fileType = fileType,
-        });
+        deps.topLevelDeps.push_back(std::move(dep));
     }
 }
-
-void InclusionDirectiveCallback::FileChanged(clang::SourceLocation Loc,
-                                             FileChangeReason Reason,
-                                             clang::SrcMgr::CharacteristicKind FileType,
-                                             clang::FileID PrevFID) {
-    if (Reason == FileChangeReason::EnterFile) {
-        if (_extIntrinsicFID.isValid()) {
-            _isInExternalIntrinsic = PrevFID == _extIntrinsicFID;
-            return;
-        }
-    }
-
-    if (Reason == FileChangeReason::ExitFile) {
-        auto thisFID = sm.getFileID(Loc);
-        if (_extIntrinsicFID.isValid() && thisFID == _extIntrinsicFID) {
-            _extIntrinsicFID = clang::FileID();
-            _isInExternalIntrinsic = false;
-            return;
-        }
-    }
-}
-
-bool InclusionDirectiveCallback::FileNotFound(clang::StringRef FileName) {
-    return _isInExternalIntrinsic;
-}
-
 }  // namespace oklt