Create FileGenerator with CustomGenerator (#213)

- FileGenerator created using CustomGenerator
- Generator removed and renamed to FileGenerator

Author: coder137

bootstrap/include/bootstrap/build_buildcc.h

@@ -28,9 +28,9 @@
 
 namespace buildcc {
 
-void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe);
+void schema_gen_cb(FileGenerator &generator, const BaseTarget &flatc_exe);
 
-void buildcc_cb(BaseTarget &target, const BaseGenerator &schema_gen,
+void buildcc_cb(BaseTarget &target, const FileGenerator &schema_gen,
                 const TargetInfo &flatbuffers_ho, const TargetInfo &fmt_ho,
                 const TargetInfo &spdlog_ho, const TargetInfo &cli11_ho,
                 const TargetInfo &taskflow_ho, const BaseTarget &tpl);
@@ -80,8 +80,8 @@ class BuildBuildCC {
   ExecutableTarget_generic &GetFlatc() {
     return storage_.Ref<ExecutableTarget_generic>(kFlatcExeName);
   }
-  BaseGenerator &GetSchemaGen() {
-    return storage_.Ref<BaseGenerator>(kSchemaGenName);
+  FileGenerator &GetSchemaGen() {
+    return storage_.Ref<FileGenerator>(kSchemaGenName);
   }
   TargetInfo &GetFlatbuffersHo() {
     return storage_.Ref<TargetInfo>(kFlatbuffersHoName);

bootstrap/main.buildcc.cpp

@@ -49,7 +49,7 @@ int main(int argc, char **argv) {
   ExecutableTarget_generic buildcc_hybrid_simple_example(
       "buildcc_hybrid_simple_example", toolchain, "example/hybrid/simple");
   Reg::Toolchain(custom_toolchain_arg.state)
-      .Func([&]() { toolchain.Verify(); })
+      //   .Func([&]() { toolchain.Verify(); })
       .BuildPackage(buildcc)
       .Build(hybrid_simple_example_cb, buildcc_hybrid_simple_example,
              buildcc_lib)

bootstrap/src/build_buildcc.cpp

@@ -18,16 +18,14 @@
 
 namespace buildcc {
 
-void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe) {
+void schema_gen_cb(FileGenerator &generator, const BaseTarget &flatc_exe) {
   generator.AddInput("{gen_root_dir}/path.fbs", "path_fbs");
   generator.AddInput("{gen_root_dir}/custom_generator.fbs",
                      "custom_generator_fbs");
-  generator.AddInput("{gen_root_dir}/generator.fbs", "generator_fbs");
   generator.AddInput("{gen_root_dir}/target.fbs", "target_fbs");
 
   generator.AddOutput("{gen_build_dir}/path_generated.h");
   generator.AddOutput("{gen_build_dir}/custom_generator_generated.h");
-  generator.AddOutput("{gen_build_dir}/generator_generated.h");
   generator.AddOutput("{gen_build_dir}/target_generated.h");
 
   generator.AddDefaultArguments({
@@ -36,12 +34,12 @@ void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe) {
   //   generator.AddCommand("{flatc_compiler} --help");
   generator.AddCommand(
       "{flatc_compiler} -o {gen_build_dir} -I {gen_root_dir} --gen-object-api "
-      "--cpp {path_fbs} {custom_generator_fbs} {generator_fbs} {target_fbs}");
+      "--cpp {path_fbs} {custom_generator_fbs} {target_fbs}");
 
   generator.Build();
 }
 
-void buildcc_cb(BaseTarget &target, const BaseGenerator &schema_gen,
+void buildcc_cb(BaseTarget &target, const FileGenerator &schema_gen,
                 const TargetInfo &flatbuffers_ho, const TargetInfo &fmt_ho,
                 const TargetInfo &spdlog_ho, const TargetInfo &cli11_ho,
                 const TargetInfo &taskflow_ho, const BaseTarget &tpl) {
@@ -201,7 +199,7 @@ void BuildBuildCC::Initialize() {
                 env_.GetTargetBuildDir()));
 
   // Schema
-  (void)storage_.Add<BaseGenerator>(
+  (void)storage_.Add<FileGenerator>(
       kSchemaGenName, kSchemaGenName,
       TargetEnv(env_.GetTargetRootDir() / "buildcc" / "schema",
                 env_.GetTargetBuildDir() / toolchain_.GetName()));

buildcc/lib/args/include/args/register.h

@@ -82,24 +82,17 @@ class Reg::Instance {
     }
   }
 
-  /**
-   * @brief Reg::Instance for Target to be built
-   */
-  template <typename C, typename... Params>
-  void Build(const C &build_cb, BaseTarget &target, Params &&...params) {
-    build_cb(target, std::forward<Params>(params)...);
-    tf::Task task = BuildTargetTask(target);
-    BuildStoreTask(target.GetUniqueId(), task);
-  }
-
-  /**
-   * @brief Reg::Instance for Generator to be built
-   */
-  template <typename C, typename... Params>
-  void Build(const C &build_cb, BaseGenerator &generator, Params &&...params) {
-    build_cb(generator, std::forward<Params>(params)...);
-    tf::Task task = BuildGeneratorTask(generator);
-    BuildStoreTask(generator.GetUniqueId(), task);
+  template <typename C, typename T, typename... Params>
+  void Build(const C &build_cb, T &builder, Params &&...params) {
+    constexpr bool is_supported_base = std::is_base_of_v<CustomGenerator, T> ||
+                                       std::is_base_of_v<BaseTarget, T>;
+    static_assert(
+        is_supported_base,
+        "Build only supports BaseTarget, CustomGenerator and derivatives");
+
+    build_cb(builder, std::forward<Params>(params)...);
+    tf::Task task = BuildTask(builder);
+    BuildStoreTask(builder.GetUniqueId(), task);
   }
 
   /**
@@ -144,8 +137,8 @@ class Reg::Instance {
 private:
 private:
   // BuildTasks
-  tf::Task BuildTargetTask(BaseTarget &target);
-  tf::Task BuildGeneratorTask(BaseGenerator &generator);
+  tf::Task BuildTask(BaseTarget &target);
+  tf::Task BuildTask(CustomGenerator &generator);
   void BuildStoreTask(const std::string &unique_id, const tf::Task &task);
 
 private:
@@ -167,11 +160,10 @@ class Reg::CallbackInstance {
     return *this;
   }
 
-  template <typename C, typename... Params>
-  CallbackInstance &Build(const C &build_cb, BaseGenerator &generator,
-                          Params &&...params) {
+  template <typename C, typename T, typename... Params>
+  CallbackInstance &Build(const C &build_cb, T &builder, Params &&...params) {
     if (condition_) {
-      Ref().Build(build_cb, generator, std::forward<Params>(params)...);
+      Ref().Build(build_cb, builder, std::forward<Params>(params)...);
     };
     return *this;
   }
@@ -191,16 +183,14 @@ class Reg::ToolchainInstance {
     return *this;
   }
 
-  template <typename C, typename... Params>
-  ToolchainInstance &Build(const C &build_cb, BaseGenerator &generator,
-                           Params &&...params) {
-    return BuildInternal(build_cb, generator, std::forward<Params>(params)...);
-  }
-  template <typename C, typename... Params>
-  ToolchainInstance &Build(const C &build_cb, BaseTarget &target,
-                           Params &&...params) {
-    return BuildInternal(build_cb, target, std::forward<Params>(params)...);
+  template <typename C, typename T, typename... Params>
+  ToolchainInstance &Build(const C &build_cb, T &builder, Params &&...params) {
+    if (condition_.build) {
+      Ref().Build(build_cb, builder, std::forward<Params>(params)...);
+    };
+    return *this;
   }
+  // TODO, Update/Change this
   template <typename P> ToolchainInstance &BuildPackage(P &package) {
     return Func([&]() { package.Setup(condition_); });
   }
@@ -209,16 +199,6 @@ class Reg::ToolchainInstance {
   ToolchainInstance &Test(const std::string &command, const BaseTarget &target,
                           const TestConfig &config = TestConfig());
 
-private:
-  template <typename C, typename T, typename... Params>
-  ToolchainInstance &BuildInternal(const C &build_cb, T &t,
-                                   Params &&...params) {
-    if (condition_.build) {
-      Ref().Build(build_cb, t, std::forward<Params>(params)...);
-    }
-    return *this;
-  }
-
 private:
   ArgToolchainState condition_;
 };

buildcc/lib/args/mock/tasks.cpp

@@ -4,14 +4,12 @@
 
 namespace buildcc {
 
-tf::Task Reg::Instance::BuildTargetTask(BaseTarget &target) {
+tf::Task Reg::Instance::BuildTask(BaseTarget &target) {
   mock().actualCall(fmt::format("BuildTask_{}", target.GetName()).c_str());
   return build_tf_.placeholder().name(target.GetUniqueId());
 }
-
-tf::Task Reg::Instance::BuildGeneratorTask(BaseGenerator &generator) {
-  mock().actualCall(
-      fmt::format("BuildGeneratorTask_{}", generator.GetName()).c_str());
+tf::Task Reg::Instance::BuildTask(CustomGenerator &generator) {
+  mock().actualCall(fmt::format("BuildTask_{}", generator.GetName()).c_str());
   return build_tf_.placeholder().name(generator.GetUniqueId());
 }
 

buildcc/lib/args/src/tasks.cpp

@@ -20,11 +20,11 @@
 
 namespace buildcc {
 
-tf::Task Reg::Instance::BuildTargetTask(BaseTarget &target) {
+tf::Task Reg::Instance::BuildTask(BaseTarget &target) {
   return build_tf_.composed_of(target.GetTaskflow()).name(target.GetUniqueId());
 }
 
-tf::Task Reg::Instance::BuildGeneratorTask(BaseGenerator &generator) {
+tf::Task Reg::Instance::BuildTask(CustomGenerator &generator) {
   return build_tf_.composed_of(generator.GetTaskflow())
       .name(generator.GetUniqueId());
 }

buildcc/lib/target/cmake/mock_target.cmake

@@ -4,11 +4,6 @@ add_library(mock_target STATIC
     mock/custom_generator/runner.cpp
     mock/custom_generator/recheck_states.cpp
 
-    # Generator mocks
-    src/generator/task.cpp
-    mock/generator/runner.cpp
-    mock/generator/recheck_states.cpp
-
     # Target mocks
     src/target/tasks.cpp
     mock/target/runner.cpp

buildcc/lib/target/cmake/target.cmake

@@ -3,9 +3,6 @@ set(TARGET_SRCS
 
     src/custom_generator/recheck_states.cpp
 
-    src/generator/task.cpp
-    src/generator/recheck_states.cpp
-
     src/target/recheck_states.cpp
     src/target/tasks.cpp
 )

buildcc/lib/target/include/target/custom_generator.h

@@ -56,19 +56,48 @@ typedef std::function<bool(CustomGeneratorContext &)> GenerateCb;
 typedef std::function<void(std::unordered_map<std::string, tf::Task> &)> DependencyCb;
 // clang-format on
 
-struct UserRelInputOutputSchema : internal::RelInputOutputSchema {
+class CustomBlobHandler {
+public:
+  bool CheckChanged(const std::vector<uint8_t> &previous,
+                    const std::vector<uint8_t> &current) const {
+    env::assert_fatal(
+        Verify(previous),
+        "Stored blob is corrupted or User verification is incorrect");
+    env::assert_fatal(
+        Verify(current),
+        "Current blob is corrupted or User verification is incorrect");
+    return !IsEqual(previous, current);
+  };
+
+  std::vector<uint8_t> GetSerializedData() {
+    auto serialized_data = Serialize();
+    env::assert_fatal(
+        Verify(serialized_data),
+        "Serialized data is corrupted or Serialize function is incorrect");
+    return serialized_data;
+  }
+
+private:
+  virtual bool Verify(const std::vector<uint8_t> &serialized_data) const = 0;
+  virtual bool IsEqual(const std::vector<uint8_t> &previous,
+                       const std::vector<uint8_t> &current) const = 0;
+  virtual std::vector<uint8_t> Serialize() const = 0;
+};
+
+struct UserGenInfo : internal::GenInfo {
   fs_unordered_set inputs;
   GenerateCb generate_cb;
+  std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
 };
 
 struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
-  std::unordered_map<std::string, UserRelInputOutputSchema> rels_map;
+  std::unordered_map<std::string, UserGenInfo> gen_info_map;
 
   void ConvertToInternal() {
-    for (auto &r_miter : rels_map) {
+    for (auto &r_miter : gen_info_map) {
       r_miter.second.internal_inputs = path_schema_convert(
           r_miter.second.inputs, internal::Path::CreateExistingPath);
-      auto p = internal_rels_map.emplace(r_miter.first, r_miter.second);
+      auto p = internal_gen_info_map.emplace(r_miter.first, r_miter.second);
       env::assert_fatal(p.second,
                         fmt::format("Could not save {}", r_miter.first));
     }
@@ -86,13 +115,17 @@ class CustomGenerator : public internal::BuilderInterface {
   virtual ~CustomGenerator() = default;
   CustomGenerator(const CustomGenerator &) = delete;
 
-  /**
-   * @brief Add default arguments for input, output and command requirements
-   *
-   * @param arguments Key-Value pair for arguments
-   */
+  // From env::Command module, forwarding here
+  // TODO, Create a Mixin
+  void AddDefaultArgument(const std::string &identifier,
+                          const std::string &pattern);
   void AddDefaultArguments(
       const std::unordered_map<std::string, std::string> &arguments);
+  std::string Construct(
+      const std::string &pattern,
+      const std::unordered_map<const char *, std::string> &arguments = {});
+  const std::string &
+  GetValueByIdentifier(const std::string &file_identifier) const;
 
   /**
    * @brief Single Generator task for inputs->generate_cb->outputs
@@ -105,7 +138,8 @@ class CustomGenerator : public internal::BuilderInterface {
    */
   void AddGenInfo(const std::string &id, const fs_unordered_set &inputs,
                   const fs_unordered_set &outputs,
-                  const GenerateCb &generate_cb);
+                  const GenerateCb &generate_cb,
+                  std::shared_ptr<CustomBlobHandler> blob_handler = nullptr);
 
   // Callbacks
   /**
@@ -127,6 +161,7 @@ class CustomGenerator : public internal::BuilderInterface {
   void Build() override;
 
   // Getters
+  const std::string &GetName() const { return name_; }
   const fs::path &GetBinaryPath() const {
     return serialization_.GetSerializedFile();
   }
@@ -140,10 +175,9 @@ class CustomGenerator : public internal::BuilderInterface {
   template <bool run> void TaskRunner(const std::string &id);
 
   void GenerateTask();
-  void BuildGenerate(std::unordered_map<std::string, UserRelInputOutputSchema>
-                         &gen_selected_map,
-                     std::unordered_map<std::string, UserRelInputOutputSchema>
-                         &dummy_gen_selected_map);
+  void BuildGenerate(
+      std::unordered_map<std::string, UserGenInfo> &gen_selected_map,
+      std::unordered_map<std::string, UserGenInfo> &dummy_gen_selected_map);
 
   // Recheck states
   void IdRemoved();
@@ -159,7 +193,7 @@ class CustomGenerator : public internal::BuilderInterface {
   UserCustomGeneratorSchema user_;
 
   std::mutex success_schema_mutex_;
-  std::unordered_map<std::string, UserRelInputOutputSchema> success_schema_;
+  std::unordered_map<std::string, UserGenInfo> success_schema_;
 
   // Internal
   env::Command command_;