Custom generator (#212)

Author: coder137

bootstrap/src/build_buildcc.cpp

@@ -20,10 +20,13 @@ namespace buildcc {
 
 void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe) {
   generator.AddInput("{gen_root_dir}/path.fbs", "path_fbs");
+  generator.AddInput("{gen_root_dir}/custom_generator.fbs",
+                     "custom_generator_fbs");
   generator.AddInput("{gen_root_dir}/generator.fbs", "generator_fbs");
   generator.AddInput("{gen_root_dir}/target.fbs", "target_fbs");
 
   generator.AddOutput("{gen_build_dir}/path_generated.h");
+  generator.AddOutput("{gen_build_dir}/custom_generator_generated.h");
   generator.AddOutput("{gen_build_dir}/generator_generated.h");
   generator.AddOutput("{gen_build_dir}/target_generated.h");
 
@@ -33,7 +36,7 @@ void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe) {
   //   generator.AddCommand("{flatc_compiler} --help");
   generator.AddCommand(
       "{flatc_compiler} -o {gen_build_dir} -I {gen_root_dir} --gen-object-api "
-      "--cpp {path_fbs} {generator_fbs} {target_fbs}");
+      "--cpp {path_fbs} {custom_generator_fbs} {generator_fbs} {target_fbs}");
 
   generator.Build();
 }
@@ -71,6 +74,7 @@ void buildcc_cb(BaseTarget &target, const BaseGenerator &schema_gen,
 
   // TARGET
   target.GlobSources("lib/target/src/common");
+  target.GlobSources("lib/target/src/custom_generator");
   target.GlobSources("lib/target/src/generator");
   target.GlobSources("lib/target/src/api");
   target.GlobSources("lib/target/src/target_info");

buildcc/buildcc.h

@@ -31,6 +31,7 @@
 
 // Base
 #include "toolchain/toolchain.h"
+#include "target/custom_generator.h"
 #include "target/generator.h"
 #include "target/target_info.h"
 #include "target/target.h"

buildcc/lib/env/src/command.cpp

@@ -61,7 +61,14 @@ std::string Command::Construct(
                 });
 
   // Construct your command
-  return fmt::vformat(pattern, store);
+  std::string ret;
+  try {
+    ret = fmt::vformat(pattern, store);
+  } catch (const std::exception &e) {
+    env::assert_fatal<false>(
+        fmt::format("Construct command failed: {}", e.what()));
+  }
+  return ret;
 }
 
 } // namespace buildcc::env

buildcc/lib/target/cmake/common_target_src.cmake

@@ -34,6 +34,8 @@ set(COMMON_TARGET_SRCS
     include/target/api/target_getter.h
 
     # Generator
+    src/custom_generator/custom_generator.cpp
+    include/target/custom_generator.h
     src/generator/generator.cpp
     include/target/generator.h
 

buildcc/lib/target/cmake/mock_target.cmake

@@ -1,5 +1,8 @@
 add_library(mock_target STATIC
     ${COMMON_TARGET_SRCS}
+    # Custom Generator mocks
+    mock/custom_generator/runner.cpp
+    mock/custom_generator/recheck_states.cpp
 
     # Generator mocks
     src/generator/task.cpp

buildcc/lib/target/cmake/target.cmake

@@ -1,6 +1,8 @@
 set(TARGET_SRCS 
     ${COMMON_TARGET_SRCS}
 
+    src/custom_generator/recheck_states.cpp
+
     src/generator/task.cpp
     src/generator/recheck_states.cpp
 

buildcc/lib/target/include/target/custom_generator.h

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2021-2022 Niket Naidu. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TARGET_CUSTOM_GENERATOR_H_
+#define TARGET_CUSTOM_GENERATOR_H_
+
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "taskflow/taskflow.hpp"
+
+#include "env/command.h"
+#include "env/task_state.h"
+
+#include "target/interface/builder_interface.h"
+
+#include "schema/custom_generator_serialization.h"
+#include "schema/path.h"
+
+#include "target/common/target_env.h"
+
+namespace buildcc {
+
+// TODO, Shift to a different file
+// TODO, Check if we need the "id" here as well
+class CustomGeneratorContext {
+public:
+  CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i,
+                         const fs_unordered_set &o)
+      : command(c), inputs(i), outputs(o) {}
+
+public:
+  const env::Command &command;
+  const fs_unordered_set &inputs;
+  const fs_unordered_set &outputs;
+};
+
+// clang-format off
+typedef std::function<bool(CustomGeneratorContext &)> GenerateCb;
+
+typedef std::function<void(std::unordered_map<std::string, tf::Task> &)> DependencyCb;
+// clang-format on
+
+struct UserRelInputOutputSchema : internal::RelInputOutputSchema {
+  fs_unordered_set inputs;
+  GenerateCb generate_cb;
+};
+
+struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
+  std::unordered_map<std::string, UserRelInputOutputSchema> rels_map;
+
+  void ConvertToInternal() {
+    for (auto &r_miter : rels_map) {
+      r_miter.second.internal_inputs = path_schema_convert(
+          r_miter.second.inputs, internal::Path::CreateExistingPath);
+      auto p = internal_rels_map.emplace(r_miter.first, r_miter.second);
+      env::assert_fatal(p.second,
+                        fmt::format("Could not save {}", r_miter.first));
+    }
+  }
+};
+
+class CustomGenerator : public internal::BuilderInterface {
+public:
+  CustomGenerator(const std::string &name, const TargetEnv &env)
+      : name_(name),
+        env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name),
+        serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)) {
+    Initialize();
+  }
+  virtual ~CustomGenerator() = default;
+  CustomGenerator(const CustomGenerator &) = delete;
+
+  /**
+   * @brief Add default arguments for input, output and command requirements
+   *
+   * @param arguments Key-Value pair for arguments
+   */
+  void AddDefaultArguments(
+      const std::unordered_map<std::string, std::string> &arguments);
+
+  /**
+   * @brief Single Generator task for inputs->generate_cb->outputs
+   *
+   * @param id Unique id associated with Generator task
+   * @param inputs File inputs
+   * @param outputs File outputs
+   * @param generate_cb User-defined generate callback to build outputs from the
+   * provided inputs
+   */
+  void AddGenInfo(const std::string &id, const fs_unordered_set &inputs,
+                  const fs_unordered_set &outputs,
+                  const GenerateCb &generate_cb);
+
+  // Callbacks
+  /**
+   * @brief Setup dependencies between Tasks using their `id`
+   * For example: `task_map["id1"].precede(task_map["id2"])`
+   *
+   * IMPORTANT: Successor tasks will not automatically run if dependent task is
+   * run.
+   * The Dependency callback only sets precedence (order in which your tasks
+   * should run)
+   * Default behaviour when dependency callback is not supplied: All task `id`s
+   * run in parallel.
+   *
+   * @param dependency_cb Unordered map of `id` and `task`
+   * The map can be safely mutated.
+   */
+  void AddDependencyCb(const DependencyCb &dependency_cb);
+
+  void Build() override;
+
+  // Getters
+  const fs::path &GetBinaryPath() const {
+    return serialization_.GetSerializedFile();
+  }
+  const fs::path &GetRootDir() const { return env_.GetTargetRootDir(); }
+  const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); }
+  tf::Taskflow &GetTaskflow() { return tf_; }
+
+private:
+  void Initialize();
+
+  template <bool run> void TaskRunner(const std::string &id);
+
+  void GenerateTask();
+  void BuildGenerate(std::unordered_map<std::string, UserRelInputOutputSchema>
+                         &gen_selected_map,
+                     std::unordered_map<std::string, UserRelInputOutputSchema>
+                         &dummy_gen_selected_map);
+
+  // Recheck states
+  void IdRemoved();
+  void IdAdded();
+  void IdUpdated();
+
+private:
+  std::string name_;
+  TargetEnv env_;
+  internal::CustomGeneratorSerialization serialization_;
+
+  // Serialization
+  UserCustomGeneratorSchema user_;
+
+  std::mutex success_schema_mutex_;
+  std::unordered_map<std::string, UserRelInputOutputSchema> success_schema_;
+
+  // Internal
+  env::Command command_;
+  tf::Taskflow tf_;
+
+  // Callbacks
+  DependencyCb dependency_cb_;
+};
+
+} // namespace buildcc
+
+#endif

buildcc/lib/target/include/target/generator.h

@@ -17,15 +17,13 @@
 #ifndef TARGET_GENERATOR_H_
 #define TARGET_GENERATOR_H_
 
-#include <functional>
 #include <mutex>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
 #include "taskflow/taskflow.hpp"
 
-#include "env/env.h"
 #include "env/task_state.h"
 
 #include "env/command.h"
@@ -54,8 +52,8 @@ class Generator : public internal::BuilderInterface {
         parallel_(parallel) {
     Initialize();
   }
-  virtual ~Generator() {}
-  Generator(const Generator &generator) = delete;
+  virtual ~Generator() = default;
+  Generator(const Generator &) = delete;
 
   /**
    * @brief Add default arguments for input, output and command requirements

buildcc/lib/target/include/target/interface/builder_interface.h

@@ -27,6 +27,66 @@
 
 namespace buildcc::internal {
 
+enum PathState {
+  kNoChange,
+  kRemoved,
+  kAdded,
+  kUpdated,
+};
+
+template <typename T>
+inline bool CheckChanged(const T &previous, const T &current) {
+  bool changed = false;
+  if (previous != current) {
+    changed = true;
+  }
+  return changed;
+}
+
+/**
+ * @brief
+ *
+ * @return PathState Returns first state found if `Removed`, `Added` or
+ * `Updated`
+ * If none of the above states are true then it returns `NoChange`
+ */
+inline PathState CheckPaths(const internal::path_unordered_set &previous_path,
+                            const internal::path_unordered_set &current_path) {
+  PathState state{PathState::kNoChange};
+
+  // * Old path is removed
+  const bool removed = std::any_of(
+      previous_path.begin(), previous_path.end(), [&](const internal::Path &p) {
+        return current_path.find(p) == current_path.end();
+      });
+  if (removed) {
+    state = PathState::kRemoved;
+  } else {
+    (void)std::any_of(current_path.cbegin(), current_path.cend(),
+                      [&](const internal::Path &p) -> bool {
+                        bool dirty = false;
+                        const auto find = previous_path.find(p);
+                        const bool added_cond = (find == previous_path.end());
+                        if (added_cond) {
+                          dirty = true;
+                          state = kAdded;
+                        } else {
+                          const bool updated_cond =
+                              (p.GetLastWriteTimestamp() >
+                               find->GetLastWriteTimestamp());
+                          if (updated_cond) {
+                            dirty = true;
+                            state = kUpdated;
+                          } else {
+                            dirty = false;
+                          }
+                        }
+                        return dirty;
+                      });
+  }
+  return state;
+}
+
 // TODO, 1. Consider updating Recheck* APIs - do not modify internal `dirty_`
 // flag
 // TODO, 2. Consider removing dependency on target/common/util.h

buildcc/lib/target/mock/custom_generator/recheck_states.cpp

@@ -0,0 +1,43 @@
+#include "target/custom_generator.h"
+
+#include "expect_custom_generator.h"
+
+#include "CppUTestExt/MockSupport.h"
+
+namespace buildcc {
+
+static constexpr const char *const ID_REMOVED_FUNCTION =
+    "CustomGenerator::IdRemoved";
+static constexpr const char *const ID_ADDED_FUNCTION =
+    "CustomGenerator::IdAdded";
+static constexpr const char *const ID_UPDATED_FUNCTION =
+    "CustomGenerator::IdUpdated";
+
+void CustomGenerator::IdRemoved() {
+  mock().actualCall(ID_REMOVED_FUNCTION).onObject(this);
+}
+void CustomGenerator::IdAdded() {
+  mock().actualCall(ID_ADDED_FUNCTION).onObject(this);
+}
+void CustomGenerator::IdUpdated() {
+  mock().actualCall(ID_UPDATED_FUNCTION).onObject(this);
+}
+
+namespace m {
+
+void CustomGeneratorExpect_IdRemoved(unsigned int calls,
+                                     CustomGenerator *generator) {
+  mock().expectNCalls(calls, ID_REMOVED_FUNCTION).onObject(generator);
+}
+void CustomGeneratorExpect_IdAdded(unsigned int calls,
+                                   CustomGenerator *generator) {
+  mock().expectNCalls(calls, ID_ADDED_FUNCTION).onObject(generator);
+}
+void CustomGeneratorExpect_IdUpdated(unsigned int calls,
+                                     CustomGenerator *generator) {
+  mock().expectNCalls(calls, ID_UPDATED_FUNCTION).onObject(generator);
+}
+
+} // namespace m
+
+} // namespace buildcc