refactor(dto): example service_create+ncnn

The Service Create call works, but most parameters doesn't.

Supported parameters are only those declared in the DTO files.

To not update all input/output connector code. This PR just
convert the DTO into APIData to pass it to connector.

/!\ But beware if a parameters is not present in the DTO it will be
ignored and will not be present in the generated APIData passed to the
connectors. /!\

That's why we need first to fill the InputConnector and OutputConnector
with all possible option from all type and all backends.

cmake .. -D USE_HTTP_SERVER_OATPP=ON -D USE_HTTP_SERVER=OFF -D USE_NCNN=ON -D USE_CAFFE=OFF -DUSE_COMMAND_LINE=OFF

Author: sileht

CMakeLists.txt

@@ -87,7 +87,7 @@ if (NOT EXISTS ${CMAKE_BINARY_DIR}/src)
 	COMMAND  bash -c "mkdir ${CMAKE_BINARY_DIR}/src")
 endif()
 
-set(CMAKE_CXX_FLAGS "-g -O2 -Wall -Wextra -fopenmp -fPIC -std=c++14 -DUSE_OPENCV -DUSE_LMDB")
+set(CMAKE_CXX_FLAGS "-g -O2 -Wall -Wextra -fopenmp -fPIC -std=c++14 -DUSE_OPENCV -DUSE_LMDB -fmax-errors=4")
 
 if(WARNING)
   string(APPEND CMAKE_CXX_FLAGS " -Werror")

src/apidata.h

@@ -298,6 +298,20 @@ namespace dd
           .getPtr();
     }
 
+    template <typename T>
+    inline static APIData fromDTO(const oatpp::Object<T> &dto)
+    {
+      std::shared_ptr<oatpp::data::mapping::ObjectMapper> object_mapper
+          = oatpp::parser::json::mapping::ObjectMapper::createShared();
+
+      oatpp::String json = object_mapper->writeToString(dto);
+      APIData ad;
+      rapidjson::Document d;
+      d.Parse<rapidjson::kParseNanAndInfFlag>(json->c_str());
+      ad.fromRapidJson(d);
+      return ad;
+    }
+
   public:
     /**
      * \brief render Mustache template based on this APIData object

src/backends/ncnn/ncnnlib.cc

@@ -30,6 +30,8 @@
 #include "net.h"
 #include <iostream>
 
+#include "dto/mllib.hpp"
+
 namespace dd
 {
   template <class TInputConnectorStrategy, class TOutputConnectorStrategy,
@@ -88,13 +90,11 @@ namespace dd
   template <class TInputConnectorStrategy, class TOutputConnectorStrategy,
             class TMLModel>
   void NCNNLib<TInputConnectorStrategy, TOutputConnectorStrategy,
-               TMLModel>::init_mllib(const APIData &ad)
+               TMLModel>::init_mllib(const oatpp::Object<DTO::MLLib> &init_dto)
   {
-    _init_dto = ad.createSharedDTO<NcnnInitDto>();
+    _init_dto = init_dto;
 
-    bool use_fp32 = (ad.has("datatype")
-                     && ad.get("datatype").get<std::string>()
-                            == "fp32"); // default is fp16
+    bool use_fp32 = (_init_dto->datatype == "fp32");
     _net->opt.use_fp16_packed = !use_fp32;
     _net->opt.use_fp16_storage = !use_fp32;
     _net->opt.use_fp16_arithmetic = !use_fp32;

src/backends/ncnn/ncnnlib.h

@@ -25,7 +25,7 @@
 #include "apidata.h"
 #include "utils/utils.hpp"
 
-#include "dto/ncnn.hpp"
+#include "dto/mllib.hpp"
 
 // NCNN
 #include "net.h"
@@ -44,7 +44,7 @@ namespace dd
     ~NCNNLib();
 
     /*- from mllib -*/
-    void init_mllib(const APIData &ad);
+    void init_mllib(const oatpp::Object<DTO::MLLib> &init_dto);
 
     void clear_mllib(const APIData &ad);
 
@@ -59,7 +59,7 @@ namespace dd
     bool _timeserie = false;
 
   private:
-    std::shared_ptr<NcnnInitDto> _init_dto;
+    oatpp::Object<DTO::MLLib> _init_dto;
     static ncnn::UnlockedPoolAllocator _blob_pool_allocator;
     static ncnn::PoolAllocator _workspace_pool_allocator;
 

src/backends/ncnn/ncnnmodel.h

@@ -25,6 +25,8 @@
 #include "dd_spdlog.h"
 #include "mlmodel.h"
 #include "apidata.h"
+#include "dto/model.hpp"
+#include "dto/service_create.hpp"
 
 namespace dd
 {
@@ -34,12 +36,13 @@ namespace dd
     NCNNModel() : MLModel()
     {
     }
-    NCNNModel(const APIData &ad, APIData &adg,
+    NCNNModel(const oatpp::Object<DTO::Model> &model_dto,
+            const oatpp::Object<DTO::ServiceCreate> &service_dto,
               const std::shared_ptr<spdlog::logger> &logger)
-        : MLModel(ad, adg, logger)
+        : MLModel(model_dto, service_dto, logger)
     {
-      if (ad.has("repository"))
-        this->_repo = ad.get("repository").get<std::string>();
+      if (model_dto->repository)
+        this->_repo = model_dto->repository->std_str();
       read_from_repository(spdlog::get("api"));
       read_corresp_file();
     }

src/dto/img_connector.hpp src/dto/input_connector.hpp

@@ -19,8 +19,8 @@
  * along with deepdetect.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef HTTP_DTO_IMG_CONNECTOR_HPP
-#define HTTP_DTO_IMG_CONNECTOR_HPP
+#ifndef DTO_INPUT_CONNECTOR_HPP
+#define DTO_INPUT_CONNECTOR_HPP
 
 #include "dd_config.h"
 #include "oatpp/core/Types.hpp"
@@ -32,10 +32,13 @@ namespace dd
   {
 #include OATPP_CODEGEN_BEGIN(DTO)
 
-    class ImgInputConnectorParameters : public oatpp::DTO
+    class InputConnector : public oatpp::DTO
     {
-      DTO_INIT(ImgInputConnectorParameters, DTO /* extends */)
+      DTO_INIT(InputConnector, DTO /* extends */)
+      // Connector type
+      DTO_FIELD(String, connector);
 
+      // IMG Input Connector
       DTO_FIELD(Int32, width);
       DTO_FIELD(Int32, height);
       DTO_FIELD(Int32, crop_width);

src/dto/mllib.hpp

@@ -0,0 +1,85 @@
+/**
+ * DeepDetect
+ * Copyright (c) 2021 Jolibrain SASU
+ * Author: Mehdi Abaakouk <[email protected]>
+ *
+ * This file is part of deepdetect.
+ *
+ * deepdetect is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * deepdetect is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with deepdetect.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DTO_MLLIB_H
+#define DTO_MLLIB_H
+
+#include "dd_config.h"
+#include "utils/utils.hpp"
+#include "oatpp/core/Types.hpp"
+#include "oatpp/core/macro/codegen.hpp"
+
+namespace dd
+{
+  namespace DTO
+  {
+#include OATPP_CODEGEN_BEGIN(DTO) ///< Begin DTO codegen section
+
+class MLLib : public oatpp::DTO
+{
+  DTO_INIT(MLLib, DTO /* extends */)
+
+  // NCNN Options
+  DTO_FIELD_INFO(nclasses)
+  {
+    info->description = "number of output classes (`supervised` service "
+                        "type), classification only";
+  };
+  DTO_FIELD(Int32, nclasses) = 0;
+
+  DTO_FIELD_INFO(threads)
+  {
+    info->description = "number of threads";
+  };
+  DTO_FIELD(Int32, threads) = dd::dd_utils::my_hardware_concurrency();
+
+  DTO_FIELD_INFO(lightmode)
+  {
+    info->description = "enable light mode";
+  };
+  DTO_FIELD(Boolean, lightmode) = true;
+
+  DTO_FIELD_INFO(inputBlob)
+  {
+    info->description = "network input blob name";
+  };
+  DTO_FIELD(String, inputBlob) = "data";
+
+  DTO_FIELD_INFO(outputBlob)
+  {
+    info->description = "network output blob name (default depends on "
+                        "network type(ie prob or "
+                        "rnn_pred or probs or detection_out)";
+  };
+  DTO_FIELD(String, outputBlob);
+
+  DTO_FIELD_INFO(datatype)
+  {
+    info->description = "fp16 or fp32";
+  };
+
+  DTO_FIELD(String, datatype) = "fp16";
+};
+#include OATPP_CODEGEN_END(DTO) ///< End DTO codegen section
+
+  }
+}
+#endif

src/dto/model.hpp

@@ -0,0 +1,49 @@
+/**
+ * DeepDetect
+ * Copyright (c) 2021 Jolibrain SASU
+ * Author: Mehdi Abaakouk <[email protected]>
+ *
+ * This file is part of deepdetect.
+ *
+ * deepdetect is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * deepdetect is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with deepdetect.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DTO_MODEL
+#define DTO_MODEL
+
+#include "oatpp/core/Types.hpp"
+#include "oatpp/core/macro/codegen.hpp"
+
+namespace dd
+{
+  namespace DTO
+  {
+#include OATPP_CODEGEN_BEGIN(DTO) ///< Begin DTO codegen section
+
+
+class Model: public oatpp::DTO
+{
+    DTO_INIT(Model, DTO /* extends */)
+    DTO_FIELD(String, repository);
+    DTO_FIELD(String, init);
+    DTO_FIELD(Boolean, create_repository) = false;
+    DTO_FIELD(Boolean, index_preload) = false;
+};
+
+#include OATPP_CODEGEN_END(DTO) ///< End DTO codegen section
+
+  }
+}
+
+#endif

src/dto/output_connector.hpp

@@ -0,0 +1,44 @@
+/**
+ * DeepDetect
+ * Copyright (c) 2021 Jolibrain SASU
+ * Author: Mehdi Abaakouk <[email protected]>
+ *
+ * This file is part of deepdetect.
+ *
+ * deepdetect is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * deepdetect is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with deepdetect.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DTO_OUTPUT_CONNECTOR_H
+#define DTO_OUTPUT_CONNECTOR_H
+
+#include "oatpp/core/Types.hpp"
+#include "oatpp/core/macro/codegen.hpp"
+
+namespace dd
+{
+  namespace DTO
+  {
+#include OATPP_CODEGEN_BEGIN(DTO) ///< Begin DTO codegen section
+
+class OutputConnector: public oatpp::DTO
+{
+    DTO_INIT(OutputConnector, DTO /* extends */)
+    DTO_FIELD(Boolean, store_config);
+};
+
+#include OATPP_CODEGEN_END(DTO) ///< End DTO codegen section
+}
+}
+
+#endif

src/dto/parameters.hpp

@@ -0,0 +1,49 @@
+/**
+ * DeepDetect
+ * Copyright (c) 2021 Jolibrain SASU
+ * Author: Mehdi Abaakouk <[email protected]>
+ *
+ * This file is part of deepdetect.
+ *
+ * deepdetect is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * deepdetect is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with deepdetect.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DTO_PARAMETERS_H
+#define DTO_PARAMETERS_H
+
+#include "oatpp/core/Types.hpp"
+#include "oatpp/core/macro/codegen.hpp"
+#include "dto/mllib.hpp"
+#include "dto/input_connector.hpp"
+#include "dto/output_connector.hpp"
+
+namespace dd
+{
+  namespace DTO
+  {
+#include OATPP_CODEGEN_BEGIN(DTO) ///< Begin DTO codegen section
+
+class Parameters: public oatpp::DTO
+{
+    DTO_INIT(Parameters, DTO /* extends */)
+    DTO_FIELD(Object<InputConnector>, input) = InputConnector::createShared();
+    DTO_FIELD(Object<MLLib>, mllib) = MLLib::createShared();
+    DTO_FIELD(Object<OutputConnector>, output) = OutputConnector::createShared();
+};
+
+#include OATPP_CODEGEN_END(DTO) ///< End DTO codegen section
+}
+}
+
+#endif