Merge branch 'PaddlePaddle:develop' into develop

Author: Zheng-Bicheng

README_CN.md

@@ -57,16 +57,11 @@
 
 - FastDeploy系列[**直播课程回放**](https://aistudio.baidu.com/aistudio/education/group/info/27800)
 
-- **2023.01.17** 发布 [**YOLOv8**](./examples/vision/detection/paddledetection/) 在FastDeploy系列硬件的部署支持。 其中包括 [**Paddle YOLOv8**](https://github.com/PaddlePaddle/PaddleYOLO/tree/release/2.5/configs/yolov8) 以及 [**社区 ultralytics YOLOv8**](https://github.com/ultralytics/ultralytics)
-    - [**Paddle YOLOv8**](https://github.com/PaddlePaddle/PaddleYOLO/tree/release/2.5/configs/yolov8) 可以部署的硬件：[**Intel CPU**](./examples/vision/detection/paddledetection/python/infer_yolov8.py)、[**NVIDIA GPU**](./examples/vision/detection/paddledetection/python/infer_yolov8.py)、[**Jetson**](./examples/vision/detection/paddledetection/python/infer_yolov8.py)、[**飞腾**](./examples/vision/detection/paddledetection/python/infer_yolov8.py)、[**昆仑芯**](./examples/vision/detection/paddledetection/python/infer_yolov8.py)、[**昇腾**](./examples/vision/detection/paddledetection/python/infer_yolov8.py)、[**ARM CPU**](./examples/vision/detection/paddledetection/cpp/infer_yolov8.cc)、[**RK3588**](./examples/vision/detection/paddledetection/rknpu2) 和 [**Sophgo TPU**](./examples/vision/detection/paddledetection/sophgo), 部分硬件包含 **Python** 部署和 **C++** 部署；
-    -  [**社区 ultralytics YOLOv8**](https://github.com/ultralytics/ultralytics) 可以部署的硬件：[**Intel CPU**](./examples/vision/detection/yolov8)、[**NVIDIA GPU**](./examples/vision/detection/yolov8)、[**Jetson**](./examples/vision/detection/yolov8)，均包含 **Python** 部署和 **C++** 部署；
-    -  FastDeploy 一行模型API切换，可以实现**YOLOv8**、 **PP-YOLOE+**、**YOLOv5** 等模型性能对比。
- - 服务化部署结合VisualDL新增支持可视化部署。在FastDeploy容器中启动VDL服务后，即可在VDL界面修改模型配置、启动/管理模型服务、查看性能数据、发送请求等，详细操作可参考相关文档
+- 服务化部署结合VisualDL新增支持可视化部署。在FastDeploy容器中启动VDL服务后，即可在VDL界面修改模型配置、启动/管理模型服务、查看性能数据、发送请求等，详细操作可参考相关文档
     - [Serving可视化部署](https://github.com/PaddlePaddle/FastDeploy/blob/develop/serving/docs/zh_CN/vdl_management.md)
     - [Serving可视化请求](https://github.com/PaddlePaddle/FastDeploy/blob/develop/serving/docs/zh_CN/client.md#%E4%BD%BF%E7%94%A8fastdeploy-client%E8%BF%9B%E8%A1%8C%E5%8F%AF%E8%A7%86%E5%8C%96%E8%AF%B7%E6%B1%82)
 
 
-
 - **✨👥✨ 社区交流**
 
     - **Slack**：Join our [Slack community](https://join.slack.com/t/fastdeployworkspace/shared_invite/zt-1o50e4voz-zbiIneCNRf_eH99eS2NVLg) and chat with other community members about ideas

examples/vision/ocr/PP-OCR/cpu-gpu/cpp/README.md

@@ -62,7 +62,7 @@ wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/dict/l
 # 运行部署示例
 # 在CPU上使用Paddle Inference推理
 ./infer_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 0
-# 在CPU上使用OenVINO推理
+# 在CPU上使用OpenVINO推理
 ./infer_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 1
 # 在CPU上使用ONNX Runtime推理
 ./infer_demo ./ch_PP-OCRv3_det_infer ./ch_ppocr_mobile_v2.0_cls_infer ./ch_PP-OCRv3_rec_infer ./ppocr_keys_v1.txt ./12.jpg 2
@@ -110,7 +110,7 @@ wget https://gitee.com/paddlepaddle/PaddleOCR/raw/release/2.6/ppocr/utils/dict/l
 |数字选项|含义|
 |:---:|:---:|
 |0| 在CPU上使用Paddle Inference推理 |
-|1| 在CPU上使用OenVINO推理 |
+|1| 在CPU上使用OpenVINO推理 |
 |2| 在CPU上使用ONNX Runtime推理 |
 |3| 在CPU上使用Paddle Lite推理 |
 |4| 在GPU上使用Paddle Inference推理 |

fastdeploy/fastdeploy_model.h

@@ -47,7 +47,7 @@ class FASTDEPLOY_DECL FastDeployModel {
   std::vector<Backend> valid_timvx_backends = {};
     /** Model's valid directml backends. This member defined all the onnxruntime directml backends have successfully tested for the model
    */
-  std::vector<Backend> valid_directml_backends = {};
+  std::vector<Backend> valid_directml_backends = {Backend::ORT};
   /** Model's valid ascend backends. This member defined all the cann backends have successfully tested for the model
    */
   std::vector<Backend> valid_ascend_backends = {};

fastdeploy/runtime/backends/ort/ort_backend.cc

@@ -98,7 +98,7 @@ bool OrtBackend::BuildOption(const OrtBackendOption& option) {
           "DML", ORT_API_VERSION, reinterpret_cast<const void**>(&ortDmlApi));
       OrtStatus* onnx_dml_status =
           ortDmlApi->SessionOptionsAppendExecutionProvider_DML(session_options_,
-                                                               0);
+                                                               option_.device_id);
       if (onnx_dml_status != nullptr) {
         FDERROR
             << "DirectML is not support in your machine, the program will exit."
@@ -260,8 +260,13 @@ bool OrtBackend::InitFromOnnx(const std::string& model_file,
     }
     char* model_content_ptr;
     int model_content_size = 0;
+#ifdef ENABLE_PADDLE2ONNX
     paddle2onnx::ConvertFP32ToFP16(model_file.c_str(), model_file.size(),
                                    &model_content_ptr, &model_content_size);
+#else
+    FDERROR << "Didn't compile with ENABLE_PADDLE2ONNX, FP16 is not supported" << std::endl;
+    return false;
+#endif
     std::string onnx_model_proto(model_content_ptr,
                                  model_content_ptr + model_content_size);
     delete[] model_content_ptr;

fastdeploy/runtime/enum_variables.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "fastdeploy/runtime/enum_variables.h"
+#include "fastdeploy/core/config.h"
 
 namespace fastdeploy {
 std::ostream& operator<<(std::ostream& out, const Backend& backend) {

fastdeploy/runtime/runtime_option.cc

@@ -141,7 +141,10 @@ void RuntimeOption::UseAscend() {
   paddle_lite_option.device = device;
 }
 
-void RuntimeOption::UseDirectML() { device = Device::DIRECTML; }
+void RuntimeOption::UseDirectML(int adapter_id) {
+  device = Device::DIRECTML;
+  device_id = adapter_id;
+}
 
 void RuntimeOption::UseSophgo() {
   device = Device::SOPHGOTPUD;

fastdeploy/runtime/runtime_option.h

@@ -82,7 +82,7 @@ struct FASTDEPLOY_DECL RuntimeOption {
   void UseAscend();
 
   /// Use onnxruntime DirectML to inference
-  void UseDirectML();
+  void UseDirectML(int adapter_id = 0);
 
   /// Use Sophgo to inference
   void UseSophgo();

fastdeploy/utils/utils.cc

@@ -15,6 +15,12 @@
 #include "fastdeploy/utils/utils.h"
 
 #include <sstream>
+#include <fstream>
+#include <string_view>
+
+#ifdef _WIN32
+#include <Windows.h>
+#endif
 
 namespace fastdeploy {
 
@@ -48,18 +54,53 @@ FDLogger& FDLogger::operator<<(std::ostream& (*os)(std::ostream&)) {
   return *this;
 }
 
-bool ReadBinaryFromFile(const std::string& file, std::string* contents) {
-  std::ifstream fin(file, std::ios::in | std::ios::binary);
-  if (!fin.is_open()) {
-    FDERROR << "Failed to open file: " << file << " to read." << std::endl;
+// using os_string = std::filesystem::path::string_type;
+#ifdef _WIN32
+using os_string = std::wstring;
+#else
+using os_string = std::string;
+#endif
+
+os_string to_osstring(std::string_view utf8_str)
+{
+#ifdef _WIN32
+    int len = MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), nullptr, 0);
+    os_string result(len, 0);
+    MultiByteToWideChar(CP_UTF8, 0, utf8_str.data(), (int)utf8_str.size(), result.data(), len);
+    return result;
+#else
+    return std::string(utf8_str);
+#endif
+}
+
+bool ReadBinaryFromFile(const std::string& path, std::string* contents)
+{
+  if (!contents) {
+    return false;
+  }
+  auto& result = *contents;
+  result.clear();
+
+  std::ifstream file(to_osstring(path), std::ios::binary | std::ios::ate);
+  if (!file.is_open()) {
     return false;
   }
-  fin.seekg(0, std::ios::end);
-  contents->clear();
-  contents->resize(fin.tellg());
-  fin.seekg(0, std::ios::beg);
-  fin.read(&(contents->at(0)), contents->size());
-  fin.close();
+
+  auto fileSize = file.tellg();
+  if (fileSize != -1) {
+    result.resize(fileSize);
+    file.seekg(0, std::ios::beg);
+    file.read(const_cast<char*>(result.data()), fileSize);
+  }
+  else {
+    // no size available, read to EOF
+    constexpr auto chunksize = 4096;
+    std::string chunk(chunksize, 0);
+    while (!file.fail()) {
+      file.read(const_cast<char*>(chunk.data()), chunksize);
+      result.insert(result.end(), chunk.data(), chunk.data() + file.gcount());
+    }
+  }
   return true;
 }
 

fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc

@@ -15,17 +15,21 @@
 #include "fastdeploy/vision/ocr/ppocr/rec_postprocessor.h"
 #include "fastdeploy/utils/perf.h"
 #include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h"
+#include "fastdeploy/utils/utils.h"
 
 namespace fastdeploy {
 namespace vision {
 namespace ocr {
 
 std::vector<std::string> ReadDict(const std::string& path) {
-  std::ifstream in(path);
+  std::string content;
+  ReadBinaryFromFile(path, &content);
+  std::stringstream in(std::move(content));
   FDASSERT(in, "Cannot open file %s to read.", path.c_str());
   std::string line;
   std::vector<std::string> m_vec;
   while (getline(in, line)) {
+    if (!line.empty() && *line.rbegin() == '\r') line.pop_back();
     m_vec.push_back(line);
   }
   m_vec.insert(m_vec.begin(), "#");  // blank char for ctc

llm/README.md

@@ -15,9 +15,9 @@
     # 挂载模型文件
     export MODEL_PATH=${PWD}/Llama-3-8B-A8W8C8
 
-    docker run --gpus all --shm-size 5G --network=host \
+    docker run --gpus all --shm-size 5G --network=host --privileged --cap-add=SYS_PTRACE \
     -v ${MODEL_PATH}:/models/ \
-    -dit registry.baidubce.com/paddlepaddle/fastdeploy:llm-serving-cuda123-cudnn9-v1.0 \
+    -dit registry.baidubce.com/paddlepaddle/fastdeploy:llm-serving-cuda123-cudnn9-v1.2 \
     bash -c 'export USE_CACHE_KV_INT8=1 && cd /opt/output/Serving && bash start_server.sh; exec bash'
   ```