[iOS][SDK] Updated Executorch with upstream main branch (NimbleEdge#169)

* updated executorch_llm_executor

* updated setup.sh

---------

Signed-off-by: Suryansh Bisen <suryansh.bisen@nimbleedgehq.ai>

Author: Suryansh1191

config.yml

@@ -11,7 +11,7 @@ android:
   cmake_args: "-DANDROID_STL=c++_shared -DEXECUTORCH_EXECUTOR=0 -DONNXBuild=1_21_0_full -DONNXgenai=0_7_0_onnx_genai -DONNXext=0_13_0_ort_extensions -DMINIMAL_BUILD=0 -DExecutorchgenai=d4906e21_executorch -DGEMINI=1"
 
 ios:
-  cmake_args: "-DIOS=1 -DEXECUTORCH_EXECUTOR=0 -DONNXBuild=1_21_0_ios_full -DONNXgenai=0_7_0_ios_onnx-genai -DONNXext=0_13_0_ort_extension -DMINIMAL_BUILD=0 -DExecutorchgenai=d4906e21_executorch"
+cmake_args: "-DIOS=1 -DONNXGENAI_EXECUTOR=0 -DEXECUTORCH_EXECUTOR=1 -DONNXBuild=1_21_0_ios_full -DONNXgenai=0_7_0_ios_onnx-genai -DONNXext=0_13_0_ort_extension -DMINIMAL_BUILD=0 -DExecutorchgenai=e9c11a40_executorch"
 
 react:
   ndk: "21.4.7075529"

coreruntime/nimblenet/llm_executors/include/executorch_llm_executor.hpp

@@ -9,7 +9,7 @@
 #include "base_llm_executor.hpp"
 #include "task.hpp"
 #ifdef EXECUTORCH_EXECUTOR
-#include "executorch/extension/llm/runner/runner.h"
+#include "executorch/extension/llm/runner/text_llm_runner.h"
 #endif  // EXECUTORCH_EXECUTOR
 #include "ne_fwd.hpp"
 #include "rigtorp/SPSCQueue.h"

coreruntime/nimblenet/llm_executors/src/executorch_llm_executor.cpp

@@ -23,8 +23,8 @@ ExecutorchLLMExecutor::ExecutorchLLMExecutor(
   _temperature = temperature;
 
   try {
-    _runner = example::Runner::create(modelDirectoryPath + "/" + pteFileName + ".pte",
-                                      modelDirectoryPath + "/" + tokenizerFileName, std::nullopt,
+    _runner = executorch::extension::llm::create_text_llm_runner(modelDirectoryPath + "/" + pteFileName + ".pte",
+                                      executorch::extension::llm::load_tokenizer(modelDirectoryPath + "/" + tokenizerFileName), std::nullopt,
                                       temperature);
     _runner->load();
     if (!_runner->is_loaded()) {
@@ -42,6 +42,11 @@ std::shared_ptr<CharStream> ExecutorchLLMExecutor::run_prompt(const std::string&
   std::lock_guard<std::mutex> lock{_mutex};
 
   stop_inference_thread();
+  if (!_runner->is_loaded()) {
+      LOG_TO_CLIENT_ERROR("LLM Runner is not loaded");
+      _runner->load();
+  }
+
   // Creating these variables again to drop ones used by previous inference
   _charStream = CharStream::construct();
   _internalQueue = std::make_shared<Queue>(_executorConfig.internalQueueSize);
@@ -82,31 +87,31 @@ void ExecutorchLLMExecutor::run_inference(const std::string& prompt) {
   try {
     executorch::extension::llm::GenerationConfig config{
         .echo = false,
-        .seq_len = _executorConfig.maxInputNumTokens,
         .temperature = _temperature,
     };
-    auto status = _runner->generate_from_pos(prompt, config, _start_pos, token_callback, {});
+    if (!_runner->is_loaded()) {
+        _runner->load();
+    }
+    auto status = _runner->generate(prompt, config, token_callback, {});
     if (status != ::executorch::runtime::Error::Ok) {
       mark_end_of_stream();
-      LOG_TO_CLIENT_ERROR("Error while running inference on LLM using executorch.");
     }
   } catch (const std::exception& e) {
     mark_end_of_stream();
-    LOG_TO_CLIENT_ERROR("Error: %s while running inference on LLM using executorch.", e.what());
   }
 }
 
 void ExecutorchLLMExecutor::add_prompt(const std::string& prompt) {
   std::lock_guard<std::mutex> lock{_mutex};
   stop_inference_thread();
-  try {
-    auto status = _runner->prefill_prompt(prompt, _start_pos, 0, 0);
-    if (!status.ok()) {
-      LOG_TO_CLIENT_ERROR("Error while setting context in LLM using executorch.");
-    }
-  } catch (const std::exception& e) {
-    LOG_TO_CLIENT_ERROR("Error: %s while setting context in LLM using executorch.", e.what());
-  }
+  // try {
+  //   auto status = _runner->prefill_prompt(prompt, _start_pos, 0, 0);
+  //   if (!status.ok()) {
+  //     LOG_TO_CLIENT_ERROR("Error while setting context in LLM using executorch.");
+  //   }
+  // } catch (const std::exception& e) {
+  //   LOG_TO_CLIENT_ERROR("Error: %s while setting context in LLM using executorch.", e.what());
+  // }
 }
 
 void ExecutorchLLMExecutor::cancel() {
@@ -133,4 +138,4 @@ void ExecutorchLLMExecutor::stop_inference_thread() {
   _internalQueue = nullptr;
 }
 
-void ExecutorchLLMExecutor::mark_end_of_stream() { _internalQueue->push('\0'); }
+void ExecutorchLLMExecutor::mark_end_of_stream() { _internalQueue->push('\0'); }

setup.sh

@@ -192,8 +192,8 @@ elif [[ "$sdk" == "ios" ]]; then
     fi
 
     if [[ $download_executorch -eq 1 ]]; then
-        mkdir -p third_party/runtime/executorch/ios/d4906e21_executorch
-        aws s3 cp s3://deliteai/build-dependencies/executorch-d4906e21/ios third_party/runtime/executorch/ios/d4906e21_executorch --recursive --no-sign-request
+        mkdir -p third_party/runtime/executorch/ios/e9c11a40_executorch
+        aws s3 cp s3://deliteai/build-dependencies/executorch-e9c11a40/ios third_party/runtime/executorch/ios/e9c11a40_executorch --recursive --no-sign-request
     fi
 fi