draft -- memory refactoring and super messy AEC attempt

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "esp32-s3-box-3/components/esp-protocols"]
 	path = esp32-s3-box-3/components/esp-protocols
 	url = https://github.com/espressif/esp-protocols.git
+[submodule "esp32-s3-box-3/esp-adf"]
+	path = esp32-s3-box-3/esp-adf
+	url = https://github.com/espressif/esp-adf.git

esp32-s3-box-3/CMakeLists.txt

@@ -7,6 +7,11 @@ if(NOT IDF_TARGET STREQUAL linux)
 
   add_compile_definitions(WIFI_SSID="$ENV{WIFI_SSID}")
   add_compile_definitions(WIFI_PASSWORD="$ENV{WIFI_PASSWORD}")
+
+  # We are not using video in this app, so set the max NALU size to very small
+  if(NOT DEFINED ENV{CONFIG_MAX_NALU_SIZE})
+    add_compile_definitions(CONFIG_MAX_NALU_SIZE=32)
+  endif()
 endif()
 
 if(NOT DEFINED ENV{PIPECAT_SMALLWEBRTC_URL})
@@ -20,7 +25,7 @@ endif()
 add_compile_definitions(PIPECAT_SMALLWEBRTC_URL="$ENV{PIPECAT_SMALLWEBRTC_URL}")
 
 set(COMPONENTS src)
-set(EXTRA_COMPONENT_DIRS "src" "components/srtp" "components/peer" "components/esp-libopus")
+set(EXTRA_COMPONENT_DIRS "src" "components/srtp" "components/peer" "components/esp-libopus" "esp-adf/components")
 
 if(IDF_TARGET STREQUAL linux)
   add_compile_definitions(LINUX_BUILD=1)

esp32-s3-box-3/dependencies.lock

@@ -1,4 +1,10 @@
 dependencies:
+  esp_codec_dev:
+    dependencies: []
+    source:
+      path: /home/khkramer/src/pipecat-esp32/esp32-s3-box-3/esp-adf/components/esp_codec_dev
+      type: local
+    version: 1.3.5
   espressif/button:
     component_hash: f53face2ab21fa0ffaf4cf0f6e513d393f56df6586bb2ad1146120f03f19ee05
     dependencies:
@@ -63,16 +69,16 @@ dependencies:
     targets:
     - esp32s3
     version: 3.0.0~1
-  espressif/esp_codec_dev:
-    component_hash: c71e2d13dad6fc41561590dd88dbc45c79e3f4ef48d5ee3575c60e8b6c8e79d5
+  espressif/esp-dsp:
+    component_hash: 619639efc18cfa361a9e423739b9b0ffc14991effc6c027f955c2f2c3bf1754b
     dependencies:
     - name: idf
       require: private
-      version: '>=4.0'
+      version: '>=4.2'
     source:
-      registry_url: https://components.espressif.com
+      registry_url: https://components.espressif.com/
       type: service
-    version: 1.3.5
+    version: 1.6.0
   espressif/esp_lcd_ili9341:
     component_hash: 31f1b793aa2110dd2ae071c21ccbff0a4eb20d9a4ee40b6294c0dc0ad9552c4e
     dependencies:
@@ -84,7 +90,7 @@ dependencies:
       require: private
       version: 0.*
     source:
-      registry_url: https://components.espressif.com
+      registry_url: https://components.espressif.com/
       type: service
     version: 1.2.0
   espressif/esp_lcd_touch:
@@ -139,6 +145,16 @@ dependencies:
       registry_url: https://components.espressif.com
       type: service
     version: 2.6.0
+  espressif/esp_websocket_client:
+    component_hash: f77326f0e1c38da4e9c97e17c5d649b0dd13027f2645e720e48db269638fd622
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.4.0
   espressif/icm42670:
     component_hash: 28b56e174f75c70037f5208aaed6c3789f0d243500d975519584bf9dc8c0836c
     dependencies:
@@ -149,6 +165,36 @@ dependencies:
       registry_url: https://components.espressif.com
       type: service
     version: 2.0.2
+  espressif/jsmn:
+    component_hash: d80350c41bbaa827c98a25b6072df00884e72f54885996fab4a4f0aebce6b6c3
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=4.3'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  espressif/nghttp:
+    component_hash: 32e513409ddce0000990ec74b0711f535462726236d72578cd70498e6bd66dc3
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.65.0
+  espressif/zlib:
+    component_hash: d901723af51f13fc8e5824f39f32239c847956e8fd951a05266588dc5cfbb9ae
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=4.4'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.3.1
   idf:
     source:
       type: idf
@@ -162,7 +208,13 @@ dependencies:
     version: 9.3.0
 direct_dependencies:
 - espressif/esp-box-3
+- espressif/esp-dsp
+- espressif/esp_lcd_ili9341
+- espressif/esp_websocket_client
+- espressif/jsmn
+- espressif/nghttp
+- espressif/zlib
 - idf
-manifest_hash: 55f6a702498de8e87201da504a38c5c958e2094ec832c4f8d6d368369d5d0625
+manifest_hash: 1891dc87058c6eec7aa77f286a1967ebe7c293709db8da5b2f1e79d2e45f57ef
 target: esp32s3
 version: 2.0.0

esp32-s3-box-3/partitions.csv

@@ -2,5 +2,5 @@
 # Name,   Type, SubType, Offset,  Size, Flags
 nvs,      data, nvs,     0x9000,  0x6000,
 phy_init, data, phy,     0xf000,  0x1000,
-factory,  app,  factory, 0x10000, 0x180000,
+factory,  app,  factory, 0x10000, 0x200000,
 

esp32-s3-box-3/sdkconfig.defaults

@@ -18,6 +18,9 @@ CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384
 # Defaults to partitions.csv
 CONFIG_PARTITION_TABLE_CUSTOM=y
 
+# Set flash size to 16MB for ESP32-S3-Box-3
+CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y
+
 # Set highest CPU Freq
 CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
 
@@ -33,3 +36,10 @@ CONFIG_COMPILER_OPTIMIZATION_PERF=y
 CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y
 
 CONFIG_CODEC_I2C_BACKWARD_COMPATIBLE=n
+
+# Use C library memory allocator for LVGL to save 64KB of BSS
+# This removes the 64KB work_mem_int buffer from BSS
+CONFIG_LV_USE_BUILTIN_MALLOC=n
+CONFIG_LV_USE_CLIB_MALLOC=y
+CONFIG_BSP_LCD_DRAW_BUF_HEIGHT=10
+CONFIG_BSP_LCD_DRAW_BUF_DOUBLE=n

esp32-s3-box-3/src/CMakeLists.txt

@@ -7,7 +7,7 @@ if(IDF_TARGET STREQUAL linux)
 else()
 	idf_component_register(
 		SRCS ${COMMON_SRC} "wifi.cpp" "media.cpp" "rtvi.cpp" "rtvi_callbacks.cpp" "screen.cpp"
-		REQUIRES driver esp_wifi nvs_flash peer esp_psram esp-libopus esp_http_client json lvgl)
+		REQUIRES driver esp_wifi nvs_flash peer esp_psram esp-libopus esp_http_client json lvgl esp-sr)
 endif()
 
 idf_component_get_property(lib peer COMPONENT_LIB)

esp32-s3-box-3/src/main.cpp

@@ -1,33 +1,56 @@
 #include "main.h"
 
+#include <inttypes.h>
 #include <esp_event.h>
 #include <esp_log.h>
 #include <peer.h>
 
 #ifndef LINUX_BUILD
 #include "nvs_flash.h"
 
+// All operations that modify the webrtc state should use this semaphore to ensure thread safety
+SemaphoreHandle_t webrtcSemaphore = NULL;
+
 extern "C" void app_main(void) {
-  esp_err_t ret = nvs_flash_init();
-  if (ret == ESP_ERR_NVS_NO_FREE_PAGES ||
-      ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
-    ESP_ERROR_CHECK(nvs_flash_erase());
-    ret = nvs_flash_init();
-  }
-  ESP_ERROR_CHECK(ret);
 
+  ESP_LOGI(LOG_TAG, "[APP] Startup..");
+  ESP_LOGI(LOG_TAG, "[APP] IDF version: %s", esp_get_idf_version());
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
+  ESP_ERROR_CHECK(nvs_flash_init());
   ESP_ERROR_CHECK(esp_event_loop_create_default());
+
+  webrtcSemaphore = xSemaphoreCreateMutex();
+
   pipecat_init_screen();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
   peer_init();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
   pipecat_init_audio_capture();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
   pipecat_init_audio_decoder();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
+  pipecat_init_aec();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
   pipecat_init_wifi();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
+
   pipecat_init_webrtc();
+  heap_caps_print_heap_info(MALLOC_CAP_INTERNAL);
 
+  ESP_LOGI(LOG_TAG, "Pipecat ESP32 client initialized");
   pipecat_screen_system_log("Pipecat ESP32 client initialized\n");
 
+  ESP_LOGI(LOG_TAG, "Starting webrtc task");
+  pipecat_webrtc_run_task();
+
   while (1) {
-    pipecat_webrtc_loop();
+    // Add some stats printout to the screen here
     vTaskDelay(pdMS_TO_TICKS(TICK_INTERVAL));
   }
 }
@@ -38,6 +61,7 @@ int main(void) {
   pipecat_webrtc();
 
   while (1) {
+    // Todo: test/fix linux build
     pipecat_webrtc_loop();
     vTaskDelay(pdMS_TO_TICKS(TICK_INTERVAL));
   }

esp32-s3-box-3/src/main.h

@@ -14,16 +14,19 @@ extern void pipecat_init_audio_decoder();
 extern void pipecat_init_audio_encoder();
 extern void pipecat_send_audio(PeerConnection *peer_connection);
 extern void pipecat_audio_decode(uint8_t *data, size_t size);
+extern void pipecat_init_aec();
 
 // WebRTC / Signalling
 extern void pipecat_init_webrtc();
 extern void pipecat_webrtc_loop();
+extern void pipecat_webrtc_run_task();
 extern void pipecat_http_request(char *offer, char *answer);
 
 // RTVI
 typedef struct {
   void (*on_bot_started_speaking)();
   void (*on_bot_stopped_speaking)();
+
   void (*on_bot_tts_text)(const char *text);
 } rtvi_callbacks_t;