Merge pull request #18 from yeoleobun/interrupt

Fix: fix and optimizing interruption event

Author: shenjinti

src/event.rs

@@ -102,13 +102,11 @@ pub enum SessionEvent {
         #[serde(rename = "trackId")]
         track_id: String,
         timestamp: u64,
-        position: u64, // current playback position at the time of interruption
-    },
-    OnInterrupt {
-        subtitle: String,
-        position: u32,
-        total_duration: u32, // ms
-        current: u32,        // current timestamp of total_duration
+        subtitle: Option<String>, // current tts text
+        position: Option<u32>, // word index in subtitle
+        #[serde(rename = "totalDuration")]
+        total_duration: u32, // whole tts duration
+        current: u32,          // elapsed time since start of tts
     },
     AsrFinal {
         #[serde(rename = "trackId")]

src/handler/tests/webrtc_test.rs

@@ -562,11 +562,12 @@ async fn test_tts_interrupt_with_config(tts_config: SynthesisOption, text: &str)
                         .await
                         .unwrap();
                 }
-                SessionEvent::OnInterrupt {
+                SessionEvent::Interruption {
                     subtitle,
                     position,
                     total_duration,
                     current,
+                    ..
                 } => {
                     interrupted_clone.store(true, Ordering::Relaxed);
                     info!(

src/media/track/tts.rs

@@ -173,6 +173,8 @@ impl Track for TtsTrack {
         let synthesize_done = Arc::new(AtomicBool::new(false));
         let synthesize_done_clone = synthesize_done.clone();
         let session_id = self.session_id.clone();
+        let current_text = Arc::new(Mutex::new(None));
+        let current_text_clone = current_text.clone();
         let subtitles = Arc::new(Mutex::new(Vec::<TTSSubtitle>::new()));
         let subtitles_clone = subtitles.clone();
         let total_audio_len = Arc::new(AtomicUsize::new(0));
@@ -181,6 +183,10 @@ impl Track for TtsTrack {
             let mut last_play_id = None;
             while let Some(command) = command_rx.recv().await {
                 let text = command.text;
+                {
+                    let mut current_text = current_text_clone.lock().await;
+                    *current_text = Some(text.clone());
+                }
                 let play_id = command.play_id;
                 let mut option = command.option;
                 if option.speaker.is_none() {
@@ -230,7 +236,6 @@ impl Track for TtsTrack {
                                     let duration = bytes_size_to_duration(audio.len(), sample_rate);
                                     total_audio_len_clone.store(audio.len(), Ordering::Relaxed);
                                     subtitles.push(TTSSubtitle::new(
-                                        &text,
                                         0,
                                         duration,
                                         0,
@@ -438,26 +443,26 @@ impl Track for TtsTrack {
                     let remaining_size = buffer.lock().await.len() * 2;
                     debug!(session_id, "total_size: {} remaining_size: {}", total_size, remaining_size);
                     let sended_size = total_size - remaining_size;
-                    let mut text = "".into();
-                    let mut position = 0;
-                    let mut current = bytes_size_to_duration(sended_size, sample_rate);
-                    let total_duration = bytes_size_to_duration(total_size, sample_rate);
+                    let text = current_text.lock().await.take();
+                    let mut position = None;
+                    let current = bytes_size_to_duration(sended_size, sample_rate);
+                    let total_duration = bytes_size_to_duration(total_size, sample_rate);                    
                     let subtitles = subtitles.lock().await;
-                    debug!("subtitles: {:?}", subtitles);
                     for subtitle in subtitles.iter().rev() {
                         if subtitle.begin_time <= current {
-                            text = subtitle.text.clone();
-                            position = subtitle.begin_index;
-                            current = subtitle.begin_time;
+                            position = Some(subtitle.begin_index);
                             break;
                         }
                     }
-                    let _ = event_sender_clone.send(SessionEvent::OnInterrupt {
+                    let event = SessionEvent::Interruption {
+                        track_id: track_id.clone(),
+                        timestamp: crate::get_timestamp(),
                         subtitle: text,
                         position,
                         total_duration,
                         current,
-                    });
+                    };
+                    let _ = event_sender_clone.send(event);
                 }
             }
             event_sender_clone

src/synthesis/aliyun.rs

@@ -233,20 +233,18 @@ impl AliyunTTSState {
         self.text
             .chars()
             .enumerate()
-            .position(|(i, _)| i == usage as usize)
+            .position(|(i, _)| i >= usage as usize)
             .unwrap_or(default as usize) as u32
     }
 
     fn move_forward(&mut self, current_usage: u32) -> Vec<TTSSubtitle> {
-        let subtitle = self.text[self.last_usage as usize..current_usage as usize].to_string();
         let begin_index = self.usage_to_index(self.last_usage, 0);
         let end_index = self.usage_to_index(current_usage, self.text.len() as u32);
         let begin_time = self.size_to_time(self.last_size);
         let end_time = self.size_to_time(self.current_size);
         self.last_size = self.current_size;
         self.last_usage = current_usage;
         vec![TTSSubtitle::new(
-            &subtitle,
             begin_time,
             end_time,
             begin_index,

src/synthesis/mod.rs

@@ -119,23 +119,20 @@ pub enum TTSEvent {
 #[allow(dead_code)]
 #[derive(Debug)]
 pub struct TTSSubtitle {
-    pub text: String,
     pub begin_time: u32,
     pub end_time: u32,
     pub begin_index: u32,
-    end_index: u32,
+    pub end_index: u32,
 }
 
 impl TTSSubtitle {
     pub fn new(
-        text: &str,
         begin_time: u32,
         end_time: u32,
         begin_index: u32,
         end_index: u32,
     ) -> Self {
         Self {
-            text: text.to_string(),
             begin_time,
             end_time,
             begin_index,

src/synthesis/tencent_cloud.rs

@@ -39,7 +39,6 @@ struct WebSocketResult {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "PascalCase")]
 struct Subtitle {
-    text: String,
     begin_time: u32,
     end_time: u32,
     begin_index: u32,
@@ -49,7 +48,6 @@ struct Subtitle {
 impl From<&Subtitle> for TTSSubtitle {
     fn from(subtitle: &Subtitle) -> Self {
         TTSSubtitle::new(
-            &subtitle.text,
             subtitle.begin_time,
             subtitle.end_time,
             subtitle.begin_index,
@@ -197,9 +195,10 @@ impl TencentCloudTtsClient {
 
                             if response.code != 0 {
                                 return Some(Err(anyhow!(
-                                    "Tencent TTS faild, Session: {}, error: {}",
+                                    "Tencent TTS faild, Session: {}, error: {}, text: {}",
                                     session_id,
-                                    response.message
+                                    response.message,
+                                    text
                                 )));
                             }