parameterized ports in the html

Author: dusty-nv

packages/llm/local_llm/agents/video_query.py

@@ -92,7 +92,7 @@ def __init__(self, model="liuhaotian/llava-v1.5-13b", nanodb=None, vision_scalin
                 path=nanodb, 
                 model=None, # disable DB's model because VLM's CLIP is used 
                 reserve=kwargs.get('nanodb_reserve'), 
-                k=8, drop_inputs=True,
+                k=12, drop_inputs=True,
             ).start().add(self.on_search)
             self.llm.add(self.on_image_embedding, channel=ChatQuery.OutputImageEmbedding)
         else:
@@ -106,14 +106,35 @@ def __init__(self, model="liuhaotian/llava-v1.5-13b", nanodb=None, vision_scalin
 
         mounts['/data/datasets/uploads'] = '/images/uploads'
 
+        video_source = self.video_source.stream.GetOptions()['resource']
+        video_output = self.video_output.stream.GetOptions()['resource']
+        
+        webrtc_args = {}
+        
+        if video_source['protocol'] == 'webrtc':
+            webrtc_args.update(dict(webrtc_input_stream=video_source['path'].strip('/'), 
+                                    webrtc_input_port=video_source['port'],
+                                    send_webrtc=True))
+        else:
+            webrtc_args.update(dict(webrtc_input_stream='input', 
+                                    webrtc_input_port=8554,
+                                    send_webrtc=False))
+        
+        if video_output['protocol'] == 'webrtc':
+            webrtc_args.update(dict(webrtc_output_stream=video_output['path'].strip('/'), 
+                                    webrtc_output_port=video_output['port']))
+        else:
+            webrtc_args.update(dict(webrtc_output_stream='output', 
+                                    webrtc_output_port=8554))
+
         self.server = WebServer(
             msg_callback=self.on_websocket, 
             index='video_query.html', 
-            send_webrtc=False, 
             title="LIVE LLAVA", 
             model=os.path.basename(model),
             mounts=mounts,
             nanodb=nanodb,
+            **webrtc_args,
             **kwargs
         )
 

packages/llm/local_llm/web/templates/video_query.html

@@ -35,7 +35,7 @@
       };
 
       function sendWebRTC() {
-        if( sendStream(getWebsocketURL('input'), document.getElementById('send-stream').value) )
+        if( sendStream(getWebsocketURL("{{ webrtc_input_stream }}", port={{ webrtc_input_port }}), document.getElementById('send-stream').value) )
           play(); // autoplay browser stream
       }
 
@@ -50,9 +50,9 @@
             for( let n=0; n < search_results.length; n++ ) {
               if( 'image' in search_results[n] ) {
                 //contents = '<div style="position: relative; text-align: center; color: white" width="50%">';
-                let contents = '<div class="me-1" style="position: relative; color: white; display: inline;">';
-                contents += `<img src=${search_results[n]['image']} style="aspect-ratio: 1.47; width: 49%;" title="&quot;METADATA&quot; : ${search_results[n]['metadata']}">`;
-                contents += `<div class="ps-1 pe-1" style="position: absolute; top: 0%; right: 0%; transform: translate(0%, -288%); background-color: rgba(0,0,0,0.5)">${search_results[n]['similarity']}</div></div>`;
+                let contents = '<div class="me-1" style="position: relative; color: white; display: inline;">';  // 49% width, -288% translate for 2x4 grid
+                contents += `<img src=${search_results[n]['image']} style="aspect-ratio: 1.47; width: 32%;" title="&quot;METADATA&quot; : ${search_results[n]['metadata']}">`;
+                contents += `<div class="ps-1 pe-1" style="position: absolute; top: 0%; right: 0%; transform: translate(0%, -168%); background-color: rgba(0,0,0,0.5)">${search_results[n]['similarity']}</div></div>`;
                 //contents += '</div>';
                 obj.append(contents);
               }
@@ -82,7 +82,7 @@
         var sendStream = document.getElementById('send-stream');
         var sendButton = document.getElementById('send-button');
 
-        playStream.value = getWebsocketURL('output');
+        playStream.value = getWebsocketURL("{{ webrtc_output_stream }}", port={{ webrtc_output_port }});
 
       {% if send_webrtc %}
         // populate the list of browser video devices (requires HTTPS)
@@ -113,7 +113,7 @@
         playWebRTC();
       {% endif %}
 
-        connectWebsocket(onWebsocketMsg, port=49000);
+        connectWebsocket(onWebsocketMsg, port={{ ws_port }});
 
         event_table = new Tabulator("#event_table", {
             columns: [
@@ -373,10 +373,10 @@ <h5 class="d-inline">NanoDB</h5>
             {% if send_webrtc %}
               <div class="row">
                 <label for="send-stream" class="col-2" style="width: 6rem;">Webcam:</label>
-                <div class="col-6">
+                <div class="col-5">
                   <select id="send-stream" name="send-stream"></select>
                 </div>
-                <div class="col-4">
+                <div class="col-4 ms-3">
                   <button id="send-button" onclick="sendWebRTC()">Send</button>
                 </div>
               </div>