Merge pull request #430 from classtranscribe/AddLLava

Allow use of POSIX shell to start Llava

ClassTranscribeDatabase/Globals.cs

@@ -87,6 +87,7 @@ public class AppSettings
         public string LLAVA_PROMPT { get; set; } = "### User: I am blind and listening to a university lecture video. What is in this image, that has been extracted from the lecture video? Be concise. Do your best to describe only the technical content of the image that is relevant to learning. Do not add opinions about the image.\n### Assistant:";
 
         public string LLAVA_LOG_STREAMS { get; set; } = "out,err";
+        public string LLAVA_USESHELL { get; set; } = "";
 
     }
 

TaskEngine/Tasks/DescribeImageTask.cs

@@ -12,6 +12,7 @@
 // using SkiaSharp;
 using System.IO;
 using System.Diagnostics;
+using Microsoft.IdentityModel.Tokens;
 
 
 
@@ -99,6 +100,7 @@ async Task<string> DescribeImage(string imagePath, string ocrtext) {
             if (!File.Exists(imagePath)) { GetLogger().LogError($"DescribeImage. Image file <{imagePath}> does not exist - nothing to do."); return ""; }
             var llavaExec = Globals.appSettings.LLAVA_PATH; //  "/llava/llava-v1.5-7b-q4.llamafile"
             var prompt = Globals.appSettings.LLAVA_PROMPT;
+            var useShell = Globals.appSettings.LLAVA_USESHELL.Trim();
             var cpuCount = Math.Max(1, Environment.ProcessorCount / 2); // don't want hyperthreading  (we are memory bandwidth bound)- and this may report logical not physical cores
                                                                         // besides we dont want monopolize the server
             var llavaArguments = Globals.appSettings.LLAVA_ARGS;
@@ -121,11 +123,11 @@ async Task<string> DescribeImage(string imagePath, string ocrtext) {
             if (args.Contains("{") || args.Contains("}") ) {
                 throw new Exception("Argument still has a curly brace - unprocessed placeholder? Only {cpuCount|prompt|imagePath} are supported." + args + ". Check LLAVA_ARGS");
             }
-
+            
             var info = new ProcessStartInfo()
-            { //  --escape = Process prompt escapes sequences (\n, \r, \t, \', \", \\)
-                FileName = llavaExec,
-                Arguments = args, // "--threads 12 --help", // ",
+            { 
+                FileName = useShell.Length > 0 ? llavaExec: useShell,
+                Arguments = useShell.Length > 0 ? $"{llavaExec} -c \"{args}\"" : args, // "--threads 12 --help", // ",
                 RedirectStandardInput = true,
                 RedirectStandardOutput = true,
                 RedirectStandardError = true,

TestRemoteLLM/Program.cs

@@ -4,7 +4,6 @@
 using Newtonsoft.Json.Linq;
 using System.Diagnostics;
 using System.Text;
-using Microsoft.AspNetCore.Routing.Constraints;
 
 internal class Program
 {
@@ -17,7 +16,7 @@ private static async Task<string> DescribeImage(string imagePath)
         /* llamafile --temp 0   --image ~/Pictures/lemurs.jpg   -m llava-v1.5-7b-Q4_K.gguf   --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf   -e -p '### User: What do you see?\n### Assistant: ' \
   --silent-prompt 2>/dev/null */
 
-        var execFile = "./llava-v1.5-7b-q4.llamafile";
+        var execFile = "./llava-v1.5-7b-q4.llamafile.exe";
         var execPath = "E:/downloads/" + execFile;
 
         if (!File.Exists(execPath)) { Console.WriteLine($"Invalid exec path:<{execPath}>"); return ""; }
@@ -30,6 +29,7 @@ private static async Task<string> DescribeImage(string imagePath)
         var prompt = "### User: What do you see in this image?\n### Assistant:"; // add single quotes and -p
         // See https://learn.microsoft.com/en-us/dotnet/api/system.diagnostics.processstartinfo.redirectstandardoutput?view=net-8.0
         var processArgs = $"{llamaOptions}  --image {imagePath} --escape -p \"{prompt}\"";  //
+
         var info = new ProcessStartInfo()
         { //  --escape = Process prompt escapes sequences (\n, \r, \t, \', \", \\)
             FileName = execPath,