Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved GetOutputDataFileName with common refactored {id}, {pid}, {t… #342

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Improved GetOutputDataFileName with common refactored {id}, {pid}, {t… #342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de607af
Improved GetOutputDataFileName with common refactored {id}, {pid}, {t…
robch Dec 3, 2024
61a4645
Merge branch 'main' into robch/2412-dec03-improved-GetOutputDataFileN…
robch Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 30 additions & 16 deletions src/ai/commands/chat_command.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -599,38 +599,30 @@ private async Task<string> GetChatCompletionsAsync(ChatClient client, List<ChatM

private void CheckWriteChatAnswerOutputFile(string completeResponse)
{
var outputAnswerFile = OutputChatAnswerFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputAnswerFile))
if (!string.IsNullOrEmpty(_outputAnswerFile))
{
var fileName = FileHelpers.GetOutputDataFileName(outputAnswerFile, _values);
FileHelpers.WriteAllText(fileName, completeResponse, Encoding.UTF8);
FileHelpers.WriteAllText(_outputAnswerFile, completeResponse, Encoding.UTF8);
}

var outputAddAnswerFile = OutputAddChatAnswerFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputAddAnswerFile))
if (!string.IsNullOrEmpty(_outputAddAnswerFile))
{
var fileName = FileHelpers.GetOutputDataFileName(outputAddAnswerFile, _values);
FileHelpers.AppendAllText(fileName, "\n" + completeResponse, Encoding.UTF8);
FileHelpers.AppendAllText(_outputAddAnswerFile, "\n" + completeResponse, Encoding.UTF8);
}
}

private async Task CheckWriteChatHistoryOutputFileAsync(Func<string, Task> saveChatHistoryToFile)
{
var outputHistoryFile = OutputChatHistoryFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputHistoryFile))
if (!string.IsNullOrEmpty(_outputChatHistoryFileName))
{
var fileName = FileHelpers.GetOutputDataFileName(outputHistoryFile, _values);
await saveChatHistoryToFile(fileName);
await saveChatHistoryToFile(_outputChatHistoryFileName);
}
}

private void CheckWriteChatHistoryOutputFile(Action<string> saveChatHistoryToFile)
{
var outputHistoryFile = OutputChatHistoryFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputHistoryFile))
if (!string.IsNullOrEmpty(_outputChatHistoryFileName))
{
var fileName = FileHelpers.GetOutputDataFileName(outputHistoryFile, _values);
saveChatHistoryToFile(fileName);
saveChatHistoryToFile(_outputChatHistoryFileName);
}
}

Expand Down Expand Up @@ -1896,6 +1888,24 @@ private void StartCommand()
// _output!.EnsureOutputAll("chat.input.id", id);
// _output!.EnsureOutputEach("chat.input.id", id);

var outputHistoryFile = OutputChatHistoryFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputHistoryFile))
{
_outputChatHistoryFileName = FileHelpers.GetOutputDataFileName(outputHistoryFile, _values, "chat.input.id");
}

var outputAnswerFile = OutputChatAnswerFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputAnswerFile))
{
_outputAnswerFile = FileHelpers.GetOutputDataFileName(outputAnswerFile, _values);
}

var outputAddAnswerFile = OutputAddChatAnswerFileToken.Data().GetOrDefault(_values);
if (!string.IsNullOrEmpty(outputAddAnswerFile))
{
_outputAddAnswerFile = FileHelpers.GetOutputDataFileName(outputAddAnswerFile, _values);
}

_lock = new SpinLock();
_lock.StartLock();
}
Expand Down Expand Up @@ -1936,5 +1946,9 @@ private float TryParse(string? s, float defaultValue)
private const float _defaultTemperature = 0.7f;
private const float _defaultFrequencyPenalty = 0.0f;
private const float _defaultPresencePenalty = 0.0f;

private string? _outputAnswerFile;
private string? _outputAddAnswerFile;
private string? _outputChatHistoryFileName;
}
}
30 changes: 2 additions & 28 deletions src/ai/helpers/output_helper_base.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,20 +196,7 @@ private void FlushOutputAllCache()
private string GetOutputAllFileName()
{
var file = _values.GetOrDefault("output.all.file.name", "output.{run.time}." + GetOutputAllFileType());

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (file.Contains("{pid}")) file = file.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (file.Contains("{time}")) file = file.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private string GetOutputAllFileType()
Expand Down Expand Up @@ -360,20 +347,7 @@ private void FlushOutputEachCacheStage2(bool overwrite = false)
private string GetOutputEachFileName()
{
var file = _values.GetOrDefault("output.each.file.name", "each.{run.time}." + GetOutputEachFileType());

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (file.Contains("{pid}")) file = file.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (file.Contains("{time}")) file = file.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private string GetOutputEachFileType()
Expand Down
14 changes: 1 addition & 13 deletions src/clis/vz/commands/image_command.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,19 +408,7 @@ public void EnsureStartLogFile()
var log = _values["diagnostics.config.log.file"];
if (!string.IsNullOrEmpty(log))
{
var id = _values.GetOrEmpty("vision.input.id");
if (log.Contains("{id}")) log = log.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (log.Contains("{pid}")) log = log.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (log.Contains("{time}")) log = log.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (log.Contains("{run.time}")) log = log.Replace("{run.time}", runTime);

log = FileHelpers.GetOutputDataFileName(log, _values);
log = FileHelpers.GetOutputDataFileName(log, _values, "vision.input.id");
FileLogger.Start(log);
}
}
Expand Down
17 changes: 0 additions & 17 deletions src/common/details/commands/command.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,23 +452,6 @@ protected string GetIdFromInputUrl(string url, string idValueName)
return id;
}

protected string ReplaceFileNameValues(string fileName, string idValueName)
{
var id = _values.GetOrEmpty(idValueName);
if (fileName.Contains("{id}")) fileName = fileName.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (fileName.Contains("{pid}")) fileName = fileName.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (fileName.Contains("{time}")) fileName = fileName.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (fileName.Contains("{run.time}")) fileName = fileName.Replace("{run.time}", runTime);

return fileName.ReplaceValues(_values);
}

protected void DisposeAfterStop()
{
try
Expand Down
33 changes: 22 additions & 11 deletions src/common/details/helpers/file_helpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ public static bool IsStandardOutputReference(string fileName)
return fileName == "-" || fileName == "stdout";
}

public static string GetOutputDataFileName(string file, INamedValues? values = null)
public static string GetOutputDataFileName(string file, INamedValues? values = null, string? idValueName = null)
{
if (file.StartsWith("@")) return GetOutputConfigFileName(file, values);
if (IsStandardOutputReference(file)) return file;
Expand All @@ -605,11 +605,32 @@ public static string GetOutputDataFileName(string file, INamedValues? values = n
? PathHelpers.Combine(outputDir, file) ?? file
: file;

outputFile = ReplaceFileNameValues(outputFile, values, idValueName);
if (Program.Debug) Console.WriteLine($"DEBUG: Output DATA file '{file}'='{outputFile}'");

return outputFile;
}

public static string ReplaceFileNameValues(string fileName, INamedValues? values = null, string? idValueName = null)
{
var id = idValueName != null ? values?.GetOrEmpty(idValueName) : null;
if (id != null && fileName.Contains("{id}")) fileName = fileName.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (fileName.Contains("{pid}")) fileName = fileName.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (fileName.Contains("{time}")) fileName = fileName.Replace("{time}", time);

var runTime = values?.GetOrEmpty("x.run.time");
if (fileName.Contains("{run.time}")) fileName = fileName.Replace("{run.time}", runTime);

return values != null
? fileName.ReplaceValues(values)
: fileName;
}


public static Stream Create(string fileName)
{
return IsStandardOutputReference(fileName)
Expand Down Expand Up @@ -876,16 +897,6 @@ public static FileInfo GetAssemblyFileInfo(Type type)
public static void LogException(ICommandValues values, Exception ex)
{
var file = $"exception.{{run.time}}.{{pid}}.{{time}}.{Program.Name}.error.log";

var runTime = values.GetOrEmpty("x.run.time");
file = file.Replace("{run.time}", runTime);

var pid = Process.GetCurrentProcess().Id.ToString();
file = file.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
file = file.Replace("{time}", time);

file = FileHelpers.GetOutputDataFileName(file, values);
FileHelpers.WriteAllText(file, ex.ToString(), Encoding.UTF8);
}
Expand Down
9 changes: 0 additions & 9 deletions src/common/details/helpers/log_helpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@ public static void EnsureStartLogFile(ICommandValues values)
var log = values["diagnostics.config.log.file"];
if (!string.IsNullOrEmpty(log))
{
var pid = Process.GetCurrentProcess().Id.ToString();
if (log.Contains("{pid}")) log = log.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (log.Contains("{time}")) log = log.Replace("{time}", time);

var runTime = values.GetOrEmpty("x.run.time");
if (log.Contains("{run.time}")) log = log.Replace("{run.time}", runTime);

try
{
log = FileHelpers.GetOutputDataFileName(log, values);
Expand Down
2 changes: 1 addition & 1 deletion src/extensions/speech_extension/commands/synthesize_command.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ private string GetInputFileFromId(string id)
}
else if (output == "file" && !string.IsNullOrEmpty(file))
{
file = ReplaceFileNameValues(file, "synthesizer.input.id");
file = FileHelpers.GetOutputDataFileName(file, _values, "synthesizer.input.id");
audioConfig = AudioOutputHelpers.CreateAudioConfigForFile(file);
}
else
Expand Down
15 changes: 1 addition & 14 deletions src/extensions/speech_extension/helpers/config_helpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,22 +138,9 @@ public static void SetupLogFile(SpeechConfig config, ICommandValues values)
var log = values["diagnostics.config.log.file"];
if (!string.IsNullOrEmpty(log))
{
var id = values.GetOrEmpty("audio.input.id");
if (log.Contains("{id}")) log = log.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (log.Contains("{pid}")) log = log.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (log.Contains("{time}")) log = log.Replace("{time}", time);

var runTime = values.GetOrEmpty("x.run.time");
if (log.Contains("{run.time}")) log = log.Replace("{run.time}", runTime);

log = FileHelpers.GetOutputDataFileName(log, values);
log = FileHelpers.GetOutputDataFileName(log, values, "audio.input.id");
config.SetProperty(PropertyId.Speech_LogFilename, log);
}
}

}
}
73 changes: 10 additions & 63 deletions src/extensions/speech_extension/helpers/output_helper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,21 @@ public void StartOutput()
{
_overwriteEach = true;
_outputEach = GetOutputEachColumns().Count() > 0;
_outputEachFileName = FileHelpers.GetOutputDataFileName(GetOutputEachFileName(), _values);
_outputEachFileName = GetOutputEachFileName();
_outputEachFileType = GetOutputEachFileType();

_outputAll = GetOutputAllColumns().Count() > 0;
_outputAllFileName = FileHelpers.GetOutputDataFileName(GetOutputAllFileName(), _values);
_outputAllFileName = GetOutputAllFileName();
_outputAllFileType = GetOutputAllFileType();

_outputBatch = _values.GetOrDefault("output.batch.json", false);
_outputBatchFileName = FileHelpers.GetOutputDataFileName(GetOutputBatchFileName(), _values);
_outputBatchFileName = GetOutputBatchFileName();

_outputSrt = _values.GetOrDefault("output.type.srt", false);
_outputSrtFileName = FileHelpers.GetOutputDataFileName(GetOutputSrtFileName(), _values);
_outputSrtFileName = GetOutputSrtFileName();

_outputVtt = _values.GetOrDefault("output.type.vtt", false);
_outputVttFileName = FileHelpers.GetOutputDataFileName(GetOutputVttFileName(), _values);
_outputVttFileName = GetOutputVttFileName();

_cacheResults = _values.Names.Any(s => s.StartsWith("check.result")) || _values.Names.Any(s => s.StartsWith("check.jmes"));

Expand Down Expand Up @@ -326,46 +326,19 @@ private void FlushOutputResultCache()
private string GetOutputBatchFileName()
{
var file = _values.GetOrDefault("output.batch.file.name", "output.{id}.{run.time}.json")!;

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (file.Contains("{pid}")) file = file.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (file.Contains("{time}")) file = file.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private string GetOutputSrtFileName()
{
var file = _values.GetOrDefault("output.srt.file.name", "output.{id}.{run.time}.srt")!;

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private string GetOutputVttFileName()
{
var file = _values.GetOrDefault("output.vtt.file.name", "output.{id}.{run.time}.vtt")!;

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private void OutputBatchJsonFile()
Expand Down Expand Up @@ -811,20 +784,7 @@ private void FlushOutputAllCache()
private string GetOutputAllFileName()
{
var file = _values.GetOrDefault("output.all.file.name", "output.{run.time}." + GetOutputAllFileType())!;

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (file.Contains("{pid}")) file = file.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (file.Contains("{time}")) file = file.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private string GetOutputAllFileType()
Expand Down Expand Up @@ -975,20 +935,7 @@ private void FlushOutputEachCacheStage2(bool overwrite = false)
private string GetOutputEachFileName()
{
var file = _values.GetOrDefault("output.each.file.name", "each.{run.time}." + GetOutputEachFileType())!;

var id = _values.GetOrEmpty("audio.input.id");
if (file.Contains("{id}")) file = file.Replace("{id}", id);

var pid = Process.GetCurrentProcess().Id.ToString();
if (file.Contains("{pid}")) file = file.Replace("{pid}", pid);

var time = DateTime.Now.ToFileTime().ToString();
if (file.Contains("{time}")) file = file.Replace("{time}", time);

var runTime = _values.GetOrEmpty("x.run.time");
if (file.Contains("{run.time}")) file = file.Replace("{run.time}", runTime);

return file.ReplaceValues(_values);
return FileHelpers.GetOutputDataFileName(file, _values, "audio.input.id");
}

private string GetOutputEachFileType()
Expand Down
Loading