From 447dc056fae0877ec38a22c99b0b4368312e7043 Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Tue, 11 Mar 2025 14:15:46 -0400 Subject: [PATCH 1/6] feat: update rust to pip install code block requirements Signed-off-by: Nick Mitchell --- examples/talk/6-code-json.pdl | 2 + pdl-live-react/src-tauri/src/cli/run.rs | 18 ++++- .../src-tauri/src/interpreter/extract.rs | 24 +++--- .../src-tauri/src/interpreter/pip.rs | 74 ++++++++++++++++--- .../src-tauri/src/interpreter/pull.rs | 12 +-- .../src-tauri/src/interpreter/shasum.rs | 4 +- pdl-live-react/src/pdl_ast.d.ts | 6 ++ src/pdl/pdl-schema.json | 19 +++++ src/pdl/pdl_ast.py | 20 ++++- src/pdl/pdl_dumper.py | 1 + 10 files changed, 147 insertions(+), 33 deletions(-) diff --git a/examples/talk/6-code-json.pdl b/examples/talk/6-code-json.pdl index b24945831..a37180531 100644 --- a/examples/talk/6-code-json.pdl +++ b/examples/talk/6-code-json.pdl @@ -26,6 +26,8 @@ text: - def: EVAL contribute: [] lang: python + requirements: + - textdistance code: | import textdistance diff --git a/pdl-live-react/src-tauri/src/cli/run.rs b/pdl-live-react/src-tauri/src/cli/run.rs index b50210149..c998a4140 100644 --- a/pdl-live-react/src-tauri/src/cli/run.rs +++ b/pdl-live-react/src-tauri/src/cli/run.rs @@ -2,10 +2,19 @@ use ::std::path::Path; use duct::cmd; use futures::executor::block_on; use yaml_rust2::yaml::LoadError; +use yaml_rust2::{ScanError, Yaml, YamlLoader}; -use crate::interpreter::pip::pip_install_interpreter_if_needed; +use crate::interpreter::pip::{ + pip_install_code_blocks_if_needed, pip_install_interpreter_if_needed, +}; use crate::interpreter::pull::pull_if_needed; +/// Read the given filesystem path and produce a potentially multi-document Yaml +fn from_path(path: &String) -> Result, ScanError> { + let content = std::fs::read_to_string(path).unwrap(); + YamlLoader::load_from_str(&content) +} + #[cfg(desktop)] pub fn run_pdl_program( source_file_path: String, @@ -20,8 +29,10 @@ pub fn run_pdl_program( ); // async the model pull and pip installs - let pull_future = pull_if_needed(&source_file_path); - let bin_path_future = pip_install_interpreter_if_needed(app_handle); + let program = &from_path(&source_file_path).unwrap()[0]; + let pull_future = pull_if_needed(&program); + let reqs_future = pip_install_code_blocks_if_needed(&app_handle, &program); + let bin_path_future = pip_install_interpreter_if_needed(&app_handle); // wait for any model pulls to finish block_on(pull_future).map_err(|e| match e { @@ -32,6 +43,7 @@ pub fn run_pdl_program( // wait for any pip installs to finish let bin_path = block_on(bin_path_future)?; + block_on(reqs_future)?; let mut args = vec![ source_file_path, diff --git a/pdl-live-react/src-tauri/src/interpreter/extract.rs b/pdl-live-react/src-tauri/src/interpreter/extract.rs index 640915e8e..1b7d2abbf 100644 --- a/pdl-live-react/src-tauri/src/interpreter/extract.rs +++ b/pdl-live-react/src-tauri/src/interpreter/extract.rs @@ -1,16 +1,18 @@ use yaml_rust2::Yaml; -/// Extract models referenced by the programs -pub fn extract_models(programs: Vec) -> Vec { - extract_values(programs, "model") +/// Extract models referenced by the program +pub fn extract_models(program: &Yaml) -> Vec { + extract_values(program, "model") +} + +/// Extract requirements.txt referenced by the program +pub fn extract_requirements(program: &Yaml) -> Vec { + extract_values(program, "requirements") } /// Take a list of Yaml fragments and produce a vector of the string-valued entries of the given field -pub fn extract_values(programs: Vec, field: &str) -> Vec { - let mut values = programs - .into_iter() - .flat_map(|p| extract_one_values(p, field)) - .collect::>(); +pub fn extract_values(program: &Yaml, field: &str) -> Vec { + let mut values = extract_one_values(program, field); // A single program may specify the same model more than once. Dedup! values.sort(); @@ -20,7 +22,7 @@ pub fn extract_values(programs: Vec, field: &str) -> Vec { } /// Take one Yaml fragment and produce a vector of the string-valued entries of the given field -fn extract_one_values(program: Yaml, field: &str) -> Vec { +fn extract_one_values(program: &Yaml, field: &str) -> Vec { let mut values: Vec = Vec::new(); match program { @@ -31,6 +33,10 @@ fn extract_one_values(program: Yaml, field: &str) -> Vec { Yaml::String(m) => { values.push(m.to_string()); } + Yaml::Array(a) => a.into_iter().for_each(|v| match v { + Yaml::String(m) => values.push(m.to_string()), + _ => {} + }), _ => {} }, _ => {} diff --git a/pdl-live-react/src-tauri/src/interpreter/pip.rs b/pdl-live-react/src-tauri/src/interpreter/pip.rs index 69be8bbf6..350bcb03a 100644 --- a/pdl-live-react/src-tauri/src/interpreter/pip.rs +++ b/pdl-live-react/src-tauri/src/interpreter/pip.rs @@ -1,23 +1,32 @@ -use ::std::fs::{copy, create_dir_all}; +use ::std::fs::{copy, create_dir_all, write}; use ::std::path::{Path, PathBuf}; use duct::cmd; +use rayon::prelude::*; use tauri::path::BaseDirectory; use tauri::Manager; +use tempfile::Builder; +use yaml_rust2::Yaml; +use crate::interpreter::extract; use crate::interpreter::shasum; #[cfg(desktop)] -pub async fn pip_install_if_needed( +fn pip_install_if_needed_with_hash( cache_path: &Path, requirements_path: &Path, + hash: String, + force: bool, ) -> Result { create_dir_all(&cache_path)?; - let hash = shasum::sha256sum(&requirements_path)?; - let venv_path = cache_path.join(hash); + let venv_path = cache_path.join("venvs").join(hash); let bin_path = venv_path.join(if cfg!(windows) { "Scripts" } else { "bin" }); + // re: force, this is part of the short-term hack to install all + // code block dependencies in the main interpreter venv. Once we + // figure out how to support a separate venv for each code block + // (that needs it), we can undo this hack. if !venv_path.exists() { println!("Creating virtual environment..."); let python = if cfg!(target_os = "macos") { @@ -27,18 +36,65 @@ pub async fn pip_install_if_needed( }; cmd!(python, "-mvenv", &venv_path).run()?; - cmd!(bin_path.join("pip"), "install", "-r", &requirements_path,).run()?; + if !force { + cmd!(bin_path.join("pip"), "install", "-r", &requirements_path).run()?; + + let cached_requirements_path = venv_path.join("requirements.txt"); + copy(requirements_path, cached_requirements_path)?; + } + } - let cached_requirements_path = venv_path.join("requirements.txt"); - copy(requirements_path, cached_requirements_path)?; + if force { + cmd!(bin_path.join("pip"), "install", "-r", &requirements_path).run()?; } Ok(bin_path.to_path_buf()) } +#[cfg(desktop)] +fn pip_install_if_needed( + cache_path: &Path, + requirements_path: &Path, +) -> Result { + let hash = shasum::sha256sum(&requirements_path)?; + pip_install_if_needed_with_hash(cache_path, requirements_path, hash, false) +} + +#[cfg(desktop)] +pub async fn pip_install_code_blocks_if_needed( + app_handle: &tauri::AppHandle, + program: &Yaml, +) -> Result<(), tauri::Error> { + let cache_path = app_handle.path().cache_dir()?.join("pdl"); + + // for now, install the requirements in the main interpreter venv + let requirements_path = app_handle + .path() + .resolve("interpreter/requirements.txt", BaseDirectory::Resource)?; + + extract::extract_requirements(program) + .into_par_iter() + .try_for_each(|req| -> Result<(), tauri::Error> { + let req_path = Builder::new() + .prefix("pdl-requirements-") + .suffix(".txt") + .tempfile()?; + // This is part of the "force" hack described above, where + // we force the code block dependencies to be installed in + // the main interpreter venv. + let hash = shasum::sha256sum(&requirements_path)?; + write(&req_path, req)?; + pip_install_if_needed_with_hash(&cache_path, &req_path.path(), hash, true)?; + Ok(()) + }) + .expect("code block requirements installed"); + + Ok(()) +} + #[cfg(desktop)] pub async fn pip_install_interpreter_if_needed( - app_handle: tauri::AppHandle, + app_handle: &tauri::AppHandle, ) -> Result { // the interpreter requirements.txt let requirements_path = app_handle @@ -47,5 +103,5 @@ pub async fn pip_install_interpreter_if_needed( let cache_path = app_handle.path().cache_dir()?.join("pdl"); - pip_install_if_needed(&cache_path, &requirements_path).await + pip_install_if_needed(&cache_path, &requirements_path) } diff --git a/pdl-live-react/src-tauri/src/interpreter/pull.rs b/pdl-live-react/src-tauri/src/interpreter/pull.rs index accdbe2ae..2519f7210 100644 --- a/pdl-live-react/src-tauri/src/interpreter/pull.rs +++ b/pdl-live-react/src-tauri/src/interpreter/pull.rs @@ -1,19 +1,13 @@ use duct::cmd; use rayon::prelude::*; use yaml_rust2::yaml::LoadError; -use yaml_rust2::{ScanError, Yaml, YamlLoader}; +use yaml_rust2::Yaml; use crate::interpreter::extract; -/// Read the given filesystem path and produce a potentially multi-document Yaml -fn from_path(path: &String) -> Result, ScanError> { - let content = std::fs::read_to_string(path).unwrap(); - YamlLoader::load_from_str(&content) -} - /// Pull models (in parallel) from the PDL program in the given filepath. -pub async fn pull_if_needed(path: &String) -> Result<(), LoadError> { - extract::extract_models(from_path(path).unwrap()) +pub async fn pull_if_needed(program: &Yaml) -> Result<(), LoadError> { + extract::extract_models(program) .into_par_iter() .try_for_each(|model| match model { m if model.starts_with("ollama/") => ollama_pull_if_needed(&m[7..]), diff --git a/pdl-live-react/src-tauri/src/interpreter/shasum.rs b/pdl-live-react/src-tauri/src/interpreter/shasum.rs index 6a16941de..2ef66f7e5 100644 --- a/pdl-live-react/src-tauri/src/interpreter/shasum.rs +++ b/pdl-live-react/src-tauri/src/interpreter/shasum.rs @@ -2,7 +2,7 @@ use ::std::fs::File; use ::std::io::{copy, Result}; use ::std::path::Path; -use base64ct::{Base64, Encoding}; +use base64ct::{Base64Url, Encoding}; use sha2::{Digest, Sha256}; pub fn sha256sum(path: &Path) -> Result { @@ -12,5 +12,5 @@ pub fn sha256sum(path: &Path) -> Result { copy(&mut file, &mut hasher)?; let hash_bytes = hasher.finalize(); - Ok(Base64::encode_string(&hash_bytes)) + Ok(Base64Url::encode_string(&hash_bytes)) } diff --git a/pdl-live-react/src/pdl_ast.d.ts b/pdl-live-react/src/pdl_ast.d.ts index aec521f7e..26690f064 100644 --- a/pdl-live-react/src/pdl_ast.d.ts +++ b/pdl-live-react/src/pdl_ast.d.ts @@ -1919,6 +1919,11 @@ export type Kind15 = "code" * */ export type Lang = "python" | "command" | "jinja" | "pdl" +/** + * Pip requirements.txt + * + */ +export type Requirements = string | string[] | null /** * Code to execute. * @@ -2837,6 +2842,7 @@ export interface CodeBlock { pdl__is_leaf?: PdlIsLeaf15 kind?: Kind15 lang: Lang + requirements?: Requirements code: Code } /** diff --git a/src/pdl/pdl-schema.json b/src/pdl/pdl-schema.json index 20570a1d6..acca252e8 100644 --- a/src/pdl/pdl-schema.json +++ b/src/pdl/pdl-schema.json @@ -1306,6 +1306,25 @@ "title": "Lang", "type": "string" }, + "requirements": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Pip requirements.txt\n ", + "title": "Requirements" + }, "code": { "anyOf": [ { diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 0eb336dd8..678fb0256 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -15,7 +15,14 @@ Union, ) -from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, RootModel +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + Field, + RootModel, + model_validator, +) from pydantic.json_schema import SkipJsonSchema from .pdl_lazy import PdlDict, PdlLazy @@ -462,10 +469,21 @@ class CodeBlock(LeafBlock): ] """Programming language of the code. """ + requirements: Optional[str | list[str]] = None + """Pip requirements.txt + """ code: "BlockType" """Code to execute. """ + @model_validator(mode="after") + def lang_is_python(self): + if self.requirements is not None and self.lang != "python": + raise ValueError( + "CodeBlock requirements field provided for non-python block" + ) + return self + class GetBlock(LeafBlock): """ diff --git a/src/pdl/pdl_dumper.py b/src/pdl/pdl_dumper.py index 0fc4abacf..80f0839dd 100644 --- a/src/pdl/pdl_dumper.py +++ b/src/pdl/pdl_dumper.py @@ -136,6 +136,7 @@ def block_to_dict( # noqa: C901 case CodeBlock(): d["lang"] = block.lang d["code"] = block_to_dict(block.code, json_compatible) + d["requirements"] = block.requirements case GetBlock(): d["get"] = block.get case DataBlock(): From b23e4e357f02861b74bb05a6b049961e9696031a Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Tue, 11 Mar 2025 14:15:46 -0400 Subject: [PATCH 2/6] feat: update rust to pip install code block requirements Signed-off-by: Nick Mitchell --- pdl-live-react/src-tauri/src/interpreter/extract.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pdl-live-react/src-tauri/src/interpreter/extract.rs b/pdl-live-react/src-tauri/src/interpreter/extract.rs index 1b7d2abbf..0e1a2db22 100644 --- a/pdl-live-react/src-tauri/src/interpreter/extract.rs +++ b/pdl-live-react/src-tauri/src/interpreter/extract.rs @@ -10,6 +10,11 @@ pub fn extract_requirements(program: &Yaml) -> Vec { extract_values(program, "requirements") } +/// Extract requirements.txt referenced by the programs +pub fn extract_requirements(programs: Vec) -> Vec { + extract_values(programs, "requirements") +} + /// Take a list of Yaml fragments and produce a vector of the string-valued entries of the given field pub fn extract_values(program: &Yaml, field: &str) -> Vec { let mut values = extract_one_values(program, field); From e4f51e0bfc8b1ccc98c883248875ae8aa4efee31 Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Tue, 11 Mar 2025 19:10:22 -0400 Subject: [PATCH 3/6] feat: keep code block venvs separate Signed-off-by: Nick Mitchell --- pdl-live-react/src-tauri/src/cli/run.rs | 44 ++++++++++++++++++- .../src-tauri/src/interpreter/extract.rs | 5 --- .../src-tauri/src/interpreter/pip.rs | 15 ++++--- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/pdl-live-react/src-tauri/src/cli/run.rs b/pdl-live-react/src-tauri/src/cli/run.rs index c998a4140..8cb5bd403 100644 --- a/pdl-live-react/src-tauri/src/cli/run.rs +++ b/pdl-live-react/src-tauri/src/cli/run.rs @@ -1,7 +1,11 @@ +use ::std::fs::{remove_file, write}; use ::std::path::Path; + use duct::cmd; use futures::executor::block_on; +use tempfile::Builder; use yaml_rust2::yaml::LoadError; +use yaml_rust2::{EmitError, YamlEmitter}; use yaml_rust2::{ScanError, Yaml, YamlLoader}; use crate::interpreter::pip::{ @@ -43,10 +47,41 @@ pub fn run_pdl_program( // wait for any pip installs to finish let bin_path = block_on(bin_path_future)?; - block_on(reqs_future)?; + // wait for code block requirements to be pulled + let updated_source_file_path = match block_on(reqs_future)? { + Some(updated_programs) => { + // We received back an updated program + println!("Updated! {:?}", updated_programs); + let mut out_str = String::new(); + let mut emitter = YamlEmitter::new(&mut out_str); + for p in updated_programs { + emitter.dump(&p).map_err(|e| match e { + EmitError::FmtError(ee) => tauri::Error::Anyhow(ee.into()), + })?; + } + match Path::new(&source_file_path).parent() { + Some(dir) => { + let tmp = Builder::new() + .prefix("pdl-program-") + .suffix(".pdl") + .tempfile_in(&dir)?; + write(&tmp, out_str)?; + let (_, path) = tmp.keep().map_err(|e| tauri::Error::Io(e.error))?; + path.display().to_string() + } + _ => { + eprintln!("Failed to find target directory for updated program"); + source_file_path.clone() + } + } + } + _ => source_file_path.clone(), + }; + + println!("SRC {} -> {}", source_file_path, updated_source_file_path); let mut args = vec![ - source_file_path, + updated_source_file_path.clone(), dashdash("--trace", trace_file), dashdash("--data", data), dashdash("--stream", stream), @@ -54,6 +89,11 @@ pub fn run_pdl_program( args.retain(|x| x.chars().count() > 0); cmd(bin_path.join("pdl"), &args).run()?; + // TODO how do we do this on all exit paths in rust? + if updated_source_file_path != source_file_path { + remove_file(updated_source_file_path)?; + } + Ok(()) } diff --git a/pdl-live-react/src-tauri/src/interpreter/extract.rs b/pdl-live-react/src-tauri/src/interpreter/extract.rs index 0e1a2db22..1b7d2abbf 100644 --- a/pdl-live-react/src-tauri/src/interpreter/extract.rs +++ b/pdl-live-react/src-tauri/src/interpreter/extract.rs @@ -10,11 +10,6 @@ pub fn extract_requirements(program: &Yaml) -> Vec { extract_values(program, "requirements") } -/// Extract requirements.txt referenced by the programs -pub fn extract_requirements(programs: Vec) -> Vec { - extract_values(programs, "requirements") -} - /// Take a list of Yaml fragments and produce a vector of the string-valued entries of the given field pub fn extract_values(program: &Yaml, field: &str) -> Vec { let mut values = extract_one_values(program, field); diff --git a/pdl-live-react/src-tauri/src/interpreter/pip.rs b/pdl-live-react/src-tauri/src/interpreter/pip.rs index 350bcb03a..2c2360e36 100644 --- a/pdl-live-react/src-tauri/src/interpreter/pip.rs +++ b/pdl-live-react/src-tauri/src/interpreter/pip.rs @@ -64,7 +64,7 @@ fn pip_install_if_needed( pub async fn pip_install_code_blocks_if_needed( app_handle: &tauri::AppHandle, program: &Yaml, -) -> Result<(), tauri::Error> { +) -> Result, tauri::Error> { let cache_path = app_handle.path().cache_dir()?.join("pdl"); // for now, install the requirements in the main interpreter venv @@ -72,9 +72,9 @@ pub async fn pip_install_code_blocks_if_needed( .path() .resolve("interpreter/requirements.txt", BaseDirectory::Resource)?; - extract::extract_requirements(program) + let n = extract::extract_requirements(&program) .into_par_iter() - .try_for_each(|req| -> Result<(), tauri::Error> { + .map(|req| -> Result { let req_path = Builder::new() .prefix("pdl-requirements-") .suffix(".txt") @@ -85,11 +85,14 @@ pub async fn pip_install_code_blocks_if_needed( let hash = shasum::sha256sum(&requirements_path)?; write(&req_path, req)?; pip_install_if_needed_with_hash(&cache_path, &req_path.path(), hash, true)?; - Ok(()) + Ok(1) }) - .expect("code block requirements installed"); + .count(); - Ok(()) + match n { + 0 => Ok(None), + _ => Ok(Some(program.clone())), + } } #[cfg(desktop)] From 17981678630bd3714ef0dcbd58e2f8ac6a7fd661 Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Wed, 12 Mar 2025 13:09:00 -0400 Subject: [PATCH 4/6] more work Signed-off-by: Nick Mitchell --- pdl-live-react/src-tauri/src/cli/run.rs | 16 +-- .../src-tauri/src/interpreter/extract.rs | 121 +++++++++++++++--- .../src-tauri/src/interpreter/pip.rs | 7 +- .../src-tauri/src/interpreter/pull.rs | 3 +- .../src-tauri/src/interpreter/shasum.rs | 8 ++ 5 files changed, 122 insertions(+), 33 deletions(-) diff --git a/pdl-live-react/src-tauri/src/cli/run.rs b/pdl-live-react/src-tauri/src/cli/run.rs index 8cb5bd403..72bad1f98 100644 --- a/pdl-live-react/src-tauri/src/cli/run.rs +++ b/pdl-live-react/src-tauri/src/cli/run.rs @@ -50,16 +50,14 @@ pub fn run_pdl_program( // wait for code block requirements to be pulled let updated_source_file_path = match block_on(reqs_future)? { - Some(updated_programs) => { + Some(updated_program) => { // We received back an updated program - println!("Updated! {:?}", updated_programs); + println!("Updated! {:?}", updated_program); let mut out_str = String::new(); let mut emitter = YamlEmitter::new(&mut out_str); - for p in updated_programs { - emitter.dump(&p).map_err(|e| match e { - EmitError::FmtError(ee) => tauri::Error::Anyhow(ee.into()), - })?; - } + emitter.dump(&updated_program).map_err(|e| match e { + EmitError::FmtError(ee) => tauri::Error::Anyhow(ee.into()), + })?; match Path::new(&source_file_path).parent() { Some(dir) => { let tmp = Builder::new() @@ -90,9 +88,9 @@ pub fn run_pdl_program( cmd(bin_path.join("pdl"), &args).run()?; // TODO how do we do this on all exit paths in rust? - if updated_source_file_path != source_file_path { + /*if updated_source_file_path != source_file_path { remove_file(updated_source_file_path)?; - } + }*/ Ok(()) } diff --git a/pdl-live-react/src-tauri/src/interpreter/extract.rs b/pdl-live-react/src-tauri/src/interpreter/extract.rs index 1b7d2abbf..daa38af68 100644 --- a/pdl-live-react/src-tauri/src/interpreter/extract.rs +++ b/pdl-live-react/src-tauri/src/interpreter/extract.rs @@ -1,63 +1,144 @@ +use ::std::fs::write; + +use tempfile::Builder; use yaml_rust2::Yaml; +use crate::interpreter::shasum; + /// Extract models referenced by the program -pub fn extract_models(program: &Yaml) -> Vec { - extract_values(program, "model") +pub fn extract_models(program: &Yaml) -> (Vec, Yaml) { + extract_values(program, "model", &|y| y.clone()) } /// Extract requirements.txt referenced by the program -pub fn extract_requirements(program: &Yaml) -> Vec { - extract_values(program, "requirements") +pub fn extract_requirements(program: &Yaml) -> (Vec, Yaml) { + let requirements = Yaml::String("requirements".to_string()); + let code = Yaml::String("code".to_string()); + let lang = Yaml::String("lang".to_string()); + let python = Yaml::String("python".to_string()); + + extract_values(program, "requirements", &|y| match y { + Yaml::Hash(h) => { + match h.contains_key(&requirements) && h.contains_key(&code) && h[&lang] == python { + true => { + let requirements_text = match &h[&requirements] { + Yaml::Array(a) => a + .into_iter() + .map(|item| match item { + Yaml::String(s) => s.to_string(), + _ => "".to_string(), + }) + .collect::>() + .join("\n"), + + Yaml::String(s) => s.to_string(), + + _ => "".to_string(), + }; + + let req_hash = shasum::sha256sum_str(requirements_text.as_str()).unwrap(); + let code_text = if let Some(c) = h[&code].as_str() { + c + } else { + "" + }; + let code_hash = shasum::sha256sum_str(code_text).unwrap(); + + let tmp = Builder::new() + .prefix(&format!("pdl-program-{}", code_hash)) + .suffix(".pdl") + .tempfile() + .unwrap(); // TODO tmpfile_in(source dir) + write(&tmp, code_text).unwrap(); + + h.remove(&requirements); + h[&code] = Yaml::String(format!( + "\"/Users/nickm/Library/Caches/pdl/venvs/{}/{}/python /tmp/{}.pdl\"", + req_hash, + if cfg!(windows) { "Scripts" } else { "bin" }, + code_hash + )); + + Yaml::Hash(h.clone()) + } + false => Yaml::Hash(h.clone()), + } + } + y => y.clone(), + }) } /// Take a list of Yaml fragments and produce a vector of the string-valued entries of the given field -pub fn extract_values(program: &Yaml, field: &str) -> Vec { - let mut values = extract_one_values(program, field); +pub fn extract_values( + program: &Yaml, + field: &str, + mutator: &impl Fn(&mut Yaml) -> Yaml, +) -> (Vec, Yaml) { + let (mut values, updated_program) = traverse(program, field, mutator); // A single program may specify the same model more than once. Dedup! values.sort(); values.dedup(); - values + (values, updated_program) } /// Take one Yaml fragment and produce a vector of the string-valued entries of the given field -fn extract_one_values(program: &Yaml, field: &str) -> Vec { +fn traverse( + program: &Yaml, + field: &str, + mutator: &impl Fn(&mut Yaml) -> Yaml, +) -> (Vec, Yaml) { let mut values: Vec = Vec::new(); - match program { + let updated_program: Yaml = match program { Yaml::Hash(h) => { + let mut hh = h.clone(); for (key, val) in h { match key { Yaml::String(f) if f == field => match &val { Yaml::String(m) => { values.push(m.to_string()); } - Yaml::Array(a) => a.into_iter().for_each(|v| match v { - Yaml::String(m) => values.push(m.to_string()), - _ => {} - }), + Yaml::Array(a) => values.push( + a.into_iter() + .map(|item| match item { + Yaml::String(s) => s.to_string(), + _ => "".to_string(), + }) + .collect::>() + .join("\n"), + ), _ => {} }, _ => {} } - for m in extract_one_values(val, field) { + let (v, p) = traverse(val, field, mutator); + hh[key] = p; + for m in v { values.push(m) } } + + mutator(&mut Yaml::Hash(hh)) } Yaml::Array(a) => { - for val in a { - for m in extract_one_values(val, field) { - values.push(m) + let mut aa = a.clone(); + for (i, val) in a.iter().enumerate() { + let (v, p) = traverse(val, field, mutator); + aa[i] = p; + for m in v { + values.push(m); } } + + mutator(&mut Yaml::Array(aa)) } - _ => {} - } + x => mutator(&mut x.clone()), + }; - values + (values, updated_program) } diff --git a/pdl-live-react/src-tauri/src/interpreter/pip.rs b/pdl-live-react/src-tauri/src/interpreter/pip.rs index 2c2360e36..2807a4ae0 100644 --- a/pdl-live-react/src-tauri/src/interpreter/pip.rs +++ b/pdl-live-react/src-tauri/src/interpreter/pip.rs @@ -72,7 +72,8 @@ pub async fn pip_install_code_blocks_if_needed( .path() .resolve("interpreter/requirements.txt", BaseDirectory::Resource)?; - let n = extract::extract_requirements(&program) + let (reqs, updated_program) = extract::extract_requirements(&program); + let n = reqs .into_par_iter() .map(|req| -> Result { let req_path = Builder::new() @@ -90,8 +91,8 @@ pub async fn pip_install_code_blocks_if_needed( .count(); match n { - 0 => Ok(None), - _ => Ok(Some(program.clone())), + 0 => Ok(None), // We did not change the program + _ => Ok(Some(updated_program)), } } diff --git a/pdl-live-react/src-tauri/src/interpreter/pull.rs b/pdl-live-react/src-tauri/src/interpreter/pull.rs index 2519f7210..ee786430b 100644 --- a/pdl-live-react/src-tauri/src/interpreter/pull.rs +++ b/pdl-live-react/src-tauri/src/interpreter/pull.rs @@ -7,7 +7,8 @@ use crate::interpreter::extract; /// Pull models (in parallel) from the PDL program in the given filepath. pub async fn pull_if_needed(program: &Yaml) -> Result<(), LoadError> { - extract::extract_models(program) + let (models, _) = extract::extract_models(program); + models .into_par_iter() .try_for_each(|model| match model { m if model.starts_with("ollama/") => ollama_pull_if_needed(&m[7..]), diff --git a/pdl-live-react/src-tauri/src/interpreter/shasum.rs b/pdl-live-react/src-tauri/src/interpreter/shasum.rs index 2ef66f7e5..4a48d1653 100644 --- a/pdl-live-react/src-tauri/src/interpreter/shasum.rs +++ b/pdl-live-react/src-tauri/src/interpreter/shasum.rs @@ -14,3 +14,11 @@ pub fn sha256sum(path: &Path) -> Result { Ok(Base64Url::encode_string(&hash_bytes)) } + +pub fn sha256sum_str(s: &str) -> Result { + let mut hasher = Sha256::new(); + hasher.update(s); + let hash_bytes = hasher.finalize(); + + Ok(Base64Url::encode_string(&hash_bytes)) +} From 6f5508f24fc0fe538d8324359743b91bdd9f3f1e Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Wed, 12 Mar 2025 14:38:49 -0400 Subject: [PATCH 5/6] more more Signed-off-by: Nick Mitchell --- pdl-live-react/src-tauri/src/cli/run.rs | 1 + .../src-tauri/src/interpreter/extract.rs | 13 ++++---- .../src-tauri/src/interpreter/pip.rs | 30 ++++--------------- src/pdl/pdl_ast.py | 3 ++ src/pdl/pdl_interpreter.py | 7 +++-- 5 files changed, 22 insertions(+), 32 deletions(-) diff --git a/pdl-live-react/src-tauri/src/cli/run.rs b/pdl-live-react/src-tauri/src/cli/run.rs index 72bad1f98..af05e5f7b 100644 --- a/pdl-live-react/src-tauri/src/cli/run.rs +++ b/pdl-live-react/src-tauri/src/cli/run.rs @@ -55,6 +55,7 @@ pub fn run_pdl_program( println!("Updated! {:?}", updated_program); let mut out_str = String::new(); let mut emitter = YamlEmitter::new(&mut out_str); + emitter.multiline_strings(true); emitter.dump(&updated_program).map_err(|e| match e { EmitError::FmtError(ee) => tauri::Error::Anyhow(ee.into()), })?; diff --git a/pdl-live-react/src-tauri/src/interpreter/extract.rs b/pdl-live-react/src-tauri/src/interpreter/extract.rs index daa38af68..936837c37 100644 --- a/pdl-live-react/src-tauri/src/interpreter/extract.rs +++ b/pdl-live-react/src-tauri/src/interpreter/extract.rs @@ -38,11 +38,11 @@ pub fn extract_requirements(program: &Yaml) -> (Vec, Yaml) { let req_hash = shasum::sha256sum_str(requirements_text.as_str()).unwrap(); let code_text = if let Some(c) = h[&code].as_str() { - c + format!("{}\nprint(result)", c) } else { - "" + "".to_string() }; - let code_hash = shasum::sha256sum_str(code_text).unwrap(); + let code_hash = shasum::sha256sum_str(&code_text.as_str()).unwrap(); let tmp = Builder::new() .prefix(&format!("pdl-program-{}", code_hash)) @@ -50,13 +50,16 @@ pub fn extract_requirements(program: &Yaml) -> (Vec, Yaml) { .tempfile() .unwrap(); // TODO tmpfile_in(source dir) write(&tmp, code_text).unwrap(); + let (_, tmp_path) = tmp.keep().unwrap(); h.remove(&requirements); + h[&lang] = Yaml::String("command".to_string()); + h.insert(Yaml::String("file".to_string()), Yaml::Boolean(true)); h[&code] = Yaml::String(format!( - "\"/Users/nickm/Library/Caches/pdl/venvs/{}/{}/python /tmp/{}.pdl\"", + "\"/Users/nickm/Library/Caches/pdl/venvs/{}/{}/python {}\"", req_hash, if cfg!(windows) { "Scripts" } else { "bin" }, - code_hash + tmp_path.display(), )); Yaml::Hash(h.clone()) diff --git a/pdl-live-react/src-tauri/src/interpreter/pip.rs b/pdl-live-react/src-tauri/src/interpreter/pip.rs index 2807a4ae0..af0f0e54f 100644 --- a/pdl-live-react/src-tauri/src/interpreter/pip.rs +++ b/pdl-live-react/src-tauri/src/interpreter/pip.rs @@ -16,17 +16,12 @@ fn pip_install_if_needed_with_hash( cache_path: &Path, requirements_path: &Path, hash: String, - force: bool, ) -> Result { create_dir_all(&cache_path)?; let venv_path = cache_path.join("venvs").join(hash); let bin_path = venv_path.join(if cfg!(windows) { "Scripts" } else { "bin" }); - // re: force, this is part of the short-term hack to install all - // code block dependencies in the main interpreter venv. Once we - // figure out how to support a separate venv for each code block - // (that needs it), we can undo this hack. if !venv_path.exists() { println!("Creating virtual environment..."); let python = if cfg!(target_os = "macos") { @@ -36,16 +31,10 @@ fn pip_install_if_needed_with_hash( }; cmd!(python, "-mvenv", &venv_path).run()?; - if !force { - cmd!(bin_path.join("pip"), "install", "-r", &requirements_path).run()?; - - let cached_requirements_path = venv_path.join("requirements.txt"); - copy(requirements_path, cached_requirements_path)?; - } - } - - if force { cmd!(bin_path.join("pip"), "install", "-r", &requirements_path).run()?; + + let cached_requirements_path = venv_path.join("requirements.txt"); + copy(requirements_path, cached_requirements_path)?; } Ok(bin_path.to_path_buf()) @@ -57,7 +46,7 @@ fn pip_install_if_needed( requirements_path: &Path, ) -> Result { let hash = shasum::sha256sum(&requirements_path)?; - pip_install_if_needed_with_hash(cache_path, requirements_path, hash, false) + pip_install_if_needed_with_hash(cache_path, requirements_path, hash) } #[cfg(desktop)] @@ -67,11 +56,6 @@ pub async fn pip_install_code_blocks_if_needed( ) -> Result, tauri::Error> { let cache_path = app_handle.path().cache_dir()?.join("pdl"); - // for now, install the requirements in the main interpreter venv - let requirements_path = app_handle - .path() - .resolve("interpreter/requirements.txt", BaseDirectory::Resource)?; - let (reqs, updated_program) = extract::extract_requirements(&program); let n = reqs .into_par_iter() @@ -80,12 +64,8 @@ pub async fn pip_install_code_blocks_if_needed( .prefix("pdl-requirements-") .suffix(".txt") .tempfile()?; - // This is part of the "force" hack described above, where - // we force the code block dependencies to be installed in - // the main interpreter venv. - let hash = shasum::sha256sum(&requirements_path)?; write(&req_path, req)?; - pip_install_if_needed_with_hash(&cache_path, &req_path.path(), hash, true)?; + pip_install_if_needed(&cache_path, &req_path.path())?; Ok(1) }) .count(); diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 678fb0256..2e0554b20 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -475,6 +475,9 @@ class CodeBlock(LeafBlock): code: "BlockType" """Code to execute. """ + file: Optional[bool] = None + """Code to execute is a file path. + """ @model_validator(mode="after") def lang_is_python(self): diff --git a/src/pdl/pdl_interpreter.py b/src/pdl/pdl_interpreter.py index a4a28144b..54c5f4cf9 100644 --- a/src/pdl/pdl_interpreter.py +++ b/src/pdl/pdl_interpreter.py @@ -1456,7 +1456,7 @@ def process_call_code( ) from exc case "command": try: - result = call_command(code_s) + result = call_command(code_s, block.file == True) background = PdlList( [ PdlDict( # type: ignore @@ -1530,7 +1530,10 @@ def call_python(code: str, scope: ScopeType) -> PdlLazy[Any]: return PdlConst(result) -def call_command(code: str) -> PdlLazy[str]: +def call_command(code: str, is_file: bool) -> PdlLazy[str]: + if is_file: + with open(code, "r") as f: + code = f.read() args = shlex.split(code) p = subprocess.run( args, capture_output=True, text=True, check=False, shell=False From 706d2df852edeac89331c23a1b1f36707bd1565d Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Wed, 12 Mar 2025 18:55:41 -0400 Subject: [PATCH 6/6] i think it's working now Signed-off-by: Nick Mitchell --- pdl-live-react/src-tauri/src/cli/run.rs | 7 +- .../src-tauri/src/interpreter/extract.rs | 26 +++--- pdl-live-react/src/pdl_ast.d.ts | 26 ++++++ pdl-live-react/src/pdl_ast_utils.ts | 8 +- src/pdl/pdl-schema.json | 82 +++++++++++++++++++ src/pdl/pdl_ast.py | 7 +- src/pdl/pdl_ast_utils.py | 11 ++- src/pdl/pdl_dumper.py | 5 +- src/pdl/pdl_interpreter.py | 38 +++++---- 9 files changed, 170 insertions(+), 40 deletions(-) diff --git a/pdl-live-react/src-tauri/src/cli/run.rs b/pdl-live-react/src-tauri/src/cli/run.rs index af05e5f7b..a41c08120 100644 --- a/pdl-live-react/src-tauri/src/cli/run.rs +++ b/pdl-live-react/src-tauri/src/cli/run.rs @@ -27,7 +27,7 @@ pub fn run_pdl_program( data: Option<&tauri_plugin_cli::ArgData>, stream: Option<&tauri_plugin_cli::ArgData>, ) -> Result<(), tauri::Error> { - println!( + eprintln!( "Running {:#?}", Path::new(&source_file_path).file_name().unwrap() ); @@ -78,7 +78,6 @@ pub fn run_pdl_program( _ => source_file_path.clone(), }; - println!("SRC {} -> {}", source_file_path, updated_source_file_path); let mut args = vec![ updated_source_file_path.clone(), dashdash("--trace", trace_file), @@ -89,9 +88,9 @@ pub fn run_pdl_program( cmd(bin_path.join("pdl"), &args).run()?; // TODO how do we do this on all exit paths in rust? - /*if updated_source_file_path != source_file_path { + if updated_source_file_path != source_file_path { remove_file(updated_source_file_path)?; - }*/ + } Ok(()) } diff --git a/pdl-live-react/src-tauri/src/interpreter/extract.rs b/pdl-live-react/src-tauri/src/interpreter/extract.rs index 936837c37..8d9830e60 100644 --- a/pdl-live-react/src-tauri/src/interpreter/extract.rs +++ b/pdl-live-react/src-tauri/src/interpreter/extract.rs @@ -1,6 +1,3 @@ -use ::std::fs::write; - -use tempfile::Builder; use yaml_rust2::Yaml; use crate::interpreter::shasum; @@ -42,25 +39,28 @@ pub fn extract_requirements(program: &Yaml) -> (Vec, Yaml) { } else { "".to_string() }; - let code_hash = shasum::sha256sum_str(&code_text.as_str()).unwrap(); + //let code_hash = shasum::sha256sum_str(&code_text.as_str()).unwrap(); - let tmp = Builder::new() + /*let tmp = Builder::new() .prefix(&format!("pdl-program-{}", code_hash)) .suffix(".pdl") .tempfile() .unwrap(); // TODO tmpfile_in(source dir) write(&tmp, code_text).unwrap(); - let (_, tmp_path) = tmp.keep().unwrap(); + let (_, tmp_path) = tmp.keep().unwrap();*/ h.remove(&requirements); h[&lang] = Yaml::String("command".to_string()); - h.insert(Yaml::String("file".to_string()), Yaml::Boolean(true)); - h[&code] = Yaml::String(format!( - "\"/Users/nickm/Library/Caches/pdl/venvs/{}/{}/python {}\"", - req_hash, - if cfg!(windows) { "Scripts" } else { "bin" }, - tmp_path.display(), - )); + //h.insert(&Yaml::String("file".to_string()), Yaml::Boolean(true)); + h[&code] = Yaml::Array(vec![ + Yaml::String(format!( + "/Users/nickm/Library/Caches/pdl/venvs/{}/{}/python", + req_hash, + if cfg!(windows) { "Scripts" } else { "bin" }, + )), + Yaml::String("-c".to_owned()), + Yaml::String(code_text), + ]); Yaml::Hash(h.clone()) } diff --git a/pdl-live-react/src/pdl_ast.d.ts b/pdl-live-react/src/pdl_ast.d.ts index 26690f064..9df38d131 100644 --- a/pdl-live-react/src/pdl_ast.d.ts +++ b/pdl-live-react/src/pdl_ast.d.ts @@ -1952,6 +1952,32 @@ export type Code = | ImportBlock | ErrorBlock | EmptyBlock + | ( + | boolean + | number + | string + | FunctionBlock + | CallBlock + | LitellmModelBlock + | GraniteioModelBlock + | CodeBlock + | GetBlock + | DataBlock + | IfBlock + | MatchBlock + | RepeatBlock + | TextBlock + | LastOfBlock + | ArrayBlock + | ObjectBlock + | MessageBlock + | ReadBlock + | IncludeBlock + | ImportBlock + | ErrorBlock + | EmptyBlock + | null + )[] | null /** * Name of the variable used to store the result of the execution of the block. diff --git a/pdl-live-react/src/pdl_ast_utils.ts b/pdl-live-react/src/pdl_ast_utils.ts index 5e68b1285..40255fb13 100644 --- a/pdl-live-react/src/pdl_ast_utils.ts +++ b/pdl-live-react/src/pdl_ast_utils.ts @@ -51,7 +51,7 @@ export function map_block_children( } }) .with({ kind: "code" }, (block) => { - const code = f(block.code) + const code = Array.isArray(block.code) ? block.code.map(f) : f(block.code) return { ...block, code: code } }) .with({ kind: "get" }, (block) => block) @@ -155,7 +155,11 @@ export function iter_block_children( if (block.input) f(block.input) }) .with({ kind: "code" }, (block) => { - f(block.code) + if (Array.isArray(block.code)) { + block.code.forEach(f) + } else { + f(block.code) + } }) .with({ kind: "get" }, () => {}) .with({ kind: "data" }, () => {}) diff --git a/src/pdl/pdl-schema.json b/src/pdl/pdl-schema.json index acca252e8..e88886b3d 100644 --- a/src/pdl/pdl-schema.json +++ b/src/pdl/pdl-schema.json @@ -1399,6 +1399,88 @@ { "$ref": "#/$defs/EmptyBlock" }, + { + "items": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "$ref": "#/$defs/FunctionBlock" + }, + { + "$ref": "#/$defs/CallBlock" + }, + { + "$ref": "#/$defs/LitellmModelBlock" + }, + { + "$ref": "#/$defs/GraniteioModelBlock" + }, + { + "$ref": "#/$defs/CodeBlock" + }, + { + "$ref": "#/$defs/GetBlock" + }, + { + "$ref": "#/$defs/DataBlock" + }, + { + "$ref": "#/$defs/IfBlock" + }, + { + "$ref": "#/$defs/MatchBlock" + }, + { + "$ref": "#/$defs/RepeatBlock" + }, + { + "$ref": "#/$defs/TextBlock" + }, + { + "$ref": "#/$defs/LastOfBlock" + }, + { + "$ref": "#/$defs/ArrayBlock" + }, + { + "$ref": "#/$defs/ObjectBlock" + }, + { + "$ref": "#/$defs/MessageBlock" + }, + { + "$ref": "#/$defs/ReadBlock" + }, + { + "$ref": "#/$defs/IncludeBlock" + }, + { + "$ref": "#/$defs/ImportBlock" + }, + { + "$ref": "#/$defs/ErrorBlock" + }, + { + "$ref": "#/$defs/EmptyBlock" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, { "type": "null" } diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 2e0554b20..46f1ac910 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -472,12 +472,9 @@ class CodeBlock(LeafBlock): requirements: Optional[str | list[str]] = None """Pip requirements.txt """ - code: "BlockType" + code: "BlockOrBlocksType" """Code to execute. """ - file: Optional[bool] = None - """Code to execute is a file path. - """ @model_validator(mode="after") def lang_is_python(self): @@ -485,6 +482,8 @@ def lang_is_python(self): raise ValueError( "CodeBlock requirements field provided for non-python block" ) + if isinstance(self.code, list) and self.lang != "command": + raise ValueError("CodeBlock code field is array for non-command block") return self diff --git a/src/pdl/pdl_ast_utils.py b/src/pdl/pdl_ast_utils.py index b66faeeb9..27872945a 100644 --- a/src/pdl/pdl_ast_utils.py +++ b/src/pdl/pdl_ast_utils.py @@ -48,7 +48,11 @@ def iter_block_children(f: Callable[[BlockType], None], block: BlockType) -> Non if block.pdl__trace is not None: f(block.pdl__trace) case CodeBlock(): - f(block.code) + if isinstance(block.code, list): + for b in block.code: + f(b) + else: + f(block.code) case GetBlock(): pass case DataBlock(): @@ -150,7 +154,10 @@ def map_block_children(f: MappedFunctions, block: BlockType) -> BlockType: if block.parameters is not None: block.parameters = f.f_expr(block.parameters) case CodeBlock(): - block.code = f.f_block(block.code) + if isinstance(block.code, list): + block.code = [f.f_block(b) for b in block.code] + else: + block.code = f.f_block(block.code) case GetBlock(): pass case DataBlock(): diff --git a/src/pdl/pdl_dumper.py b/src/pdl/pdl_dumper.py index 80f0839dd..7eddbd37f 100644 --- a/src/pdl/pdl_dumper.py +++ b/src/pdl/pdl_dumper.py @@ -135,7 +135,10 @@ def block_to_dict( # noqa: C901 d["modelResponse"] = block.modelResponse case CodeBlock(): d["lang"] = block.lang - d["code"] = block_to_dict(block.code, json_compatible) + if isinstance(block.code, list): + d["code"] = [block_to_dict(b, json_compatible) for b in block.code] + else: + d["code"] = block_to_dict(block.code, json_compatible) d["requirements"] = block.requirements case GetBlock(): d["get"] = block.get diff --git a/src/pdl/pdl_interpreter.py b/src/pdl/pdl_interpreter.py index 54c5f4cf9..1dfb5552c 100644 --- a/src/pdl/pdl_interpreter.py +++ b/src/pdl/pdl_interpreter.py @@ -1431,14 +1431,24 @@ def process_call_code( state: InterpreterState, scope: ScopeType, block: CodeBlock, loc: PdlLocationType ) -> tuple[PdlLazy[Any], LazyMessages, ScopeType, CodeBlock]: background: LazyMessages - code_, _, _, block = process_block_of( - block, - "code", - state.with_yield_result(False).with_yield_background(False), - scope, - loc, - ) - code_s = code_.result() + code_a = None + if isinstance(block.code, list): + code_s = "" + code_a, _, _, _ = process_block( + state.with_yield_result(False).with_yield_background(False), + scope, + ArrayBlock(array=block.code), + loc, + ) + else: + code_, _, _, block = process_block_of( + block, + "code", + state.with_yield_result(False).with_yield_background(False), + scope, + loc, + ) + code_s = code_.result() match block.lang: case "python": try: @@ -1456,7 +1466,7 @@ def process_call_code( ) from exc case "command": try: - result = call_command(code_s, block.file == True) + result = call_command(code_s, code_a) background = PdlList( [ PdlDict( # type: ignore @@ -1530,11 +1540,11 @@ def call_python(code: str, scope: ScopeType) -> PdlLazy[Any]: return PdlConst(result) -def call_command(code: str, is_file: bool) -> PdlLazy[str]: - if is_file: - with open(code, "r") as f: - code = f.read() - args = shlex.split(code) +def call_command(code: str, code_a: PdlLazy[list[str]] | None) -> PdlLazy[str]: + if code_a is not None and isinstance(code_a.result(), list): + args = code_a.result() + else: + args = shlex.split(code) p = subprocess.run( args, capture_output=True, text=True, check=False, shell=False ) # nosec B603