From 15a74df025d5582cd9c67de5174de6c60a41c33e Mon Sep 17 00:00:00 2001 From: Sandeep Narendranath Karjala Date: Tue, 1 Jul 2025 13:43:43 -0700 Subject: [PATCH 01/14] Add --all-ranks CLI infrastructure and core processing logic --- src/cli.rs | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 2 + 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index 3a4b243..fc940bb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -47,10 +47,18 @@ pub struct Cli { /// For inductor provenance tracking highlighter #[arg(short, long)] inductor_provenance: bool, + /// Parse all ranks and generate a single unified page + #[arg(long)] + all_ranks: bool, } fn main() -> anyhow::Result<()> { let cli = Cli::parse(); + + if cli.all_ranks { + return handle_all_ranks(&cli); + } + let path = if cli.latest { let input_path = cli.path; // Path should be a directory @@ -92,11 +100,12 @@ fn main() -> anyhow::Result<()> { strict: cli.strict, strict_compile_id: cli.strict_compile_id, custom_parsers: Vec::new(), - custom_header_html: cli.custom_header_html, + custom_header_html: cli.custom_header_html.clone(), verbose: cli.verbose, plain_text: cli.plain_text, export: cli.export, inductor_provenance: cli.inductor_provenance, + all_ranks: cli.all_ranks, }; let output = parse_path(&path, config)?; @@ -114,3 +123,118 @@ fn main() -> anyhow::Result<()> { } Ok(()) } + +// handle_all_ranks function with placeholder landing page +fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { + let input_path = &cli.path; + + if !input_path.is_dir() { + bail!( + "Input path {} must be a directory when using --all-ranks", + input_path.display() + ); + } + + let out_path = &cli.out; + + if out_path.exists() { + if !cli.overwrite { + bail!( + "Directory {} already exists, use -o OUTDIR to write to another location or pass --overwrite to overwrite the old contents", + out_path.display() + ); + } + fs::remove_dir_all(&out_path)?; + } + fs::create_dir(&out_path)?; + + // Find all rank log files in the directory + let rank_files: Vec<_> = std::fs::read_dir(input_path) + .with_context(|| format!("Couldn't access directory {}", input_path.display()))? + .flatten() + .filter(|entry| { + let path = entry.path(); + path.is_file() && + path.file_name() + .and_then(|name| name.to_str()) + .map(|name| name.contains("rank_") && name.ends_with(".log")) + .unwrap_or(false) + }) + .collect(); + + if rank_files.is_empty() { + bail!("No rank log files found in directory {}", input_path.display()); + } + + let mut rank_links = Vec::new(); + + // Process each rank file + for rank_file in rank_files { + let rank_path = rank_file.path(); + let rank_name = rank_path + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("unknown"); + + // Extract rank number from filename + let rank_num = if let Some(pos) = rank_name.find("rank_") { + let after_rank = &rank_name[pos + 5..]; + after_rank.chars().take_while(|c| c.is_ascii_digit()).collect::() + } else { + "unknown".to_string() + }; + + println!("Processing rank {} from file: {}", rank_num, rank_path.display()); + + // Create subdirectory for this rank + let rank_out_dir = out_path.join(format!("rank_{}", rank_num)); + fs::create_dir(&rank_out_dir)?; + + let config = ParseConfig { + strict: cli.strict, + strict_compile_id: cli.strict_compile_id, + custom_parsers: Vec::new(), + custom_header_html: cli.custom_header_html.clone(), + verbose: cli.verbose, + plain_text: cli.plain_text, + export: cli.export, + inductor_provenance: cli.inductor_provenance, + all_ranks: false, + }; + + let output = parse_path(&rank_path, config)?; + + // Write output files to rank subdirectory + for (filename, content) in output { + let out_file = rank_out_dir.join(filename); + if let Some(dir) = out_file.parent() { + fs::create_dir_all(dir)?; + } + fs::write(out_file, content)?; + } + + // Add link to this rank's page + rank_links.push((rank_num.clone(), format!("rank_{}/index.html", rank_num))); + } + + // Sort rank links by rank number + rank_links.sort_by(|a, b| { + let a_num: i32 = a.0.parse().unwrap_or(999); + let b_num: i32 = b.0.parse().unwrap_or(999); + a_num.cmp(&b_num) + }); + + // Core logic complete - no HTML generation yet + // TODO - Add landing page HTML generation using template system + + println!("Generated multi-rank report with {} ranks", rank_links.len()); + println!("Individual rank reports available in:"); + for (rank_num, _) in &rank_links { + println!(" - rank_{}/index.html", rank_num); + } + + // No browser opening since no landing page yet + // TODO - Generate landing page and open browser + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 7f5b4c1..2d5722a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ pub struct ParseConfig { pub plain_text: bool, pub export: bool, pub inductor_provenance: bool, + pub all_ranks: bool, } impl Default for ParseConfig { @@ -43,6 +44,7 @@ impl Default for ParseConfig { plain_text: false, export: false, inductor_provenance: false, + all_ranks: false, } } } From cae8159e228d4b5ac16e89be1ce6c29f84972399 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Thu, 3 Jul 2025 16:08:48 -0700 Subject: [PATCH 02/14] Fix Lint Error --- src/cli.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index fc940bb..673cb01 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -187,7 +187,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { println!("Processing rank {} from file: {}", rank_num, rank_path.display()); // Create subdirectory for this rank - let rank_out_dir = out_path.join(format!("rank_{}", rank_num)); + let rank_out_dir = out_path.join(format!("rank_{rank_num}")); fs::create_dir(&rank_out_dir)?; let config = ParseConfig { @@ -214,7 +214,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { } // Add link to this rank's page - rank_links.push((rank_num.clone(), format!("rank_{}/index.html", rank_num))); + rank_links.push((rank_num.clone(), format!("rank_{rank_num}/index.html"))); } // Sort rank links by rank number @@ -230,7 +230,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { println!("Generated multi-rank report with {} ranks", rank_links.len()); println!("Individual rank reports available in:"); for (rank_num, _) in &rank_links { - println!(" - rank_{}/index.html", rank_num); + println!(" - rank_{rank_num}/index.html"); } // No browser opening since no landing page yet From 43e2cb3730b16c8966299d58ebaa3e07293cbab4 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Thu, 3 Jul 2025 20:05:24 -0700 Subject: [PATCH 03/14] Update cli.rs --- src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index 673cb01..9f28e7d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -230,7 +230,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { println!("Generated multi-rank report with {} ranks", rank_links.len()); println!("Individual rank reports available in:"); for (rank_num, _) in &rank_links { - println!(" - rank_{rank_num}/index.html"); + println!(" - rank_{}/index.html", rank_num); } // No browser opening since no landing page yet From 4ba11f44af83377122f7ff8a4a1104ad45d6ccca Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Thu, 3 Jul 2025 20:09:05 -0700 Subject: [PATCH 04/14] Update cli.rs --- src/cli.rs | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 9f28e7d..86a7c83 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -154,16 +154,20 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { .flatten() .filter(|entry| { let path = entry.path(); - path.is_file() && - path.file_name() - .and_then(|name| name.to_str()) - .map(|name| name.contains("rank_") && name.ends_with(".log")) - .unwrap_or(false) + path.is_file() + && path + .file_name() + .and_then(|name| name.to_str()) + .map(|name| name.contains("rank_") && name.ends_with(".log")) + .unwrap_or(false) }) .collect(); if rank_files.is_empty() { - bail!("No rank log files found in directory {}", input_path.display()); + bail!( + "No rank log files found in directory {}", + input_path.display() + ); } let mut rank_links = Vec::new(); @@ -179,12 +183,19 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { // Extract rank number from filename let rank_num = if let Some(pos) = rank_name.find("rank_") { let after_rank = &rank_name[pos + 5..]; - after_rank.chars().take_while(|c| c.is_ascii_digit()).collect::() + after_rank + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect::() } else { "unknown".to_string() }; - println!("Processing rank {} from file: {}", rank_num, rank_path.display()); + println!( + "Processing rank {} from file: {}", + rank_num, + rank_path.display() + ); // Create subdirectory for this rank let rank_out_dir = out_path.join(format!("rank_{rank_num}")); @@ -227,7 +238,10 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { // Core logic complete - no HTML generation yet // TODO - Add landing page HTML generation using template system - println!("Generated multi-rank report with {} ranks", rank_links.len()); + println!( + "Generated multi-rank report with {} ranks", + rank_links.len() + ); println!("Individual rank reports available in:"); for (rank_num, _) in &rank_links { println!(" - rank_{}/index.html", rank_num); From eaff4df8b687a847bc01a0ca564a690bc95d3943 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:03:06 -0700 Subject: [PATCH 05/14] Fix cli.rs to Address PR feedback --- src/cli.rs | 143 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 61 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 86a7c83..124e1be 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,6 +6,24 @@ use std::path::PathBuf; use tlparse::{parse_path, ParseConfig}; +// Main output filename used by both single rank and multi-rank processing +const MAIN_OUTPUT_FILENAME: &str = "index.html"; + +// Helper function to setup output directory (handles overwrite logic) +fn setup_output_directory(out_path: &PathBuf, overwrite: bool) -> anyhow::Result<()> { + if out_path.exists() { + if !overwrite { + bail!( + "Directory {} already exists, use -o OUTDIR to write to another location or pass --overwrite to overwrite the old contents", + out_path.display() + ); + } + fs::remove_dir_all(&out_path)?; + } + fs::create_dir(&out_path)?; + Ok(()) +} + #[derive(Parser)] #[command(author, version, about, long_about = None)] #[command(propagate_version = true)] @@ -47,15 +65,15 @@ pub struct Cli { /// For inductor provenance tracking highlighter #[arg(short, long)] inductor_provenance: bool, - /// Parse all ranks and generate a single unified page + /// Parse all ranks and generate a single unified HTML page #[arg(long)] - all_ranks: bool, + all_ranks_html: bool, } fn main() -> anyhow::Result<()> { let cli = Cli::parse(); - if cli.all_ranks { + if cli.all_ranks_html { return handle_all_ranks(&cli); } @@ -84,17 +102,7 @@ fn main() -> anyhow::Result<()> { }; let out_path = cli.out; - - if out_path.exists() { - if !cli.overwrite { - bail!( - "Directory {} already exists, use -o OUTDIR to write to another location or pass --overwrite to overwrite the old contents", - out_path.display() - ); - } - fs::remove_dir_all(&out_path)?; - } - fs::create_dir(&out_path)?; + setup_output_directory(&out_path, cli.overwrite)?; let config = ParseConfig { strict: cli.strict, @@ -105,7 +113,7 @@ fn main() -> anyhow::Result<()> { plain_text: cli.plain_text, export: cli.export, inductor_provenance: cli.inductor_provenance, - all_ranks: cli.all_ranks, + all_ranks: cli.all_ranks_html, }; let output = parse_path(&path, config)?; @@ -124,29 +132,61 @@ fn main() -> anyhow::Result<()> { Ok(()) } +// Helper function to handle parsing and writing output for a single rank +// Returns the relative path to the main output file within the rank directory +fn handle_one_rank( + rank_path: &PathBuf, + rank_out_dir: &PathBuf, + cli: &Cli, +) -> anyhow::Result { + fs::create_dir(rank_out_dir)?; + + let config = ParseConfig { + strict: cli.strict, + strict_compile_id: cli.strict_compile_id, + custom_parsers: Vec::new(), + custom_header_html: cli.custom_header_html.clone(), + verbose: cli.verbose, + plain_text: cli.plain_text, + export: cli.export, + inductor_provenance: cli.inductor_provenance, + all_ranks: false, + }; + + let output = parse_path(rank_path, config)?; + + let mut main_output_path = None; + + // Write output files to rank subdirectory + for (filename, content) in output { + let out_file = rank_out_dir.join(&filename); + if let Some(dir) = out_file.parent() { + fs::create_dir_all(dir)?; + } + fs::write(out_file, content)?; + + // Track the main output file (typically index.html) + if filename.file_name().and_then(|name| name.to_str()) == Some(MAIN_OUTPUT_FILENAME) { + main_output_path = Some(filename); + } + } + + Ok(main_output_path.unwrap_or_else(|| PathBuf::from(MAIN_OUTPUT_FILENAME))) +} + // handle_all_ranks function with placeholder landing page fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { let input_path = &cli.path; if !input_path.is_dir() { bail!( - "Input path {} must be a directory when using --all-ranks", + "Input path {} must be a directory when using --all-ranks-html", input_path.display() ); } let out_path = &cli.out; - - if out_path.exists() { - if !cli.overwrite { - bail!( - "Directory {} already exists, use -o OUTDIR to write to another location or pass --overwrite to overwrite the old contents", - out_path.display() - ); - } - fs::remove_dir_all(&out_path)?; - } - fs::create_dir(&out_path)?; + setup_output_directory(out_path, cli.overwrite)?; // Find all rank log files in the directory let rank_files: Vec<_> = std::fs::read_dir(input_path) @@ -181,14 +221,17 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { .unwrap_or("unknown"); // Extract rank number from filename - let rank_num = if let Some(pos) = rank_name.find("rank_") { - let after_rank = &rank_name[pos + 5..]; - after_rank + let rank_num = if let Some(after_rank) = rank_name.strip_prefix("rank_") { + let num_str = after_rank .chars() .take_while(|c| c.is_ascii_digit()) - .collect::() + .collect::(); + if num_str.is_empty() { + bail!("Could not extract rank number from filename: {}", rank_name); + } + num_str } else { - "unknown".to_string() + bail!("Filename does not contain 'rank_': {}", rank_name); }; println!( @@ -197,41 +240,19 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { rank_path.display() ); - // Create subdirectory for this rank + // Create subdirectory for this rank and handle parsing let rank_out_dir = out_path.join(format!("rank_{rank_num}")); - fs::create_dir(&rank_out_dir)?; - - let config = ParseConfig { - strict: cli.strict, - strict_compile_id: cli.strict_compile_id, - custom_parsers: Vec::new(), - custom_header_html: cli.custom_header_html.clone(), - verbose: cli.verbose, - plain_text: cli.plain_text, - export: cli.export, - inductor_provenance: cli.inductor_provenance, - all_ranks: false, - }; - - let output = parse_path(&rank_path, config)?; - - // Write output files to rank subdirectory - for (filename, content) in output { - let out_file = rank_out_dir.join(filename); - if let Some(dir) = out_file.parent() { - fs::create_dir_all(dir)?; - } - fs::write(out_file, content)?; - } + let main_output_path = handle_one_rank(&rank_path, &rank_out_dir, cli)?; - // Add link to this rank's page - rank_links.push((rank_num.clone(), format!("rank_{rank_num}/index.html"))); + // Add link to this rank's page using the actual output path + let rank_link = format!("rank_{rank_num}/{}", main_output_path.display()); + rank_links.push((rank_num.clone(), rank_link)); } // Sort rank links by rank number rank_links.sort_by(|a, b| { - let a_num: i32 = a.0.parse().unwrap_or(999); - let b_num: i32 = b.0.parse().unwrap_or(999); + let a_num: i32 = a.0.parse().expect(&format!("Failed to parse rank number from '{}'", a.0)); + let b_num: i32 = b.0.parse().expect(&format!("Failed to parse rank number from '{}'", b.0)); a_num.cmp(&b_num) }); From 52d8260584a74e064295f1daa0a9a6a104d93277 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:15:09 -0700 Subject: [PATCH 06/14] Update cli.rs --- src/cli.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 124e1be..21c81c0 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -194,12 +194,20 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { .flatten() .filter(|entry| { let path = entry.path(); - path.is_file() - && path - .file_name() - .and_then(|name| name.to_str()) - .map(|name| name.contains("rank_") && name.ends_with(".log")) - .unwrap_or(false) + if !path.is_file() { + return false; + } + + let Some(filename) = path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + + if !filename.starts_with("rank_") || !filename.ends_with(".log") { + return false; + } + + let middle = &filename[5..filename.len()-4]; // Remove "rank_" and ".log" + !middle.is_empty() && middle.chars().all(|c| c.is_ascii_digit()) }) .collect(); From 8f4b1b6813f1831ac27782e4ef8ea821675343ba Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:36:59 -0700 Subject: [PATCH 07/14] Update cli.rs From 5bbf21c52fc41956fbad94ae7b0efb9f91576a93 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:40:09 -0700 Subject: [PATCH 08/14] Update cli.rs to fix Lint error From 24c52cd13789d632f2c75061856e6d604c2be238 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:49:05 -0700 Subject: [PATCH 09/14] Update cli.rs --- src/cli.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 21c81c0..fb98d02 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -206,7 +206,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { return false; } - let middle = &filename[5..filename.len()-4]; // Remove "rank_" and ".log" + let middle = &filename[5..filename.len() - 4]; // Remove "rank_" and ".log" !middle.is_empty() && middle.chars().all(|c| c.is_ascii_digit()) }) .collect(); @@ -259,8 +259,12 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { // Sort rank links by rank number rank_links.sort_by(|a, b| { - let a_num: i32 = a.0.parse().expect(&format!("Failed to parse rank number from '{}'", a.0)); - let b_num: i32 = b.0.parse().expect(&format!("Failed to parse rank number from '{}'", b.0)); + let a_num: i32 = + a.0.parse() + .expect(&format!("Failed to parse rank number from '{}'", a.0)); + let b_num: i32 = + b.0.parse() + .expect(&format!("Failed to parse rank number from '{}'", b.0)); a_num.cmp(&b_num) }); From d649d8471456677c5cf913b3f8c2d09d2f0a103e Mon Sep 17 00:00:00 2001 From: Sandeep Narendranath Karjala Date: Mon, 7 Jul 2025 17:35:01 -0700 Subject: [PATCH 10/14] Generate unified landing page with links to indiv. rank reports --- src/cli.rs | 39 ++++++++++++++++++++++++++++++--------- src/lib.rs | 17 +++++++++++++++++ src/templates.rs | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index fb98d02..eed1fb3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -248,7 +248,6 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { rank_path.display() ); - // Create subdirectory for this rank and handle parsing let rank_out_dir = out_path.join(format!("rank_{rank_num}")); let main_output_path = handle_one_rank(&rank_path, &rank_out_dir, cli)?; @@ -268,20 +267,42 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { a_num.cmp(&b_num) }); - // Core logic complete - no HTML generation yet - // TODO - Add landing page HTML generation using template system + // Generate landing page HTML using template system + use tinytemplate::TinyTemplate; + use tlparse::{MultiRankContext, RankInfo, CSS, JAVASCRIPT, TEMPLATE_MULTI_RANK_INDEX}; + + let mut tt = TinyTemplate::new(); + tt.add_formatter("format_unescaped", tinytemplate::format_unescaped); + tt.add_template("multi_rank_index.html", TEMPLATE_MULTI_RANK_INDEX)?; + + let ranks: Vec = rank_links + .iter() + .map(|(rank_num, link)| RankInfo { + number: rank_num.clone(), + link: link.clone(), + }) + .collect(); + + let context = MultiRankContext { + css: CSS, + javascript: JAVASCRIPT, + custom_header_html: cli.custom_header_html.clone(), + rank_count: rank_links.len(), + ranks, + }; + + let landing_html = tt.render("multi_rank_index.html", &context)?; + + fs::write(out_path.join("index.html"), landing_html)?; println!( "Generated multi-rank report with {} ranks", rank_links.len() ); - println!("Individual rank reports available in:"); - for (rank_num, _) in &rank_links { - println!(" - rank_{}/index.html", rank_num); - } - // No browser opening since no landing page yet - // TODO - Generate landing page and open browser + if !cli.no_browser { + opener::open(out_path.join("index.html"))?; + } Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index ff11115..6155656 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,8 @@ pub mod parsers; mod templates; mod types; +pub use crate::templates::{CSS, JAVASCRIPT, TEMPLATE_MULTI_RANK_INDEX}; + #[derive(Debug)] enum ParserResult { NoPayload, @@ -40,6 +42,21 @@ pub struct ParseConfig { pub all_ranks: bool, } +#[derive(serde::Serialize)] +pub struct RankInfo { + pub number: String, + pub link: String, +} + +#[derive(serde::Serialize)] +pub struct MultiRankContext { + pub css: &'static str, + pub javascript: &'static str, + pub custom_header_html: String, + pub rank_count: usize, + pub ranks: Vec, +} + impl Default for ParseConfig { fn default() -> Self { Self { diff --git a/src/templates.rs b/src/templates.rs index 4f31e0c..32fdd1f 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -157,7 +157,7 @@ and avoid inlining the function in the first place.

When compiled autograd is enabled, the compile id will include a prefix signifier [!a/x/y], -where a is the compiled autograd id. For instance, [!0/-/-] refers +where a is the compiled autograd id. For instance, [!0/-/-] refers to the first graph captured by compiled autograd. It is then traced by torch.compile as [!0/x/y_z].

@@ -486,7 +486,7 @@ you may address them. {{ for failure in failures }} - + @@ -529,3 +529,34 @@ pub static TEMPLATE_SYMBOLIC_GUARD_INFO: &str = r#" pub static PROVENANCE_CSS: &str = include_str!("provenance.css"); pub static PROVENANCE_JS: &str = include_str!("provenance.js"); pub static TEMPLATE_PROVENANCE_TRACKING: &str = include_str!("provenance.html"); + +pub static TEMPLATE_MULTI_RANK_INDEX: &str = r#" + + + + + + +
+{custom_header_html | format_unescaped} +

Multi-Rank TLParse Report

+

+This report contains compilation information from {rank_count} distributed training rank(s). +Each rank ran independently and generated its own compilation artifacts. Click on any rank below +to view its detailed compilation report, including stack traces, IR dumps, and performance metrics. +

+

+Ranks processed: +

+ +
+ +"#; From 62501412daea7500f18c2931a0c9ffe856cfdc29 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:21:17 -0700 Subject: [PATCH 11/14] Update cli.rs --- src/cli.rs | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index eed1fb3..90ecad3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -202,12 +202,19 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { return false; }; - if !filename.starts_with("rank_") || !filename.ends_with(".log") { + // Only support PyTorch TORCH_TRACE files: dedicated_log_torch_trace_rank_0_hash.log + if !filename.starts_with("dedicated_log_torch_trace_rank_") || !filename.ends_with(".log") { return false; } - let middle = &filename[5..filename.len() - 4]; // Remove "rank_" and ".log" - !middle.is_empty() && middle.chars().all(|c| c.is_ascii_digit()) + // Extract rank number from the pattern + let after_prefix = &filename[31..]; // Remove "dedicated_log_torch_trace_rank_" + if let Some(underscore_pos) = after_prefix.find('_') { + let rank_part = &after_prefix[..underscore_pos]; + return !rank_part.is_empty() && rank_part.chars().all(|c| c.is_ascii_digit()); + } + + false }) .collect(); @@ -228,18 +235,19 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { .and_then(|name| name.to_str()) .unwrap_or("unknown"); - // Extract rank number from filename - let rank_num = if let Some(after_rank) = rank_name.strip_prefix("rank_") { - let num_str = after_rank - .chars() - .take_while(|c| c.is_ascii_digit()) - .collect::(); - if num_str.is_empty() { - bail!("Could not extract rank number from filename: {}", rank_name); + // Extract rank number from PyTorch TORCH_TRACE filename + let rank_num = if let Some(after_prefix) = rank_name.strip_prefix("dedicated_log_torch_trace_rank_") { + if let Some(underscore_pos) = after_prefix.find('_') { + let rank_part = &after_prefix[..underscore_pos]; + if rank_part.is_empty() || !rank_part.chars().all(|c| c.is_ascii_digit()) { + bail!("Could not extract rank number from TORCH_TRACE filename: {}", rank_name); + } + rank_part.to_string() + } else { + bail!("Invalid TORCH_TRACE filename format: {}", rank_name); } - num_str } else { - bail!("Filename does not contain 'rank_': {}", rank_name); + bail!("Filename does not match PyTorch TORCH_TRACE pattern: {}", rank_name); }; println!( From b828232fb62a55aec2cab5fe63d6ce44a51372c9 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:49:05 -0700 Subject: [PATCH 12/14] Update cli.rs --- src/cli.rs | 115 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 39 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 90ecad3..cddb827 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -78,7 +78,7 @@ fn main() -> anyhow::Result<()> { } let path = if cli.latest { - let input_path = cli.path; + let input_path = &cli.path; // Path should be a directory if !input_path.is_dir() { bail!( @@ -87,7 +87,7 @@ fn main() -> anyhow::Result<()> { ); } - let last_modified_file = std::fs::read_dir(&input_path) + let last_modified_file = std::fs::read_dir(input_path) .with_context(|| format!("Couldn't access directory {}", input_path.display()))? .flatten() .filter(|f| f.metadata().unwrap().is_file()) @@ -98,33 +98,14 @@ fn main() -> anyhow::Result<()> { }; last_modified_file.path() } else { - cli.path + cli.path.clone() }; - let out_path = cli.out; + let out_path = cli.out.clone(); setup_output_directory(&out_path, cli.overwrite)?; - let config = ParseConfig { - strict: cli.strict, - strict_compile_id: cli.strict_compile_id, - custom_parsers: Vec::new(), - custom_header_html: cli.custom_header_html.clone(), - verbose: cli.verbose, - plain_text: cli.plain_text, - export: cli.export, - inductor_provenance: cli.inductor_provenance, - all_ranks: cli.all_ranks_html, - }; - - let output = parse_path(&path, config)?; - - for (filename, path) in output { - let out_file = out_path.join(filename); - if let Some(dir) = out_file.parent() { - fs::create_dir_all(dir)?; - } - fs::write(out_file, path)?; - } + // Use handle_one_rank for single rank processing (don't create directory since it already exists) + handle_one_rank(&path, &out_path, &cli, false)?; if !cli.no_browser { opener::open(out_path.join("index.html"))?; @@ -138,8 +119,11 @@ fn handle_one_rank( rank_path: &PathBuf, rank_out_dir: &PathBuf, cli: &Cli, + create_output_dir: bool, ) -> anyhow::Result { - fs::create_dir(rank_out_dir)?; + if create_output_dir { + fs::create_dir(rank_out_dir)?; + } let config = ParseConfig { strict: cli.strict, @@ -203,7 +187,9 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { }; // Only support PyTorch TORCH_TRACE files: dedicated_log_torch_trace_rank_0_hash.log - if !filename.starts_with("dedicated_log_torch_trace_rank_") || !filename.ends_with(".log") { + if !filename.starts_with("dedicated_log_torch_trace_rank_") + || !filename.ends_with(".log") + { return false; } @@ -214,6 +200,14 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { return !rank_part.is_empty() && rank_part.chars().all(|c| c.is_ascii_digit()); } + false + // Extract rank number from the pattern + let after_prefix = &filename[31..]; // Remove "dedicated_log_torch_trace_rank_" + if let Some(underscore_pos) = after_prefix.find('_') { + let rank_part = &after_prefix[..underscore_pos]; + return !rank_part.is_empty() && rank_part.chars().all(|c| c.is_ascii_digit()); + } + false }) .collect(); @@ -236,19 +230,26 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { .unwrap_or("unknown"); // Extract rank number from PyTorch TORCH_TRACE filename - let rank_num = if let Some(after_prefix) = rank_name.strip_prefix("dedicated_log_torch_trace_rank_") { - if let Some(underscore_pos) = after_prefix.find('_') { - let rank_part = &after_prefix[..underscore_pos]; - if rank_part.is_empty() || !rank_part.chars().all(|c| c.is_ascii_digit()) { - bail!("Could not extract rank number from TORCH_TRACE filename: {}", rank_name); + let rank_num = + if let Some(after_prefix) = rank_name.strip_prefix("dedicated_log_torch_trace_rank_") { + if let Some(underscore_pos) = after_prefix.find('_') { + let rank_part = &after_prefix[..underscore_pos]; + if rank_part.is_empty() || !rank_part.chars().all(|c| c.is_ascii_digit()) { + bail!( + "Could not extract rank number from TORCH_TRACE filename: {}", + rank_name + ); + } + rank_part.to_string() + } else { + bail!("Invalid TORCH_TRACE filename format: {}", rank_name); } - rank_part.to_string() } else { - bail!("Invalid TORCH_TRACE filename format: {}", rank_name); - } - } else { - bail!("Filename does not match PyTorch TORCH_TRACE pattern: {}", rank_name); - }; + bail!( + "Filename does not match PyTorch TORCH_TRACE pattern: {}", + rank_name + ); + }; println!( "Processing rank {} from file: {}", @@ -257,7 +258,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { ); let rank_out_dir = out_path.join(format!("rank_{rank_num}")); - let main_output_path = handle_one_rank(&rank_path, &rank_out_dir, cli)?; + let main_output_path = handle_one_rank(&rank_path, &rank_out_dir, cli, true)?; // Add link to this rank's page using the actual output path let rank_link = format!("rank_{rank_num}/{}", main_output_path.display()); @@ -266,6 +267,12 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { // Sort rank links by rank number rank_links.sort_by(|a, b| { + let a_num: i32 = + a.0.parse() + .expect(&format!("Failed to parse rank number from '{}'", a.0)); + let b_num: i32 = + b.0.parse() + .expect(&format!("Failed to parse rank number from '{}'", b.0)); let a_num: i32 = a.0.parse() .expect(&format!("Failed to parse rank number from '{}'", a.0)); @@ -301,6 +308,33 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { let landing_html = tt.render("multi_rank_index.html", &context)?; + fs::write(out_path.join("index.html"), landing_html)?; + // Generate landing page HTML using template system + use tinytemplate::TinyTemplate; + use tlparse::{MultiRankContext, RankInfo, CSS, JAVASCRIPT, TEMPLATE_MULTI_RANK_INDEX}; + + let mut tt = TinyTemplate::new(); + tt.add_formatter("format_unescaped", tinytemplate::format_unescaped); + tt.add_template("multi_rank_index.html", TEMPLATE_MULTI_RANK_INDEX)?; + + let ranks: Vec = rank_links + .iter() + .map(|(rank_num, link)| RankInfo { + number: rank_num.clone(), + link: link.clone(), + }) + .collect(); + + let context = MultiRankContext { + css: CSS, + javascript: JAVASCRIPT, + custom_header_html: cli.custom_header_html.clone(), + rank_count: rank_links.len(), + ranks, + }; + + let landing_html = tt.render("multi_rank_index.html", &context)?; + fs::write(out_path.join("index.html"), landing_html)?; println!( @@ -308,6 +342,9 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { rank_links.len() ); + if !cli.no_browser { + opener::open(out_path.join("index.html"))?; + if !cli.no_browser { opener::open(out_path.join("index.html"))?; } From 41cddb88c8e06ecebf8b24c5364d605c5da298a1 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Wed, 9 Jul 2025 13:49:35 -0700 Subject: [PATCH 13/14] Update cli.rs to fix duplicate issue --- src/cli.rs | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index cddb827..15ca9fd 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -200,14 +200,6 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { return !rank_part.is_empty() && rank_part.chars().all(|c| c.is_ascii_digit()); } - false - // Extract rank number from the pattern - let after_prefix = &filename[31..]; // Remove "dedicated_log_torch_trace_rank_" - if let Some(underscore_pos) = after_prefix.find('_') { - let rank_part = &after_prefix[..underscore_pos]; - return !rank_part.is_empty() && rank_part.chars().all(|c| c.is_ascii_digit()); - } - false }) .collect(); @@ -267,12 +259,6 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { // Sort rank links by rank number rank_links.sort_by(|a, b| { - let a_num: i32 = - a.0.parse() - .expect(&format!("Failed to parse rank number from '{}'", a.0)); - let b_num: i32 = - b.0.parse() - .expect(&format!("Failed to parse rank number from '{}'", b.0)); let a_num: i32 = a.0.parse() .expect(&format!("Failed to parse rank number from '{}'", a.0)); @@ -308,33 +294,6 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { let landing_html = tt.render("multi_rank_index.html", &context)?; - fs::write(out_path.join("index.html"), landing_html)?; - // Generate landing page HTML using template system - use tinytemplate::TinyTemplate; - use tlparse::{MultiRankContext, RankInfo, CSS, JAVASCRIPT, TEMPLATE_MULTI_RANK_INDEX}; - - let mut tt = TinyTemplate::new(); - tt.add_formatter("format_unescaped", tinytemplate::format_unescaped); - tt.add_template("multi_rank_index.html", TEMPLATE_MULTI_RANK_INDEX)?; - - let ranks: Vec = rank_links - .iter() - .map(|(rank_num, link)| RankInfo { - number: rank_num.clone(), - link: link.clone(), - }) - .collect(); - - let context = MultiRankContext { - css: CSS, - javascript: JAVASCRIPT, - custom_header_html: cli.custom_header_html.clone(), - rank_count: rank_links.len(), - ranks, - }; - - let landing_html = tt.render("multi_rank_index.html", &context)?; - fs::write(out_path.join("index.html"), landing_html)?; println!( @@ -342,9 +301,6 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { rank_links.len() ); - if !cli.no_browser { - opener::open(out_path.join("index.html"))?; - if !cli.no_browser { opener::open(out_path.join("index.html"))?; } From 4492fb7040f78cabf30de0505b50963565ae7063 Mon Sep 17 00:00:00 2001 From: Sandeep Karjala <68705118+skarjala@users.noreply.github.com> Date: Thu, 10 Jul 2025 09:54:15 -0700 Subject: [PATCH 14/14] Update cli.rs to fix clone & reference issues --- src/cli.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 15ca9fd..6c645d6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -74,7 +74,7 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); if cli.all_ranks_html { - return handle_all_ranks(&cli); + return handle_all_ranks(cli); } let path = if cli.latest { @@ -159,7 +159,7 @@ fn handle_one_rank( } // handle_all_ranks function with placeholder landing page -fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { +fn handle_all_ranks(cli: Cli) -> anyhow::Result<()> { let input_path = &cli.path; if !input_path.is_dir() { @@ -250,7 +250,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { ); let rank_out_dir = out_path.join(format!("rank_{rank_num}")); - let main_output_path = handle_one_rank(&rank_path, &rank_out_dir, cli, true)?; + let main_output_path = handle_one_rank(&rank_path, &rank_out_dir, &cli, true)?; // Add link to this rank's page using the actual output path let rank_link = format!("rank_{rank_num}/{}", main_output_path.display()); @@ -287,7 +287,7 @@ fn handle_all_ranks(cli: &Cli) -> anyhow::Result<()> { let context = MultiRankContext { css: CSS, javascript: JAVASCRIPT, - custom_header_html: cli.custom_header_html.clone(), + custom_header_html: cli.custom_header_html, rank_count: rank_links.len(), ranks, };
Failure Type Reason Additional Info
{failure.failure_type | format_unescaped} {failure.reason | format_unescaped} {failure.additional_info | format_unescaped}