diff --git a/README.md b/README.md index f5a597c..524d06d 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 --no-tui ``` -The CLI downloads and extracts the `.ncu-rep` file, prints a clickable terminal -link to the report, and ends with a macOS command that opens it in Nsight -Compute: +The CLI downloads and extracts `ncu-details.txt` and `ncu-details.csv` for +agent-readable analysis. It also extracts the optional `.ncu-rep` GUI report and +ends with a macOS command that opens it in Nsight Compute: ```bash open -a "NVIDIA Nsight Compute" profile.0-.../profile.ncu-rep diff --git a/docs/linalg-qr-b200.md b/docs/linalg-qr-b200.md index 258aea2..47c7a1e 100644 --- a/docs/linalg-qr-b200.md +++ b/docs/linalg-qr-b200.md @@ -26,9 +26,9 @@ export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v popcorn submit --leaderboard qr_v2 --profile-brev --benchmark-index 0 submission.py ``` -The CLI downloads a `.zip`, extracts `profile.ncu-rep`, and prints an -`open -a "NVIDIA Nsight Compute" ...` command. See -[profiling.md](profiling.md) for the complete QR v2 profiling flow. +The CLI downloads a `.zip` and extracts `ncu-details.txt`, `ncu-details.csv`, +and the optional `profile.ncu-rep` GUI report. See [profiling.md](profiling.md) +for the complete QR v2 profiling flow. Submit to the leaderboard: diff --git a/docs/profiling.md b/docs/profiling.md index ac0762d..be7df04 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -1,7 +1,8 @@ # QR v2 Nsight Compute Profiling This profiles the GPU Mode QR v2 problem from `reference-kernels` and downloads -an Nsight Compute `.ncu-rep` report that you can open locally. +Nsight Compute details that AI agents can read directly. The full `.ncu-rep` +GUI report is still included for local inspection. ## 1. Install and Register @@ -45,16 +46,21 @@ The first QR v2 benchmark shape is: batch: 20; n: 32; cond: 1; seed: 43214 ``` -## 4. Open the Report +## 4. Read the Details After the run finishes, the CLI downloads and extracts files like: ```text profile.0-batch-20-n-32-cond-1-seed-43214.zip -profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep +profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.txt +profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.csv +profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep # optional GUI report ``` -The last line printed by the CLI opens the report on macOS: +Use `ncu-details.txt` or `ncu-details.csv` as the default artifact for AI +analysis. The CLI prints clickable links for these detail files. + +The last line printed by the CLI opens the optional GUI report on macOS: ```bash open -a "NVIDIA Nsight Compute" 'profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep' @@ -72,8 +78,8 @@ popcorn submit submission.py \ ``` This profiles every entry in the `benchmarks:` list in QR v2 `task.yml`, not -the `tests:` list. It will produce one zip and one extracted `.ncu-rep` per -benchmark shape. +the `tests:` list. It will produce one zip plus extracted details and optional +`.ncu-rep` files per benchmark shape. ## Normal Submit Commands @@ -88,4 +94,3 @@ For leaderboard submission: ```bash popcorn submit submission.py --leaderboard qr_v2 --gpu B200 --mode leaderboard --no-tui ``` - diff --git a/src/cmd/submit.rs b/src/cmd/submit.rs index 29364f6..52d6cec 100644 --- a/src/cmd/submit.rs +++ b/src/cmd/submit.rs @@ -812,7 +812,14 @@ pub async fn run_submit_plain( struct ProfileReportLink { file_url: String, label: String, - open_command: String, + open_command: Option, + kind: ProfileArtifactKind, +} + +#[derive(Debug)] +enum ProfileArtifactKind { + Details, + Report, } fn profile_report_links(content: &str) -> Vec { @@ -828,6 +835,24 @@ fn profile_report_links(content: &str) -> Vec { let mut links = Vec::new(); for artifact in artifacts { + if let Some(details) = artifact.get("details").and_then(|value| value.as_array()) { + for detail in details { + let Some(file_url) = detail.get("file_url").and_then(|value| value.as_str()) else { + continue; + }; + let label = detail + .get("path") + .and_then(|value| value.as_str()) + .unwrap_or("ncu-details.txt"); + links.push(ProfileReportLink { + file_url: file_url.to_string(), + label: label.to_string(), + open_command: None, + kind: ProfileArtifactKind::Details, + }); + } + } + let Some(reports) = artifact.get("reports").and_then(|value| value.as_array()) else { continue; }; @@ -849,7 +874,8 @@ fn profile_report_links(content: &str) -> Vec { links.push(ProfileReportLink { file_url: file_url.to_string(), label: label.to_string(), - open_command, + open_command: Some(open_command), + kind: ProfileArtifactKind::Report, }); } } @@ -858,11 +884,23 @@ fn profile_report_links(content: &str) -> Vec { fn print_profile_report_links(links: Vec) { for link in links { - println!( - "\nOpen in Nsight Compute: {}", - terminal_link(&link.file_url, &link.label) - ); - println!("{}", link.open_command); + match link.kind { + ProfileArtifactKind::Details => { + println!( + "\nNCU details for agents: {}", + terminal_link(&link.file_url, &link.label) + ); + } + ProfileArtifactKind::Report => { + println!( + "\nOpen in Nsight Compute: {}", + terminal_link(&link.file_url, &link.label) + ); + if let Some(open_command) = link.open_command { + println!("{}", open_command); + } + } + } } } diff --git a/src/service/mod.rs b/src/service/mod.rs index 538a9e5..66976ca 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -795,11 +795,22 @@ pub async fn profile_brev_solution>( #[derive(Debug)] struct DownloadedProfileArtifact { zip_path: PathBuf, + details: Vec, reports: Vec, } impl DownloadedProfileArtifact { fn to_json(&self) -> Value { + let details: Vec = self + .details + .iter() + .map(|path| { + serde_json::json!({ + "path": path.display().to_string(), + "file_url": file_url(path), + }) + }) + .collect(); let reports: Vec = self .reports .iter() @@ -818,6 +829,7 @@ impl DownloadedProfileArtifact { serde_json::json!({ "zip_path": self.zip_path.display().to_string(), + "details": details, "reports": reports, }) } @@ -859,13 +871,23 @@ async fn download_profile_artifacts( let zip_path = PathBuf::from(name); std::fs::write(&zip_path, bytes.as_ref()) .map_err(|e| anyhow!("Failed to write profile artifact {}: {}", name, e))?; - let reports = extract_ncu_reports(&zip_path, bytes.as_ref())?; - saved.push(DownloadedProfileArtifact { zip_path, reports }); + let extracted = extract_profile_artifacts(&zip_path, bytes.as_ref())?; + saved.push(DownloadedProfileArtifact { + zip_path, + details: extracted.details, + reports: extracted.reports, + }); } Ok(saved) } -fn extract_ncu_reports(zip_path: &Path, bytes: &[u8]) -> Result> { +#[derive(Debug)] +struct ExtractedProfileArtifacts { + details: Vec, + reports: Vec, +} + +fn extract_profile_artifacts(zip_path: &Path, bytes: &[u8]) -> Result { let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|e| { anyhow!( "Failed to read profile artifact {}: {}", @@ -882,6 +904,7 @@ fn extract_ncu_reports(zip_path: &Path, bytes: &[u8]) -> Result> { ) })?; + let mut details = Vec::new(); let mut reports = Vec::new(); for idx in 0..archive.len() { let mut entry = archive.by_index(idx).map_err(|e| { @@ -891,29 +914,41 @@ fn extract_ncu_reports(zip_path: &Path, bytes: &[u8]) -> Result> { e ) })?; - if !entry.name().ends_with(".ncu-rep") { + if !entry.name().ends_with(".ncu-rep") + && !entry.name().ends_with("ncu-details.txt") + && !entry.name().ends_with("ncu-details.csv") + { continue; } let file_name = Path::new(entry.name()) .file_name() .ok_or_else(|| anyhow!("Profile artifact contains an invalid report path"))?; - let mut report_path = extract_dir.join(file_name); - if report_path.exists() { - let stem = report_path + let mut output_path = extract_dir.join(file_name); + if output_path.exists() { + let stem = output_path .file_stem() .and_then(|s| s.to_str()) .unwrap_or("profile"); - report_path = extract_dir.join(format!("{}-{}.ncu-rep", stem, idx)); + let extension = output_path + .extension() + .and_then(|s| s.to_str()) + .map(|ext| format!(".{}", ext)) + .unwrap_or_default(); + output_path = extract_dir.join(format!("{}-{}{}", stem, idx, extension)); } - let mut output = StdFile::create(&report_path) - .map_err(|e| anyhow!("Failed to create {}: {}", report_path.display(), e))?; + let mut output = StdFile::create(&output_path) + .map_err(|e| anyhow!("Failed to create {}: {}", output_path.display(), e))?; std::io::copy(&mut entry, &mut output) - .map_err(|e| anyhow!("Failed to extract {}: {}", report_path.display(), e))?; - reports.push(report_path); + .map_err(|e| anyhow!("Failed to extract {}: {}", output_path.display(), e))?; + if output_path.extension().and_then(|s| s.to_str()) == Some("ncu-rep") { + reports.push(output_path); + } else { + details.push(output_path); + } } - Ok(reports) + Ok(ExtractedProfileArtifacts { details, reports }) } fn file_url(path: &Path) -> String {