Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v
popcorn submit submission.py --leaderboard qr_v2 --profile-brev --benchmark-index 0 --no-tui
```

The CLI downloads and extracts the `.ncu-rep` file, prints a clickable terminal
link to the report, and ends with a macOS command that opens it in Nsight
Compute:
The CLI downloads and extracts `ncu-details.txt` and `ncu-details.csv` for
agent-readable analysis. It also extracts the optional `.ncu-rep` GUI report and
ends with a macOS command that opens it in Nsight Compute:

```bash
open -a "NVIDIA Nsight Compute" profile.0-.../profile.ncu-rep
Expand Down
6 changes: 3 additions & 3 deletions docs/linalg-qr-b200.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ export POPCORN_BREV_PROFILER_URL=https://http--brev-profiler-proxy--dxfjds728w5v
popcorn submit --leaderboard qr_v2 --profile-brev --benchmark-index 0 submission.py
```

The CLI downloads a `.zip`, extracts `profile.ncu-rep`, and prints an
`open -a "NVIDIA Nsight Compute" ...` command. See
[profiling.md](profiling.md) for the complete QR v2 profiling flow.
The CLI downloads a `.zip` and extracts `ncu-details.txt`, `ncu-details.csv`,
and the optional `profile.ncu-rep` GUI report. See [profiling.md](profiling.md)
for the complete QR v2 profiling flow.

Submit to the leaderboard:

Expand Down
19 changes: 12 additions & 7 deletions docs/profiling.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# QR v2 Nsight Compute Profiling

This profiles the GPU Mode QR v2 problem from `reference-kernels` and downloads
an Nsight Compute `.ncu-rep` report that you can open locally.
Nsight Compute details that AI agents can read directly. The full `.ncu-rep`
GUI report is still included for local inspection.

## 1. Install and Register

Expand Down Expand Up @@ -45,16 +46,21 @@ The first QR v2 benchmark shape is:
batch: 20; n: 32; cond: 1; seed: 43214
```

## 4. Open the Report
## 4. Read the Details

After the run finishes, the CLI downloads and extracts files like:

```text
profile.0-batch-20-n-32-cond-1-seed-43214.zip
profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep
profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.txt
profile.0-batch-20-n-32-cond-1-seed-43214/ncu-details.csv
profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep # optional GUI report
```

The last line printed by the CLI opens the report on macOS:
Use `ncu-details.txt` or `ncu-details.csv` as the default artifact for AI
analysis. The CLI prints clickable links for these detail files.

The last line printed by the CLI opens the optional GUI report on macOS:

```bash
open -a "NVIDIA Nsight Compute" 'profile.0-batch-20-n-32-cond-1-seed-43214/profile.ncu-rep'
Expand All @@ -72,8 +78,8 @@ popcorn submit submission.py \
```

This profiles every entry in the `benchmarks:` list in QR v2 `task.yml`, not
the `tests:` list. It will produce one zip and one extracted `.ncu-rep` per
benchmark shape.
the `tests:` list. It will produce one zip plus extracted details and optional
`.ncu-rep` files per benchmark shape.

## Normal Submit Commands

Expand All @@ -88,4 +94,3 @@ For leaderboard submission:
```bash
popcorn submit submission.py --leaderboard qr_v2 --gpu B200 --mode leaderboard --no-tui
```

52 changes: 45 additions & 7 deletions src/cmd/submit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -812,7 +812,14 @@ pub async fn run_submit_plain(
struct ProfileReportLink {
file_url: String,
label: String,
open_command: String,
open_command: Option<String>,
kind: ProfileArtifactKind,
}

#[derive(Debug)]
enum ProfileArtifactKind {
Details,
Report,
}

fn profile_report_links(content: &str) -> Vec<ProfileReportLink> {
Expand All @@ -828,6 +835,24 @@ fn profile_report_links(content: &str) -> Vec<ProfileReportLink> {

let mut links = Vec::new();
for artifact in artifacts {
if let Some(details) = artifact.get("details").and_then(|value| value.as_array()) {
for detail in details {
let Some(file_url) = detail.get("file_url").and_then(|value| value.as_str()) else {
continue;
};
let label = detail
.get("path")
.and_then(|value| value.as_str())
.unwrap_or("ncu-details.txt");
links.push(ProfileReportLink {
file_url: file_url.to_string(),
label: label.to_string(),
open_command: None,
kind: ProfileArtifactKind::Details,
});
}
}

let Some(reports) = artifact.get("reports").and_then(|value| value.as_array()) else {
continue;
};
Expand All @@ -849,7 +874,8 @@ fn profile_report_links(content: &str) -> Vec<ProfileReportLink> {
links.push(ProfileReportLink {
file_url: file_url.to_string(),
label: label.to_string(),
open_command,
open_command: Some(open_command),
kind: ProfileArtifactKind::Report,
});
}
}
Expand All @@ -858,11 +884,23 @@ fn profile_report_links(content: &str) -> Vec<ProfileReportLink> {

fn print_profile_report_links(links: Vec<ProfileReportLink>) {
for link in links {
println!(
"\nOpen in Nsight Compute: {}",
terminal_link(&link.file_url, &link.label)
);
println!("{}", link.open_command);
match link.kind {
ProfileArtifactKind::Details => {
println!(
"\nNCU details for agents: {}",
terminal_link(&link.file_url, &link.label)
);
}
ProfileArtifactKind::Report => {
println!(
"\nOpen in Nsight Compute: {}",
terminal_link(&link.file_url, &link.label)
);
if let Some(open_command) = link.open_command {
println!("{}", open_command);
}
}
}
}
}

Expand Down
61 changes: 48 additions & 13 deletions src/service/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -795,11 +795,22 @@ pub async fn profile_brev_solution<P: AsRef<Path>>(
#[derive(Debug)]
struct DownloadedProfileArtifact {
zip_path: PathBuf,
details: Vec<PathBuf>,
reports: Vec<PathBuf>,
}

impl DownloadedProfileArtifact {
fn to_json(&self) -> Value {
let details: Vec<Value> = self
.details
.iter()
.map(|path| {
serde_json::json!({
"path": path.display().to_string(),
"file_url": file_url(path),
})
})
.collect();
let reports: Vec<Value> = self
.reports
.iter()
Expand All @@ -818,6 +829,7 @@ impl DownloadedProfileArtifact {

serde_json::json!({
"zip_path": self.zip_path.display().to_string(),
"details": details,
"reports": reports,
})
}
Expand Down Expand Up @@ -859,13 +871,23 @@ async fn download_profile_artifacts(
let zip_path = PathBuf::from(name);
std::fs::write(&zip_path, bytes.as_ref())
.map_err(|e| anyhow!("Failed to write profile artifact {}: {}", name, e))?;
let reports = extract_ncu_reports(&zip_path, bytes.as_ref())?;
saved.push(DownloadedProfileArtifact { zip_path, reports });
let extracted = extract_profile_artifacts(&zip_path, bytes.as_ref())?;
saved.push(DownloadedProfileArtifact {
zip_path,
details: extracted.details,
reports: extracted.reports,
});
}
Ok(saved)
}

fn extract_ncu_reports(zip_path: &Path, bytes: &[u8]) -> Result<Vec<PathBuf>> {
#[derive(Debug)]
struct ExtractedProfileArtifacts {
details: Vec<PathBuf>,
reports: Vec<PathBuf>,
}

fn extract_profile_artifacts(zip_path: &Path, bytes: &[u8]) -> Result<ExtractedProfileArtifacts> {
let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|e| {
anyhow!(
"Failed to read profile artifact {}: {}",
Expand All @@ -882,6 +904,7 @@ fn extract_ncu_reports(zip_path: &Path, bytes: &[u8]) -> Result<Vec<PathBuf>> {
)
})?;

let mut details = Vec::new();
let mut reports = Vec::new();
for idx in 0..archive.len() {
let mut entry = archive.by_index(idx).map_err(|e| {
Expand All @@ -891,29 +914,41 @@ fn extract_ncu_reports(zip_path: &Path, bytes: &[u8]) -> Result<Vec<PathBuf>> {
e
)
})?;
if !entry.name().ends_with(".ncu-rep") {
if !entry.name().ends_with(".ncu-rep")
&& !entry.name().ends_with("ncu-details.txt")
&& !entry.name().ends_with("ncu-details.csv")
{
continue;
}

let file_name = Path::new(entry.name())
.file_name()
.ok_or_else(|| anyhow!("Profile artifact contains an invalid report path"))?;
let mut report_path = extract_dir.join(file_name);
if report_path.exists() {
let stem = report_path
let mut output_path = extract_dir.join(file_name);
if output_path.exists() {
let stem = output_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("profile");
report_path = extract_dir.join(format!("{}-{}.ncu-rep", stem, idx));
let extension = output_path
.extension()
.and_then(|s| s.to_str())
.map(|ext| format!(".{}", ext))
.unwrap_or_default();
output_path = extract_dir.join(format!("{}-{}{}", stem, idx, extension));
}

let mut output = StdFile::create(&report_path)
.map_err(|e| anyhow!("Failed to create {}: {}", report_path.display(), e))?;
let mut output = StdFile::create(&output_path)
.map_err(|e| anyhow!("Failed to create {}: {}", output_path.display(), e))?;
std::io::copy(&mut entry, &mut output)
.map_err(|e| anyhow!("Failed to extract {}: {}", report_path.display(), e))?;
reports.push(report_path);
.map_err(|e| anyhow!("Failed to extract {}: {}", output_path.display(), e))?;
if output_path.extension().and_then(|s| s.to_str()) == Some("ncu-rep") {
reports.push(output_path);
} else {
details.push(output_path);
}
}
Ok(reports)
Ok(ExtractedProfileArtifacts { details, reports })
}

fn file_url(path: &Path) -> String {
Expand Down
Loading