2a7ca101d7
Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream: Added: - python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc. Includes 204 .py files with full test coverage for all 27 algorithms - ci/ (163 files) — Build/test infrastructure build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml Directly maps to our [INFRA-CI] and [INFRA-BUILD] items - .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md - docs/ (491 files) — Official CCCL documentation CI references, CMake guides, Python compute docs, libcudacxx PTX docs - test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar) - Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml - CLAUDE.md symlink → AGENTS.md (NVIDIA's standard) cccl_upstream now mirrors full NVIDIA/cccl structure: Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks) After: 53M (+python +ci +docs +.agent +test +configs) This completes the CCCL base needed for: - [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds - [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations - [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh - Agent workflow: .agent/skills/ for consistent style and test patterns
1732 lines
55 KiB
Python
1732 lines
55 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import csv
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from ci.compile_time import summarize_tus
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
SUMMARY_SCRIPT = REPO_ROOT / "ci" / "compile_time" / "summarize_events.py"
|
|
PREPARE_SCRIPT = REPO_ROOT / "ci" / "compile_time" / "prepare_traces.py"
|
|
PARSE_MATRIX_SCRIPT = REPO_ROOT / "ci" / "compile_time" / "parse_matrix.py"
|
|
RENDER_COMMENT_SCRIPT = REPO_ROOT / "ci" / "compile_time" / "render_pr_comment.py"
|
|
WRAPPER_SCRIPT = REPO_ROOT / "ci" / "build_compile_time_bench.sh"
|
|
PULL_REQUEST_WORKFLOW = (
|
|
REPO_ROOT / ".github" / "workflows" / "ci-workflow-pull-request.yml"
|
|
)
|
|
|
|
|
|
def csv_rows(path: Path) -> list[dict[str, str]]:
|
|
with path.open(newline="", encoding="utf-8") as f:
|
|
return list(csv.DictReader(f))
|
|
|
|
|
|
class TraceBuilder:
|
|
def __init__(self, root: Path):
|
|
self.root = root
|
|
|
|
def project_detail(self, rel: str) -> str:
|
|
return (self.root / rel).as_posix()
|
|
|
|
def event(
|
|
self,
|
|
name: str,
|
|
detail: str,
|
|
ts: int,
|
|
dur: int,
|
|
*,
|
|
pid: int = 1,
|
|
tid: int = 1,
|
|
) -> dict:
|
|
return {
|
|
"ph": "X",
|
|
"pid": pid,
|
|
"tid": tid,
|
|
"name": name,
|
|
"ts": ts,
|
|
"dur": dur,
|
|
"args": {"detail": detail},
|
|
}
|
|
|
|
def write_trace(self, path: Path, events: list[dict], input_name: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"traceEvents": events,
|
|
"otherData": {
|
|
"inputFiles": [f"/generated/headers/target/{input_name}.cu"]
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
class SummarizeEventsBaselineCompareTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
self.work = Path(self.tempdir.name)
|
|
self.traces = TraceBuilder(REPO_ROOT)
|
|
self.wrapper_infix = f"compile-time-test-{self.work.name}"
|
|
self.wrapper_preset = "all-dev"
|
|
self.wrapper_build_dir = (
|
|
REPO_ROOT / "build" / self.wrapper_infix / self.wrapper_preset
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
shutil.rmtree(REPO_ROOT / "build" / self.wrapper_infix, ignore_errors=True)
|
|
self.tempdir.cleanup()
|
|
|
|
def comparison_csv(self, output_dir: Path, name: str) -> Path:
|
|
return output_dir / "comparison" / name
|
|
|
|
def wrapper_trace_dir(self) -> Path:
|
|
return self.wrapper_build_dir / "compile_time" / "raw_traces"
|
|
|
|
def wrapper_output_dir(self) -> Path:
|
|
return self.wrapper_build_dir / "compile_time" / "event_reports"
|
|
|
|
def assert_empty_csv(self, path: Path) -> None:
|
|
self.assertTrue(path.exists())
|
|
self.assertEqual(csv_rows(path), [])
|
|
|
|
def run_summary(
|
|
self,
|
|
current: Path,
|
|
baseline: Path,
|
|
output_dir: Path,
|
|
*args: str,
|
|
baseline_repo_root: Path | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
command = [
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
current.as_posix(),
|
|
"--baseline-dir",
|
|
baseline.as_posix(),
|
|
"-o",
|
|
output_dir.as_posix(),
|
|
]
|
|
if baseline_repo_root is not None:
|
|
command.extend(["--baseline-repo-root", baseline_repo_root.as_posix()])
|
|
command.extend(args)
|
|
|
|
return subprocess.run(
|
|
command,
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
def run_wrapper(
|
|
self,
|
|
*args: str,
|
|
no_tu_csv: bool = True,
|
|
common_args: tuple[str, ...] = (),
|
|
) -> subprocess.CompletedProcess[str]:
|
|
command = [
|
|
"bash",
|
|
WRAPPER_SCRIPT.as_posix(),
|
|
"-skip-configure",
|
|
"-skip-build",
|
|
"-no-prepare-perfetto",
|
|
*common_args,
|
|
]
|
|
if no_tu_csv:
|
|
command.append("-no-tu-csv")
|
|
command.extend(["--", *args])
|
|
env = os.environ.copy()
|
|
env["CCCL_BUILD_INFIX"] = self.wrapper_infix
|
|
return subprocess.run(
|
|
command,
|
|
cwd=REPO_ROOT,
|
|
env=env,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
def test_comparison_outputs_and_unmatched_children(self) -> None:
|
|
baseline = self.work / "baseline"
|
|
current = self.work / "current"
|
|
output = self.work / "reports"
|
|
|
|
parent = self.traces.project_detail("libcudacxx/include/cuda/std/parent.h")
|
|
matched_child = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/matched_child.h"
|
|
)
|
|
better = self.traces.project_detail("libcudacxx/include/cuda/std/better.h")
|
|
baseline_only_child = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/baseline_only_child.h"
|
|
)
|
|
baseline_only = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/only_baseline.h"
|
|
)
|
|
current_only = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/only_current.h"
|
|
)
|
|
|
|
self.traces.write_trace(
|
|
baseline / "target" / "match.json",
|
|
[
|
|
self.traces.event("Parent", parent, 0, 100),
|
|
self.traces.event("Child", matched_child, 10, 30),
|
|
self.traces.event("Child", baseline_only_child, 50, 20),
|
|
self.traces.event("Better", better, 200, 70),
|
|
self.traces.event("OnlyBaseline", baseline_only, 300, 20),
|
|
],
|
|
"match",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "match.json",
|
|
[
|
|
self.traces.event("Parent", parent, 0, 150),
|
|
self.traces.event("Child", matched_child, 10, 40),
|
|
self.traces.event("Better", better, 200, 40),
|
|
self.traces.event("OnlyCurrent", current_only, 300, 90),
|
|
],
|
|
"match",
|
|
)
|
|
self.traces.write_trace(
|
|
baseline / "target" / "baseline_only_trace.json",
|
|
[self.traces.event("Parent", parent, 0, 999)],
|
|
"baseline_only_trace",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "current_only_trace.json",
|
|
[self.traces.event("Parent", parent, 0, 999)],
|
|
"current_only_trace",
|
|
)
|
|
|
|
self.run_summary(
|
|
current,
|
|
baseline,
|
|
output,
|
|
"-f",
|
|
"all",
|
|
"-e",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"10",
|
|
"--tag",
|
|
"synthetic",
|
|
)
|
|
|
|
worse = csv_rows(
|
|
self.comparison_csv(
|
|
output, "top-10-all-exclusive-all-by-total-worse-synthetic.csv"
|
|
)
|
|
)
|
|
better_rows = csv_rows(
|
|
self.comparison_csv(
|
|
output, "top-10-all-exclusive-all-by-total-better-synthetic.csv"
|
|
)
|
|
)
|
|
|
|
parent_row = next(row for row in worse if row["event_name"] == "Parent")
|
|
self.assertEqual(parent_row["baseline_selected_s"], "0.000070")
|
|
self.assertEqual(parent_row["current_selected_s"], "0.000110")
|
|
self.assertEqual(parent_row["impact_magnitude_s"], "0.000040")
|
|
|
|
better_row = next(row for row in better_rows if row["event_name"] == "Better")
|
|
self.assertEqual(better_row["baseline_selected_s"], "0.000070")
|
|
self.assertEqual(better_row["current_selected_s"], "0.000040")
|
|
self.assertEqual(better_row["impact_magnitude_s"], "0.000030")
|
|
|
|
all_comparison_keys = {row["event_key"] for row in worse + better_rows}
|
|
self.assertFalse(any("only_baseline" in key for key in all_comparison_keys))
|
|
self.assertFalse(any("only_current" in key for key in all_comparison_keys))
|
|
|
|
def test_comparison_threshold_filters_small_changes(self) -> None:
|
|
baseline = self.work / "baseline"
|
|
current = self.work / "current"
|
|
output = self.work / "reports"
|
|
|
|
small = self.traces.project_detail("libcudacxx/include/cuda/std/small.h")
|
|
large = self.traces.project_detail("libcudacxx/include/cuda/std/large.h")
|
|
self.traces.write_trace(
|
|
baseline / "target" / "match.json",
|
|
[
|
|
self.traces.event("Same", small, 0, 10),
|
|
self.traces.event("Same", large, 100, 10),
|
|
],
|
|
"match",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "match.json",
|
|
[
|
|
self.traces.event("Same", small, 0, 12),
|
|
self.traces.event("Same", large, 100, 20),
|
|
],
|
|
"match",
|
|
)
|
|
|
|
self.run_summary(
|
|
current,
|
|
baseline,
|
|
output,
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"10",
|
|
"--threshold",
|
|
"0.000005",
|
|
"--tag",
|
|
"threshold",
|
|
)
|
|
|
|
worse = csv_rows(
|
|
self.comparison_csv(
|
|
output, "top-10-all-inclusive-by-total-worse-threshold.csv"
|
|
)
|
|
)
|
|
self.assertEqual(len(worse), 1)
|
|
self.assertTrue(worse[0]["event_key"].endswith("large.h"))
|
|
self.assertEqual(worse[0]["impact_magnitude_s"], "0.000010")
|
|
|
|
def test_comparison_ranks_by_total_impact_not_selected_metric(self) -> None:
|
|
baseline = self.work / "baseline"
|
|
current = self.work / "current"
|
|
output = self.work / "reports"
|
|
|
|
repeated = self.traces.project_detail("libcudacxx/include/cuda/std/repeated.h")
|
|
one_trace = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/one_trace.h"
|
|
)
|
|
self.traces.write_trace(
|
|
baseline / "target" / "first.json",
|
|
[
|
|
self.traces.event("Same", repeated, 0, 10),
|
|
self.traces.event("Same", one_trace, 100, 10),
|
|
],
|
|
"first",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "first.json",
|
|
[
|
|
self.traces.event("Same", repeated, 0, 16),
|
|
self.traces.event("Same", one_trace, 100, 20),
|
|
],
|
|
"first",
|
|
)
|
|
self.traces.write_trace(
|
|
baseline / "target" / "second.json",
|
|
[self.traces.event("Same", repeated, 0, 10)],
|
|
"second",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "second.json",
|
|
[self.traces.event("Same", repeated, 0, 16)],
|
|
"second",
|
|
)
|
|
|
|
self.run_summary(
|
|
current,
|
|
baseline,
|
|
output,
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"max",
|
|
"-n",
|
|
"10",
|
|
"--tag",
|
|
"impact",
|
|
)
|
|
|
|
worse = csv_rows(
|
|
self.comparison_csv(output, "top-10-all-inclusive-by-max-worse-impact.csv")
|
|
)
|
|
self.assertTrue(worse[0]["event_key"].endswith("repeated.h"))
|
|
self.assertEqual(worse[0]["impact_magnitude_s"], "0.000012")
|
|
self.assertEqual(worse[0]["selected_magnitude_s"], "0.000006")
|
|
|
|
def test_threshold_requires_comparison_mode(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
same = self.traces.project_detail("libcudacxx/include/cuda/std/same.h")
|
|
self.traces.write_trace(
|
|
traces / "target" / "same.json",
|
|
[self.traces.event("Same", same, 0, 10)],
|
|
"same",
|
|
)
|
|
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"--threshold",
|
|
"0.000001",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertNotEqual(completed.returncode, 0)
|
|
self.assertIn(
|
|
"--threshold can only be used together with --baseline-dir",
|
|
completed.stderr,
|
|
)
|
|
|
|
def test_file_processing_same_filter_keeps_unmatched_child_in_parent_cost(
|
|
self,
|
|
) -> None:
|
|
baseline = self.work / "baseline"
|
|
current = self.work / "current"
|
|
output = self.work / "reports"
|
|
|
|
parent = self.traces.project_detail("libcudacxx/include/cuda/std/parent.h")
|
|
matched_child = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/matched_child.h"
|
|
)
|
|
unmatched_child = self.traces.project_detail(
|
|
"libcudacxx/include/cuda/std/baseline_only_child.h"
|
|
)
|
|
|
|
self.traces.write_trace(
|
|
baseline / "target" / "same.json",
|
|
[
|
|
self.traces.event("Processing Header File", parent, 0, 100),
|
|
self.traces.event("Processing Header File", matched_child, 10, 30),
|
|
self.traces.event("Processing Header File", unmatched_child, 50, 20),
|
|
],
|
|
"same",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "same.json",
|
|
[
|
|
self.traces.event("Processing Header File", parent, 0, 130),
|
|
self.traces.event("Processing Header File", matched_child, 10, 40),
|
|
],
|
|
"same",
|
|
)
|
|
|
|
self.run_summary(
|
|
current,
|
|
baseline,
|
|
output,
|
|
"-f",
|
|
"file-processing",
|
|
"-e",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"file-processing",
|
|
)
|
|
worse = csv_rows(
|
|
self.comparison_csv(
|
|
output,
|
|
"top-5-file-processing-exclusive-same-filter-by-total-worse-file-processing.csv",
|
|
)
|
|
)
|
|
parent_row = next(row for row in worse if row["event_key"].endswith("parent.h"))
|
|
self.assertEqual(parent_row["baseline_selected_s"], "0.000070")
|
|
self.assertEqual(parent_row["current_selected_s"], "0.000090")
|
|
|
|
def test_same_filter_exclusive_uses_direct_children_only(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
parent = self.traces.project_detail("libcudacxx/include/cuda/std/parent.h")
|
|
nested = self.traces.project_detail("libcudacxx/include/cuda/std/nested.h")
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "direct.json",
|
|
[
|
|
self.traces.event("Processing Header File", parent, 0, 100),
|
|
self.traces.event("Scanning Function Body", "not-a-header", 10, 80),
|
|
self.traces.event("Processing Header File", nested, 20, 10),
|
|
],
|
|
"direct",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"file-processing",
|
|
"-e",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"direct",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(
|
|
output / "top-5-file-processing-exclusive-same-filter-by-total-direct.csv"
|
|
)
|
|
parent_row = next(row for row in rows if row["event_key"].endswith("parent.h"))
|
|
self.assertEqual(parent_row["selected_total_s"], "0.000100")
|
|
|
|
def test_empty_comparisons_still_write_csvs(self) -> None:
|
|
baseline = self.work / "baseline"
|
|
current = self.work / "current"
|
|
output = self.work / "reports"
|
|
|
|
foo = self.traces.project_detail("libcudacxx/include/cuda/std/foo.h")
|
|
bar = self.traces.project_detail("libcudacxx/include/cuda/std/bar.h")
|
|
self.traces.write_trace(
|
|
baseline / "target" / "same.json",
|
|
[self.traces.event("Foo", foo, 0, 10)],
|
|
"same",
|
|
)
|
|
self.traces.write_trace(
|
|
current / "target" / "same.json",
|
|
[self.traces.event("Bar", bar, 0, 20)],
|
|
"same",
|
|
)
|
|
self.run_summary(
|
|
current,
|
|
baseline,
|
|
output,
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"empty",
|
|
)
|
|
|
|
self.assert_empty_csv(
|
|
self.comparison_csv(output, "top-5-all-inclusive-by-total-worse-empty.csv")
|
|
)
|
|
self.assert_empty_csv(
|
|
self.comparison_csv(output, "top-5-all-inclusive-by-total-better-empty.csv")
|
|
)
|
|
with (output / "summary.json").open(encoding="utf-8") as f:
|
|
manifest = json.load(f)
|
|
self.assertIn(
|
|
"no comparable event keys",
|
|
" ".join(manifest["slices"][0]["warnings"]),
|
|
)
|
|
|
|
def test_wrapper_forwarding_writes_current_report(self) -> None:
|
|
current = self.wrapper_trace_dir()
|
|
output = self.wrapper_output_dir()
|
|
|
|
same = self.traces.project_detail("libcudacxx/include/cuda/std/same.h")
|
|
self.traces.write_trace(
|
|
current / "target" / "current_name.json",
|
|
[self.traces.event("Same", same, 0, 100)],
|
|
"current_name",
|
|
)
|
|
|
|
self.run_wrapper(
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"wrapper",
|
|
common_args=("-arch", "70"),
|
|
)
|
|
|
|
self.assertTrue((output / "top-5-all-inclusive-by-total-wrapper.csv").exists())
|
|
|
|
def test_comparison_matches_paths_from_distinct_repo_roots(self) -> None:
|
|
baseline_repo = self.work / "baseline-src"
|
|
baseline = self.work / "baseline"
|
|
current = self.work / "current"
|
|
output = self.work / "reports"
|
|
|
|
baseline_traces = TraceBuilder(baseline_repo)
|
|
baseline_same = baseline_traces.project_detail(
|
|
"libcudacxx/include/cuda/std/same.h"
|
|
)
|
|
current_same = self.traces.project_detail("libcudacxx/include/cuda/std/same.h")
|
|
self.traces.write_trace(
|
|
current / "target" / "same.json",
|
|
[self.traces.event("Same", current_same, 0, 12)],
|
|
"same",
|
|
)
|
|
baseline_traces.write_trace(
|
|
baseline / "target" / "same.json",
|
|
[baseline_traces.event("Same", baseline_same, 0, 10)],
|
|
"same",
|
|
)
|
|
|
|
self.run_summary(
|
|
current,
|
|
baseline,
|
|
output,
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
baseline_repo_root=baseline_repo,
|
|
)
|
|
|
|
worse = csv_rows(
|
|
self.comparison_csv(output, "top-5-all-inclusive-by-total-worse.csv")
|
|
)
|
|
self.assertEqual(worse[0]["event_key"], "libcudacxx/include/cuda/std/same.h")
|
|
self.assertEqual(worse[0]["impact_magnitude_s"], "0.000002")
|
|
|
|
def test_wrapper_uses_computed_event_output_dir(self) -> None:
|
|
current = self.wrapper_trace_dir()
|
|
wrapper_default = self.wrapper_output_dir()
|
|
explicit_output = self.work / "explicit-output"
|
|
|
|
same = self.traces.project_detail("libcudacxx/include/cuda/std/same.h")
|
|
self.traces.write_trace(
|
|
current / "target" / "same.json",
|
|
[self.traces.event("Same", same, 0, 12)],
|
|
"same",
|
|
)
|
|
|
|
self.run_wrapper(
|
|
f"--output-dir={explicit_output.as_posix()}",
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"equals-output",
|
|
)
|
|
|
|
self.assertTrue(
|
|
(
|
|
wrapper_default / "top-5-all-inclusive-by-total-equals-output.csv"
|
|
).exists()
|
|
)
|
|
self.assertFalse(
|
|
(
|
|
explicit_output / "top-5-all-inclusive-by-total-equals-output.csv"
|
|
).exists()
|
|
)
|
|
|
|
def test_sort_by_average_per_root_tu(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
one_tu = self.traces.project_detail("libcudacxx/include/cuda/std/one_tu.h")
|
|
two_tus = self.traces.project_detail("libcudacxx/include/cuda/std/two_tus.h")
|
|
self.traces.write_trace(
|
|
traces / "target" / "first.json",
|
|
[
|
|
self.traces.event("Same", one_tu, 0, 80),
|
|
self.traces.event("Same", two_tus, 100, 60),
|
|
],
|
|
"first",
|
|
)
|
|
self.traces.write_trace(
|
|
traces / "target" / "second.json",
|
|
[self.traces.event("Same", two_tus, 0, 40)],
|
|
"second",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"avg-root-tu",
|
|
"-n",
|
|
"2",
|
|
"--tag",
|
|
"avg-root-tu",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(output / "top-2-all-inclusive-by-avg-root-tu-avg-root-tu.csv")
|
|
self.assertTrue(rows[0]["event_key"].endswith("one_tu.h"))
|
|
self.assertEqual(rows[0]["selected_avg_per_root_tu_s"], "0.000080")
|
|
self.assertTrue(rows[1]["event_key"].endswith("two_tus.h"))
|
|
self.assertEqual(rows[1]["selected_avg_per_root_tu_s"], "0.000050")
|
|
|
|
def test_host_compiler_filter_matches_host_phase_events(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "host.json",
|
|
[
|
|
self.traces.event(
|
|
"g++ (preprocessing 1)", "g++ (preprocessing 1)", 0, 10
|
|
),
|
|
self.traces.event("gcc (compiling)", "gcc (compiling)", 20, 30),
|
|
self.traces.event("CUDA C++ Front-End", "frontend", 60, 40),
|
|
],
|
|
"host",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"host-compiler",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"host",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(output / "top-5-host-compiler-inclusive-by-total-host.csv")
|
|
self.assertEqual(
|
|
{row["event_name"] for row in rows},
|
|
{"g++ (preprocessing 1)", "gcc (compiling)"},
|
|
)
|
|
|
|
def test_scope_filter_defaults_to_cccl_demangled_symbols(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "symbols.json",
|
|
[
|
|
self.traces.event(
|
|
"Scanning Function Body",
|
|
"std::basic_string_view::size() const noexcept",
|
|
0,
|
|
100,
|
|
),
|
|
self.traces.event(
|
|
"Scanning Function Body",
|
|
"cuda::std::__4::basic_string_view::size() const noexcept",
|
|
200,
|
|
80,
|
|
),
|
|
self.traces.event(
|
|
"Scanning Function Body",
|
|
'cuda::std::literals::operator ""sv(const char *, unsigned long)',
|
|
400,
|
|
60,
|
|
),
|
|
],
|
|
"symbols",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"scanning-function-body",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"scope-default",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(
|
|
output / "top-5-scanning-function-body-inclusive-by-total-scope-default.csv"
|
|
)
|
|
self.assertEqual(len(rows), 2)
|
|
self.assertIn("cuda::std", rows[0]["event_key"])
|
|
self.assertIn("operator", rows[1]["event_key"])
|
|
|
|
def test_empty_scope_filter_disables_symbol_scope_filtering(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "symbols.json",
|
|
[
|
|
self.traces.event(
|
|
"Scanning Function Body",
|
|
"std::basic_string_view::size() const noexcept",
|
|
0,
|
|
100,
|
|
),
|
|
self.traces.event(
|
|
"Scanning Function Body",
|
|
"cuda::std::__4::basic_string_view::size() const noexcept",
|
|
200,
|
|
80,
|
|
),
|
|
],
|
|
"symbols",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"scanning-function-body",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--scope-filter",
|
|
"",
|
|
"--tag",
|
|
"scope-disabled",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(
|
|
output
|
|
/ "top-5-scanning-function-body-inclusive-by-total-scope-disabled.csv"
|
|
)
|
|
self.assertEqual(len(rows), 2)
|
|
self.assertIn("std::basic_string_view", rows[0]["event_key"])
|
|
self.assertIn("cuda::std", rows[1]["event_key"])
|
|
|
|
def test_scope_filter_uses_reported_template_symbol_scope(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "templates.json",
|
|
[
|
|
self.traces.event(
|
|
"Instantiating Template Function",
|
|
(
|
|
"nvtx3::v1::domain::get "
|
|
"[nvtx3::v1::domain::get<cuda::__4::__nvtx_cccl_domain>()]"
|
|
),
|
|
0,
|
|
100,
|
|
),
|
|
self.traces.event(
|
|
"Instantiating Template Function",
|
|
"cub::detail::load [cub::detail::load<int>()]",
|
|
200,
|
|
80,
|
|
),
|
|
],
|
|
"templates",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"template-instantiation",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"template-scope",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(
|
|
output
|
|
/ "top-5-template-instantiation-inclusive-by-total-template-scope.csv"
|
|
)
|
|
self.assertEqual(len(rows), 1)
|
|
self.assertIn("cub::detail::load", rows[0]["event_key"])
|
|
|
|
def test_scope_filter_matches_mangled_cccl_namespaces(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "mangled.json",
|
|
[
|
|
self.traces.event(
|
|
"Generating Function IR",
|
|
"_ZN4cuda3std3__43fooEv",
|
|
0,
|
|
100,
|
|
),
|
|
self.traces.event(
|
|
"Generating Function IR",
|
|
"_ZN6thrust6detail3barEv",
|
|
200,
|
|
80,
|
|
),
|
|
self.traces.event(
|
|
"Generating Function IR",
|
|
"_ZNSt6vectorIiE4sizeEv",
|
|
400,
|
|
200,
|
|
),
|
|
],
|
|
"mangled",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"code-generation",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"mangled-scope",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rows = csv_rows(
|
|
output / "top-5-code-generation-inclusive-by-total-mangled-scope.csv"
|
|
)
|
|
self.assertEqual(
|
|
{row["event_key"] for row in rows},
|
|
{
|
|
"_ZN4cuda3std3__43fooEv",
|
|
"_ZN6thrust6detail3barEv",
|
|
},
|
|
)
|
|
|
|
def test_total_compilation_filter_uses_trace_wall_span(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
|
|
self.traces.write_trace(
|
|
traces / "target" / "total.json",
|
|
[
|
|
self.traces.event(
|
|
"g++ (preprocessing 1)", "g++ (preprocessing 1)", 10, 20
|
|
),
|
|
self.traces.event("CUDA C++ Front-End", "frontend", 50, 40),
|
|
],
|
|
"total",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"total-compilation",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"total",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
total_rows = csv_rows(
|
|
output / "top-5-total-compilation-inclusive-by-total-total.csv"
|
|
)
|
|
self.assertEqual(len(total_rows), 1)
|
|
self.assertEqual(total_rows[0]["event_name"], "Total Compilation Time")
|
|
self.assertEqual(total_rows[0]["selected_total_s"], "0.000080")
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"total",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"all-no-total",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
all_rows = csv_rows(output / "top-5-all-inclusive-by-total-all-no-total.csv")
|
|
self.assertFalse(
|
|
any(row["event_name"] == "Total Compilation Time" for row in all_rows)
|
|
)
|
|
|
|
def test_same_dir_comparison_is_empty(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
same = self.traces.project_detail("libcudacxx/include/cuda/std/same.h")
|
|
self.traces.write_trace(
|
|
traces / "target" / "same.json",
|
|
[self.traces.event("Same", same, 0, 10)],
|
|
"same",
|
|
)
|
|
|
|
self.run_summary(
|
|
traces,
|
|
traces,
|
|
output,
|
|
"-f",
|
|
"all",
|
|
"-i",
|
|
"--sort",
|
|
"max",
|
|
"-n",
|
|
"5",
|
|
"--tag",
|
|
"same",
|
|
)
|
|
self.assert_empty_csv(
|
|
self.comparison_csv(output, "top-5-all-inclusive-by-max-worse-same.csv")
|
|
)
|
|
self.assert_empty_csv(
|
|
self.comparison_csv(output, "top-5-all-inclusive-by-max-better-same.csv")
|
|
)
|
|
|
|
def test_multi_slice_writes_manifest_and_allows_empty_slice(self) -> None:
|
|
traces = self.work / "traces"
|
|
output = self.work / "reports"
|
|
slices = self.work / "slices.json"
|
|
same = self.traces.project_detail("libcudacxx/include/cuda/std/same.h")
|
|
self.traces.write_trace(
|
|
traces / "target" / "same.json",
|
|
[self.traces.event("Same", same, 0, 10)],
|
|
"same",
|
|
)
|
|
slices.write_text(
|
|
json.dumps(
|
|
{
|
|
"slices": [
|
|
{
|
|
"id": "all-events",
|
|
"title": "All events",
|
|
"filter": "all",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"top": 5,
|
|
"threshold": 0,
|
|
},
|
|
{
|
|
"id": "empty-events",
|
|
"title": "Empty events",
|
|
"filter": "does-not-match",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"top": 5,
|
|
"threshold": 0,
|
|
},
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
SUMMARY_SCRIPT.as_posix(),
|
|
traces.as_posix(),
|
|
"-o",
|
|
output.as_posix(),
|
|
"--slices",
|
|
slices.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
with (output / "summary.json").open(encoding="utf-8") as f:
|
|
manifest = json.load(f)
|
|
|
|
self.assertEqual(
|
|
[item["id"] for item in manifest["slices"]], ["all-events", "empty-events"]
|
|
)
|
|
self.assertEqual(manifest["slices"][0]["reports"]["current"]["row_count"], 1)
|
|
self.assertEqual(manifest["slices"][1]["reports"]["current"]["row_count"], 0)
|
|
self.assertTrue(
|
|
(
|
|
output
|
|
/ "empty-events"
|
|
/ "top-5-regex-does-not-match-inclusive-by-total.csv"
|
|
).exists()
|
|
)
|
|
|
|
|
|
class CompileTimeMatrixAndCommentTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
self.work = Path(self.tempdir.name)
|
|
|
|
def tearDown(self) -> None:
|
|
self.tempdir.cleanup()
|
|
|
|
def test_parse_matrix_disabled_when_section_missing(self) -> None:
|
|
matrix = self.work / "matrix.yaml"
|
|
matrix.write_text("workflows: {}\n", encoding="utf-8")
|
|
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
PARSE_MATRIX_SCRIPT.as_posix(),
|
|
matrix.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertEqual(json.loads(completed.stdout), {"include": []})
|
|
|
|
def test_parse_matrix_valid_config(self) -> None:
|
|
matrix = self.work / "matrix.yaml"
|
|
matrix.write_text(
|
|
"""
|
|
compile_time:
|
|
pull_request:
|
|
- id: public-headers
|
|
name: Public headers
|
|
gpu: rtx2080
|
|
launch_args: "--cuda 13.3 --host gcc13"
|
|
baseline_ref: origin/main
|
|
preset: all-dev
|
|
targets: [cub.headers.base]
|
|
args: "-arch native"
|
|
slices:
|
|
- id: total-compilation
|
|
title: TU total compilation
|
|
filter: total-compilation
|
|
timing: inclusive
|
|
sort: total
|
|
top: 15
|
|
threshold: 0.001
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
PARSE_MATRIX_SCRIPT.as_posix(),
|
|
matrix.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
include = json.loads(completed.stdout)["include"]
|
|
self.assertEqual(len(include), 1)
|
|
self.assertEqual(
|
|
include[0]["comment_header"], "compile-time-bench-public-headers"
|
|
)
|
|
self.assertEqual(json.loads(include[0]["targets_json"]), ["cub.headers.base"])
|
|
self.assertEqual(
|
|
json.loads(include[0]["slices_json"])["slices"][0]["id"],
|
|
"total-compilation",
|
|
)
|
|
|
|
def test_parse_matrix_rejects_duplicate_slice_ids(self) -> None:
|
|
matrix = self.work / "matrix.yaml"
|
|
matrix.write_text(
|
|
"""
|
|
compile_time:
|
|
pull_request:
|
|
- id: public-headers
|
|
name: Public headers
|
|
gpu: rtx2080
|
|
launch_args: "--cuda 13.3 --host gcc13"
|
|
baseline_ref: origin/main
|
|
preset: all-dev
|
|
targets: [cub.headers.base]
|
|
slices:
|
|
- id: repeated
|
|
title: First
|
|
filter: all
|
|
timing: inclusive
|
|
sort: total
|
|
top: 15
|
|
threshold: 0
|
|
- id: repeated
|
|
title: Second
|
|
filter: all
|
|
timing: inclusive
|
|
sort: total
|
|
top: 15
|
|
threshold: 0
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
PARSE_MATRIX_SCRIPT.as_posix(),
|
|
matrix.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertNotEqual(completed.returncode, 0)
|
|
self.assertIn("duplicate slice id", completed.stderr)
|
|
|
|
def test_parse_matrix_rejects_bool_numeric_fields(self) -> None:
|
|
matrix = self.work / "matrix.yaml"
|
|
matrix.write_text(
|
|
"""
|
|
compile_time:
|
|
pull_request:
|
|
- id: public-headers
|
|
name: Public headers
|
|
gpu: rtx2080
|
|
launch_args: "--cuda 13.3 --host gcc13"
|
|
baseline_ref: origin/main
|
|
preset: all-dev
|
|
targets: [cub.headers.base]
|
|
artifact_retention_days: true
|
|
slices:
|
|
- id: total-compilation
|
|
title: TU total compilation
|
|
filter: total-compilation
|
|
timing: inclusive
|
|
sort: total
|
|
top: true
|
|
threshold: false
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
PARSE_MATRIX_SCRIPT.as_posix(),
|
|
matrix.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertNotEqual(completed.returncode, 0)
|
|
self.assertIn("positive integer", completed.stderr)
|
|
|
|
def test_pull_request_workflow_supports_compile_time_bench_skip_tag(self) -> None:
|
|
workflow = PULL_REQUEST_WORKFLOW.read_text(encoding="utf-8")
|
|
|
|
self.assertIn("[skip-compile-time-bench]", workflow)
|
|
self.assertIn("compile_time_enabled=false", workflow)
|
|
self.assertIn('compile_time_matrix={"include":[]}', workflow)
|
|
|
|
def test_render_comment_omits_empty_sections_and_splits_directions(self) -> None:
|
|
summary = self.work / "summary.json"
|
|
config = self.work / "config.json"
|
|
output = self.work / "comment.md"
|
|
summary.write_text(
|
|
json.dumps(
|
|
{
|
|
"slices": [
|
|
{
|
|
"id": "nonempty",
|
|
"title": "Nonempty",
|
|
"filter": "all",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"comparison": {
|
|
"worse": {
|
|
"rows": [
|
|
{
|
|
"rank": 1,
|
|
"event_name": "Scanning Function Body",
|
|
"event_key": "cuda::std::__4::same<char>(int)",
|
|
"baseline_selected_s": "0.000001",
|
|
"current_selected_s": "0.000003",
|
|
"impact_magnitude_s": "0.000010",
|
|
"selected_delta_s": "0.000002",
|
|
"matched_trace_count": 1,
|
|
}
|
|
]
|
|
},
|
|
"better": {"rows": []},
|
|
},
|
|
"children": [
|
|
{
|
|
"id": "empty-child",
|
|
"title": "Empty child",
|
|
"filter": "all",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"comparison": {
|
|
"worse": {"rows": []},
|
|
"better": {"rows": []},
|
|
},
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
config.write_text(
|
|
json.dumps(
|
|
{
|
|
"id": "public-headers",
|
|
"name": "Public headers",
|
|
"baseline_ref": "origin/main",
|
|
"preset": "all-dev",
|
|
"targets": ["cub.headers.base"],
|
|
"gpu": "rtx2080",
|
|
"launch_args": "--cuda 13.3 --host gcc13",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
RENDER_COMMENT_SCRIPT.as_posix(),
|
|
"--summary",
|
|
summary.as_posix(),
|
|
"--config",
|
|
config.as_posix(),
|
|
"--artifacts-url",
|
|
"https://example.test/artifacts",
|
|
"-o",
|
|
output.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rendered = output.read_text(encoding="utf-8")
|
|
self.assertIn("<!-- cccl-compile-time-bench: public-headers -->", rendered)
|
|
self.assertIn("Regressions", rendered)
|
|
self.assertIn("Regression impact", rendered)
|
|
self.assertIn(
|
|
"Scanning Function Body: `cuda::std::__4::same<char>(int)`",
|
|
rendered,
|
|
)
|
|
self.assertIn("0.000010", rendered)
|
|
self.assertNotIn("Improvements</strong>", rendered)
|
|
self.assertNotIn("Empty child", rendered)
|
|
self.assertIn("https://example.test/artifacts", rendered)
|
|
|
|
def test_render_comment_reports_empty_slice_warnings(self) -> None:
|
|
summary = self.work / "summary.json"
|
|
config = self.work / "config.json"
|
|
output = self.work / "comment.md"
|
|
summary.write_text(
|
|
json.dumps(
|
|
{
|
|
"slices": [
|
|
{
|
|
"id": "empty",
|
|
"title": "Empty slice",
|
|
"filter": "typo-filter",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"warnings": [
|
|
"baseline report matched no events for this slice"
|
|
],
|
|
"comparison": {
|
|
"worse": {"rows": []},
|
|
"better": {"rows": []},
|
|
},
|
|
"children": [],
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
config.write_text(
|
|
json.dumps(
|
|
{
|
|
"id": "public-headers",
|
|
"name": "Public headers",
|
|
"baseline_ref": "origin/main",
|
|
"preset": "all-dev",
|
|
"targets": ["cub.headers.base"],
|
|
"gpu": "rtx2080",
|
|
"launch_args": "--cuda 13.3 --host gcc13",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
RENDER_COMMENT_SCRIPT.as_posix(),
|
|
"--summary",
|
|
summary.as_posix(),
|
|
"--config",
|
|
config.as_posix(),
|
|
"--artifacts-url",
|
|
"https://example.test/artifacts",
|
|
"-o",
|
|
output.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rendered = output.read_text(encoding="utf-8")
|
|
self.assertIn("1 warning(s)", rendered)
|
|
self.assertIn("Empty slice — Warnings", rendered)
|
|
self.assertIn("baseline report matched no events", rendered)
|
|
self.assertNotIn(
|
|
"No compile-time benchmark changes exceeded the configured thresholds.",
|
|
rendered,
|
|
)
|
|
|
|
def test_render_comment_separates_top_level_slice_sections(self) -> None:
|
|
summary = self.work / "summary.json"
|
|
config = self.work / "config.json"
|
|
output = self.work / "comment.md"
|
|
row = {
|
|
"rank": 1,
|
|
"event_name": "Same",
|
|
"event_key": "cuda/std/same",
|
|
"baseline_selected_s": "0.000001",
|
|
"current_selected_s": "0.000003",
|
|
"impact_magnitude_s": "0.000010",
|
|
"selected_delta_s": "0.000002",
|
|
"matched_trace_count": 1,
|
|
}
|
|
summary.write_text(
|
|
json.dumps(
|
|
{
|
|
"slices": [
|
|
{
|
|
"id": "first",
|
|
"title": "First",
|
|
"filter": "all",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"comparison": {
|
|
"worse": {"rows": [row]},
|
|
"better": {"rows": []},
|
|
},
|
|
"children": [],
|
|
},
|
|
{
|
|
"id": "second",
|
|
"title": "Second",
|
|
"filter": "all",
|
|
"timing": "inclusive",
|
|
"sort": "total",
|
|
"comparison": {
|
|
"worse": {"rows": [row]},
|
|
"better": {"rows": []},
|
|
},
|
|
"children": [],
|
|
},
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
config.write_text(
|
|
json.dumps(
|
|
{
|
|
"id": "public-headers",
|
|
"name": "Public headers",
|
|
"baseline_ref": "origin/main",
|
|
"preset": "all-dev",
|
|
"targets": ["cub.headers.base"],
|
|
"gpu": "rtx2080",
|
|
"launch_args": "--cuda 13.3 --host gcc13",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
RENDER_COMMENT_SCRIPT.as_posix(),
|
|
"--summary",
|
|
summary.as_posix(),
|
|
"--config",
|
|
config.as_posix(),
|
|
"--artifacts-url",
|
|
"https://example.test/artifacts",
|
|
"-o",
|
|
output.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
rendered = output.read_text(encoding="utf-8")
|
|
self.assertIn("</details>\n\n### Second", rendered)
|
|
self.assertNotIn("</details>\n### Second", rendered)
|
|
|
|
|
|
class PrepareTracesTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
self.work = Path(self.tempdir.name)
|
|
|
|
def tearDown(self) -> None:
|
|
self.tempdir.cleanup()
|
|
|
|
def test_promotes_project_details_into_perfetto_event_names(self) -> None:
|
|
input_dir = self.work / "raw"
|
|
output_dir = self.work / "perfetto"
|
|
trace_path = input_dir / "target" / "trace.json"
|
|
trace_path.parent.mkdir(parents=True, exist_ok=True)
|
|
trace_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"traceEvents": [
|
|
{
|
|
"ph": "X",
|
|
"name": "Processing Header File",
|
|
"args": {
|
|
"detail": (
|
|
REPO_ROOT
|
|
/ "libcudacxx/include/cuda/std/string_view"
|
|
).as_posix()
|
|
},
|
|
},
|
|
{
|
|
"ph": "X",
|
|
"name": "Other Event",
|
|
"args": {"detail": "not promoted"},
|
|
},
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
PREPARE_SCRIPT.as_posix(),
|
|
"--input",
|
|
input_dir.as_posix(),
|
|
"--output",
|
|
output_dir.as_posix(),
|
|
"--repo-root",
|
|
REPO_ROOT.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
output_trace = output_dir / "target" / "trace.perfetto.json"
|
|
with output_trace.open(encoding="utf-8") as f:
|
|
events = json.load(f)["traceEvents"]
|
|
|
|
self.assertEqual(
|
|
events[0]["name"], "Processing Header File: cuda/std/string_view"
|
|
)
|
|
self.assertEqual(events[0]["args"]["original_name"], "Processing Header File")
|
|
self.assertEqual(events[1]["name"], "Other Event")
|
|
|
|
def test_single_file_input_accepts_output_directory(self) -> None:
|
|
input_trace = self.work / "trace.json"
|
|
output_dir = self.work / "perfetto"
|
|
input_trace.write_text(
|
|
json.dumps(
|
|
{
|
|
"traceEvents": [
|
|
{
|
|
"ph": "X",
|
|
"name": "Processing Header File",
|
|
"args": {"detail": "cuda/std/string_view"},
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
PREPARE_SCRIPT.as_posix(),
|
|
"--input",
|
|
input_trace.as_posix(),
|
|
"--output",
|
|
output_dir.as_posix(),
|
|
"--repo-root",
|
|
REPO_ROOT.as_posix(),
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
self.assertTrue((output_dir / "trace.perfetto.json").exists())
|
|
|
|
|
|
class SummarizeTusTest(unittest.TestCase):
|
|
def test_generated_tu_input(self) -> None:
|
|
build_dir = Path("/tmp/build")
|
|
pp_path = (
|
|
build_dir
|
|
/ "libcudacxx/test"
|
|
/ "headers"
|
|
/ "libcudacxx.test.public_headers"
|
|
/ "cuda/std/string_view.cpp4.ii"
|
|
)
|
|
|
|
self.assertEqual(
|
|
summarize_tus.generated_tu_input(pp_path.with_name("string_view.cpp")),
|
|
"cuda/std/string_view",
|
|
)
|
|
|
|
def test_finds_nvcc_and_clang_preprocessed_tus(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
build_dir = Path(tmp)
|
|
header_dir = build_dir / "target" / "headers" / "generated"
|
|
header_dir.mkdir(parents=True)
|
|
nvcc_tu = header_dir / "cuda_std_span.cpp4.ii"
|
|
clang_tu = header_dir / "cuda_std_string_view.ii"
|
|
nvcc_tu.write_text("", encoding="utf-8")
|
|
clang_tu.write_text("", encoding="utf-8")
|
|
|
|
self.assertEqual(
|
|
summarize_tus.find_preprocessed_tus(build_dir),
|
|
[nvcc_tu, clang_tu],
|
|
)
|
|
self.assertEqual(
|
|
summarize_tus.tu_source_for_preprocessed_tu(nvcc_tu),
|
|
header_dir / "cuda_std_span",
|
|
)
|
|
self.assertEqual(
|
|
summarize_tus.tu_source_for_preprocessed_tu(clang_tu),
|
|
header_dir / "cuda_std_string_view",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|