under test, not sure no errors

qwen3_6_scripts/bi100_env.py

@@ -0,0 +1,26 @@
+import os
+
+
+def env_bool(name: str, default: bool = False) -> bool:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    if raw in ("1", "true", "True", "yes", "YES", "on", "ON"):
+        return True
+    if raw in ("0", "false", "False", "no", "NO", "off", "OFF"):
+        return False
+    raise RuntimeError(f"{name} must be boolean, got {raw!r}")
+
+
+def env_int(name: str, default: int, min_value: int, max_value: int) -> int:
+    raw = os.getenv(name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError as exc:
+        raise RuntimeError(f"{name} must be int, got {raw!r}") from exc
+    if not (min_value <= value <= max_value):
+        raise RuntimeError(
+            f"{name}={value} outside [{min_value}, {max_value}]")
+    return value

qwen3_6_scripts/bi100_profile.py

@@ -0,0 +1,237 @@
+import contextlib
+import fnmatch
+import functools
+import json
+import os
+import re
+import threading
+import time
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+_EVENT_SCHEMA = "bi100-profile-event-v1"
+_EVENT_VERSION = 1
+_NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,63}$")
+_FILTER_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.*?-]{0,63}$")
+
+
+def _strict_bool(name: str, default: str = "0") -> bool:
+    value = os.getenv(name, default).strip()
+    if value not in {"0", "1"}:
+        raise RuntimeError(f"{name} must be exactly 0 or 1, got {value!r}")
+    return value == "1"
+
+
+_ENABLED = _strict_bool("BI100_PROFILE")
+_INCLUDE_STARTUP = _strict_bool("BI100_PROFILE_INCLUDE_STARTUP")
+_MODE = os.getenv("BI100_PROFILE_MODE", "sync").strip().lower()
+_FILTERS = tuple(
+    item.strip()
+    for item in os.getenv("BI100_PROFILE_FILTER", "").split(",")
+    if item.strip()
+)
+if _ENABLED and _MODE not in {"sync", "event"}:
+    raise RuntimeError(f"unsupported BI100_PROFILE_MODE={_MODE!r}")
+if _ENABLED and any(_FILTER_RE.fullmatch(pattern) is None
+                    for pattern in _FILTERS):
+    raise RuntimeError("BI100_PROFILE_FILTER contains an invalid pattern")
+
+_EVENT_RECORDS = []
+_COUNTERS = {}
+_LOCK = threading.Lock()
+_FORWARD_INDEX = 0
+_LAST_FLUSH_NS = None
+_ACTIVE_FORWARD_TOKEN = None
+_NEXT_FORWARD_TOKEN = 0
+
+
+def _enabled_for(name: str) -> bool:
+    return (_ENABLED
+            and (not _FILTERS
+                 or any(fnmatch.fnmatchcase(name, pattern)
+                        for pattern in _FILTERS)))
+
+
+def _skip_startup() -> bool:
+    return (not _INCLUDE_STARTUP
+            and os.getenv("BI100_IN_STARTUP_PROFILE") == "1")
+
+
+def bi100_profile_event_enabled() -> bool:
+    return _ENABLED and _MODE == "event" and not _skip_startup()
+
+
+def _begin_profile_forward():
+    global _ACTIVE_FORWARD_TOKEN, _NEXT_FORWARD_TOKEN
+    if not bi100_profile_event_enabled():
+        return None
+    with _LOCK:
+        _EVENT_RECORDS.clear()
+        _COUNTERS.clear()
+        token = _NEXT_FORWARD_TOKEN
+        _NEXT_FORWARD_TOKEN += 1
+        _ACTIVE_FORWARD_TOKEN = token
+    return token
+
+
+def _abort_profile_forward(token) -> None:
+    global _ACTIVE_FORWARD_TOKEN
+    if token is None:
+        return
+    with _LOCK:
+        if _ACTIVE_FORWARD_TOKEN != token:
+            return
+        _EVENT_RECORDS.clear()
+        _COUNTERS.clear()
+        _ACTIVE_FORWARD_TOKEN = None
+
+
+def bi100_profile_transaction(function):
+    """Keep one top-level model forward isolated from failed forwards."""
+    @functools.wraps(function)
+    def wrapped(*args, **kwargs):
+        token = _begin_profile_forward()
+        if token is None:
+            return function(*args, **kwargs)
+        try:
+            result = function(*args, **kwargs)
+        except BaseException:
+            _abort_profile_forward(token)
+            raise
+        with _LOCK:
+            was_flushed = _ACTIVE_FORWARD_TOKEN != token
+        if not was_flushed:
+            _abort_profile_forward(token)
+            raise RuntimeError(
+                "BI100 profile transaction completed without a flush")
+        return result
+
+    return wrapped
+
+
+def _normalize_metadata(metadata):
+    normalized = {}
+    for key, value in metadata.items():
+        if not isinstance(key, str) or _NAME_RE.fullmatch(key) is None:
+            raise TypeError("profile metadata keys must be bounded names")
+        if isinstance(value, bool):
+            normalized[key] = value
+        elif isinstance(value, int) and not isinstance(value, bool):
+            normalized[key] = value
+        elif isinstance(value, str) and len(value) <= 64:
+            normalized[key] = value
+        else:
+            raise TypeError(
+                "profile metadata values must be bool, int, or short strings")
+    return normalized
+
+
+def bi100_profile_count(name: str, **metadata) -> None:
+    """Record privacy-safe path metadata for the current model forward."""
+    if not bi100_profile_event_enabled() or not _enabled_for(name):
+        return
+    if not isinstance(name, str) or _NAME_RE.fullmatch(name) is None:
+        raise TypeError("profile counter name must be a bounded name")
+    normalized = _normalize_metadata(metadata)
+    encoded = json.dumps(
+        {"name": name, **normalized}, sort_keys=True, separators=(",", ":"))
+    with _LOCK:
+        _COUNTERS[encoded] = _COUNTERS.get(encoded, 0) + 1
+
+
+@contextlib.contextmanager
+def bi100_timer(name: str):
+    if not _enabled_for(name) or _skip_startup():
+        yield
+        return
+    import torch
+
+    if _MODE == "event":
+        started = torch.cuda.Event(enable_timing=True)
+        finished = torch.cuda.Event(enable_timing=True)
+        host_started_ns = time.monotonic_ns()
+        started.record()
+        try:
+            yield
+        finally:
+            finished.record()
+            with _LOCK:
+                _EVENT_RECORDS.append(
+                    (name, started, finished, host_started_ns))
+        return
+
+    torch.cuda.synchronize()
+    t0 = time.perf_counter()
+    try:
+        yield
+    finally:
+        torch.cuda.synchronize()
+        logger.info("[BI100_PROFILE] %s %.3f ms", name,
+                    (time.perf_counter() - t0) * 1000)
+
+
+def bi100_profile_flush(*, tp_rank, **metadata):
+    """Synchronize once and emit one aggregate event record per model forward."""
+    global _ACTIVE_FORWARD_TOKEN, _FORWARD_INDEX, _LAST_FLUSH_NS
+    if not bi100_profile_event_enabled():
+        return None
+    if (not isinstance(tp_rank, int) or isinstance(tp_rank, bool)
+            or not 0 <= tp_rank < 256):
+        raise TypeError("profile TP rank must be an integer in [0, 255]")
+    normalized_metadata = _normalize_metadata(metadata)
+
+    with _LOCK:
+        records = list(_EVENT_RECORDS)
+        counters = dict(_COUNTERS)
+        _EVENT_RECORDS.clear()
+        _COUNTERS.clear()
+        _ACTIVE_FORWARD_TOKEN = None
+    if not records:
+        return None
+
+    import torch
+
+    torch.cuda.synchronize()
+    flushed_ns = time.monotonic_ns()
+    regions = {}
+    model_started_ns = []
+    for name, started, finished, host_started_ns in records:
+        stats = regions.setdefault(name, {"count": 0, "total_ms": 0.0})
+        stats["count"] += 1
+        stats["total_ms"] += float(started.elapsed_time(finished))
+        if name == "model.forward":
+            model_started_ns.append(host_started_ns)
+
+    counter_rows = []
+    for encoded, count in sorted(counters.items()):
+        row = json.loads(encoded)
+        row["count"] = count
+        counter_rows.append(row)
+
+    first_model_started_ns = (
+        min(model_started_ns) if model_started_ns else None)
+    payload = {
+        "schema": _EVENT_SCHEMA,
+        "version": _EVENT_VERSION,
+        "tp_rank": tp_rank,
+        "forward_index": _FORWARD_INDEX,
+        "metadata": normalized_metadata,
+        "event_count": len(records),
+        "model_forward_event_count": len(model_started_ns),
+        "regions": regions,
+        "counters": counter_rows,
+        "host_model_start_to_flush_ms": (
+            (flushed_ns - first_model_started_ns) / 1_000_000
+            if first_model_started_ns is not None else None),
+        "host_gap_since_previous_flush_ms": (
+            (first_model_started_ns - _LAST_FLUSH_NS) / 1_000_000
+            if first_model_started_ns is not None
+            and _LAST_FLUSH_NS is not None
+            else None),
+    }
+    _FORWARD_INDEX += 1
+    _LAST_FLUSH_NS = flushed_ns
+    logger.info("[BI100_PROFILE_EVENT] %s",
+                json.dumps(payload, sort_keys=True, separators=(",", ":")))
+    return payload

qwen3_6_scripts/block_major_kv_cache.py

@@ -0,0 +1,398 @@
+from __future__ import annotations
+
+import os
+import time
+from collections.abc import Mapping
+
+import torch
+
+from vllm.logger import init_logger
+
+
+logger = init_logger(__name__)
+
+ENABLE_ENV = "BI100_BLOCK_MAJOR_CPU_KV"
+TRACE_ENV = "BI100_BLOCK_MAJOR_CPU_KV_TRACE"
+CPU_OFFLOAD_ENV = "BI100_CPU_KV_OFFLOAD"
+HYBRID_ACCOUNTING_ENV = "BI100_HYBRID_KV_ACCOUNTING"
+NUM_ATTENTION_LAYERS = 10
+KV_PLANES = 2
+ELEMENTS_PER_PLANE_BLOCK = 4096
+STAGING_BLOCKS = 512
+STAGING_BUFFER_COUNT = 2
+BYTES_PER_BLOCK = (
+    NUM_ATTENTION_LAYERS * KV_PLANES * ELEMENTS_PER_PLANE_BLOCK * 2
+)
+GPU_STAGING_BYTES = STAGING_BLOCKS * STAGING_BUFFER_COUNT * BYTES_PER_BLOCK
+
+
+def _strict_binary_selector(
+    name: str,
+    environ: Mapping[str, str] | None = None,
+) -> bool:
+    source = os.environ if environ is None else environ
+    raw = source.get(name, "0")
+    if raw == "0":
+        return False
+    if raw == "1":
+        return True
+    raise RuntimeError(f"{name} must be exactly '0' or '1', got {raw!r}")
+
+
+def block_major_cpu_kv_enabled(
+    environ: Mapping[str, str] | None = None,
+) -> bool:
+    return _strict_binary_selector(ENABLE_ENV, environ)
+
+
+def block_major_cpu_kv_trace_enabled(
+    environ: Mapping[str, str] | None = None,
+) -> bool:
+    return _strict_binary_selector(TRACE_ENV, environ)
+
+
+def _require_block_major_runtime(
+    environ: Mapping[str, str] | None = None,
+) -> None:
+    source = os.environ if environ is None else environ
+    if source.get(CPU_OFFLOAD_ENV, "0") != "1":
+        raise RuntimeError(
+            f"{ENABLE_ENV}=1 requires {CPU_OFFLOAD_ENV}=1")
+    if source.get(HYBRID_ACCOUNTING_ENV, "legacy40") != "full_attention":
+        raise RuntimeError(
+            f"{ENABLE_ENV}=1 requires "
+            f"{HYBRID_ACCOUNTING_ENV}=full_attention")
+
+
+def reserve_block_major_gpu_blocks(
+    num_gpu_blocks: int,
+    cache_block_size: int,
+    environ: Mapping[str, str] | None = None,
+) -> int:
+    if (not isinstance(num_gpu_blocks, int)
+            or isinstance(num_gpu_blocks, bool)
+            or num_gpu_blocks < 0):
+        raise ValueError("num_gpu_blocks must be a non-negative integer")
+    if not block_major_cpu_kv_enabled(environ):
+        return num_gpu_blocks
+
+    _require_block_major_runtime(environ)
+    if cache_block_size != BYTES_PER_BLOCK:
+        raise RuntimeError(
+            f"{ENABLE_ENV}=1 requires cache block size "
+            f"{BYTES_PER_BLOCK}, got {cache_block_size}")
+    reserved_blocks = (
+        GPU_STAGING_BYTES + cache_block_size - 1
+    ) // cache_block_size
+    remaining_blocks = num_gpu_blocks - reserved_blocks
+    if remaining_blocks <= 0:
+        raise RuntimeError(
+            "block-major GPU staging leaves no usable GPU KV blocks")
+    logger.info(
+        "[BI100 BLOCK KV] capacity reserve blocks=%d bytes=%d "
+        "profiled_blocks=%d usable_blocks=%d",
+        reserved_blocks,
+        GPU_STAGING_BYTES,
+        num_gpu_blocks,
+        remaining_blocks,
+    )
+    return remaining_blocks
+
+
+def validate_block_mapping(
+    mapping: torch.Tensor,
+    source_limit: int,
+    destination_limit: int,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    if not isinstance(mapping, torch.Tensor):
+        raise TypeError("block mapping must be a torch.Tensor")
+    if mapping.device.type != "cpu":
+        raise ValueError("block mapping must be on CPU")
+    if mapping.dtype != torch.int64:
+        raise ValueError("block mapping must use torch.int64")
+    if not mapping.is_contiguous():
+        raise ValueError("block mapping must be contiguous")
+    if mapping.dim() != 2 or mapping.shape[1] != 2:
+        raise ValueError("block mapping must have shape [N, 2]")
+    if source_limit <= 0 or destination_limit <= 0:
+        raise ValueError("block mapping limits must be positive")
+
+    sources: set[int] = set()
+    destinations: set[int] = set()
+    for row, pair in enumerate(mapping.tolist()):
+        source, destination = pair
+        if not 0 <= source < source_limit:
+            raise ValueError(
+                f"source block out of range at row {row}: {source}")
+        if not 0 <= destination < destination_limit:
+            raise ValueError(
+                f"destination block out of range at row {row}: "
+                f"{destination}")
+        if source in sources:
+            raise ValueError(f"duplicate source block: {source}")
+        if destination in destinations:
+            raise ValueError(f"duplicate destination block: {destination}")
+        sources.add(source)
+        destinations.add(destination)
+
+    return mapping[:, 0].contiguous(), mapping[:, 1].contiguous()
+
+
+class BlockMajorCpuKVCache:
+
+    def __init__(
+        self,
+        gpu_cache: list[torch.Tensor],
+        num_cpu_blocks: int,
+        pin_memory: bool,
+    ) -> None:
+        self._validate_gpu_cache(gpu_cache)
+        if block_major_cpu_kv_enabled():
+            _require_block_major_runtime()
+        if num_cpu_blocks <= 0:
+            raise RuntimeError(
+                f"{ENABLE_ENV}=1 requires a positive CPU block count")
+        if not pin_memory:
+            raise RuntimeError(
+                f"{ENABLE_ENV}=1 requires pinned CPU memory")
+
+        try:
+            from vllm import corex_block_major_kv_transfer as extension
+        except ImportError as exc:
+            raise RuntimeError(
+                "block-major CoreX extension is unavailable") from exc
+
+        self.extension = extension
+        self.gpu_cache = gpu_cache
+        self.device = gpu_cache[0].device
+        self.dtype = gpu_cache[0].dtype
+        self.num_gpu_blocks = gpu_cache[0].shape[1]
+        self.num_cpu_blocks = num_cpu_blocks
+        self.trace_enabled = block_major_cpu_kv_trace_enabled()
+
+        self.cpu_pool = torch.zeros(
+            (
+                num_cpu_blocks,
+                NUM_ATTENTION_LAYERS,
+                KV_PLANES,
+                ELEMENTS_PER_PLANE_BLOCK,
+            ),
+            dtype=self.dtype,
+            device="cpu",
+            pin_memory=True,
+        )
+        if not self.cpu_pool.is_pinned():
+            raise RuntimeError("block-major CPU pool is not pinned")
+
+        # Preserve the public CacheEngine shape without allocating a second
+        # layer-major CPU cache. Transfer methods use cpu_pool directly.
+        self.layer_views = [
+            self.cpu_pool[:, layer, :, :].permute(1, 0, 2)
+            for layer in range(NUM_ATTENTION_LAYERS)
+        ]
+        self.cpu_staging = [
+            torch.empty(
+                (
+                    STAGING_BLOCKS,
+                    NUM_ATTENTION_LAYERS,
+                    KV_PLANES,
+                    ELEMENTS_PER_PLANE_BLOCK,
+                ),
+                dtype=self.dtype,
+                device="cpu",
+                pin_memory=True,
+            )
+            for _ in range(STAGING_BUFFER_COUNT)
+        ]
+        if not all(staging.is_pinned() for staging in self.cpu_staging):
+            raise RuntimeError("block-major CPU staging is not pinned")
+
+        with torch.cuda.device(self.device):
+            self.gpu_staging = [
+                torch.empty_like(staging, device=self.device)
+                for staging in self.cpu_staging
+            ]
+            self.events = [
+                torch.cuda.Event(enable_timing=False)
+                for _ in range(STAGING_BUFFER_COUNT)
+            ]
+            self.error_flag = torch.zeros(
+                1, dtype=torch.int32, device=self.device)
+
+        logger.info(
+            "[BI100 BLOCK KV] enabled device=%s gpu_blocks=%d cpu_blocks=%d "
+            "layers=%d block_bytes=%d staging_blocks=%d staging_buffers=%d",
+            self.device,
+            self.num_gpu_blocks,
+            self.num_cpu_blocks,
+            NUM_ATTENTION_LAYERS,
+            BYTES_PER_BLOCK,
+            STAGING_BLOCKS,
+            STAGING_BUFFER_COUNT,
+        )
+
+    @staticmethod
+    def _validate_gpu_cache(gpu_cache: list[torch.Tensor]) -> None:
+        if len(gpu_cache) != NUM_ATTENTION_LAYERS:
+            raise RuntimeError(
+                f"{ENABLE_ENV}=1 requires exactly "
+                f"{NUM_ATTENTION_LAYERS} GPU attention caches, got "
+                f"{len(gpu_cache)}")
+        first = gpu_cache[0]
+        if first.device.type != "cuda":
+            raise RuntimeError("block-major GPU cache must be on CUDA")
+        if first.dtype != torch.float16:
+            raise RuntimeError("block-major GPU cache must use float16")
+        if (first.dim() != 3 or first.shape[0] != KV_PLANES
+                or first.shape[2] != ELEMENTS_PER_PLANE_BLOCK):
+            raise RuntimeError(
+                "block-major GPU cache must have shape [2, blocks, 4096]")
+        if not first.is_contiguous():
+            raise RuntimeError("block-major GPU cache must be contiguous")
+
+        for layer, tensor in enumerate(gpu_cache):
+            if tensor.device != first.device:
+                raise RuntimeError(
+                    f"GPU cache layer {layer} is on a different device")
+            if tensor.dtype != first.dtype or tensor.shape != first.shape:
+                raise RuntimeError(
+                    f"GPU cache layer {layer} has inconsistent geometry")
+            if not tensor.is_contiguous():
+                raise RuntimeError(
+                    f"GPU cache layer {layer} is not contiguous")
+
+    def _to_gpu_ids(self, block_ids: torch.Tensor) -> torch.Tensor:
+        return block_ids.to(
+            device=self.device,
+            dtype=torch.int32,
+            non_blocking=False,
+        )
+
+    @staticmethod
+    def _chunks(
+        source: torch.Tensor,
+        destination: torch.Tensor,
+        gpu_ids: torch.Tensor,
+    ):
+        for start in range(0, source.numel(), STAGING_BLOCKS):
+            end = min(start + STAGING_BLOCKS, source.numel())
+            yield (
+                source[start:end],
+                destination[start:end],
+                gpu_ids[start:end],
+                end - start,
+            )
+
+    def _begin(self) -> None:
+        self.error_flag.zero_()
+
+    def _finish(
+        self,
+        direction: str,
+        block_count: int,
+        started: float | None,
+    ) -> None:
+        # check_error performs the final stream synchronization. This also
+        # makes every staging slot safe to reuse in the next CacheEngine call.
+        self.extension.check_error(self.error_flag)
+        if started is not None:
+            elapsed_ms = (time.perf_counter() - started) * 1000.0
+            logger.info(
+                "[BI100 BLOCK KV TRACE] direction=%s blocks=%d bytes=%d "
+                "elapsed_ms=%.3f",
+                direction,
+                block_count,
+                block_count * BYTES_PER_BLOCK,
+                elapsed_ms,
+            )
+
+    def swap_out(self, mapping: torch.Tensor) -> None:
+        started = time.perf_counter() if self.trace_enabled else None
+        source_gpu, destination_cpu = validate_block_mapping(
+            mapping,
+            source_limit=self.num_gpu_blocks,
+            destination_limit=self.num_cpu_blocks,
+        )
+        block_count = source_gpu.numel()
+        if block_count == 0:
+            return
+        source_gpu_ids = self._to_gpu_ids(source_gpu)
+
+        self._begin()
+        pending: tuple[int, torch.Tensor, int] | None = None
+        for index, (_, destination, gpu_ids, count) in enumerate(
+                self._chunks(
+                    source_gpu, destination_cpu, source_gpu_ids)):
+            slot = index % STAGING_BUFFER_COUNT
+            self.extension.pack(
+                self.gpu_cache,
+                gpu_ids,
+                self.gpu_staging[slot],
+                self.error_flag,
+                count,
+            )
+            self.cpu_staging[slot][:count].copy_(
+                self.gpu_staging[slot][:count],
+                non_blocking=True,
+            )
+            self.events[slot].record()
+            if pending is not None:
+                pending_slot, pending_destination, pending_count = pending
+                self.events[pending_slot].synchronize()
+                self.extension.cpu_scatter(
+                    self.cpu_staging[pending_slot],
+                    self.cpu_pool,
+                    pending_destination,
+                    pending_count,
+                )
+            pending = (slot, destination, count)
+
+        if pending is not None:
+            pending_slot, pending_destination, pending_count = pending
+            self.events[pending_slot].synchronize()
+            self.extension.cpu_scatter(
+                self.cpu_staging[pending_slot],
+                self.cpu_pool,
+                pending_destination,
+                pending_count,
+            )
+        self._finish("d2h", block_count, started)
+
+    def swap_in(self, mapping: torch.Tensor) -> None:
+        started = time.perf_counter() if self.trace_enabled else None
+        source_cpu, destination_gpu = validate_block_mapping(
+            mapping,
+            source_limit=self.num_cpu_blocks,
+            destination_limit=self.num_gpu_blocks,
+        )
+        block_count = source_cpu.numel()
+        if block_count == 0:
+            return
+        destination_gpu_ids = self._to_gpu_ids(destination_gpu)
+
+        self._begin()
+        for index, (source, _, gpu_ids, count) in enumerate(
+                self._chunks(
+                    source_cpu, destination_gpu, destination_gpu_ids)):
+            slot = index % STAGING_BUFFER_COUNT
+            if index >= STAGING_BUFFER_COUNT:
+                self.events[slot].synchronize()
+            self.extension.cpu_gather(
+                self.cpu_pool,
+                source,
+                self.cpu_staging[slot],
+                count,
+            )
+            self.gpu_staging[slot][:count].copy_(
+                self.cpu_staging[slot][:count],
+                non_blocking=True,
+            )
+            self.extension.scatter(
+                self.gpu_staging[slot],
+                gpu_ids,
+                self.gpu_cache,
+                self.error_flag,
+                count,
+            )
+            self.events[slot].record()
+        self._finish("h2d", block_count, started)

qwen3_6_scripts/build_cccl_moe_sort_scatter.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Build cccl_moe_sort_scatter — split compilation
+#
+# Step 1: Compile .cu with CCCL headers (no torch) → .o
+# Step 2: Compile _pybind.cpp with torch headers (no CCCL) → .o
+# Step 3: Link both → .so
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INC="${SCRIPT_DIR}/cccl_preload/include"
+CU_SRC="${SCRIPT_DIR}/cccl_moe_sort_scatter.cu"
+PY_SRC="${SCRIPT_DIR}/cccl_moe_sort_scatter_pybind.cpp"
+OUT="${1:-${SCRIPT_DIR}/prebuilt/corex-3.2.3-ivcore10/cccl_moe_sort_scatter.so}"
+
+# Find corex clang++
+CXX=""
+for c in /usr/local/corex-3.2.3/bin/clang++ /usr/local/corex/bin/clang++; do
+    [[ -x "$c" ]] && CXX="$c" && break
+done
+[[ -n "${CXX}" ]] || { echo "no corex clang++"; exit 2; }
+
+# Find torch paths
+TORCH_INC=$(python3 -c "from torch.utils.cpp_extension import include_paths; print(include_paths()[0])")
+TORCH_LIB=$(python3 -c "import torch; import os; print(os.path.join(os.path.dirname(torch.__file__), 'lib'))")
+PYTHON_INC=$(python3 -c "from sysconfig import get_paths; print(get_paths()['include'])")
+CUDA_INC="/usr/local/corex/include"
+
+echo "[build] CXX=${CXX}"
+echo "[build] CCCL=${INC}"
+echo "[build] torch=${TORCH_INC}"
+
+# Step 1: Compile CUDA kernels (CCCL headers, no torch)
+echo "[build] Step 1: compile CUDA kernels..."
+"${CXX}" \
+    -fPIC -O3 -std=c++17 \
+    -I"${INC}" \
+    -I"${CUDA_INC}" \
+    -DCCCL_IGNORE_DEPRECATED_CUDA_BELOW_12 \
+    -DCUB_WRAPPED_NAMESPACE=cccl_moe \
+    --cuda-gpu-arch=ivcore10 \
+    --cuda-path=/usr/local/corex \
+    -c "${CU_SRC}" -o /tmp/cccl_moe_kernels.o \
+    2>&1
+
+# Step 2: Compile pybind wrapper (torch headers, no CCCL)
+echo "[build] Step 2: compile pybind wrapper..."
+"${CXX}" \
+    -fPIC -O2 -std=c++17 \
+    -I"${TORCH_INC}" \
+    -I"${TORCH_INC}/torch/csrc/api/include" \
+    -I"${PYTHON_INC}" \
+    -I"${CUDA_INC}" \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=cccl_moe_sort_scatter \
+    -x c++ \
+    -c "${PY_SRC}" -o /tmp/cccl_moe_pybind.o \
+    2>&1
+
+# Step 3: Link
+echo "[build] Step 3: link..."
+mkdir -p "$(dirname "${OUT}")"
+"${CXX}" \
+    -shared -fPIC \
+    /tmp/cccl_moe_kernels.o \
+    /tmp/cccl_moe_pybind.o \
+    -L"${TORCH_LIB}" \
+    -ltorch -lc10 -ltorch_cpu -ltorch_cuda \
+    -L/usr/local/corex/lib64 -lcudart \
+    -Wl,-rpath,"${TORCH_LIB}" \
+    -o "${OUT}" \
+    2>&1
+
+SIZE=$(stat -c%s "${OUT}" 2>/dev/null || echo "?")
+echo "[build] SUCCESS: ${OUT} (${SIZE} bytes)"

qwen3_6_scripts/build_corex_attn_head_rms_norm.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_attn_head_rms_norm.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_attn_head_rms_norm.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" \
+    --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_attn_head_rms_norm \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" \
+    -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_attn_head_rms_norm.cu" \
+    -L"${TORCH_ROOT}/lib" \
+    -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" \
+    -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart \
+    -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX attention head RMSNorm extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_batched_gemm.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Build corex_batched_gemm.so — CUTLASS batched GEMM pybind for MoE decode
+#
+# Verified: 2.462ms for 8-expert decode (issue #68)
+#
+# Usage: bash build_corex_batched_gemm.sh VLLM_ROOT
+#   or:  bash build_corex_batched_gemm.sh  (outputs to prebuilt/)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJ_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+if [ ! -d "$COREX_ROOT" ]; then
+    COREX_ROOT=/usr/local/corex
+fi
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+if [ ! -d "$TORCH_ROOT" ]; then
+    TORCH_ROOT=$(python3 -c "import torch; import os; print(os.path.dirname(torch.__file__))" 2>/dev/null || echo "/usr/local/corex/lib/python3/dist-packages/torch")
+fi
+
+CUTLASS_INCLUDE="$COREX_ROOT/lib64/python3/dist-packages/tensorflow/include/third_party/gpus/cuda/include"
+if [ ! -f "$CUTLASS_INCLUDE/cutlass/cutlass.h" ]; then
+    # Fallback: search
+    CUTLASS_INCLUDE=$(find "$COREX_ROOT" -path "*/cutlass/cutlass.h" -printf '%h\n' 2>/dev/null | head -1 | sed 's|/cutlass$||')
+    if [ -z "$CUTLASS_INCLUDE" ]; then
+        echo "[build] ERROR: cannot find cutlass/cutlass.h under $COREX_ROOT"
+        exit 1
+    fi
+fi
+
+# Output path
+if [ -n "${1:-}" ]; then
+    OUTPUT="${1}/corex_batched_gemm.so"
+else
+    OUTPUT="$SCRIPT_DIR/prebuilt/corex-3.2.3-ivcore10/corex_batched_gemm.so"
+fi
+
+# Source files
+BIND_CPP="$PROJ_ROOT/ex_engine/xllm_kernels/cuda/bindings/corex_batched_gemm_bind.cpp"
+KERNEL_CU="$PROJ_ROOT/ex_engine/xllm_kernels/cuda/corex_batched_gemm_kernel.cu"
+
+echo "[build] COREX_ROOT=$COREX_ROOT"
+echo "[build] TORCH_ROOT=$TORCH_ROOT"
+echo "[build] CUTLASS_INCLUDE=$CUTLASS_INCLUDE"
+echo "[build] OUTPUT=$OUTPUT"
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" \
+    --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_batched_gemm \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" \
+    -I"${TORCH_ROOT}/include/THC" \
+    -I"${CUTLASS_INCLUDE}" \
+    -I/usr/local/include/python3.10 \
+    "${KERNEL_CU}" "${BIND_CPP}" \
+    -L"${TORCH_ROOT}/lib" \
+    -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" \
+    -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart \
+    -o "${OUTPUT}"
+
+echo "[build] ✓ built ${OUTPUT}"
+echo "[build] size: $(du -h "${OUTPUT}" | cut -f1)"
+
+python3 -c "
+import importlib.util
+spec = importlib.util.spec_from_file_location('corex_batched_gemm', '${OUTPUT}')
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+print('[build] ✓ import OK:', [x for x in dir(mod) if not x.startswith('_')])
+" 2>&1 || echo "[build] import test skipped"
+
+echo "[build] done"

qwen3_6_scripts/build_corex_block_major_kv_transfer.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_block_major_kv_transfer.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_block_major_kv_transfer.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" \
+    --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_block_major_kv_transfer \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" \
+    -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_block_major_kv_transfer.cu" \
+    -L"${TORCH_ROOT}/lib" \
+    -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" \
+    -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart \
+    -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX block-major KV transfer extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_fused_paged_prefill_split4.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_fused_paged_prefill_split4.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_fused_paged_prefill_split4.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_fused_paged_prefill \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_fused_paged_prefill_split4.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcublas -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX split4 fused paged-prefill extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_gdn_beta_decay.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_gdn_beta_decay.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_gdn_beta_decay.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" \
+    --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_gdn_beta_decay \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" \
+    -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_gdn_beta_decay.cu" \
+    -L"${TORCH_ROOT}/lib" \
+    -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" \
+    -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart \
+    -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX GDN beta/decay extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_gdn_causal_conv.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_gdn_causal_conv.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_gdn_causal_conv.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" \
+    --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_gdn_causal_conv \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" \
+    -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_gdn_causal_conv.cu" \
+    -L"${TORCH_ROOT}/lib" \
+    -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" \
+    -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart \
+    -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX GDN causal conv extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_gdn_chunk_recurrent.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_gdn_chunk_recurrent.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_gdn_chunk_recurrent.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_gdn_chunk_recurrent \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    -I"${COREX_ROOT}/include" \
+    -I"${SCRIPT_DIR}" \
+    "${SCRIPT_DIR}/corex_gdn_chunk_recurrent.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX GDN chunk+recurrent C++ extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_gdn_gated_norm.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_gdn_gated_norm.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_gdn_gated_norm.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" \
+    --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_gdn_gated_norm \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" \
+    -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_gdn_gated_norm.cu" \
+    -L"${TORCH_ROOT}/lib" \
+    -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" \
+    -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart \
+    -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX GDN gated norm extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_gdn_packed_decode.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_gdn_packed_decode.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_gdn_packed_decode.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_gdn_packed_decode \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_gdn_packed_decode.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX GDN packed decode extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_gdn_qk_map.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_gdn_qk_map.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_gdn_qk_map.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_gdn_qk_map \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_gdn_qk_map.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX GDN q/k map extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_moe_direct_routed.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_moe_direct_routed.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_moe_direct_routed.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_moe_direct_routed \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_moe_direct_routed.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX direct routed-expert extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_moe_exact_reduce.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_moe_exact_reduce.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_moe_exact_reduce.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_moe_exact_reduce \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_moe_exact_reduce.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX MoE exact reduce extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_moe_index_combine.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_moe_index_combine.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_moe_index_combine.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_moe_index_combine \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    -I"${COREX_ROOT}/include" \
+    -I"${SCRIPT_DIR}" \
+    "${SCRIPT_DIR}/corex_moe_index_combine.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX MoE index+combine extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_moe_topk_softmax.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_moe_topk_softmax.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_moe_topk_softmax.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_moe_topk_softmax \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    -I"${COREX_ROOT}/include" \
+    -I"${SCRIPT_DIR}" \
+    "${SCRIPT_DIR}/corex_moe_topk_softmax.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX MoE topk+softmax extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_moe_weight_gather.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_moe_weight_gather.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_moe_weight_gather.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_moe_weight_gather \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_moe_weight_gather.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX MoE selected-weight gather extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_corex_paged_kv_gather.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VLLM_ROOT=${1:?usage: build_corex_paged_kv_gather.sh VLLM_ROOT}
+COREX_ROOT=${COREX_ROOT:-/usr/local/corex-3.2.3}
+TORCH_ROOT=${TORCH_ROOT:-${COREX_ROOT}/lib64/python3/dist-packages/torch}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT=${VLLM_ROOT}/corex_paged_kv_gather.so
+
+"${COREX_ROOT}/bin/clang++" \
+    -std=c++17 -O3 -shared -fPIC \
+    --cuda-path="${COREX_ROOT}" --cuda-gpu-arch=ivcore10 \
+    --no-cuda-version-check -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=corex_paged_kv_gather \
+    -DTORCH_API_INCLUDE_EXTENSION_H \
+    -I"${TORCH_ROOT}/include" \
+    -I"${TORCH_ROOT}/include/torch/csrc/api/include" \
+    -I"${TORCH_ROOT}/include/TH" -I"${TORCH_ROOT}/include/THC" \
+    -I/usr/local/include/python3.10 \
+    "${SCRIPT_DIR}/corex_paged_kv_gather.cu" \
+    -L"${TORCH_ROOT}/lib" -L"${COREX_ROOT}/lib64" \
+    -Wl,-rpath,"${TORCH_ROOT}/lib" -Wl,-rpath,"${COREX_ROOT}/lib64" \
+    -ltorch_python -ltorch_cuda -ltorch_cpu -ltorch \
+    -lc10_cuda -lc10 -lcudart -o "${OUTPUT}"
+
+test -s "${OUTPUT}"
+printf '[ok] CoreX paged K/V gather extension %s\n' "${OUTPUT}"

qwen3_6_scripts/build_ix_attn_bridge.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# build_ix_attn_bridge.sh — Build ix_attn_bridge.so on real BI-V100
+#
+# Compiles ix_attn_bridge.cpp → prebuilt .so for Docker deployment.
+# Functions: prefill_attention, decode_attention, linear, residual_rms_norm
+#
+# Run on real machine: bash qwen3_6_scripts/build_ix_attn_bridge.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+CPP_SOURCE="${SCRIPT_DIR}/ix_attn_bridge.cpp"
+PREBUILT_DIR="${SCRIPT_DIR}/prebuilt/corex-3.2.3-ivcore10"
+
+if [ ! -f "$CPP_SOURCE" ]; then
+    echo "ERROR: ix_attn_bridge.cpp not found at $CPP_SOURCE"
+    exit 1
+fi
+
+echo "=== Building ix_attn_bridge.so ==="
+echo "Source: $CPP_SOURCE"
+
+python3 -c "
+import os, sys, glob, shutil
+from torch.utils.cpp_extension import load
+
+cpp_source = '$CPP_SOURCE'
+extra_ldflags = []
+
+try:
+    import ixformer
+    ixf_dir = os.path.dirname(ixformer.__file__)
+    for so in glob.glob(os.path.join(ixf_dir, '*.so')):
+        extra_ldflags.append(so)
+    extra_ldflags.append(f'-Wl,-rpath,{ixf_dir}')
+except ImportError:
+    pass
+
+corex_lib = '/usr/local/corex/lib64'
+if os.path.isdir(corex_lib):
+    for lib in ['libixattn.so', 'libixformer.so', 'libcublas.so']:
+        p = os.path.join(corex_lib, lib)
+        if os.path.isfile(p):
+            extra_ldflags.append(p)
+    extra_ldflags.append(f'-Wl,-rpath,{corex_lib}')
+
+print(f'Linking: {extra_ldflags}')
+
+mod = load(
+    name='ix_attn_bridge',
+    sources=[cpp_source],
+    extra_cflags=['-O2', '-std=c++17'],
+    extra_ldflags=extra_ldflags,
+    verbose=True,
+)
+
+import torch.utils.cpp_extension as ext
+build_dir = ext._get_build_directory('ix_attn_bridge', verbose=False)
+
+for f in glob.glob(os.path.join(build_dir, '*.so')):
+    dst = '$PREBUILT_DIR/ix_attn_bridge.so'
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    shutil.copy2(f, dst)
+    sz = os.path.getsize(dst)
+    print(f'✓ ix_attn_bridge.so ({sz} bytes) → {dst}')
+    break
+
+fns = [x for x in dir(mod) if not x.startswith('_')]
+print(f'Functions: {fns}')
+print('=== Build SUCCESS ===')
+"

qwen3_6_scripts/build_ix_bridge.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# Build ix_full_bridge.so — bridges ixformer_torch_ext C++ symbols to Python
+set -eo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+VLLM_ROOT="${1:?usage: build_ix_bridge.sh VLLM_ROOT}"
+BRIDGE_SRC="${SCRIPT_DIR}/ix_full_bridge.cpp"
+
+if [ ! -f "$BRIDGE_SRC" ]; then
+    echo "[bridge] SKIP: $BRIDGE_SRC not found"
+    exit 0
+fi
+
+# Find ixformer .so with the real symbols
+IX_TORCH_SO=""
+for f in /usr/local/corex/lib/python3/dist-packages/ixformer/_ixformer_torch*.so; do
+    if [ -f "$f" ]; then
+        IX_TORCH_SO="$f"
+        break
+    fi
+done
+
+IX_LIB_SO=""
+for f in /usr/local/corex/lib/python3/dist-packages/ixformer/libixformer.so; do
+    if [ -f "$f" ]; then
+        IX_LIB_SO="$f"
+        break
+    fi
+done
+
+TORCH_LIB=$(python3 -c "import torch; print(torch.__path__[0] + '/lib')" 2>/dev/null)
+IX_DIR=$(python3 -c "import ixformer; import os; print(os.path.dirname(ixformer.__file__))" 2>/dev/null)
+
+echo "[bridge] IX_TORCH_SO=${IX_TORCH_SO}"
+echo "[bridge] IX_LIB_SO=${IX_LIB_SO}"
+echo "[bridge] TORCH_LIB=${TORCH_LIB}"
+
+# Clear cached build (namespace changed)
+rm -rf /root/.cache/torch_extensions/py310_cu102/ix_full_bridge
+
+python3 << PYEOF
+import torch
+from torch.utils.cpp_extension import load
+import shutil, os
+
+extra_ldflags = []
+
+# Link against _ixformer_torch .so (has ixformer_torch_ext:: symbols)
+ix_torch = "${IX_TORCH_SO}"
+if ix_torch and os.path.exists(ix_torch):
+    extra_ldflags.append(ix_torch)
+    extra_ldflags.append(f"-Wl,-rpath,{os.path.dirname(ix_torch)}")
+
+# Also link libixformer.so (has launcher symbols)
+ix_lib = "${IX_LIB_SO}"
+if ix_lib and os.path.exists(ix_lib):
+    extra_ldflags.append(ix_lib)
+
+# torch lib rpath
+torch_lib = "${TORCH_LIB}"
+if torch_lib and os.path.isdir(torch_lib):
+    extra_ldflags.append(f"-Wl,-rpath,{torch_lib}")
+
+print(f"[bridge] ldflags: {extra_ldflags}")
+
+mod = load(
+    name="ix_full_bridge",
+    sources=["${BRIDGE_SRC}"],
+    extra_cflags=["-O2", "-std=c++17"],
+    extra_ldflags=extra_ldflags,
+    verbose=True,
+)
+
+# Find the compiled .so and copy to VLLM_ROOT
+import importlib
+spec = importlib.util.find_spec("ix_full_bridge")
+if spec and spec.origin:
+    dest = os.path.join("${VLLM_ROOT}", "ix_full_bridge.so")
+    shutil.copy2(spec.origin, dest)
+    print(f"[bridge] SUCCESS: {dest}")
+    fns = [x for x in dir(mod) if not x.startswith("_")]
+    print(f"[bridge] functions: {fns}")
+else:
+    # Search in cache
+    import glob
+    for so in glob.glob(os.path.expanduser("~/.cache/torch_extensions/**/ix_full_bridge*.so"), recursive=True):
+        dest = os.path.join("${VLLM_ROOT}", "ix_full_bridge.so")
+        shutil.copy2(so, dest)
+        print(f"[bridge] SUCCESS: {so} -> {dest}")
+        break
+    else:
+        print("[bridge] WARNING: could not find compiled .so")
+PYEOF
+
+echo "[bridge] Build complete"

qwen3_6_scripts/build_ix_moe_bridge.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# build_ix_moe_bridge.sh — Build ix_moe_bridge.so on real BI-V100
+#
+# This compiles ex_engine/csrc/ix_moe_bridge.cpp into a prebuilt .so
+# that can be deployed without JIT compilation in Docker.
+#
+# Run on real machine: bash qwen3_6_scripts/build_ix_moe_bridge.sh
+# Output: qwen3_6_scripts/prebuilt/corex-3.2.3-ivcore10/ix_moe_bridge.so
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+CPP_SOURCE="${PROJECT_DIR}/ex_engine/csrc/ix_moe_bridge.cpp"
+PREBUILT_DIR="${SCRIPT_DIR}/prebuilt/corex-3.2.3-ivcore10"
+
+if [ ! -f "$CPP_SOURCE" ]; then
+    # Also try the local copy
+    CPP_SOURCE="${SCRIPT_DIR}/ix_moe_bridge.cpp"
+fi
+
+if [ ! -f "$CPP_SOURCE" ]; then
+    echo "ERROR: ix_moe_bridge.cpp not found"
+    exit 1
+fi
+
+echo "=== Building ix_moe_bridge.so ==="
+echo "Source: $CPP_SOURCE"
+echo "Output: $PREBUILT_DIR/ix_moe_bridge.so"
+
+python3 -c "
+import os, sys, glob
+from torch.utils.cpp_extension import load
+
+cpp_source = '$CPP_SOURCE'
+extra_ldflags = []
+
+# Find ixformer .so to link against
+try:
+    import ixformer
+    ixf_dir = os.path.dirname(ixformer.__file__)
+    for so in glob.glob(os.path.join(ixf_dir, '*.so')):
+        extra_ldflags.append(so)
+    extra_ldflags.append(f'-Wl,-rpath,{ixf_dir}')
+except ImportError:
+    pass
+
+corex_lib = '/usr/local/corex/lib64'
+if os.path.isdir(corex_lib):
+    for lib in ['libixattn.so', 'libixformer.so', 'libcublas.so']:
+        p = os.path.join(corex_lib, lib)
+        if os.path.isfile(p):
+            extra_ldflags.append(p)
+    extra_ldflags.append(f'-Wl,-rpath,{corex_lib}')
+
+print(f'Linking: {extra_ldflags}')
+
+mod = load(
+    name='ix_moe_bridge',
+    sources=[cpp_source],
+    extra_cflags=['-O2', '-std=c++17'],
+    extra_ldflags=extra_ldflags,
+    verbose=True,
+)
+
+# Find the compiled .so and copy to prebuilt
+import torch.utils.cpp_extension as ext
+build_dir = ext._get_build_directory('ix_moe_bridge', verbose=False)
+print(f'Build dir: {build_dir}')
+
+import shutil
+for f in glob.glob(os.path.join(build_dir, '*.so')):
+    dst = '$PREBUILT_DIR/ix_moe_bridge.so'
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    shutil.copy2(f, dst)
+    sz = os.path.getsize(dst)
+    print(f'✓ ix_moe_bridge.so ({sz} bytes) → {dst}')
+    break
+
+# Verify
+fns = [x for x in dir(mod) if not x.startswith('_')]
+print(f'Functions: {fns}')
+print('=== Build SUCCESS ===')
+"

qwen3_6_scripts/build_xllm_kernels.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# build_xllm_kernels.sh — Compile xllm CUDA kernels into .so on BI-V100
+#
+# Uses corex CUB (/usr/local/corex/include/cub/) NOT cccl_upstream
+# Each .so = kernel .cu + pybind11 binding .cpp
+#
+# Run: bash qwen3_6_scripts/build_xllm_kernels.sh
+
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+CUDA_DIR="${PROJECT_DIR}/ex_engine/xllm_kernels/cuda"
+HEADER_DIR="${CUDA_DIR}/headers"
+BIND_DIR="${CUDA_DIR}/bindings"
+PREBUILT_DIR="${SCRIPT_DIR}/prebuilt/corex-3.2.3-ivcore10"
+mkdir -p "$PREBUILT_DIR"
+
+build_kernel() {
+    local name="$1"
+    shift
+    local sources="$@"
+    echo "=== Building ${name}.so ==="
+    python3 -c "
+import os, glob, shutil
+from torch.utils.cpp_extension import load
+
+sources = '${sources}'.split()
+mod = load(
+    name='${name}',
+    sources=sources,
+    extra_cflags=['-std=c++17'],
+    extra_include_paths=['${HEADER_DIR}', '/usr/local/corex/include'],
+    verbose=True,
+)
+
+import torch.utils.cpp_extension as ext
+build_dir = ext._get_build_directory('${name}', verbose=False)
+for f in glob.glob(os.path.join(build_dir, '*.so')):
+    dst = '${PREBUILT_DIR}/${name}.so'
+    shutil.copy2(f, dst)
+    sz = os.path.getsize(dst)
+    print(f'✓ ${name}.so ({sz} bytes) → {dst}')
+    break
+
+fns = [x for x in dir(mod) if not x.startswith('_')]
+print(f'Functions: {fns}')
+"
+    echo ""
+}
+
+echo "Building xllm CUDA kernels for BI-V100 (ivcore10)"
+echo "Using corex CUB: /usr/local/corex/include/cub/"
+echo ""
+
+build_kernel "xllm_norm" \
+    "${CUDA_DIR}/norm.cu" "${BIND_DIR}/xllm_norm_bind.cpp"
+
+build_kernel "xllm_activation" \
+    "${CUDA_DIR}/activation.cu" "${BIND_DIR}/xllm_activation_bind.cpp"
+
+build_kernel "xllm_rope" \
+    "${CUDA_DIR}/rope.cu" "${BIND_DIR}/xllm_rope_bind.cpp"
+
+build_kernel "xllm_cache" \
+    "${CUDA_DIR}/reshape_paged_cache.cu" "${CUDA_DIR}/block_copy.cu" "${BIND_DIR}/xllm_cache_bind.cpp"
+
+build_kernel "xllm_moe" \
+    "${CUDA_DIR}/moe/moe_fused_topk.cu" "${CUDA_DIR}/moe/moe_compute_index.cu" "${CUDA_DIR}/moe/moe_combine.cu" "${BIND_DIR}/xllm_moe_bind.cpp"
+
+echo "=== All kernels built ==="
+ls -lh "${PREBUILT_DIR}"/xllm_*.so 2>/dev/null || echo "No .so files found"

qwen3_6_scripts/cat_cutlass_cu10.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# cat_cutlass_cu10.sh — Cat the critical Cu10 CUTLASS files into cat_files/
+
+SAMPLES="/usr/local/corex-samples-3.2.3_x86_64/samples/cutlass"
+OUTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/cat_files"
+mkdir -p "$OUTDIR"
+
+echo "Output dir: $OUTDIR"
+
+cp /usr/local/corex/include/crt/iluvatar_mma.hpp "$OUTDIR/iluvatar_mma.hpp"
+echo "✓ iluvatar_mma.hpp"
+
+cp "${SAMPLES}/include/cutlass/arch/mma_cu10.h" "$OUTDIR/mma_cu10.h"
+echo "✓ mma_cu10.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/threadblock/default_mma_core_cu10.h" "$OUTDIR/default_mma_core_cu10.h"
+echo "✓ default_mma_core_cu10.h"
+
+cp "${SAMPLES}/examples/05_batched_gemm/batched_gemm.cu" "$OUTDIR/batched_gemm.cu"
+echo "✓ batched_gemm.cu"
+
+cp "${SAMPLES}/include/cutlass/gemm/device/gemm_universal.h" "$OUTDIR/gemm_universal.h"
+echo "✓ gemm_universal.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/device/gemm_batched.h" "$OUTDIR/gemm_batched.h"
+echo "✓ gemm_batched.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/warp/mma_tensor_op.h" "$OUTDIR/mma_tensor_op.h"
+echo "✓ mma_tensor_op.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/warp/mma_tensor_op_policy.h" "$OUTDIR/mma_tensor_op_policy.h"
+echo "✓ mma_tensor_op_policy.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/warp/mma_tensor_op_tile_iterator.h" "$OUTDIR/mma_tensor_op_tile_iterator.h"
+echo "✓ mma_tensor_op_tile_iterator.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/warp/default_mma_tensor_op.h" "$OUTDIR/default_mma_tensor_op.h"
+echo "✓ default_mma_tensor_op.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/threadblock/default_mma_core.h" "$OUTDIR/default_mma_core.h"
+echo "✓ default_mma_core.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/device/default_gemm_configuration.h" "$OUTDIR/default_gemm_configuration.h"
+echo "✓ default_gemm_configuration.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/kernel/default_gemm.h" "$OUTDIR/default_gemm.h"
+echo "✓ default_gemm.h"
+
+cp "${SAMPLES}/include/cutlass/gemm/kernel/default_gemm_universal.h" "$OUTDIR/default_gemm_universal.h"
+echo "✓ default_gemm_universal.h"
+
+# Also grab the ixinfer.h
+cp /usr/local/corex/include/ixinfer.h "$OUTDIR/ixinfer.h" 2>/dev/null && echo "✓ ixinfer.h"
+
+# Full tree
+find "${SAMPLES}" -type f | sort > "$OUTDIR/cutlass_samples_tree.txt"
+echo "✓ cutlass_samples_tree.txt"
+
+echo ""
+echo "=== Files saved ==="
+ls -lh "$OUTDIR/"
+echo ""
+echo "=== Commit these with: git add cat_files/ && git commit && git push ==="

qwen3_6_scripts/cat_cutlass_cu10_part2.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# cat_cutlass_cu10_part2.sh — Cat tensorop examples and arch files
+SAMPLES="/usr/local/corex-samples-3.2.3_x86_64/samples/cutlass"
+OUTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/cat_files"
+mkdir -p "$OUTDIR"
+
+cp "${SAMPLES}/examples/08_turing_tensorop_gemm/turing_tensorop_gemm.cu" "$OUTDIR/turing_tensorop_gemm.cu"
+echo "✓ turing_tensorop_gemm.cu"
+
+cp "${SAMPLES}/examples/00_basic_gemm/basic_gemm.cu" "$OUTDIR/basic_gemm.cu"
+echo "✓ basic_gemm.cu"
+
+cp "${SAMPLES}/include/cutlass/arch/arch.h" "$OUTDIR/arch.h" 2>/dev/null
+echo "✓ arch.h"
+
+# Get the Cu10 arch tag definition
+grep -rl "struct Cu10" "${SAMPLES}/include/" 2>/dev/null | while read f; do
+    base=$(basename "$f")
+    cp "$f" "$OUTDIR/arch_${base}"
+    echo "✓ arch_${base} (contains Cu10 definition)"
+done
+
+# Get the cutlass.h to see CUTLASS_ARCH_CU10_SUPPORTED
+cp "${SAMPLES}/include/cutlass/cutlass.h" "$OUTDIR/cutlass.h"
+echo "✓ cutlass.h"
+
+# Get gemm_batched.h full (we only had head before)
+cp "${SAMPLES}/include/cutlass/gemm/device/gemm_batched.h" "$OUTDIR/gemm_batched_full.h"
+echo "✓ gemm_batched_full.h"
+
+# Get gemm.h (device level)
+cp "${SAMPLES}/include/cutlass/gemm/device/gemm.h" "$OUTDIR/gemm_device.h"
+echo "✓ gemm_device.h"
+
+# Get the CMakeLists for batched_gemm and tensorop examples
+cp "${SAMPLES}/examples/05_batched_gemm/CMakeLists.txt" "$OUTDIR/CMakeLists_batched_gemm.txt"
+cp "${SAMPLES}/examples/08_turing_tensorop_gemm/CMakeLists.txt" "$OUTDIR/CMakeLists_tensorop_gemm.txt"
+echo "✓ CMakeLists"
+
+echo ""
+ls -lh "$OUTDIR/"
+echo ""
+echo "git add cat_files/ && git commit -m 'data: Cu10 CUTLASS part 2' && git push"

qwen3_6_scripts/cccl_moe_sort_scatter.cu

@@ -0,0 +1,103 @@
+// cccl_moe_sort_scatter.cu — CCCL CUB device-level MoE token dispatch
+//
+// Split compilation: this file uses CCCL headers only (no torch).
+// Pybind wrapper in cccl_moe_sort_scatter_pybind.cpp links against this.
+//
+// Build pattern (same as cccl_allocator_preload.cu):
+//   clang++ -I cccl_preload/include -DCCCL_IGNORE_DEPRECATED_CUDA_BELOW_12
+//           -DCUB_WRAPPED_NAMESPACE=cccl_moe ...
+
+// Suppress CUDA <12 check — corex 10.2 works for block-level CUB
+#define CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12
+
+// Isolate from corex CUB
+#define CUB_WRAPPED_NAMESPACE cccl_moe
+
+#include <cub/block/block_scan.cuh>
+#include <cuda_runtime.h>
+#include <cstdint>
+
+// ========================================================================
+// Kernels
+// ========================================================================
+
+static constexpr int32_t kBlock = 256;
+
+__global__ void moe_histogram_kernel(
+    const int32_t* __restrict__ expert_id,
+    int32_t* __restrict__ expert_sizes,
+    int64_t num_elements,
+    int32_t num_experts) {
+  int64_t tid = int64_t(blockIdx.x) * kBlock + threadIdx.x;
+  if (tid < num_elements) {
+    int32_t eid = expert_id[tid];
+    if (eid >= 0 && eid < num_experts) {
+      atomicAdd(&expert_sizes[eid], 1);
+    }
+  }
+}
+
+__global__ void moe_prefix_sum_kernel(
+    const int32_t* __restrict__ expert_sizes,
+    int32_t* __restrict__ expert_offsets,
+    int32_t num_experts) {
+  using BlockScan = cccl_moe::cub::BlockScan<int32_t, 256>;
+  __shared__ typename BlockScan::TempStorage s_scan;
+
+  int32_t val = (threadIdx.x < num_experts) ? expert_sizes[threadIdx.x] : 0;
+  int32_t offset;
+  BlockScan(s_scan).ExclusiveSum(val, offset);
+  __syncthreads();
+
+  if (threadIdx.x < num_experts) {
+    expert_offsets[threadIdx.x] = offset;
+  }
+}
+
+__global__ void moe_place_kernel(
+    const int32_t* __restrict__ expert_id,
+    int32_t* __restrict__ expert_offsets,
+    int32_t* __restrict__ dst_src,
+    int32_t* __restrict__ src_dst,
+    int64_t num_elements,
+    int32_t num_experts) {
+  int64_t flat_idx = int64_t(blockIdx.x) * kBlock + threadIdx.x;
+  if (flat_idx >= num_elements) return;
+
+  int32_t eid = expert_id[flat_idx];
+  if (eid < 0 || eid >= num_experts) return;
+
+  int32_t pos = atomicAdd(&expert_offsets[eid], 1);
+  dst_src[pos] = static_cast<int32_t>(flat_idx);
+  src_dst[flat_idx] = pos;
+}
+
+// ========================================================================
+// C API — called from pybind wrapper
+// ========================================================================
+
+extern "C" {
+
+void cccl_moe_launch_histogram(
+    const int32_t* expert_id, int32_t* expert_sizes,
+    int64_t N, int32_t E, cudaStream_t stream) {
+  int64_t grid = (N + kBlock - 1) / kBlock;
+  moe_histogram_kernel<<<grid, kBlock, 0, stream>>>(expert_id, expert_sizes, N, E);
+}
+
+void cccl_moe_launch_prefix_sum(
+    const int32_t* expert_sizes, int32_t* expert_offsets,
+    int32_t E, cudaStream_t stream) {
+  moe_prefix_sum_kernel<<<1, kBlock, 0, stream>>>(expert_sizes, expert_offsets, E);
+}
+
+void cccl_moe_launch_place(
+    const int32_t* expert_id, int32_t* expert_offsets,
+    int32_t* dst_src, int32_t* src_dst,
+    int64_t N, int32_t E, cudaStream_t stream) {
+  int64_t grid = (N + kBlock - 1) / kBlock;
+  moe_place_kernel<<<grid, kBlock, 0, stream>>>(
+      expert_id, expert_offsets, dst_src, src_dst, N, E);
+}
+
+} // extern "C"

qwen3_6_scripts/cccl_moe_sort_scatter_pybind.cpp

@@ -0,0 +1,62 @@
+// cccl_moe_sort_scatter_pybind.cpp — Torch pybind wrapper
+//
+// Links against cccl_moe_sort_scatter.so (built separately with CCCL headers).
+// This file only includes torch headers — no CCCL, no namespace conflict.
+
+#include <torch/extension.h>
+#include <c10/cuda/CUDAStream.h>
+#include <cuda_runtime.h>
+
+// C API from cccl_moe_sort_scatter.so
+extern "C" {
+void cccl_moe_launch_histogram(
+    const int32_t* expert_id, int32_t* expert_sizes,
+    int64_t N, int32_t E, cudaStream_t stream);
+void cccl_moe_launch_prefix_sum(
+    const int32_t* expert_sizes, int32_t* expert_offsets,
+    int32_t E, cudaStream_t stream);
+void cccl_moe_launch_place(
+    const int32_t* expert_id, int32_t* expert_offsets,
+    int32_t* dst_src, int32_t* src_dst,
+    int64_t N, int32_t E, cudaStream_t stream);
+}
+
+std::tuple<torch::Tensor, torch::Tensor, torch::Tensor>
+moe_sort_scatter(const torch::Tensor& expert_id, int64_t num_experts) {
+  TORCH_CHECK(expert_id.is_cuda(), "expert_id must be on CUDA");
+  auto stream = at::cuda::getCurrentCUDAStream();
+  int64_t N = expert_id.numel();
+  int32_t E = static_cast<int32_t>(num_experts);
+
+  auto expert_id_i32 = expert_id.to(torch::kInt32).contiguous();
+  auto opt_i32 = expert_id_i32.options();
+
+  auto expert_sizes = torch::zeros({num_experts}, opt_i32);
+  auto expert_offsets = torch::empty({num_experts}, opt_i32);
+  auto dst_src = torch::empty({N}, opt_i32);
+  auto src_dst = torch::empty({N}, opt_i32);
+
+  cccl_moe_launch_histogram(
+      expert_id_i32.data_ptr<int32_t>(),
+      expert_sizes.data_ptr<int32_t>(),
+      N, E, stream);
+
+  cccl_moe_launch_prefix_sum(
+      expert_sizes.data_ptr<int32_t>(),
+      expert_offsets.data_ptr<int32_t>(),
+      E, stream);
+
+  cccl_moe_launch_place(
+      expert_id_i32.data_ptr<int32_t>(),
+      expert_offsets.data_ptr<int32_t>(),
+      dst_src.data_ptr<int32_t>(),
+      src_dst.data_ptr<int32_t>(),
+      N, E, stream);
+
+  return std::make_tuple(src_dst, dst_src, expert_sizes);
+}
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+  m.def("moe_sort_scatter", &moe_sort_scatter,
+        "CCCL CUB-based MoE token dispatch (histogram+prefix_sum+scatter)");
+}

qwen3_6_scripts/cccl_preload/build_cccl_preload.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Build libcccl_allocator.so
+#
+# Full CCCL dependency chain (288 headers) in ./include/
+# Source: cccl_upstream/cub/cub/util_allocator.cuh + transitive deps
+#
+# Usage:
+#   bash build_cccl_preload.sh [output_dir]
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUTPUT_DIR="${1:-${SCRIPT_DIR}}"
+SRC="${SCRIPT_DIR}/cccl_allocator_preload.cu"
+INC="${SCRIPT_DIR}/include"
+OUT="${OUTPUT_DIR}/libcccl_allocator.so"
+
+[[ -d "${INC}/cub" ]] || { echo "CCCL include tree missing: ${INC}/cub"; exit 2; }
+[[ -d "${INC}/cuda" ]] || { echo "CCCL include tree missing: ${INC}/cuda"; exit 2; }
+
+# Find compiler
+CXX=""
+for candidate in \
+    /usr/local/corex-3.2.3/bin/clang++ \
+    /usr/local/corex/bin/clang++ \
+    /usr/local/corex/lib64/clang/16/bin/clang++ \
+    ; do
+    if [[ -x "${candidate}" ]]; then
+        CXX="${candidate}"
+        break
+    fi
+done
+[[ -n "${CXX}" ]] || { CXX=g++; echo "[build] no CoreX clang++, falling back to g++"; }
+echo "[build] CXX=${CXX}"
+
+# Find CUDA headers (for cuda_runtime_api.h)
+CUDA_INC=""
+for candidate in \
+    /usr/local/corex/include \
+    /usr/local/cuda/include \
+    ; do
+    if [[ -f "${candidate}/cuda_runtime_api.h" ]]; then
+        CUDA_INC="${candidate}"
+        break
+    fi
+done
+
+# Find CUDA libs
+CUDA_LIB=""
+for candidate in \
+    /usr/local/corex/lib64 \
+    /usr/local/cuda/lib64 \
+    ; do
+    if [[ -f "${candidate}/libcudart.so" ]]; then
+        CUDA_LIB="${candidate}"
+        break
+    fi
+done
+
+echo "[build] CUDA include: ${CUDA_INC:-system}"
+echo "[build] CUDA lib:     ${CUDA_LIB:-system}"
+echo "[build] CCCL include: ${INC} ($(find "${INC}" -type f | wc -l) files)"
+echo "[build] Source:        ${SRC}"
+echo "[build] Output:        ${OUT}"
+
+COMMON_FLAGS=(
+    -shared -fPIC -O2 -std=c++17
+    -I"${INC}"
+    ${CUDA_INC:+-I"${CUDA_INC}"}
+    ${CUDA_LIB:+-L"${CUDA_LIB}"}
+    -lcudart -ldl
+    # Suppress CCCL warnings that don't affect correctness
+    -Wno-unused-function
+    -Wno-unknown-pragmas
+    # CUB needs this for non-NVCC compilers
+    -D__CUDA_ARCH_LIST__=700
+    -DCUB_DISABLE_NAMESPACE_MAGIC
+    -DCUB_WRAPPED_NAMESPACE=cccl_preload
+)
+
+if [[ "${CXX}" == *clang++* ]]; then
+    "${CXX}" "${COMMON_FLAGS[@]}" -x c++ -o "${OUT}" "${SRC}" 2>&1
+else
+    "${CXX}" "${COMMON_FLAGS[@]}" -x c++ -o "${OUT}" "${SRC}" 2>&1
+fi
+
+if [[ -f "${OUT}" ]]; then
+    SIZE=$(stat -c%s "${OUT}" 2>/dev/null || echo "?")
+    echo ""
+    echo "[build] SUCCESS: ${OUT} (${SIZE} bytes)"
+    echo ""
+    echo "Test:"
+    echo "  LD_PRELOAD=${OUT} CCCL_ALLOC_DEBUG=1 \\"
+    echo "  PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\"
+    echo "  python3 verify_preload.py"
+else
+    echo "[build] FAILED"
+    exit 1
+fi

qwen3_6_scripts/cccl_preload/cccl_allocator_preload.cu

@@ -0,0 +1,154 @@
+/*
+ * cccl_allocator_preload.cu
+ *
+ * LD_PRELOAD .so — CUB CachingDeviceAllocator from CCCL upstream.
+ * Full dependency chain (288 files) extracted into include/.
+ *
+ * Intercepts cudaMalloc/cudaFree, routes through CUB's geometric-bin
+ * caching allocator. Strips expandable_segments from
+ * PYTORCH_CUDA_ALLOC_CONF before libtorch reads it.
+ *
+ * Source: CCCL cub/cub/util_allocator.cuh (BSD-3, NVIDIA)
+ * Build:  bash build_cccl_preload.sh
+ */
+
+/* ---- CCCL include chain (288 files from cccl_upstream) ---- */
+#include <cub/util_allocator.cuh>
+
+/* ---- System ---- */
+#include <dlfcn.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+/* ========================================================================
+ * Configuration for BI-V100 (32GB × 4 cards)
+ *
+ * CUB CachingDeviceAllocator parameters:
+ *   bin_growth = 2   (power-of-2 bins: 256B, 512B, 1KB, ... 4GB)
+ *   min_bin    = 8   (2^8 = 256B minimum allocation)
+ *   max_bin    = 32  (2^32 = 4GB maximum cached bin)
+ *   max_cached = 8GB per device
+ *
+ * More granular bins (growth=2) than CUB default (growth=8) because
+ * PyTorch tensor sizes vary widely in inference.
+ * ======================================================================== */
+
+static constexpr unsigned int ALLOC_BIN_GROWTH   = 2;
+static constexpr unsigned int ALLOC_MIN_BIN      = 8;   /* 256 bytes */
+static constexpr unsigned int ALLOC_MAX_BIN      = 32;  /* 4 GB */
+static constexpr size_t       ALLOC_MAX_CACHED   = (size_t)8 * 1024 * 1024 * 1024; /* 8GB */
+
+/* ---- Global allocator singleton ---- */
+static cccl_preload::cub::CachingDeviceAllocator& get_allocator() {
+    static cccl_preload::cub::CachingDeviceAllocator instance(
+        ALLOC_BIN_GROWTH,
+        ALLOC_MIN_BIN,
+        ALLOC_MAX_BIN,
+        ALLOC_MAX_CACHED,
+        true  /* skip_cleanup: CoreX may tear down CUDA before our dtor */
+    );
+    return instance;
+}
+
+static bool g_preload_active = false;
+static bool g_debug = false;
+
+/* ---- Real cudaMalloc/cudaFree via dlsym(RTLD_NEXT) ---- */
+using RealMalloc_t = cudaError_t (*)(void**, size_t);
+using RealFree_t   = cudaError_t (*)(void*);
+
+static RealMalloc_t get_real_malloc() {
+    static RealMalloc_t fn = (RealMalloc_t)dlsym(RTLD_NEXT, "cudaMalloc");
+    return fn;
+}
+static RealFree_t get_real_free() {
+    static RealFree_t fn = (RealFree_t)dlsym(RTLD_NEXT, "cudaFree");
+    return fn;
+}
+
+/* ========================================================================
+ * Constructor: runs at LD_PRELOAD load time
+ * ======================================================================== */
+__attribute__((constructor))
+static void cccl_preload_init() {
+    const char* debug_env = getenv("CCCL_ALLOC_DEBUG");
+    g_debug = (debug_env && atoi(debug_env) > 0);
+
+    const char* disable_env = getenv("CCCL_ALLOC_DISABLE");
+    if (disable_env && atoi(disable_env) > 0) {
+        fprintf(stderr, "[cccl_alloc] DISABLED by CCCL_ALLOC_DISABLE=1\n");
+        return;
+    }
+
+    /* Strip expandable_segments from PYTORCH_CUDA_ALLOC_CONF */
+    const char* alloc_conf = getenv("PYTORCH_CUDA_ALLOC_CONF");
+    if (alloc_conf) {
+        std::string conf(alloc_conf);
+        std::string clean;
+        size_t pos = 0;
+        while (pos < conf.size()) {
+            size_t comma = conf.find(',', pos);
+            if (comma == std::string::npos) comma = conf.size();
+            std::string token = conf.substr(pos, comma - pos);
+            if (token.find("expandable_segments") == std::string::npos) {
+                if (!clean.empty()) clean += ",";
+                clean += token;
+            }
+            pos = comma + 1;
+        }
+        if (clean.empty())
+            unsetenv("PYTORCH_CUDA_ALLOC_CONF");
+        else
+            setenv("PYTORCH_CUDA_ALLOC_CONF", clean.c_str(), 1);
+
+        fprintf(stderr, "[cccl_alloc] PYTORCH_CUDA_ALLOC_CONF: \"%s\" -> \"%s\"\n",
+                alloc_conf, clean.empty() ? "(unset)" : clean.c_str());
+    }
+
+    /* Initialize allocator */
+    auto& alloc = get_allocator();
+    if (g_debug) {
+        alloc.debug = true;
+    }
+
+    g_preload_active = true;
+    fprintf(stderr,
+        "[cccl_alloc] LD_PRELOAD active — CUB CachingDeviceAllocator "
+        "(growth=%u, bins=[%u..%u], max_cached=%.1fGB)\n",
+        ALLOC_BIN_GROWTH, ALLOC_MIN_BIN, ALLOC_MAX_BIN,
+        (double)ALLOC_MAX_CACHED / (1024.0*1024.0*1024.0));
+}
+
+/* ========================================================================
+ * cudaMalloc / cudaFree intercepts
+ *
+ * CUB's DeviceAllocate internally calls cudaMalloc on cache miss.
+ * We must detect this reentrant call and forward to the real function,
+ * otherwise we get infinite recursion → segfault.
+ * ======================================================================== */
+
+static thread_local bool g_in_allocator = false;
+
+extern "C" cudaError_t cudaMalloc(void** devPtr, size_t size)
+{
+    if (!g_preload_active || g_in_allocator) {
+        return get_real_malloc()(devPtr, size);
+    }
+    g_in_allocator = true;
+    cudaError_t err = get_allocator().DeviceAllocate(devPtr, size);
+    g_in_allocator = false;
+    return err;
+}
+
+extern "C" cudaError_t cudaFree(void* devPtr)
+{
+    if (!g_preload_active || devPtr == nullptr || g_in_allocator) {
+        return get_real_free()(devPtr);
+    }
+    g_in_allocator = true;
+    cudaError_t err = get_allocator().DeviceFree(devPtr);
+    g_in_allocator = false;
+    return err;
+}

qwen3_6_scripts/cccl_preload/include/cub/config.cuh

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/**
+ * \file
+ * Static configuration header for the CUB project.
+ */
+
+#pragma once
+
+// For _CCCL_IMPLICIT_SYSTEM_HEADER
+#include <cuda/__cccl_config> // IWYU pragma: export
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cub/util_arch.cuh> // IWYU pragma: export
+#include <cub/util_cpp_dialect.cuh> // IWYU pragma: export
+#include <cub/util_macro.cuh> // IWYU pragma: export
+#include <cub/util_namespace.cuh> // IWYU pragma: export
+
+#if !_CCCL_COMPILER(NVRTC)
+#  include <cuda/__nvtx/nvtx.h>
+#endif // !_CCCL_COMPILER(NVRTC)

qwen3_6_scripts/cccl_preload/include/cub/detail/detect_cuda_runtime.cuh

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/**
+ * @file
+ * Utilities for CUDA dynamic parallelism.
+ */
+
+#pragma once
+
+// We cannot use `cub/config.cuh` here due to circular dependencies
+#include <cuda/__cccl_config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes:
+//! Defined if RDC is enabled and CUB_DISABLE_CDP is not defined.
+//! Deprecated [Since 3.2]
+#  define CUB_RDC_ENABLED
+
+//! If defined, support for device-side usage of CUB is disabled.
+//! Deprecated [Since 3.2]. Use CCCL_DISABLE_CDP instead.
+#  define CUB_DISABLE_CDP
+
+//! Execution space for functions that use the CUDA runtime API, e.g. to launch kernels. Such functions are `__host__
+//! __device__` when compiling with RDC, otherwise only `__host__`.
+//! Deprecated [Since 3.2]
+#  define CUB_RUNTIME_FUNCTION
+#else // Non-doxygen pass:
+
+#  if _CCCL_HAS_CDP()
+#    define CUB_RDC_ENABLED
+#  endif // _CCCL_HAS_CDP()
+
+#  ifndef CUB_RUNTIME_FUNCTION
+#    define CUB_RUNTIME_FUNCTION _CCCL_CDP_API
+#  endif // CUB_RUNTIME_FUNCTION predefined
+#endif // Do not document

qwen3_6_scripts/cccl_preload/include/cub/util_allocator.cuh

@@ -0,0 +1,901 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/******************************************************************************
+ * Simple caching allocator for device memory allocations. The allocator is
+ * thread-safe and capable of managing device allocations on multiple devices.
+ ******************************************************************************/
+
+#pragma once
+
+#include <cub/config.cuh>
+
+#ifndef CCCL_DISABLE_NVRTC_COMPATIBILITY_CHECK
+#  if _CCCL_COMPILER(NVRTC)
+#    error \
+      "Including <cub/util_allocator.cuh> is not supported when compiling with NVRTC, which supports device code only. You can define CCCL_DISABLE_NVRTC_COMPATIBILITY_CHECK to disable this warning."
+#  endif // _CCCL_COMPILER(NVRTC)
+#endif // CCCL_DISABLE_NVRTC_COMPATIBILITY_CHECK
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cub/util_debug.cuh>
+#include <cub/util_namespace.cuh>
+
+#include <cuda/std/__host_stdlib/math.h>
+
+#include <map>
+#include <mutex>
+#include <set>
+
+CUB_NAMESPACE_BEGIN
+
+/******************************************************************************
+ * CachingDeviceAllocator (host use)
+ ******************************************************************************/
+
+/**
+ * @brief A simple caching allocator for device memory allocations.
+ *
+ * @par Overview
+ * The allocator is thread-safe and stream-safe and is capable of managing cached
+ * device allocations on multiple devices.  It behaves as follows:
+ *
+ * @par
+ * - Allocations from the allocator are associated with an @p active_stream. Once freed,
+ *   the allocation becomes available immediately for reuse within the @p active_stream
+ *   with which it was associated with during allocation, and it becomes available for
+ *   reuse within other streams when all prior work submitted to @p active_stream has completed.
+ * - Allocations are categorized and cached by bin size. A new allocation request of
+ *   a given size will only consider cached allocations within the corresponding bin.
+ * - Bin limits progress geometrically in accordance with the growth factor
+ *   @p bin_growth provided during construction. Unused device allocations within
+ *   a larger bin cache are not reused for allocation requests that categorize to
+ *   smaller bin sizes.
+ * - Allocation requests below ( @p bin_growth ^ @p min_bin ) are rounded up to
+ *   ( @p bin_growth ^ @p min_bin ).
+ * - Allocations above ( @p bin_growth ^ @p max_bin ) are not rounded up to the nearest
+ *   bin and are simply freed when they are deallocated instead of being returned
+ *   to a bin-cache.
+ * - If the total storage of cached allocations on a given device will exceed
+ *   @p max_cached_bytes, allocations for that device are simply freed when they are
+ *   deallocated instead of being returned to their bin-cache.
+ *
+ * @par
+ * For example, the default-constructed CachingDeviceAllocator is configured with:
+ * - @p bin_growth          = 8
+ * - @p min_bin             = 3
+ * - @p max_bin             = 7
+ * - @p max_cached_bytes    = 6MB - 1B
+ *
+ * @par
+ * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB
+ * and sets a maximum of 6,291,455 cached bytes per device
+ *
+ */
+struct CachingDeviceAllocator
+{
+  //---------------------------------------------------------------------
+  // Constants
+  //---------------------------------------------------------------------
+
+  /// Out-of-bounds bin
+  static constexpr unsigned int INVALID_BIN = (unsigned int) -1;
+
+  /// Invalid size
+  static constexpr size_t INVALID_SIZE = (size_t) -1;
+
+#ifndef _CCCL_DOXYGEN_INVOKED // Do not document
+
+  /// Invalid device ordinal
+  static constexpr int INVALID_DEVICE_ORDINAL = -1;
+
+  //---------------------------------------------------------------------
+  // Type definitions and helper types
+  //---------------------------------------------------------------------
+
+  /**
+   * Descriptor for device memory allocations
+   */
+  struct BlockDescriptor
+  {
+    // Device pointer
+    void* d_ptr;
+
+    // Size of allocation in bytes
+    size_t bytes;
+
+    // Bin enumeration
+    unsigned int bin;
+
+    // device ordinal
+    int device;
+
+    // Associated associated_stream
+    cudaStream_t associated_stream;
+
+    // Signal when associated stream has run to the point at which this block was freed
+    cudaEvent_t ready_event;
+
+    // Constructor (suitable for searching maps for a specific block, given its pointer and
+    // device)
+    BlockDescriptor(void* d_ptr, int device)
+        : d_ptr(d_ptr)
+        , bytes(0)
+        , bin(INVALID_BIN)
+        , device(device)
+        , associated_stream(nullptr)
+        , ready_event(nullptr)
+    {}
+
+    // Constructor (suitable for searching maps for a range of suitable blocks, given a device)
+    BlockDescriptor(int device)
+        : d_ptr(nullptr)
+        , bytes(0)
+        , bin(INVALID_BIN)
+        , device(device)
+        , associated_stream(nullptr)
+        , ready_event(nullptr)
+    {}
+
+    // Comparison functor for comparing device pointers
+    static bool PtrCompare(const BlockDescriptor& a, const BlockDescriptor& b)
+    {
+      if (a.device == b.device)
+      {
+        return (a.d_ptr < b.d_ptr);
+      }
+      else
+      {
+        return (a.device < b.device);
+      }
+    }
+
+    // Comparison functor for comparing allocation sizes
+    static bool SizeCompare(const BlockDescriptor& a, const BlockDescriptor& b)
+    {
+      if (a.device == b.device)
+      {
+        return (a.bytes < b.bytes);
+      }
+      else
+      {
+        return (a.device < b.device);
+      }
+    }
+  };
+
+  /// BlockDescriptor comparator function interface
+  using Compare = bool (*)(const BlockDescriptor&, const BlockDescriptor&);
+
+  class TotalBytes
+  {
+  public:
+    size_t free;
+    size_t live;
+    TotalBytes()
+    {
+      free = live = 0;
+    }
+  };
+
+  /// Set type for cached blocks (ordered by size)
+  using CachedBlocks = std::multiset<BlockDescriptor, Compare>;
+
+  /// Set type for live blocks (ordered by ptr)
+  using BusyBlocks = std::multiset<BlockDescriptor, Compare>;
+
+  /// Map type of device ordinals to the number of cached bytes cached by each device
+  using GpuCachedBytes = std::map<int, TotalBytes>;
+
+  //---------------------------------------------------------------------
+  // Utility functions
+  //---------------------------------------------------------------------
+
+  /**
+   * Integer pow function for unsigned base and exponent
+   */
+  static unsigned int IntPow(unsigned int base, unsigned int exp)
+  {
+    unsigned int retval = 1;
+    while (exp > 0)
+    {
+      if (exp & 1)
+      {
+        retval = retval * base; // multiply the result by the current base
+      }
+      base = base * base; // square the base
+      exp  = exp >> 1; // divide the exponent in half
+    }
+    return retval;
+  }
+
+  /**
+   * Round up to the nearest power-of
+   */
+  void NearestPowerOf(unsigned int& power, size_t& rounded_bytes, unsigned int base, size_t value)
+  {
+    power         = 0;
+    rounded_bytes = 1;
+
+    if (value * base < value)
+    {
+      // Overflow
+      power         = sizeof(size_t) * 8;
+      rounded_bytes = size_t(0) - 1;
+      return;
+    }
+
+    while (rounded_bytes < value)
+    {
+      rounded_bytes *= base;
+      power++;
+    }
+  }
+
+  //---------------------------------------------------------------------
+  // Fields
+  //---------------------------------------------------------------------
+
+  /// Mutex for thread-safety
+  std::mutex mutex;
+
+  /// Geometric growth factor for bin-sizes
+  unsigned int bin_growth;
+
+  /// Minimum bin enumeration
+  unsigned int min_bin;
+
+  /// Maximum bin enumeration
+  unsigned int max_bin;
+
+  /// Minimum bin size
+  size_t min_bin_bytes;
+
+  /// Maximum bin size
+  size_t max_bin_bytes;
+
+  /// Maximum aggregate cached bytes per device
+  size_t max_cached_bytes;
+
+  /// Whether or not to skip a call to FreeAllCached() when destructor is called.
+  /// (The CUDA runtime may have already shut down for statically declared allocators)
+  const bool skip_cleanup;
+
+  /// Whether or not to print (de)allocation events to stdout
+  bool debug;
+
+  /// Map of device ordinal to aggregate cached bytes on that device
+  GpuCachedBytes cached_bytes;
+
+  /// Set of cached device allocations available for reuse
+  CachedBlocks cached_blocks;
+
+  /// Set of live device allocations currently in use
+  BusyBlocks live_blocks;
+
+#endif // _CCCL_DOXYGEN_INVOKED
+
+  //---------------------------------------------------------------------
+  // Methods
+  //---------------------------------------------------------------------
+
+  /**
+   * @brief Constructor.
+   *
+   * @param bin_growth
+   *   Geometric growth factor for bin-sizes
+   *
+   * @param min_bin
+   *   Minimum bin (default is bin_growth ^ 1)
+   *
+   * @param max_bin
+   *   Maximum bin (default is no max bin)
+   *
+   * @param max_cached_bytes
+   *   Maximum aggregate cached bytes per device (default is no limit)
+   *
+   * @param skip_cleanup
+   *   Whether or not to skip a call to @p FreeAllCached() when the destructor is called (default
+   *   is to deallocate)
+   */
+  CachingDeviceAllocator(
+    unsigned int bin_growth,
+    unsigned int min_bin    = 1,
+    unsigned int max_bin    = INVALID_BIN,
+    size_t max_cached_bytes = INVALID_SIZE,
+    bool skip_cleanup       = false)
+      : bin_growth(bin_growth)
+      , min_bin(min_bin)
+      , max_bin(max_bin)
+      , min_bin_bytes(IntPow(bin_growth, min_bin))
+      , max_bin_bytes(IntPow(bin_growth, max_bin))
+      , max_cached_bytes(max_cached_bytes)
+      , skip_cleanup(skip_cleanup)
+      , debug(false)
+      , cached_blocks(BlockDescriptor::SizeCompare)
+      , live_blocks(BlockDescriptor::PtrCompare)
+  {}
+
+  /**
+   * @brief Default constructor.
+   *
+   * Configured with:
+   * @par
+   * - @p bin_growth          = 8
+   * - @p min_bin             = 3
+   * - @p max_bin             = 7
+   * - @p max_cached_bytes    = ( @p bin_growth ^ @p max_bin) * 3 ) - 1 = 6,291,455 bytes
+   *
+   * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB and
+   * sets a maximum of 6,291,455 cached bytes per device
+   */
+  CachingDeviceAllocator(bool skip_cleanup = false, bool debug = false)
+      : bin_growth(8)
+      , min_bin(3)
+      , max_bin(7)
+      , min_bin_bytes(IntPow(bin_growth, min_bin))
+      , max_bin_bytes(IntPow(bin_growth, max_bin))
+      , max_cached_bytes((max_bin_bytes * 3) - 1)
+      , skip_cleanup(skip_cleanup)
+      , debug(debug)
+      , cached_blocks(BlockDescriptor::SizeCompare)
+      , live_blocks(BlockDescriptor::PtrCompare)
+  {}
+
+  /**
+   * @brief Sets the limit on the number bytes this allocator is allowed to cache per device.
+   *
+   * Changing the ceiling of cached bytes does not cause any allocations (in-use or
+   * cached-in-reserve) to be freed.  See \p FreeAllCached().
+   */
+  cudaError_t SetMaxCachedBytes(size_t max_cached_bytes_)
+  {
+    // Lock
+    mutex.lock();
+
+#ifdef CUB_DEBUG_LOG
+    _CubLog(
+      "Changing max_cached_bytes (%lld -> %lld)\n", (long long) this->max_cached_bytes, (long long) max_cached_bytes_);
+#endif
+
+    this->max_cached_bytes = max_cached_bytes_;
+
+    // Unlock
+    mutex.unlock();
+
+    return cudaSuccess;
+  }
+
+  /**
+   * @brief Provides a suitable allocation of device memory for the given size on the specified
+   *        device.
+   *
+   * Once freed, the allocation becomes available immediately for reuse within the @p
+   * active_stream with which it was associated with during allocation, and it becomes available
+   * for reuse within other streams when all prior work submitted to @p active_stream has
+   * completed.
+   *
+   * @param[in] device
+   *   Device on which to place the allocation
+   *
+   * @param[out] d_ptr
+   *   Reference to pointer to the allocation
+   *
+   * @param[in] bytes
+   *   Minimum number of bytes for the allocation
+   *
+   * @param[in] active_stream
+   *   The stream to be associated with this allocation
+   */
+  cudaError_t DeviceAllocate(int device, void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr)
+  {
+    *d_ptr                = nullptr;
+    int entrypoint_device = INVALID_DEVICE_ORDINAL;
+    cudaError_t error     = cudaSuccess;
+
+    if (device == INVALID_DEVICE_ORDINAL)
+    {
+      error = CubDebug(cudaGetDevice(&entrypoint_device));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+
+      device = entrypoint_device;
+    }
+
+    // Create a block descriptor for the requested allocation
+    bool found = false;
+    BlockDescriptor search_key(device);
+    search_key.associated_stream = active_stream;
+    NearestPowerOf(search_key.bin, search_key.bytes, bin_growth, bytes);
+
+    if (search_key.bin > max_bin)
+    {
+      // Bin is greater than our maximum bin: allocate the request
+      // exactly and give out-of-bounds bin.  It will not be cached
+      // for reuse when returned.
+      search_key.bin   = INVALID_BIN;
+      search_key.bytes = bytes;
+    }
+    else
+    {
+      // Search for a suitable cached allocation: lock
+      mutex.lock();
+
+      if (search_key.bin < min_bin)
+      {
+        // Bin is less than minimum bin: round up
+        search_key.bin   = min_bin;
+        search_key.bytes = min_bin_bytes;
+      }
+
+      // Iterate through the range of cached blocks on the same device in the same bin
+      CachedBlocks::iterator block_itr = cached_blocks.lower_bound(search_key);
+      while ((block_itr != cached_blocks.end()) && (block_itr->device == device) && (block_itr->bin == search_key.bin))
+      {
+        // To prevent races with reusing blocks returned by the host but still
+        // in use by the device, only consider cached blocks that are
+        // either (from the active stream) or (from an idle stream)
+        bool is_reusable = false;
+        if (active_stream == block_itr->associated_stream)
+        {
+          is_reusable = true;
+        }
+        else
+        {
+          const cudaError_t event_status = cudaEventQuery(block_itr->ready_event);
+          if (event_status != cudaErrorNotReady)
+          {
+            CubDebug(event_status);
+            is_reusable = true;
+          }
+        }
+
+        if (is_reusable)
+        {
+          // Reuse existing cache block.  Insert into live blocks.
+          found                        = true;
+          search_key                   = *block_itr;
+          search_key.associated_stream = active_stream;
+          live_blocks.insert(search_key);
+
+          // Remove from free blocks
+          cached_bytes[device].free -= search_key.bytes;
+          cached_bytes[device].live += search_key.bytes;
+
+#ifdef CUB_DEBUG_LOG
+          _CubLog("\tDevice %d reused cached block at %p (%lld bytes) for stream %lld (previously associated with "
+                  "stream %lld).\n",
+                  device,
+                  search_key.d_ptr,
+                  (long long) search_key.bytes,
+                  (long long) search_key.associated_stream,
+                  (long long) block_itr->associated_stream);
+#endif
+
+          cached_blocks.erase(block_itr);
+
+          break;
+        }
+        block_itr++;
+      }
+
+      // Done searching: unlock
+      mutex.unlock();
+    }
+
+    // Allocate the block if necessary
+    if (!found)
+    {
+      // Set runtime's current device to specified device (entrypoint may not be set)
+      if (device != entrypoint_device)
+      {
+        error = CubDebug(cudaGetDevice(&entrypoint_device));
+        if (cudaSuccess != error)
+        {
+          return error;
+        }
+
+        error = CubDebug(cudaSetDevice(device));
+        if (cudaSuccess != error)
+        {
+          return error;
+        }
+      }
+
+      // Attempt to allocate
+      error = CubDebug(cudaMalloc(&search_key.d_ptr, search_key.bytes));
+      if (error == cudaErrorMemoryAllocation)
+      {
+        // The allocation attempt failed: free all cached blocks on device and retry
+#ifdef CUB_DEBUG_LOG
+        _CubLog("\tDevice %d failed to allocate %lld bytes for stream %lld, retrying after freeing cached allocations",
+                device,
+                (long long) search_key.bytes,
+                (long long) search_key.associated_stream);
+#endif
+
+        error = cudaSuccess; // Reset the error we will return
+        cudaGetLastError(); // Reset CUDART's error
+
+        // Lock
+        mutex.lock();
+
+        // Iterate the range of free blocks on the same device
+        BlockDescriptor free_key(device);
+        CachedBlocks::iterator block_itr = cached_blocks.lower_bound(free_key);
+
+        while ((block_itr != cached_blocks.end()) && (block_itr->device == device))
+        {
+          // No need to worry about synchronization with the device: cudaFree is
+          // blocking and will synchronize across all kernels executing
+          // on the current device
+
+          // Free device memory and destroy stream event.
+          error = CubDebug(cudaFree(block_itr->d_ptr));
+          if (cudaSuccess != error)
+          {
+            break;
+          }
+
+          error = CubDebug(cudaEventDestroy(block_itr->ready_event));
+          if (cudaSuccess != error)
+          {
+            break;
+          }
+
+          // Reduce balance and erase entry
+          cached_bytes[device].free -= block_itr->bytes;
+
+#ifdef CUB_DEBUG_LOG
+          _CubLog("\tDevice %d freed %lld bytes.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks "
+                  "(%lld bytes) outstanding.\n",
+                  device,
+                  (long long) block_itr->bytes,
+                  (long long) cached_blocks.size(),
+                  (long long) cached_bytes[device].free,
+                  (long long) live_blocks.size(),
+                  (long long) cached_bytes[device].live);
+#endif
+
+          block_itr = cached_blocks.erase(block_itr);
+        }
+
+        // Unlock
+        mutex.unlock();
+
+        // Return under error
+        if (error)
+        {
+          return error;
+        }
+
+        // Try to allocate again
+        error = CubDebug(cudaMalloc(&search_key.d_ptr, search_key.bytes));
+        if (cudaSuccess != error)
+        {
+          return error;
+        }
+      }
+
+      // Create ready event
+      error = CubDebug(cudaEventCreateWithFlags(&search_key.ready_event, cudaEventDisableTiming));
+
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+
+      // Insert into live blocks
+      mutex.lock();
+      live_blocks.insert(search_key);
+      cached_bytes[device].live += search_key.bytes;
+      mutex.unlock();
+
+#ifdef CUB_DEBUG_LOG
+      _CubLog("\tDevice %d allocated new device block at %p (%lld bytes associated with stream %lld).\n",
+              device,
+              search_key.d_ptr,
+              (long long) search_key.bytes,
+              (long long) search_key.associated_stream);
+#endif
+
+      // Attempt to revert back to previous device if necessary
+      if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device))
+      {
+        error = CubDebug(cudaSetDevice(entrypoint_device));
+        if (cudaSuccess != error)
+        {
+          return error;
+        }
+      }
+    }
+
+    // Copy device pointer to output parameter
+    *d_ptr = search_key.d_ptr;
+
+#ifdef CUB_DEBUG_LOG
+    if (debug)
+    {
+      _CubLog("\t\t%lld available blocks cached (%lld bytes), %lld live blocks outstanding(%lld bytes).\n",
+              (long long) cached_blocks.size(),
+              (long long) cached_bytes[device].free,
+              (long long) live_blocks.size(),
+              (long long) cached_bytes[device].live);
+    }
+#endif
+
+    return error;
+  }
+
+  /**
+   * @brief Provides a suitable allocation of device memory for the given size on the current
+   *        device.
+   *
+   * Once freed, the allocation becomes available immediately for reuse within the @p
+   * active_stream with which it was associated with during allocation, and it becomes available
+   * for reuse within other streams when all prior work submitted to @p active_stream has
+   * completed.
+   *
+   * @param[out] d_ptr
+   *   Reference to pointer to the allocation
+   *
+   * @param[in] bytes
+   *   Minimum number of bytes for the allocation
+   *
+   * @param[in] active_stream
+   *   The stream to be associated with this allocation
+   */
+  cudaError_t DeviceAllocate(void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr)
+  {
+    return DeviceAllocate(INVALID_DEVICE_ORDINAL, d_ptr, bytes, active_stream);
+  }
+
+  /**
+   * @brief Frees a live allocation of device memory on the specified device, returning it to the
+   *        allocator.
+   *
+   * Once freed, the allocation becomes available immediately for reuse within the
+   * @p active_stream with which it was associated with during allocation, and it becomes
+   * available for reuse within other streams when all prior work submitted to @p active_stream
+   * has completed.
+   */
+  cudaError_t DeviceFree(int device, void* d_ptr)
+  {
+    int entrypoint_device = INVALID_DEVICE_ORDINAL;
+    cudaError_t error     = cudaSuccess;
+
+    if (device == INVALID_DEVICE_ORDINAL)
+    {
+      error = CubDebug(cudaGetDevice(&entrypoint_device));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+      device = entrypoint_device;
+    }
+
+    // Lock
+    mutex.lock();
+
+    // Find corresponding block descriptor
+    bool recached = false;
+    BlockDescriptor search_key(d_ptr, device);
+    BusyBlocks::iterator block_itr = live_blocks.find(search_key);
+    if (block_itr != live_blocks.end())
+    {
+      // Remove from live blocks
+      search_key = *block_itr;
+      live_blocks.erase(block_itr);
+      cached_bytes[device].live -= search_key.bytes;
+
+      // Keep the returned allocation if bin is valid and we won't exceed the max cached threshold
+      if ((search_key.bin != INVALID_BIN) && (cached_bytes[device].free + search_key.bytes <= max_cached_bytes))
+      {
+        // Insert returned allocation into free blocks
+        recached = true;
+        cached_blocks.insert(search_key);
+        cached_bytes[device].free += search_key.bytes;
+
+#ifdef CUB_DEBUG_LOG
+        _CubLog("\tDevice %d returned %lld bytes from associated stream %lld.\n\t\t %lld available blocks cached (%lld "
+                "bytes), %lld live blocks outstanding. (%lld bytes)\n",
+                device,
+                (long long) search_key.bytes,
+                (long long) search_key.associated_stream,
+                (long long) cached_blocks.size(),
+                (long long) cached_bytes[device].free,
+                (long long) live_blocks.size(),
+                (long long) cached_bytes[device].live);
+#endif
+      }
+    }
+
+    // Unlock
+    mutex.unlock();
+
+    // First set to specified device (entrypoint may not be set)
+    if (device != entrypoint_device)
+    {
+      error = CubDebug(cudaGetDevice(&entrypoint_device));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+
+      error = CubDebug(cudaSetDevice(device));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+    }
+
+    if (recached)
+    {
+      // Insert the ready event in the associated stream (must have current device set properly)
+      error = CubDebug(cudaEventRecord(search_key.ready_event, search_key.associated_stream));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+    }
+
+    if (!recached)
+    {
+      // Free the allocation from the runtime and cleanup the event.
+      error = CubDebug(cudaFree(d_ptr));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+
+      error = CubDebug(cudaEventDestroy(search_key.ready_event));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+
+#ifdef CUB_DEBUG_LOG
+      _CubLog("\tDevice %d freed %lld bytes from associated stream %lld.\n\t\t  %lld available blocks cached (%lld "
+              "bytes), %lld live blocks (%lld bytes) outstanding.\n",
+              device,
+              (long long) search_key.bytes,
+              (long long) search_key.associated_stream,
+              (long long) cached_blocks.size(),
+              (long long) cached_bytes[device].free,
+              (long long) live_blocks.size(),
+              (long long) cached_bytes[device].live);
+#endif
+    }
+
+    // Reset device
+    if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device))
+    {
+      error = CubDebug(cudaSetDevice(entrypoint_device));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+    }
+
+    return error;
+  }
+
+  /**
+   * @brief Frees a live allocation of device memory on the current device, returning it to the
+   *        allocator.
+   *
+   * Once freed, the allocation becomes available immediately for reuse within the @p
+   * active_stream with which it was associated with during allocation, and it becomes available
+   * for reuse within other streams when all prior work submitted to @p active_stream has
+   * completed.
+   */
+  cudaError_t DeviceFree(void* d_ptr)
+  {
+    return DeviceFree(INVALID_DEVICE_ORDINAL, d_ptr);
+  }
+
+  /**
+   * @brief Frees all cached device allocations on all devices
+   */
+  cudaError_t FreeAllCached()
+  {
+    cudaError_t error     = cudaSuccess;
+    int entrypoint_device = INVALID_DEVICE_ORDINAL;
+    int current_device    = INVALID_DEVICE_ORDINAL;
+
+    mutex.lock();
+
+    while (!cached_blocks.empty())
+    {
+      // Get first block
+      CachedBlocks::iterator begin = cached_blocks.begin();
+
+      // Get entry-point device ordinal if necessary
+      if (entrypoint_device == INVALID_DEVICE_ORDINAL)
+      {
+        error = CubDebug(cudaGetDevice(&entrypoint_device));
+        if (cudaSuccess != error)
+        {
+          break;
+        }
+      }
+
+      // Set current device ordinal if necessary
+      if (begin->device != current_device)
+      {
+        error = CubDebug(cudaSetDevice(begin->device));
+        if (cudaSuccess != error)
+        {
+          break;
+        }
+        current_device = begin->device;
+      }
+
+      // Free device memory
+      error = CubDebug(cudaFree(begin->d_ptr));
+      if (cudaSuccess != error)
+      {
+        break;
+      }
+
+      error = CubDebug(cudaEventDestroy(begin->ready_event));
+      if (cudaSuccess != error)
+      {
+        break;
+      }
+
+      // Reduce balance and erase entry
+      const size_t block_bytes = begin->bytes;
+      cached_bytes[current_device].free -= block_bytes;
+      cached_blocks.erase(begin);
+
+#ifdef CUB_DEBUG_LOG
+      _CubLog("\tDevice %d freed %lld bytes.\n\t\t  %lld available blocks cached (%lld bytes), %lld live blocks (%lld "
+              "bytes) outstanding.\n",
+              current_device,
+              (long long) block_bytes,
+              (long long) cached_blocks.size(),
+              (long long) cached_bytes[current_device].free,
+              (long long) live_blocks.size(),
+              (long long) cached_bytes[current_device].live);
+#endif
+    }
+
+    mutex.unlock();
+
+    // Attempt to revert back to entry-point device if necessary
+    if (entrypoint_device != INVALID_DEVICE_ORDINAL)
+    {
+      error = CubDebug(cudaSetDevice(entrypoint_device));
+      if (cudaSuccess != error)
+      {
+        return error;
+      }
+    }
+
+    return error;
+  }
+
+  /**
+   * @brief Destructor
+   */
+  virtual ~CachingDeviceAllocator()
+  {
+    if (!skip_cleanup)
+    {
+      FreeAllCached();
+    }
+  }
+};
+
+CUB_NAMESPACE_END

qwen3_6_scripts/cccl_preload/include/cub/util_arch.cuh

@@ -0,0 +1,219 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/**
+ * \file
+ * Static architectural properties by SM version.
+ */
+
+#pragma once
+
+#include <cub/config.cuh>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cub/util_cpp_dialect.cuh> // IWYU pragma: export
+#include <cub/util_macro.cuh>
+#include <cub/util_namespace.cuh>
+
+#include <cuda/__cmath/ceil_div.h>
+#include <cuda/__cmath/round_up.h>
+#include <cuda/__device/compute_capability.h>
+#include <cuda/std/__algorithm/clamp.h>
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__algorithm/min.h>
+
+// Legacy include; this functionality used to be defined in here.
+#include <cub/detail/detect_cuda_runtime.cuh>
+
+CUB_NAMESPACE_BEGIN
+
+#ifndef _CCCL_DOXYGEN_INVOKED // Do not document
+
+/// In device code, CUB_PTX_ARCH expands to the PTX version for which we are
+/// compiling. In host code, CUB_PTX_ARCH's value is implementation defined.
+#  ifndef CUB_PTX_ARCH
+// deprecated in 3.1
+#    if _CCCL_CUDA_COMPILER(NVHPC)
+// NV_TARGET_MINIMUM_SM_INTEGER is the oldest target PTX version, and is defined when compiling both host code and
+// device code.
+#      define CUB_PTX_ARCH (NV_TARGET_MINIMUM_SM_INTEGER * 10)
+#    else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv
+#      define CUB_PTX_ARCH _CCCL_PTX_ARCH()
+#    endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^
+#  endif
+
+/// Maximum number of devices supported.
+#  ifndef CUB_MAX_DEVICES
+//! Deprecated [Since 3.0]
+#    define CUB_MAX_DEVICES (128)
+#  endif
+static_assert(CUB_MAX_DEVICES > 0, "CUB_MAX_DEVICES must be greater than 0.");
+
+/// Number of threads per warp
+#  ifndef CUB_LOG_WARP_THREADS
+//! Deprecated [Since 3.0]
+#    define CUB_LOG_WARP_THREADS(unused) (5)
+//! Deprecated [Since 3.0]
+#    define CUB_WARP_THREADS(unused) (1 << CUB_LOG_WARP_THREADS(0))
+
+//! Deprecated [Since 3.0]
+#    define CUB_PTX_WARP_THREADS CUB_WARP_THREADS(0)
+//! Deprecated [Since 3.0]
+#    define CUB_PTX_LOG_WARP_THREADS CUB_LOG_WARP_THREADS(0)
+#  endif
+
+/// Number of smem banks
+#  ifndef CUB_LOG_SMEM_BANKS
+//! Deprecated [Since 3.0]
+#    define CUB_LOG_SMEM_BANKS(unused) (5)
+//! Deprecated [Since 3.0]
+#    define CUB_SMEM_BANKS(unused) (1 << CUB_LOG_SMEM_BANKS(0))
+
+//! Deprecated [Since 3.0]
+#    define CUB_PTX_LOG_SMEM_BANKS CUB_LOG_SMEM_BANKS(0)
+//! Deprecated [Since 3.0]
+#    define CUB_PTX_SMEM_BANKS CUB_SMEM_BANKS
+#  endif
+
+/// Oversubscription factor
+#  ifndef CUB_SUBSCRIPTION_FACTOR
+//! Deprecated [Since 3.0]
+#    define CUB_SUBSCRIPTION_FACTOR(unused) (5)
+//! Deprecated [Since 3.0]
+#    define CUB_PTX_SUBSCRIPTION_FACTOR CUB_SUBSCRIPTION_FACTOR(0)
+#  endif
+
+/// Prefer padding overhead vs X-way conflicts greater than this threshold
+#  ifndef CUB_PREFER_CONFLICT_OVER_PADDING
+//! Deprecated [Since 3.0]
+#    define CUB_PREFER_CONFLICT_OVER_PADDING(unused) (1)
+//! Deprecated [Since 3.0]
+#    define CUB_PTX_PREFER_CONFLICT_OVER_PADDING CUB_PREFER_CONFLICT_OVER_PADDING(0)
+#  endif
+
+namespace detail
+{
+inline constexpr int max_devices       = CUB_MAX_DEVICES;
+inline constexpr int warp_threads      = CUB_PTX_WARP_THREADS;
+inline constexpr int log2_warp_threads = CUB_PTX_LOG_WARP_THREADS;
+inline constexpr int smem_banks        = CUB_SMEM_BANKS(0);
+inline constexpr int log2_smem_banks   = CUB_PTX_LOG_SMEM_BANKS;
+
+inline constexpr int subscription_factor           = CUB_PTX_SUBSCRIPTION_FACTOR;
+inline constexpr bool prefer_conflict_over_padding = CUB_PTX_PREFER_CONFLICT_OVER_PADDING;
+
+// The maximum amount of shared memory available per thread block for eternity. Every current and future CUDA
+// architecture has and will have at least this amount of shared memory. This is also the maximum size of total static
+// shared memory in a kernel. Note that dynamic shared memory may be larger than this amount.
+static constexpr ::cuda::std::size_t max_smem_per_block = 48 * 1024;
+
+// The size in bytes of the largest machine word that can be atomically read/written in a single instruction, so we can
+// use it to pass messages from one thread to another using strong loads (acquire) and stores (release).
+inline constexpr int largest_atomic_message_size = 16;
+
+struct scaling_result
+{
+  int items_per_thread;
+  int threads_per_block;
+};
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API inline constexpr auto
+scale_reg_bound(int nominal_4B_threads_per_block, int nominal_4B_items_per_thread, int target_type_size)
+  -> scaling_result
+{
+  const int items_per_thread =
+    (::cuda::std::max) (1, nominal_4B_items_per_thread * 4 / (::cuda::std::max) (4, target_type_size));
+  const int threads_per_block =
+    (::cuda::std::min) (nominal_4B_threads_per_block,
+                        ::cuda::ceil_div(int{max_smem_per_block} / (target_type_size * items_per_thread), 32) * 32);
+  return {items_per_thread, threads_per_block};
+}
+
+template <int Nominal4ByteThreadsPerBlock, int Nominal4ByteItemsPerThread, typename T>
+struct RegBoundScaling
+{
+private:
+  static constexpr auto result =
+    scale_reg_bound(Nominal4ByteThreadsPerBlock, Nominal4ByteItemsPerThread, int{sizeof(T)});
+
+public:
+  static constexpr int ITEMS_PER_THREAD = result.items_per_thread;
+  static constexpr int BLOCK_THREADS    = result.threads_per_block;
+};
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API inline constexpr auto
+scale_mem_bound(int nominal_4B_threads_per_block, int nominal_4B_items_per_thread, int target_type_size)
+  -> scaling_result
+{
+  const int items_per_thread =
+    ::cuda::std::clamp(nominal_4B_items_per_thread * 4 / target_type_size, 1, nominal_4B_items_per_thread * 2);
+  const int threads_per_block =
+    (::cuda::std::min) (nominal_4B_threads_per_block,
+                        ::cuda::round_up(int{max_smem_per_block} / (target_type_size * items_per_thread), 32));
+  return {items_per_thread, threads_per_block};
+}
+
+template <int Nominal4ByteThreadsPerBlock, int Nominal4ByteItemsPerThread, typename T>
+struct MemBoundScaling
+{
+private:
+  static constexpr auto result =
+    scale_mem_bound(Nominal4ByteThreadsPerBlock, Nominal4ByteItemsPerThread, int{sizeof(T)});
+
+public:
+  static constexpr int ITEMS_PER_THREAD = result.items_per_thread;
+  static constexpr int BLOCK_THREADS    = result.threads_per_block;
+};
+
+template <int Nominal4ByteThreadsPerBlock, int Nominal4ByteItemsPerThread, typename = void>
+struct NoScaling
+{
+  static constexpr int ITEMS_PER_THREAD = Nominal4ByteItemsPerThread;
+  static constexpr int BLOCK_THREADS    = Nominal4ByteThreadsPerBlock;
+};
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::compute_capability current_tuning_cc() noexcept
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC)
+  return ::cuda::compute_capability(NV_TARGET_MINIMUM_SM_INTEGER);
+#  elif _CCCL_DEVICE_COMPILATION()
+  return ::cuda::device::current_compute_capability();
+#  else
+  // clang 22+ supports __CUDA_ARCH_LIST__ and also instantiates tuning policies inside kernels during the **host**
+  // pass (e.g. to compute the value for __launch_bounds__), where we rely on current_tuning_cc(), which is then passed
+  // to the policy selector. In the rare case that the policy selector is an adapter over a policy hub and invokes
+  // ChainedPolicy (e.g. test cub.test.device.histogram_custom_policy_hub.lid_0), it will fail to compile during
+  // constant evaluation, since it cannot find a policy for a PTX version of zero. As a workaround, we return the oldest
+  // CC we are compiling for during the host pass. And for consistency, we do the same for all compilers.
+#    if _CCCL_CUDA_COMPILER(CLANG)
+  return ::cuda::__target_compute_capabilities().front();
+#    else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv
+  return {};
+#    endif // ^^^ !_CCCL_CUDA_COMPILER(CLANG) ^^^
+#  endif
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class PolicySelector>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto select_policy(::cuda::compute_capability cc)
+{
+  return PolicySelector{}(cc);
+}
+
+template <class PolicySelector>
+[[nodiscard]] _CCCL_DEVICE_API constexpr auto current_policy()
+{
+  return select_policy<PolicySelector>(current_tuning_cc());
+}
+} // namespace detail
+#endif // Do not document
+
+CUB_NAMESPACE_END

qwen3_6_scripts/cccl_preload/include/cub/util_cpp_dialect.cuh

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+//! @file
+//! Detect the version of the C++ standard used by the compiler.
+
+#pragma once
+
+#include <cuda/__cccl_config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#ifndef _CCCL_DOXYGEN_INVOKED // Do not document
+
+// Deprecation warnings may be silenced by defining the following macros. These
+// may be combined.
+// - CCCL_IGNORE_DEPRECATED_COMPILER
+//   Ignore deprecation warnings when using deprecated compilers. Compiling
+//   with deprecated C++ dialects will still issue warnings.
+
+//! Deprecated [Since 3.0]
+#  define CUB_CPP_DIALECT _CCCL_STD_VER
+
+// Define CUB_COMPILER_DEPRECATION macro:
+#  if _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
+#    define CUB_COMP_DEPR_IMPL(msg) _CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " #msg))
+#  else // clang / gcc:
+#    define CUB_COMP_DEPR_IMPL(msg) _CCCL_PRAGMA(GCC warning #msg)
+#  endif
+
+// Compiler checks:
+// clang-format off
+#  define CUB_COMPILER_DEPRECATION(REQ) \
+    CUB_COMP_DEPR_IMPL(CUB requires at least REQ. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message.)
+
+#  define CUB_COMPILER_DEPRECATION_SOFT(REQ, CUR)                                                        \
+    CUB_COMP_DEPR_IMPL(                                                                                  \
+      CUB requires at least REQ. CUR is deprecated but still supported. CUR support will be removed in a \
+        future release. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message.)
+// clang-format on
+
+#  ifndef CCCL_IGNORE_DEPRECATED_COMPILER
+#    if _CCCL_COMPILER(GCC, <, 7)
+CUB_COMPILER_DEPRECATION(GCC 7.0);
+#    elif _CCCL_COMPILER(CLANG, <, 7)
+CUB_COMPILER_DEPRECATION(Clang 7.0);
+#    elif _CCCL_COMPILER(MSVC, <, 19, 10)
+// <2017. Hard upgrade message:
+CUB_COMPILER_DEPRECATION(MSVC 2019(19.20 / 16.0 / 14.20));
+#    endif
+#  endif // CCCL_IGNORE_DEPRECATED_COMPILER
+
+#  undef CUB_COMPILER_DEPRECATION_SOFT
+#  undef CUB_COMPILER_DEPRECATION
+
+// C++17 dialect check:
+#  ifndef CCCL_IGNORE_DEPRECATED_CPP_DIALECT
+#    if _CCCL_STD_VER < 2017
+#      error CUB requires at least C++17. Define CCCL_IGNORE_DEPRECATED_CPP_DIALECT to suppress this message.
+#    endif // _CCCL_STD_VER < 2017
+#  endif
+
+#  undef CUB_COMP_DEPR_IMPL
+
+#endif // !_CCCL_DOXYGEN_INVOKED

qwen3_6_scripts/cccl_preload/include/cub/util_debug.cuh

@@ -0,0 +1,187 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/**
+ * \file
+ * Error and event logging routines.
+ *
+ * The following macros definitions are supported:
+ * - \p CUB_LOG.  Simple event messages are printed to \p stdout.
+ */
+
+#pragma once
+
+#include <cub/config.cuh>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <nv/target>
+
+#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes:
+
+/**
+ * @def CUB_DEBUG_LOG
+ *
+ * Causes kernel launch configurations to be printed to the console
+ */
+#  define CUB_DEBUG_LOG
+
+/**
+ * @def CUB_DEBUG_SYNC
+ *
+ * Causes synchronization of the stream after every kernel launch to check
+ * for errors. Also causes kernel launch configurations to be printed to the
+ * console.
+ */
+#  define CUB_DEBUG_SYNC
+
+/**
+ * @def CUB_DEBUG_ALL
+ *
+ * Causes host and device-side precondition assertions to be checked. Apart
+ * from that, causes synchronization of the stream after every kernel launch to
+ * check for errors. Also causes kernel launch configurations to be printed to
+ * the console.
+ */
+#  define CUB_DEBUG_ALL
+
+#endif // _CCCL_DOXYGEN_INVOKED
+
+// CUB_DEBUG_SYNC also enables CUB_DEBUG_LOG
+#ifdef CUB_DEBUG_SYNC
+#  ifndef CUB_DEBUG_LOG
+#    define CUB_DEBUG_LOG
+#  endif
+#endif
+
+// CUB_DEBUG_ALL = CUB_DEBUG_LOG + CUB_DEBUG_SYNC
+#ifdef CUB_DEBUG_ALL
+#  ifndef CUB_DEBUG_LOG
+#    define CUB_DEBUG_LOG
+#  endif // CUB_DEBUG_LOG
+#  ifndef CUB_DEBUG_SYNC
+#    define CUB_DEBUG_SYNC
+#  endif // CUB_DEBUG_SYNC
+#endif // CUB_DEBUG_ALL
+
+/// CUB error reporting macro (prints error messages to stderr)
+#if (defined(DEBUG) || defined(_DEBUG)) && !defined(CUB_STDERR)
+#  define CUB_STDERR
+#endif
+
+#if defined(CUB_STDERR) || defined(CUB_DEBUG_LOG)
+#  include <cuda/std/__host_stdlib/cstdio>
+#endif
+
+CUB_NAMESPACE_BEGIN
+
+/**
+ * \brief %If \p CUB_STDERR is defined and \p error is not \p cudaSuccess, the
+ * corresponding error message is printed to \p stderr (or \p stdout in device
+ * code) along with the supplied source context.
+ *
+ * \return The CUDA error.
+ */
+_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t
+Debug(cudaError_t error, [[maybe_unused]] const char* filename, [[maybe_unused]] int line)
+{
+  // Clear the global CUDA error state which may have been set by the last
+  // call. Otherwise, errors may "leak" to unrelated kernel launches.
+
+  // clang-format off
+  #ifndef CUB_RDC_ENABLED
+  #define CUB_TEMP_DEVICE_CODE
+  #else
+  #define CUB_TEMP_DEVICE_CODE last_error = cudaGetLastError()
+  #endif
+
+  cudaError_t last_error = cudaSuccess;
+
+  NV_IF_ELSE_TARGET(
+    NV_IS_HOST,
+    (last_error = cudaGetLastError();),
+    (CUB_TEMP_DEVICE_CODE;)
+  );
+
+  #undef CUB_TEMP_DEVICE_CODE
+  // clang-format on
+
+  if (error == cudaSuccess && last_error != cudaSuccess)
+  {
+    error = last_error;
+  }
+
+#ifdef CUB_STDERR
+  if (error)
+  {
+    NV_IF_ELSE_TARGET(
+      NV_IS_HOST,
+      (fprintf(stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error));
+       fflush(stderr);),
+      (printf("CUDA error %d [block (%d,%d,%d) thread (%d,%d,%d), %s, %d]\n",
+              error,
+              blockIdx.z,
+              blockIdx.y,
+              blockIdx.x,
+              threadIdx.z,
+              threadIdx.y,
+              threadIdx.x,
+              filename,
+              line);));
+  }
+#endif
+
+  return error;
+}
+
+/**
+ * \brief Debug macro
+ */
+#ifndef CubDebug
+#  define CubDebug(e) CUB_NS_QUALIFIER::Debug((cudaError_t) (e), __FILE__, __LINE__)
+#endif
+
+/**
+ * \brief Debug macro with exit
+ */
+#ifndef CubDebugExit
+#  define CubDebugExit(e)                                               \
+    if (CUB_NS_QUALIFIER::Debug((cudaError_t) (e), __FILE__, __LINE__)) \
+    {                                                                   \
+      exit(1);                                                          \
+    }
+#endif
+
+/**
+ * \brief Log macro for printf statements.
+ */
+#if !defined(_CubLog)
+#  if _CCCL_HOSTJIT()
+#    define _CubLog(format, ...) (void(0))
+#  else // ^^^ _CCCL_HOSTJIT() ^^^ / vvv !_CCCL_HOSTJIT() vvv
+#    define _CubLog(format, ...)                                    \
+      do                                                            \
+      {                                                             \
+        NV_IF_ELSE_TARGET(                                          \
+          NV_IS_HOST,                                               \
+          (printf(format, __VA_ARGS__);),                           \
+          (printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, \
+                  blockIdx.z,                                       \
+                  blockIdx.y,                                       \
+                  blockIdx.x,                                       \
+                  threadIdx.z,                                      \
+                  threadIdx.y,                                      \
+                  threadIdx.x,                                      \
+                  __VA_ARGS__);));                                  \
+      } while (false)
+#  endif // !_CCCL_HOSTJIT()
+#endif // !defined(_CubLog)
+
+CUB_NAMESPACE_END

qwen3_6_scripts/cccl_preload/include/cub/util_macro.cuh

@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/******************************************************************************
+ * Common C/C++ macro utilities
+ ******************************************************************************/
+
+#pragma once
+
+#include <cuda/__cccl_config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cub/detail/detect_cuda_runtime.cuh> // IWYU pragma: export
+#include <cub/util_namespace.cuh> // IWYU pragma: export
+
+CUB_NAMESPACE_BEGIN
+
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION
+#endif
+
+/**
+ * @def CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION
+ * If defined, the default suppression of kernel visibility attribute warning is disabled.
+ */
+#if !defined(CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION)
+_CCCL_DIAG_SUPPRESS_GCC("-Wattributes")
+_CCCL_DIAG_SUPPRESS_CLANG("-Wattributes")
+#  if !_CCCL_CUDA_COMPILER(NVHPC)
+_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage)
+#  endif // !_CCCL_CUDA_COMPILER(NVHPC)
+#endif // !CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION
+
+#ifndef CUB_DEFINE_KERNEL_GETTER
+#  define CUB_DEFINE_KERNEL_GETTER(name, ...)                                               \
+    _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr decltype(&__VA_ARGS__) name() \
+    {                                                                                       \
+      return &__VA_ARGS__;                                                                  \
+    }
+#endif
+
+// TODO(bgruber): drop in CCCL 4.0 when we drop the public dispatchers
+#ifndef CUB_DEFINE_SUB_POLICY_GETTER
+#  define CUB_DEFINE_SUB_POLICY_GETTER(name)                            \
+    _CCCL_HOST_DEVICE static constexpr auto name()                      \
+    {                                                                   \
+      return MakePolicyWrapper(typename StaticPolicyT::name##Policy()); \
+    }
+#endif
+
+#if defined(CUB_DEFINE_RUNTIME_POLICIES)
+#  define CUB_DETAIL_STATIC_ISH_ASSERT(expr, msg) _CCCL_ASSERT(expr, msg)
+#  define CUB_DETAIL_CONSTEXPR_ISH
+#else // ^^^ CUB_DEFINE_RUNTIME_POLICIES ^^^ / vvv !CUB_DEFINE_RUNTIME_POLICIES vvv
+#  define CUB_DETAIL_STATIC_ISH_ASSERT(expr, msg) static_assert(expr, msg);
+#  define CUB_DETAIL_CONSTEXPR_ISH                constexpr
+#endif // !(CUB_DEFINE_RUNTIME_POLICIES)
+
+CUB_NAMESPACE_END

qwen3_6_scripts/cccl_preload/include/cub/util_namespace.cuh

@@ -0,0 +1,172 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/**
+ * \file util_namespace.cuh
+ * \brief Utilities that allow `cub::` to be placed inside an
+ * application-specific namespace.
+ */
+
+#pragma once
+
+// This is not used by this file; this is a hack so that we can detect the
+// CUB version from Thrust on older versions of CUB that did not have
+// version.cuh.
+#include <cub/version.cuh>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cub/detail/detect_cuda_runtime.cuh>
+
+// Prior to 1.13.1, only the PREFIX/POSTFIX macros were used. Notify users
+// that they must now define the qualifier macro, too.
+#if (defined(CUB_NS_PREFIX) || defined(CUB_NS_POSTFIX)) && !defined(CUB_NS_QUALIFIER)
+#  error CUB requires a definition of CUB_NS_QUALIFIER when CUB_NS_PREFIX/POSTFIX are defined.
+#endif
+
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define THRUST_CUB_WRAPPED_NAMESPACE
+#endif
+
+/**
+ * \def THRUST_CUB_WRAPPED_NAMESPACE
+ * If defined, this value will be used as the name of a namespace that wraps the
+ * `thrust::` and `cub::` namespaces.
+ * This macro should not be used with any other CUB namespace macros.
+ */
+#ifdef THRUST_CUB_WRAPPED_NAMESPACE
+#  define CUB_WRAPPED_NAMESPACE THRUST_CUB_WRAPPED_NAMESPACE
+#endif
+
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CUB_WRAPPED_NAMESPACE
+#endif
+
+/**
+ * \def CUB_WRAPPED_NAMESPACE
+ * If defined, this value will be used as the name of a namespace that wraps the
+ * `cub::` namespace.
+ * If THRUST_CUB_WRAPPED_NAMESPACE is set, this will inherit that macro's value.
+ * This macro should not be used with any other CUB namespace macros.
+ */
+#ifdef CUB_WRAPPED_NAMESPACE
+#  define CUB_NS_PREFIX             \
+    namespace CUB_WRAPPED_NAMESPACE \
+    {
+#  define CUB_NS_POSTFIX }
+
+#  define CUB_NS_QUALIFIER ::CUB_WRAPPED_NAMESPACE::cub
+#endif
+
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CUB_NS_PREFIX
+#endif
+
+/**
+ * \def CUB_NS_PREFIX
+ * This macro is inserted prior to all `namespace cub { ... }` blocks. It is
+ * derived from CUB_WRAPPED_NAMESPACE, if set, and will be empty otherwise.
+ * It may be defined by users, in which case CUB_NS_PREFIX,
+ * CUB_NS_POSTFIX, and CUB_NS_QUALIFIER must all be set consistently.
+ */
+#ifndef CUB_NS_PREFIX
+#  define CUB_NS_PREFIX
+#endif
+
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CUB_NS_POSTFIX
+#endif
+
+/**
+ * \def CUB_NS_POSTFIX
+ * This macro is inserted following the closing braces of all
+ * `namespace cub { ... }` block. It is defined appropriately when
+ * CUB_WRAPPED_NAMESPACE is set, and will be empty otherwise. It may be
+ * defined by users, in which case CUB_NS_PREFIX, CUB_NS_POSTFIX, and
+ * CUB_NS_QUALIFIER must all be set consistently.
+ */
+#ifndef CUB_NS_POSTFIX
+#  define CUB_NS_POSTFIX
+#endif
+
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CUB_NS_QUALIFIER
+#endif
+
+/**
+ * \def CUB_NS_QUALIFIER
+ * This macro is used to qualify members of cub:: when accessing them from
+ * outside of their namespace. By default, this is just `::cub`, and will be
+ * set appropriately when CUB_WRAPPED_NAMESPACE is defined. This macro may be
+ * defined by users, in which case CUB_NS_PREFIX, CUB_NS_POSTFIX, and
+ * CUB_NS_QUALIFIER must all be set consistently.
+ */
+#ifndef CUB_NS_QUALIFIER
+#  define CUB_NS_QUALIFIER ::cub
+#endif
+
+#if defined(CUB_DISABLE_NAMESPACE_MAGIC) || defined(CUB_WRAPPED_NAMESPACE)
+#  if !defined(CUB_WRAPPED_NAMESPACE)
+#    if !defined(CUB_IGNORE_NAMESPACE_MAGIC_ERROR)
+#      error "Disabling namespace magic is unsafe without wrapping namespace"
+#    endif // !defined(CUB_IGNORE_NAMESPACE_MAGIC_ERROR)
+#  endif // !defined(CUB_WRAPPED_NAMESPACE)
+#  define CUB_DETAIL_MAGIC_NS_BEGIN
+#  define CUB_DETAIL_MAGIC_NS_END
+#else // not defined(CUB_DISABLE_NAMESPACE_MAGIC)
+#  if defined(_NVHPC_CUDA)
+#    define CUB_DETAIL_MAGIC_NS_BEGIN                                                                                  \
+      inline namespace _CCCL_PP_CAT(                                                                                   \
+        _CCCL_PP_CAT(_CCCL_PP_CAT(_V_, CUB_VERSION), _CCCL_PP_SPLICE_WITH(_, _SM, NV_TARGET_SM_INTEGER_LIST)), _NVHPC) \
+      {
+#    define CUB_DETAIL_MAGIC_NS_END }
+#  else // not defined(_NVHPC_CUDA)
+#    define CUB_DETAIL_MAGIC_NS_BEGIN                                                                                 \
+      inline namespace _CCCL_PP_CAT(_CCCL_PP_CAT(_V_, CUB_VERSION), _CCCL_PP_SPLICE_WITH(_, _SM, __CUDA_ARCH_LIST__)) \
+      {
+#    define CUB_DETAIL_MAGIC_NS_END }
+#  endif // not defined(_NVHPC_CUDA)
+#endif // not defined(CUB_DISABLE_NAMESPACE_MAGIC)
+
+/**
+ * \def CUB_NAMESPACE_BEGIN
+ * This macro is used to open a `cub::` namespace block, along with any
+ * enclosing namespaces requested by CUB_WRAPPED_NAMESPACE, etc.
+ * This macro is defined by CUB and may not be overridden.
+ */
+#define CUB_NAMESPACE_BEGIN \
+  CUB_NS_PREFIX             \
+  namespace cub             \
+  {                         \
+  CUB_DETAIL_MAGIC_NS_BEGIN
+
+/**
+ * \def CUB_NAMESPACE_END
+ * This macro is used to close a `cub::` namespace block, along with any
+ * enclosing namespaces requested by CUB_WRAPPED_NAMESPACE, etc.
+ * This macro is defined by CUB and may not be overridden.
+ */
+#define CUB_NAMESPACE_END   \
+  CUB_DETAIL_MAGIC_NS_END   \
+  } /* end namespace cub */ \
+  CUB_NS_POSTFIX
+
+// Declare these namespaces here for the purpose of Doxygenating them
+CUB_NS_PREFIX
+
+/*! \namespace cub
+ *  \brief \p cub is the top-level namespace which contains all CUB
+ *         functions and types.
+ */
+namespace cub
+{
+}
+
+CUB_NS_POSTFIX

qwen3_6_scripts/cccl_preload/include/cub/version.cuh

@@ -0,0 +1,65 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+/*! \file version.cuh
+ *  \brief Compile-time macros encoding CUB release version
+ *
+ *         <cub/version.h> is the only CUB header that is guaranteed to
+ *         change with every CUB release.
+ *
+ */
+
+#pragma once
+
+// For _CCCL_IMPLICIT_SYSTEM_HEADER
+#include <cuda/__cccl_config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/version>
+
+/*! \def CUB_VERSION
+ *  \brief The preprocessor macro \p CUB_VERSION encodes the version
+ *         number of the CUB library as MMMmmmpp.
+ *
+ *  \note CUB_VERSION is formatted as `MMMmmmpp`, which differs from `CCCL_VERSION` that uses `MMMmmmppp`.
+ *
+ *         <tt>CUB_VERSION % 100</tt> is the sub-minor version.
+ *         <tt>CUB_VERSION / 100 % 1000</tt> is the minor version.
+ *         <tt>CUB_VERSION / 100000</tt> is the major version.
+ */
+#define CUB_VERSION 300500 // macro expansion with ## requires this to be a single value
+
+/*! \def CUB_MAJOR_VERSION
+ *  \brief The preprocessor macro \p CUB_MAJOR_VERSION encodes the
+ *         major version number of the CUB library.
+ */
+#define CUB_MAJOR_VERSION (CUB_VERSION / 100000)
+
+/*! \def CUB_MINOR_VERSION
+ *  \brief The preprocessor macro \p CUB_MINOR_VERSION encodes the
+ *         minor version number of the CUB library.
+ */
+#define CUB_MINOR_VERSION (CUB_VERSION / 100 % 1000)
+
+/*! \def CUB_SUBMINOR_VERSION
+ *  \brief The preprocessor macro \p CUB_SUBMINOR_VERSION encodes the
+ *         sub-minor version number of the CUB library.
+ */
+#define CUB_SUBMINOR_VERSION (CUB_VERSION % 100)
+
+/*! \def CUB_PATCH_NUMBER
+ *  \brief The preprocessor macro \p CUB_PATCH_NUMBER encodes the
+ *         patch number of the CUB library.
+ */
+#define CUB_PATCH_NUMBER 0
+
+static_assert(CUB_MAJOR_VERSION == CCCL_MAJOR_VERSION);
+static_assert(CUB_MINOR_VERSION == CCCL_MINOR_VERSION);
+static_assert(CUB_SUBMINOR_VERSION == CCCL_PATCH_VERSION);

qwen3_6_scripts/cccl_preload/include/cuda/__cccl_config

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA__CCCL_CONFIG
+#define _CUDA__CCCL_CONFIG
+
+#include <cuda/std/__cccl/architecture.h> // IWYU pragma: export
+#include <cuda/std/__cccl/assert.h> // IWYU pragma: export
+#include <cuda/std/__cccl/attributes.h> // IWYU pragma: export
+#include <cuda/std/__cccl/builtin.h> // IWYU pragma: export
+#include <cuda/std/__cccl/compiler.h> // IWYU pragma: export
+#include <cuda/std/__cccl/cuda_capabilities.h> // IWYU pragma: export
+#include <cuda/std/__cccl/cuda_toolkit.h> // IWYU pragma: export
+#include <cuda/std/__cccl/deprecated.h> // IWYU pragma: export
+#include <cuda/std/__cccl/diagnostic.h> // IWYU pragma: export
+#include <cuda/std/__cccl/dialect.h> // IWYU pragma: export
+#include <cuda/std/__cccl/exceptions.h> // IWYU pragma: export
+#include <cuda/std/__cccl/execution_space.h> // IWYU pragma: export
+#include <cuda/std/__cccl/extended_data_types.h> // IWYU pragma: export
+#include <cuda/std/__cccl/host_std_lib.h> // IWYU pragma: export
+#include <cuda/std/__cccl/os.h> // IWYU pragma: export
+#include <cuda/std/__cccl/preprocessor.h> // IWYU pragma: export
+#include <cuda/std/__cccl/ptx_isa.h> // IWYU pragma: export
+#include <cuda/std/__cccl/rtti.h> // IWYU pragma: export
+#include <cuda/std/__cccl/sequence_access.h> // IWYU pragma: export
+#include <cuda/std/__cccl/system_header.h> // IWYU pragma: export
+#include <cuda/std/__cccl/unreachable.h> // IWYU pragma: export
+#include <cuda/std/__cccl/version.h> // IWYU pragma: export
+#include <cuda/std/__cccl/visibility.h> // IWYU pragma: export
+
+#endif // _CUDA__CCCL_CONFIG

qwen3_6_scripts/cccl_preload/include/cuda/__cmath/ceil_div.h

@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___CMATH_CEIL_DIV_H
+#define _CUDA___CMATH_CEIL_DIV_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/min.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/is_enum.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/make_unsigned.h>
+#include <cuda/std/__type_traits/underlying_type.h>
+#include <cuda/std/__utility/to_underlying.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder
+//! @param __a The dividend
+//! @param __b The divisor
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>)
+[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, _Up> ceil_div(const _Tp __a, const _Up __b) noexcept
+{
+  _CCCL_ASSERT(__b > _Up{0}, "cuda::ceil_div: 'b' must be positive");
+  if constexpr (::cuda::std::is_signed_v<_Tp>)
+  {
+    _CCCL_ASSERT(__a >= _Tp{0}, "cuda::ceil_div: 'a' must be non negative");
+  }
+  using _Common = ::cuda::std::common_type_t<_Tp, _Up>;
+  using _Prom   = decltype(_Tp{} / _Up{});
+  using _UProm  = ::cuda::std::make_unsigned_t<_Prom>;
+  auto __a1     = static_cast<_UProm>(__a);
+  auto __b1     = static_cast<_UProm>(__b);
+  if constexpr (::cuda::std::is_signed_v<_Prom>)
+  {
+    return static_cast<_Common>((__a1 + __b1 - 1) / __b1);
+  }
+  else
+  {
+    _CCCL_IF_CONSTEVAL_DEFAULT
+    {
+      const auto __res = __a1 / __b1;
+      return static_cast<_Common>(__res + (__res * __b1 != __a1));
+    }
+    else
+    {
+      // the ::min method is faster even if __b is a compile-time constant
+      NV_IF_ELSE_TARGET(NV_IS_DEVICE,
+                        (return static_cast<_Common>(::cuda::std::min(__a1, 1 + ((__a1 - 1) / __b1)));),
+                        (const auto __res = __a1 / __b1; //
+                         return static_cast<_Common>(__res + (__res * __b1 != __a1));))
+    }
+  }
+}
+
+//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder, \p __b is an enum
+//! @param __a The dividend
+//! @param __b The divisor
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>)
+[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, ::cuda::std::underlying_type_t<_Up>>
+ceil_div(const _Tp __a, const _Up __b) noexcept
+{
+  return ::cuda::ceil_div(__a, ::cuda::std::to_underlying(__b));
+}
+
+//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder, \p __b is an enum
+//! @param __a The dividend
+//! @param __b The divisor
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>)
+[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, _Up>
+ceil_div(const _Tp __a, const _Up __b) noexcept
+{
+  return ::cuda::ceil_div(::cuda::std::to_underlying(__a), __b);
+}
+
+//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder, \p __b is an enum
+//! @param __a The dividend
+//! @param __b The divisor
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>)
+[[nodiscard]]
+_CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, ::cuda::std::underlying_type_t<_Up>>
+ceil_div(const _Tp __a, const _Up __b) noexcept
+{
+  return ::cuda::ceil_div(::cuda::std::to_underlying(__a), ::cuda::std::to_underlying(__b));
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___CMATH_CEIL_DIV_H

qwen3_6_scripts/cccl_preload/include/cuda/__cmath/round_up.h

@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___CMATH_ROUND_UP_H
+#define _CUDA___CMATH_ROUND_UP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/ceil_div.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/is_enum.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/make_unsigned.h>
+#include <cuda/std/__utility/to_underlying.h>
+#include <cuda/std/limits>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Round the number \p __a to the next multiple of \p __b
+//! @param __a The input number
+//! @param __b The multiplicand
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>)
+[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, _Up> round_up(const _Tp __a, const _Up __b) noexcept
+{
+  _CCCL_ASSERT(__b > _Up{0}, "cuda::round_up: 'b' must be positive");
+  if constexpr (::cuda::std::is_signed_v<_Tp>)
+  {
+    _CCCL_ASSERT(__a >= _Tp{0}, "cuda::round_up: 'a' must be non negative");
+  }
+  using _Common = ::cuda::std::common_type_t<_Tp, _Up>;
+  using _Prom   = decltype(_Tp{} / _Up{});
+  auto __c      = ::cuda::ceil_div(static_cast<_Prom>(__a), static_cast<_Prom>(__b));
+  _CCCL_ASSERT(static_cast<_Common>(__c) <= ::cuda::std::numeric_limits<_Common>::max() / static_cast<_Common>(__b),
+               "cuda::round_up: result overflow");
+  return static_cast<_Common>(static_cast<_Prom>(__c) * static_cast<_Prom>(__b));
+}
+
+//! @brief Round the number \p __a to the next multiple of \p __b
+//! @param __a The input number
+//! @param __b The multiplicand
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>)
+[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, ::cuda::std::underlying_type_t<_Up>>
+round_up(const _Tp __a, const _Up __b) noexcept
+{
+  return ::cuda::round_up(__a, ::cuda::std::to_underlying(__b));
+}
+
+//! @brief Round the number \p __a to the next multiple of \p __b
+//! @param __a The input number
+//! @param __b The multiplicand
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>)
+[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, _Up>
+round_up(const _Tp __a, const _Up __b) noexcept
+{
+  return ::cuda::round_up(::cuda::std::to_underlying(__a), __b);
+}
+
+//! @brief Round the number \p __a to the next multiple of \p __b
+//! @param __a The input number
+//! @param __b The multiplicand
+//! @pre \p __a must be non-negative
+//! @pre \p __b must be positive
+_CCCL_TEMPLATE(class _Tp, class _Up)
+_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>)
+[[nodiscard]]
+_CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, ::cuda::std::underlying_type_t<_Up>>
+round_up(const _Tp __a, const _Up __b) noexcept
+{
+  return ::cuda::round_up(::cuda::std::to_underlying(__a), ::cuda::std::to_underlying(__b));
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___CMATH_ROUND_UP_H

qwen3_6_scripts/cccl_preload/include/cuda/__device/compute_capability.h

@@ -0,0 +1,272 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___DEVICE_COMPUTE_CAPABILITY_H
+#define _CUDA___DEVICE_COMPUTE_CAPABILITY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__fwd/devices.h>
+#include <cuda/std/__fwd/format.h>
+#include <cuda/std/__type_traits/always_false.h>
+#include <cuda/std/__utility/to_underlying.h>
+#include <cuda/std/array>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Type representing the CUDA compute capability.
+class compute_capability
+{
+public:
+  int __cc_{}; //!< The stored compute capability in format 10 * major + minor.
+
+  _CCCL_HIDE_FROM_ABI constexpr compute_capability() noexcept = default;
+
+  //! @brief Constructs the object from compute capability \c __cc. The expected format is 10 * major + minor.
+  //!
+  //! @param __cc Compute capability.
+  _CCCL_HOST_DEVICE_API explicit constexpr compute_capability(int __cc) noexcept
+      : __cc_{__cc}
+  {}
+
+  //! @brief Constructs the object by combining the \c __major and \c __minor compute capability.
+  //!
+  //! @param __major The major compute capability.
+  //! @param __minor The minor compute capability. Must be less than 10.
+  _CCCL_HOST_DEVICE_API constexpr compute_capability(int __major, int __minor) noexcept
+      : __cc_{10 * __major + __minor}
+  {
+    _CCCL_ASSERT(__minor < 10, "invalid minor compute capability");
+  }
+
+  //! @brief Constructs the object from the architecture id.
+  //!
+  //! @param __arch_id The architecture id.
+  _CCCL_HOST_DEVICE_API explicit constexpr compute_capability(arch_id __arch_id) noexcept
+  {
+    const auto __val = ::cuda::std::to_underlying(__arch_id);
+    if (__val > __arch_specific_id_multiplier)
+    {
+      __cc_ = __val / __arch_specific_id_multiplier;
+    }
+    else
+    {
+      __cc_ = __val;
+    }
+  }
+
+  _CCCL_HIDE_FROM_ABI constexpr compute_capability(const compute_capability&) noexcept = default;
+
+  _CCCL_HIDE_FROM_ABI constexpr compute_capability& operator=(const compute_capability& __other) noexcept = default;
+
+  //! @brief Gets the stored compute capability.
+  //!
+  //! @return The stored compute capability in format 10 * major + minor.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int get() const noexcept
+  {
+    return __cc_;
+  }
+
+  //! @brief Gets the major compute capability.
+  //!
+  //! @return Major compute capability.
+  //!
+  //! @deprecated This symbol is deprecated because it collides with major(...) macro defined in <sys/sysmacros.h> and
+  //! will be removed in next major release. Use cc.major_cap() instead.
+  [[nodiscard]]
+  CCCL_DEPRECATED_BECAUSE("This symbol is deprecated because it collides with major(...) macro defined in "
+                          "<sys/sysmacros.h> and will be removed in next major release. Use cc.major_cap() instead.")
+  _CCCL_HOST_DEVICE_API constexpr int major() const noexcept
+  {
+    return major_cap();
+  }
+
+  //! @brief Gets the major compute capability.
+  //!
+  //! @return Major compute capability.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int major_cap() const noexcept
+  {
+    return __cc_ / 10;
+  }
+
+  //! @brief Gets the minor compute capability.
+  //!
+  //! @return Minor compute capability. The value is always less than 10.
+  //!
+  //! @deprecated This symbol is deprecated because it collides with minor(...) macro defined in <sys/sysmacros.h> and
+  //! will be removed in next major release. Use cc.minor_cap() instead.
+  [[nodiscard]]
+  CCCL_DEPRECATED_BECAUSE("This symbol is deprecated because it collides with minor(...) macro defined in "
+                          "<sys/sysmacros.h> and will be removed in next major release. Use cc.minor_cap() instead.")
+  _CCCL_HOST_DEVICE_API constexpr int minor() const noexcept
+  {
+    return minor_cap();
+  }
+
+  //! @brief Gets the minor compute capability.
+  //!
+  //! @return Minor compute capability. The value is always less than 10.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int minor_cap() const noexcept
+  {
+    return __cc_ % 10;
+  }
+
+  //! @brief Conversion operator to \c int.
+  //!
+  //! @return The stored compute capability in format 10 * major + minor.
+  _CCCL_HOST_DEVICE_API explicit constexpr operator int() const noexcept
+  {
+    return __cc_;
+  }
+
+  //! @brief Equality operator.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator==(compute_capability __lhs, compute_capability __rhs) noexcept
+  {
+    return __lhs.__cc_ == __rhs.__cc_;
+  }
+
+  //! @brief Inequality operator.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator!=(compute_capability __lhs, compute_capability __rhs) noexcept
+  {
+    return __lhs.__cc_ != __rhs.__cc_;
+  }
+
+  //! @brief Less than operator.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator<(compute_capability __lhs, compute_capability __rhs) noexcept
+  {
+    return __lhs.__cc_ < __rhs.__cc_;
+  }
+
+  //! @brief Less than or equal to operator.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator<=(compute_capability __lhs, compute_capability __rhs) noexcept
+  {
+    return __lhs.__cc_ <= __rhs.__cc_;
+  }
+
+  //! @brief Greater than operator.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator>(compute_capability __lhs, compute_capability __rhs) noexcept
+  {
+    return __lhs.__cc_ > __rhs.__cc_;
+  }
+
+  //! @brief Greater than or equal to operator.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator>=(compute_capability __lhs, compute_capability __rhs) noexcept
+  {
+    return __lhs.__cc_ >= __rhs.__cc_;
+  }
+};
+
+template <int... _Vs>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __make_all_compute_capabilities() noexcept
+{
+  return ::cuda::std::array{compute_capability{_Vs}...};
+}
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __all_compute_capabilities() noexcept
+{
+  return ::cuda::__make_all_compute_capabilities<_CCCL_KNOWN_CUDA_ARCH_LIST>();
+}
+
+#if _CCCL_CUDA_COMPILATION()
+template <int... _Vs>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __make_cc_list() noexcept
+{
+#  if defined(__CUDA_ARCH_LIST__)
+  return ::cuda::std::array{compute_capability{_Vs / 10}...};
+#  elif defined(NV_TARGET_SM_INTEGER_LIST)
+  return ::cuda::std::array{compute_capability{_Vs}...};
+#  else // ^^^ has arch list ^^^ / vvv no arch list vvv
+  static_assert(::cuda::std::__always_false_v<decltype(sizeof...(_Vs))>,
+                "This function can be instantiated only when __CUDA_ARCH_LIST__ or NV_TARGET_SM_INTEGER_LIST are "
+                "defined");
+#  endif // ^^^ no arch list ^^^
+}
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __target_compute_capabilities() noexcept
+{
+#  if defined(__CUDA_ARCH_LIST__)
+  return ::cuda::__make_cc_list<__CUDA_ARCH_LIST__>();
+#  elif defined(NV_TARGET_SM_INTEGER_LIST)
+  return ::cuda::__make_cc_list<NV_TARGET_SM_INTEGER_LIST>();
+#  else // ^^^ has arch list ^^^ / vvv no arch list vvv
+  // Fallback to a list of all compute capabilities.
+  return ::cuda::__all_compute_capabilities();
+#  endif // ^^^ no arch list ^^^
+}
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_END_NAMESPACE_CUDA
+
+#if __cpp_lib_format >= 201907L
+_CCCL_BEGIN_NAMESPACE_STD
+
+template <class _CharT>
+struct formatter<::cuda::compute_capability, _CharT> : private formatter<int, _CharT>
+{
+  template <class _ParseCtx>
+  _CCCL_HOST_API constexpr auto parse(_ParseCtx& __ctx)
+  {
+    return __ctx.begin();
+  }
+
+  template <class _FmtCtx>
+  _CCCL_HOST_API auto format(const ::cuda::compute_capability& __cc, _FmtCtx& __ctx) const
+  {
+    return formatter<int, _CharT>::format(__cc.get(), __ctx);
+  }
+};
+
+_CCCL_END_NAMESPACE_STD
+#endif // __cpp_lib_format >= 201907L
+
+// todo: specialize cuda::std::formatter for cuda::compute_capability
+
+#if _CCCL_CUDA_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+
+//! @brief Returns the \c cuda::compute_capability that is currently being compiled.
+//!
+//! @note This API cannot be used in constexpr context when compiling with nvc++ in CUDA mode.
+[[nodiscard]] _CCCL_DEVICE_API inline _CCCL_TARGET_CONSTEXPR ::cuda::compute_capability
+current_compute_capability() noexcept
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC)
+  return ::cuda::compute_capability{__builtin_current_device_sm()};
+#  elif _CCCL_DEVICE_COMPILATION()
+  return ::cuda::compute_capability{__CUDA_ARCH__ / 10};
+#  else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
+  return {};
+#  endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^
+}
+
+_CCCL_END_NAMESPACE_CUDA_DEVICE
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___DEVICE_COMPUTE_CAPABILITY_H

qwen3_6_scripts/cccl_preload/include/cuda/__fwd/complex.h

@@ -0,0 +1,48 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FWD_COMPLEX_H
+#define _CUDA___FWD_COMPLEX_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <class _Tp>
+class _CCCL_TYPE_VISIBILITY_DEFAULT complex;
+
+// __is_cuda_complex_v
+
+template <class _Tp>
+inline constexpr bool __is_cuda_complex_v = false;
+template <class _Tp>
+inline constexpr bool __is_cuda_complex_v<const _Tp> = __is_cuda_complex_v<_Tp>;
+template <class _Tp>
+inline constexpr bool __is_cuda_complex_v<volatile _Tp> = __is_cuda_complex_v<_Tp>;
+template <class _Tp>
+inline constexpr bool __is_cuda_complex_v<const volatile _Tp> = __is_cuda_complex_v<_Tp>;
+template <class _Tp>
+inline constexpr bool __is_cuda_complex_v<complex<_Tp>> = true;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FWD_COMPLEX_H

qwen3_6_scripts/cccl_preload/include/cuda/__fwd/devices.h

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FWD_DEVICES_H
+#define _CUDA___FWD_DEVICES_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/span.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+#if _CCCL_HAS_CTK()
+class __physical_device;
+class device_ref;
+template <::cudaDeviceAttr _Attr>
+struct __dev_attr;
+#endif // _CCCL_HAS_CTK()
+
+struct arch_traits_t;
+class compute_capability;
+enum class arch_id : int;
+
+inline constexpr int __arch_specific_id_multiplier = 100000;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FWD_DEVICES_H

qwen3_6_scripts/cccl_preload/include/cuda/__memory/address_space.h

@@ -0,0 +1,259 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_ADDRESS_SPACE_H
+#define _CUDA___MEMORY_ADDRESS_SPACE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_CUDA_COMPILATION()
+
+#  include <cuda/std/__memory/addressof.h>
+#  include <cuda/std/__utility/to_underlying.h>
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+
+//! @brief Address space enumeration for CUDA device code.
+//!
+//!        See https://docs.nvidia.com/cuda/parallel-thread-execution/#state-spaces for more details.
+enum class address_space
+{
+  global, //!< Global state space
+  shared, //!< Shared state space
+  constant, //!< Constant state space
+  local, //!< Local state space
+  grid_constant, //!< Kernel function parameter in the parameter state space
+  cluster_shared, //!< Cluster shared window within the shared state space
+  __max,
+};
+
+[[nodiscard]] _CCCL_DEVICE_API constexpr bool __cccl_is_valid_address_space(address_space __space) noexcept
+{
+  const auto __v = ::cuda::std::to_underlying(__space);
+  return __v >= 0 && __v < ::cuda::std::to_underlying(address_space::__max);
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline bool __is_smem_valid_ptr(const void* __ptr) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90, (return __ptr != nullptr;), (return true;));
+}
+
+//! @brief Checks if the given pointer is from the specified address state space.
+//! @param __ptr The address to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the pointer is from the specified address space, `false` otherwise.
+[[nodiscard]] _CCCL_DEVICE_API inline bool __internal_is_address_from(const void* __ptr, address_space __space) noexcept
+{
+  _CCCL_ASSERT(::cuda::device::__cccl_is_valid_address_space(__space), "invalid address space");
+  // NVCC and NVRTC < 12.3 have problems tracking the address space of pointers, fallback to inline PTX for them
+  switch (__space)
+  {
+    case address_space::global: {
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.global p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      bool __p = static_cast<bool>(::__isGlobal(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    case address_space::constant: {
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.const p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      bool __p = static_cast<bool>(::__isConstant(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    case address_space::local: {
+      // __isLocal is buggy until CUDA 13.1, see nvbug 5254298
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 13, 1) || _CCCL_CUDA_COMPILER(NVRTC, <, 13, 1)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.local p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 13, 1) || _CCCL_CUDA_COMPILER(NVRTC, <, 13, 1) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) vvv
+      bool __p = static_cast<bool>(::__isLocal(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) ^^^
+    }
+    case address_space::grid_constant: {
+#  if _CCCL_CUDA_COMPILER(NVCC, >=, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 3)
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_70,
+        (bool __p = static_cast<bool>(::__isGridConstant(__ptr)); //
+         if (__p) //
+         { //
+           _CCCL_ASSUME(__p); //
+         } //
+         return __p;),
+        (return false;))
+#  else // ^^^ has functional __isGridConstant() ^^^ / vvv no functional __isGridConstant() vvv
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_70,
+        (unsigned __ret; //
+         asm volatile("{\n\t"
+                      "  .reg .pred p;\n\t"
+                      "  isspacep.param p, %1;\n\t"
+                      "  selp.u32 %0, 1, 0, p;\n\t"
+                      "}\n\t" : "=r"(__ret) : "l"(__ptr));
+         return static_cast<bool>(__ret);),
+        (return false;))
+#  endif // ^^^ no functional __isGridConstant() ^^^
+    }
+    case address_space::cluster_shared: {
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_90,
+        (unsigned __ret; //
+         asm volatile("{\n\t"
+                      "  .reg .pred p;\n\t"
+                      "  isspacep.shared::cluster p, %1;\n\t"
+                      "  selp.u32 %0, 1, 0, p;\n\t"
+                      "}\n\t" : "=r"(__ret) : "l"(__ptr));
+         return static_cast<bool>(__ret);),
+        ([[fallthrough]]; /* to `case shared:` */))
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_90,
+        (bool __p = static_cast<bool>(::__isClusterShared(__ptr)); //
+         if (__p) //
+         { //
+           _CCCL_ASSUME(__p); //
+         } //
+         return __p;),
+        ([[fallthrough]]; /* to `case shared:` */))
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    case address_space::shared: {
+      // smem can start at address 0x0 before sm_90
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.shared p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      bool __p = static_cast<bool>(::__isShared(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    default:
+      return false;
+  }
+}
+
+//! @brief Checks if the given pointer is from the specified address state space.
+//! @param __ptr The address to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the pointer is from the specified address space, `false` otherwise.
+[[nodiscard]] _CCCL_DEVICE_API inline bool is_address_from(const void* __ptr, address_space __space) noexcept
+{
+  // The debug assertions intentionally differ but compile out in release builds.
+  // NOLINTBEGIN(bugprone-branch-clone)
+  if (__space == address_space::shared)
+  {
+    _CCCL_ASSERT(::cuda::device::__is_smem_valid_ptr(__ptr), "invalid pointer");
+  }
+  else
+  {
+    _CCCL_ASSERT(__ptr != nullptr, "invalid pointer");
+  }
+  // NOLINTEND(bugprone-branch-clone)
+  return ::cuda::device::__internal_is_address_from(__ptr, __space);
+}
+
+//! @brief Checks if the given pointer is from the specified address state space.
+//! @param __ptr The address to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the pointer is from the specified address space, `false` otherwise.
+[[nodiscard]] _CCCL_DEVICE_API inline bool is_address_from(const volatile void* __ptr, address_space __space) noexcept
+{
+  return ::cuda::device::is_address_from(const_cast<const void*>(__ptr), __space);
+}
+
+//! @brief Checks if the given object is from the specified address state space.
+//! @param __obj The object to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the object is from the specified address space, `false` otherwise.
+template <class _Tp>
+[[nodiscard]] _CCCL_DEVICE_API inline bool is_object_from(_Tp& __obj, address_space __space) noexcept
+{
+  return ::cuda::device::is_address_from(::cuda::std::addressof(__obj), __space);
+}
+
+_CCCL_END_NAMESPACE_CUDA_DEVICE
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+#endif // _CUDA___MEMORY_ADDRESS_SPACE_H

qwen3_6_scripts/cccl_preload/include/cuda/__memory/check_address.h

@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_IS_VALID_ADDRESS
+#define _CUDA___MEMORY_IS_VALID_ADDRESS
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+#if _CCCL_CUDA_COMPILATION()
+#  include <cuda/__memory/address_space.h>
+#  include <cuda/__ptx/instructions/get_sreg.h>
+#endif // _CCCL_CUDA_COMPILATION()
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_CUDA_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+
+[[nodiscard]] _CCCL_DEVICE_API inline bool
+__is_smem_valid_address_range(const void* __ptr, ::cuda::std::size_t __n) noexcept
+{
+  if (!::cuda::device::__is_smem_valid_ptr(__ptr))
+  {
+    return false;
+  }
+  if (!::cuda::device::__internal_is_address_from(__ptr, ::cuda::device::address_space::shared))
+  {
+    return false;
+  }
+  // if __ptr is a shared memory pointer, __ptr + __n must also be a valid shared memory pointer
+  if (!::cuda::device::__internal_is_address_from(
+        reinterpret_cast<const char*>(__ptr) + __n, ::cuda::device::address_space::shared))
+  {
+    return false;
+  }
+  return (__n <= ::cuda::ptx::get_sreg_total_smem_size());
+}
+
+_CCCL_END_NAMESPACE_CUDA_DEVICE
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+[[nodiscard]] _CCCL_API inline bool __is_valid_address_range(const void* __ptr, ::cuda::std::size_t __n) noexcept
+{
+  if (__n == 0)
+  {
+    return false;
+  }
+
+  // use (~::cuda::std::uintptr_t{0}) instead of cuda::std::numeric_limits<cuda::std::uintptr_t>::max() to avoid
+  // circular dependency because:
+  //    numeric_limits -> bit_cast -> cstring -> check_address
+  // <cuda/std/__utility/cmp.h> also includes cuda/std/limits
+  const auto __limit = (~::cuda::std::uintptr_t{0}) - static_cast<::cuda::std::uintptr_t>(__n);
+
+  if (reinterpret_cast<::cuda::std::uintptr_t>(__ptr) > __limit)
+  {
+    return false;
+  }
+  NV_IF_TARGET(NV_IS_DEVICE, ({
+                 if (::cuda::device::__internal_is_address_from(__ptr, ::cuda::device::address_space::shared)
+                     && !::cuda::device::__is_smem_valid_address_range(__ptr, __n))
+                 {
+                   return false;
+                 }
+               }));
+  return (__ptr != nullptr);
+}
+
+[[nodiscard]] _CCCL_API inline bool __is_valid_address(const void* __ptr) noexcept
+{
+  return ::cuda::__is_valid_address_range(__ptr, 0);
+}
+
+[[nodiscard]] _CCCL_API inline bool
+__are_ptrs_overlapping(const void* __ptr_lhs, const void* __ptr_rhs, ::cuda::std::size_t __n) noexcept
+{
+  const auto __ptr1_start = static_cast<const char*>(__ptr_lhs);
+  const auto __ptr2_start = static_cast<const char*>(__ptr_rhs);
+  const auto __ptr1_end   = __ptr1_start + __n;
+  const auto __ptr2_end   = __ptr2_start + __n;
+  return __ptr1_start < __ptr2_end && __ptr2_start < __ptr1_end;
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_IS_VALID_ADDRESS

qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx.h

@@ -0,0 +1,150 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___NVTX_NVTX_H
+#define _CUDA___NVTX_NVTX_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes:
+//! When this macro is defined, no NVTX ranges are emitted by CCCL
+#  define CCCL_DISABLE_NVTX
+#endif // _CCCL_DOXYGEN_INVOKED
+
+#define _CCCL_HAS_NVTX3() 0
+
+// Enable the functionality of this header if:
+// * The NVTX3 C API is available in CTK
+// * NVTX is not explicitly disabled (via CCCL_DISABLE_NVTX or NVTX_DISABLE)
+// * the compiler is not nvc++ (NVTX3 uses module as an identifier, which trips up NVHPC, fixed in CTK >= 13.0)
+// * the compiler is not NVRTC
+#if __has_include(<nvtx3/nvToolsExt.h>) && !defined(CCCL_DISABLE_NVTX) && !defined(NVTX_DISABLE) \
+                      && (!_CCCL_COMPILER(NVHPC) || _CCCL_CTK_AT_LEAST(13, 0))                       \
+                      && !_CCCL_COMPILER(NVRTC)
+
+// Since NVTX 3.2, the NVTX headers can declare themselves as system headers by declaring the following macro:
+#  ifdef NVTX_AS_SYSTEM_HEADER
+#    define NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER
+#  else // NVTX_AS_SYSTEM_HEADER
+#    define NVTX_AS_SYSTEM_HEADER
+#  endif // NVTX_AS_SYSTEM_HEADER
+
+// Include our NVTX3 C++ wrapper if not available from the CTK or not provided by the user
+// Note: NVTX3 is available in the CTK since 12.9, so we can drop our copy once this is the minimum supported version
+#  if __has_include(<nvtx3/nvtx3.hpp>)
+#    include <nvtx3/nvtx3.hpp>
+#  else // __has_include(<nvtx3/nvtx3.hpp>)
+#    include <cuda/__nvtx/nvtx3.h>
+#  endif // __has_include(<nvtx3/nvtx3.hpp>)
+
+#  ifndef NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER
+#    undef NVTX_AS_SYSTEM_HEADER
+#  endif // NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER
+#  undef NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER
+
+// We expect the NVTX3 V1 C++ API to be available when nvtx3.hpp is available. This should work, because newer versions
+// of NVTX3 will continue to declare previous API versions. See also:
+// https://github.com/NVIDIA/NVTX/blob/release-v3/c/include/nvtx3/nvtx3.hpp#L2835-L2841.
+#  ifdef NVTX3_CPP_DEFINITIONS_V1_0
+#    undef _CCCL_HAS_NVTX3
+#    define _CCCL_HAS_NVTX3() 1
+#  else // NVTX3_CPP_DEFINITIONS_V1_0
+// If this happens NVTX3 changed in a way we did not anticipate, and we need to get in touch with them
+#    if _CCCL_COMPILER(MSVC)
+#      pragma message( \
+        "warning: nvtx3.h is available but does not define the V1 API. This is odd. Please open a GitHub issue at: https://github.com/NVIDIA/cccl/issues.")
+#    else
+#      warning nvtx3.h is available but does not define the V1 API. This is odd. Please open a GitHub issue at: https://github.com/NVIDIA/cccl/issues.
+#    endif
+#  endif // NVTX3_CPP_DEFINITIONS_V1_0
+#endif // __has_include(<nvtx3/nvToolsExt.h>) && !defined(CCCL_DISABLE_NVTX) && !defined(NVTX_DISABLE) &&
+       // (!_CCCL_COMPILER(NVHPC)) && !_CCCL_COMPILER(NVRTC)
+
+#if _CCCL_HAS_NVTX3()
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+struct __nvtx_cccl_domain
+{
+  static constexpr const char* name{"CCCL"};
+};
+
+using __nvtx_cccl_range = ::nvtx3::v1::scoped_range_in<__nvtx_cccl_domain>;
+
+// this type ensures that no NVTX range code is emitted in device code
+struct __nvtx_cccl_optional_range_host_only
+{
+  bool __engaged = false;
+  alignas(__nvtx_cccl_range) unsigned char __storage[sizeof(__nvtx_cccl_range)];
+
+  __nvtx_cccl_optional_range_host_only() = default;
+
+  _CCCL_HOST_API void __start(const ::nvtx3::v1::event_attributes& __attributes)
+  {
+    ::new (__storage) __nvtx_cccl_range(__attributes);
+    __engaged = true;
+  }
+
+  _CCCL_API ~__nvtx_cccl_optional_range_host_only()
+  {
+    NV_IF_TARGET(NV_IS_HOST, ({
+                   if (__engaged)
+                   {
+                     reinterpret_cast<__nvtx_cccl_range*>(__storage)->~__nvtx_cccl_range();
+                   }
+                 }));
+  }
+};
+_CCCL_END_NAMESPACE_CUDA
+
+// Hook for the NestedNVTXRangeGuard from the unit tests
+#  ifndef _CCCL_BEFORE_NVTX_RANGE_SCOPE
+#    define _CCCL_BEFORE_NVTX_RANGE_SCOPE(name)
+#  endif // !CCCL_DETAIL_BEFORE_NVTX_RANGE_SCOPE
+
+#  if _CCCL_HOST_COMPILATION()
+// Conditionally inserts a NVTX range starting here until the end of the current function scope in host code. Does
+// nothing in device code.
+// The __nvtx_cccl_optional_range_host_only type (a simplified optional<T>) is needed to defer the construction of the
+// NVTX range and message string registration (static variables) into a region running only on the host, while
+// preserving the semantic scope where the range is declared.
+#    define _CCCL_NVTX_RANGE_SCOPE_IF(condition, name)                                                             \
+      _CCCL_BEFORE_NVTX_RANGE_SCOPE(name)                                                                          \
+      ::cuda::__nvtx_cccl_optional_range_host_only __cuda_nvtx3_range;                                             \
+      NV_IF_TARGET(                                                                                                \
+        NV_IS_HOST, ({                                                                                             \
+          static const ::nvtx3::v1::registered_string_in<::cuda::__nvtx_cccl_domain> __cuda_nvtx3_func_name{name}; \
+          static const ::nvtx3::v1::event_attributes __cuda_nvtx3_func_attr{__cuda_nvtx3_func_name};               \
+          if (condition)                                                                                           \
+          {                                                                                                        \
+            __cuda_nvtx3_range.__start(__cuda_nvtx3_func_attr);                                                    \
+          }                                                                                                        \
+        }))
+#  else // ^^^ _CCCL_HOST_COMPILATION() ^^^ / vvv !_CCCL_HOST_COMPILATION() vvv
+#    define _CCCL_NVTX_RANGE_SCOPE_IF(condition, name)
+#  endif // ^^^ !_CCCL_HOST_COMPILATION() ^^^
+
+#  define _CCCL_NVTX_RANGE_SCOPE(name) _CCCL_NVTX_RANGE_SCOPE_IF(true, name)
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#else // _CCCL_HAS_NVTX3()
+#  define _CCCL_NVTX_RANGE_SCOPE_IF(condition, name)
+#  define _CCCL_NVTX_RANGE_SCOPE(name)
+#endif // _CCCL_HAS_NVTX3()
+
+#endif // _CUDA___NVTX_NVTX_H

qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/get_sreg.h

@@ -0,0 +1,949 @@
+// This file was automatically generated. Do not edit.
+
+#ifndef _CUDA_PTX_GENERATED_GET_SREG_H_
+#define _CUDA_PTX_GENERATED_GET_SREG_H_
+
+/*
+// mov.u32 sreg_value, %%tid.x; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_tid_x();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_x()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%tid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%tid.y; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_tid_y();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_y()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%tid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%tid.z; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_tid_z();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_z()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%tid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%ntid.x; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_ntid_x();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_x()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%ntid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%ntid.y; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_ntid_y();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_y()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%ntid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%ntid.z; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_ntid_z();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_z()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%ntid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%laneid; // PTX ISA 13
+template <typename = void>
+__device__ static inline uint32_t get_sreg_laneid();
+*/
+#if __cccl_ptx_isa >= 130
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_laneid()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%laneid;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 130
+
+/*
+// mov.u32 sreg_value, %%warpid; // PTX ISA 13
+template <typename = void>
+__device__ static inline uint32_t get_sreg_warpid();
+*/
+#if __cccl_ptx_isa >= 130
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_warpid()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%warpid;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 130
+
+/*
+// mov.u32 sreg_value, %%nwarpid; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nwarpid();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nwarpid()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%nwarpid;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%ctaid.x; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_ctaid_x();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_x()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%ctaid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%ctaid.y; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_ctaid_y();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_y()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%ctaid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%ctaid.z; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_ctaid_z();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_z()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%ctaid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%nctaid.x; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nctaid_x();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_x()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%nctaid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%nctaid.y; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nctaid_y();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_y()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%nctaid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%nctaid.z; // PTX ISA 20
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nctaid_z();
+*/
+#if __cccl_ptx_isa >= 200
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_z()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%nctaid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%smid; // PTX ISA 13
+template <typename = void>
+__device__ static inline uint32_t get_sreg_smid();
+*/
+#if __cccl_ptx_isa >= 130
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_smid()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%smid;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 130
+
+/*
+// mov.u32 sreg_value, %%nsmid; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nsmid();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nsmid()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%nsmid;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u64 sreg_value, %%gridid; // PTX ISA 30
+template <typename = void>
+__device__ static inline uint64_t get_sreg_gridid();
+*/
+#if __cccl_ptx_isa >= 300
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_gridid()
+{
+  ::cuda::std::uint64_t __sreg_value;
+  asm("mov.u64 %0, %%gridid;" : "=l"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 300
+
+/*
+// mov.pred sreg_value, %%is_explicit_cluster; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline bool get_sreg_is_explicit_cluster();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline bool get_sreg_is_explicit_cluster()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("{\n\t .reg .pred P_OUT; \n\t"
+      "mov.pred P_OUT, %%is_explicit_cluster;\n\t"
+      "selp.b32 %0, 1, 0, P_OUT; \n"
+      "}"
+      : "=r"(__sreg_value)
+      :
+      :);
+  return static_cast<bool>(__sreg_value);
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__();
+  return false;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%clusterid.x; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_clusterid_x();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_x()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%clusterid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%clusterid.y; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_clusterid_y();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_y()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%clusterid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%clusterid.z; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_clusterid_z();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_z()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%clusterid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%nclusterid.x; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nclusterid_x();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_x()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%nclusterid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%nclusterid.y; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nclusterid_y();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_y()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%nclusterid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%nclusterid.z; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_nclusterid_z();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_z()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%nclusterid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_ctaid.x; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_ctaid_x();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_x()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_ctaid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_ctaid.y; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_ctaid_y();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_y()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_ctaid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_ctaid.z; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_ctaid_z();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_z()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_ctaid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_nctaid.x; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_nctaid_x();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_x()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_nctaid.x;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_nctaid.y; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_nctaid_y();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_y()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_nctaid.y;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_nctaid.z; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_nctaid_z();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_z()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_nctaid.z;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_ctarank; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_ctarank();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctarank()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_ctarank;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%cluster_nctarank; // PTX ISA 78, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_cluster_nctarank();
+*/
+#if __cccl_ptx_isa >= 780
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctarank()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%cluster_nctarank;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 780
+
+/*
+// mov.u32 sreg_value, %%lanemask_eq; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_lanemask_eq();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_eq()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%lanemask_eq;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%lanemask_le; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_lanemask_le();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_le()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%lanemask_le;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%lanemask_lt; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_lanemask_lt();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_lt()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%lanemask_lt;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%lanemask_ge; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_lanemask_ge();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_ge()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%lanemask_ge;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%lanemask_gt; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_lanemask_gt();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_gt()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%lanemask_gt;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u32 sreg_value, %%clock; // PTX ISA 10
+template <typename = void>
+__device__ static inline uint32_t get_sreg_clock();
+*/
+#if __cccl_ptx_isa >= 100
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clock()
+{
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%clock;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+}
+#endif // __cccl_ptx_isa >= 100
+
+/*
+// mov.u32 sreg_value, %%clock_hi; // PTX ISA 50, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_clock_hi();
+*/
+#if __cccl_ptx_isa >= 500
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clock_hi()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%clock_hi;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 500
+
+/*
+// mov.u64 sreg_value, %%clock64; // PTX ISA 20, SM_35
+template <typename = void>
+__device__ static inline uint64_t get_sreg_clock64();
+*/
+#if __cccl_ptx_isa >= 200
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_clock64()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint64_t __sreg_value;
+  asm volatile("mov.u64 %0, %%clock64;" : "=l"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 200
+
+/*
+// mov.u64 sreg_value, %%globaltimer; // PTX ISA 31, SM_35
+template <typename = void>
+__device__ static inline uint64_t get_sreg_globaltimer();
+*/
+#if __cccl_ptx_isa >= 310
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_globaltimer()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint64_t __sreg_value;
+  asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 310
+
+/*
+// mov.u32 sreg_value, %%globaltimer_lo; // PTX ISA 31, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_globaltimer_lo();
+*/
+#if __cccl_ptx_isa >= 310
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_globaltimer_lo()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%globaltimer_lo;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 310
+
+/*
+// mov.u32 sreg_value, %%globaltimer_hi; // PTX ISA 31, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_globaltimer_hi();
+*/
+#if __cccl_ptx_isa >= 310
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_globaltimer_hi()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm volatile("mov.u32 %0, %%globaltimer_hi;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 310
+
+/*
+// mov.u32 sreg_value, %%total_smem_size; // PTX ISA 41, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_total_smem_size();
+*/
+#if __cccl_ptx_isa >= 410
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_total_smem_size()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%total_smem_size;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 410
+
+/*
+// mov.u32 sreg_value, %%aggr_smem_size; // PTX ISA 81, SM_90
+template <typename = void>
+__device__ static inline uint32_t get_sreg_aggr_smem_size();
+*/
+#if __cccl_ptx_isa >= 810
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_aggr_smem_size()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%aggr_smem_size;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 810
+
+/*
+// mov.u32 sreg_value, %%dynamic_smem_size; // PTX ISA 41, SM_35
+template <typename = void>
+__device__ static inline uint32_t get_sreg_dynamic_smem_size();
+*/
+#if __cccl_ptx_isa >= 410
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_dynamic_smem_size()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350
+  ::cuda::std::uint32_t __sreg_value;
+  asm("mov.u32 %0, %%dynamic_smem_size;" : "=r"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 410
+
+/*
+// mov.u64 sreg_value, %%current_graph_exec; // PTX ISA 80, SM_50
+template <typename = void>
+__device__ static inline uint64_t get_sreg_current_graph_exec();
+*/
+#if __cccl_ptx_isa >= 800
+extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__();
+template <typename = void>
+_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_current_graph_exec()
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500
+  ::cuda::std::uint64_t __sreg_value;
+  asm("mov.u64 %0, %%current_graph_exec;" : "=l"(__sreg_value) : :);
+  return __sreg_value;
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__();
+  return 0;
+#  endif
+}
+#endif // __cccl_ptx_isa >= 800
+
+#endif // _CUDA_PTX_GENERATED_GET_SREG_H_

qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/get_sreg.h

@@ -0,0 +1,43 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_PTX_GET_SREG_H_
+#define _CUDA_PTX_GET_SREG_H_
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__ptx/ptx_dot_variants.h>
+#include <cuda/__ptx/ptx_helper_functions.h>
+#include <cuda/std/cstdint>
+
+#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_PTX
+
+// 10. Special Registers
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#special-registers
+#include <cuda/__ptx/instructions/generated/get_sreg.h>
+
+_CCCL_END_NAMESPACE_CUDA_PTX
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_PTX_GET_SREG_H_

qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_dot_variants.h

@@ -0,0 +1,230 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// WARNING: The source of truth for this file is libcuda-ptx. Do not modify without syncing with libcuda-ptx.
+
+#ifndef _CUDA_PTX_DOT_VARIANTS_H_
+#define _CUDA_PTX_DOT_VARIANTS_H_
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__type_traits/integral_constant.h>
+
+/*
+ * Public integral constant types and values for ".variant"s:
+ *
+ * - .sem:     acquire, release, ..
+ * - .space:   global, shared, constant, ..
+ * - .scope:   cta, cluster, gpu, ..
+ * - .op:      add, min, cas, ..
+ *
+ * For each .variant, the code below defines:
+ * - An enum `dot_variant` with each possible value
+ * - A type template `variant_t<dot_variant>`
+ * - Types `variant_A_t`, ..., `variant_Z_t`
+ * - Constexpr values `variant_A` of type `variant_A_t`
+ *
+ * These types enable specifying fine-grained overloads of a PTX binding. If a
+ * binding can handle multiple variants, then it is defined as:
+ *
+ * template <dot_variant var>
+ * [...] void ptx_binding(variant_t<var> __v) { ... }
+ *
+ * If it only handles a single variant, then it is defined as:
+ *
+ * [...] void ptx_binding(variant_A __v) { ... }
+ *
+ * If two variants have different behaviors or return types (see .space
+ * overloads of mbarrier.arrive.expect_tx for an example), then these can be
+ * provided as separate overloads of the same function:
+ *
+ * [...] void ptx_binding(variant_A __v) { ... }
+ * [...] int ptx_binding(variant_B __v) { ... }
+ *
+ */
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_PTX
+
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#operation-types
+enum class dot_sem
+{
+  acq_rel,
+  acquire,
+  relaxed,
+  release,
+  sc,
+  weak
+};
+
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#state-spaces
+enum class dot_space
+{
+  global,
+  cluster, // The PTX spelling is shared::cluster
+  shared, // The PTX spelling is shared::cta
+
+  // The following state spaces are unlikely to be used in cuda::ptx in the near
+  // future, so they are not exposed:
+
+  // reg,
+  // sreg,
+  // const_mem, // Using const_mem as `const` is reserved in C++.
+  // local,
+  // param,
+  // tex // deprecated
+};
+
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#scope
+enum class dot_scope
+{
+  cta,
+  cluster,
+  gpu,
+  sys
+};
+
+enum class dot_op
+{
+  add,
+  dec,
+  inc,
+  max,
+  min,
+  and_op, // Using and_op, as `and, or, xor` are reserved in C++.
+  or_op,
+  xor_op,
+  cas,
+  exch
+};
+
+enum class dot_cta_group
+{
+  cta_group_1,
+  cta_group_2
+};
+
+enum class dot_kind
+{
+  f16,
+  f8f6f4,
+  i8,
+  mxf4,
+  mxf4nvf4,
+  mxf8f6f4,
+  tf32
+};
+
+template <dot_sem __sem>
+using sem_t         = ::cuda::std::integral_constant<dot_sem, __sem>;
+using sem_acq_rel_t = sem_t<dot_sem::acq_rel>;
+using sem_acquire_t = sem_t<dot_sem::acquire>;
+using sem_relaxed_t = sem_t<dot_sem::relaxed>;
+using sem_release_t = sem_t<dot_sem::release>;
+using sem_sc_t      = sem_t<dot_sem::sc>;
+using sem_weak_t    = sem_t<dot_sem::weak>;
+
+[[maybe_unused]] static constexpr sem_acq_rel_t sem_acq_rel{};
+[[maybe_unused]] static constexpr sem_acquire_t sem_acquire{};
+[[maybe_unused]] static constexpr sem_relaxed_t sem_relaxed{};
+[[maybe_unused]] static constexpr sem_release_t sem_release{};
+[[maybe_unused]] static constexpr sem_sc_t sem_sc{};
+[[maybe_unused]] static constexpr sem_weak_t sem_weak{};
+
+template <dot_space __spc>
+using space_t         = ::cuda::std::integral_constant<dot_space, __spc>;
+using space_global_t  = space_t<dot_space::global>;
+using space_shared_t  = space_t<dot_space::shared>;
+using space_cluster_t = space_t<dot_space::cluster>;
+
+[[maybe_unused]] static constexpr space_global_t space_global{};
+[[maybe_unused]] static constexpr space_shared_t space_shared{};
+[[maybe_unused]] static constexpr space_cluster_t space_cluster{};
+
+template <dot_scope __scope>
+using scope_t         = ::cuda::std::integral_constant<dot_scope, __scope>;
+using scope_cluster_t = scope_t<dot_scope::cluster>;
+using scope_cta_t     = scope_t<dot_scope::cta>;
+using scope_gpu_t     = scope_t<dot_scope::gpu>;
+using scope_sys_t     = scope_t<dot_scope::sys>;
+
+[[maybe_unused]] static constexpr scope_cluster_t scope_cluster{};
+[[maybe_unused]] static constexpr scope_cta_t scope_cta{};
+[[maybe_unused]] static constexpr scope_gpu_t scope_gpu{};
+[[maybe_unused]] static constexpr scope_sys_t scope_sys{};
+
+template <dot_op __op>
+using op_t        = ::cuda::std::integral_constant<dot_op, __op>;
+using op_add_t    = op_t<dot_op::add>;
+using op_dec_t    = op_t<dot_op::dec>;
+using op_inc_t    = op_t<dot_op::inc>;
+using op_max_t    = op_t<dot_op::max>;
+using op_min_t    = op_t<dot_op::min>;
+using op_and_op_t = op_t<dot_op::and_op>;
+using op_or_op_t  = op_t<dot_op::or_op>;
+using op_xor_op_t = op_t<dot_op::xor_op>;
+using op_cas_t    = op_t<dot_op::cas>;
+using op_exch_t   = op_t<dot_op::exch>;
+
+[[maybe_unused]] static constexpr op_add_t op_add{};
+[[maybe_unused]] static constexpr op_dec_t op_dec{};
+[[maybe_unused]] static constexpr op_inc_t op_inc{};
+[[maybe_unused]] static constexpr op_max_t op_max{};
+[[maybe_unused]] static constexpr op_min_t op_min{};
+[[maybe_unused]] static constexpr op_and_op_t op_and_op{};
+[[maybe_unused]] static constexpr op_or_op_t op_or_op{};
+[[maybe_unused]] static constexpr op_xor_op_t op_xor_op{};
+[[maybe_unused]] static constexpr op_cas_t op_cas{};
+[[maybe_unused]] static constexpr op_exch_t op_exch{};
+
+template <dot_cta_group __cta_group>
+using cta_group_t   = ::cuda::std::integral_constant<dot_cta_group, __cta_group>;
+using cta_group_1_t = cta_group_t<dot_cta_group::cta_group_1>;
+using cta_group_2_t = cta_group_t<dot_cta_group::cta_group_2>;
+
+[[maybe_unused]] static constexpr cta_group_1_t cta_group_1{};
+[[maybe_unused]] static constexpr cta_group_2_t cta_group_2{};
+
+template <dot_kind __kind>
+using kind_t          = ::cuda::std::integral_constant<dot_kind, __kind>;
+using kind_f16_t      = kind_t<dot_kind::f16>;
+using kind_f8f6f4_t   = kind_t<dot_kind::f8f6f4>;
+using kind_i8_t       = kind_t<dot_kind::i8>;
+using kind_mxf4_t     = kind_t<dot_kind::mxf4>;
+using kind_mxf4nvf4_t = kind_t<dot_kind::mxf4nvf4>;
+using kind_mxf8f6f4_t = kind_t<dot_kind::mxf8f6f4>;
+using kind_tf32_t     = kind_t<dot_kind::tf32>;
+
+[[maybe_unused]] static constexpr kind_f16_t kind_f16{};
+[[maybe_unused]] static constexpr kind_f8f6f4_t kind_f8f6f4{};
+[[maybe_unused]] static constexpr kind_i8_t kind_i8{};
+[[maybe_unused]] static constexpr kind_mxf4_t kind_mxf4{};
+[[maybe_unused]] static constexpr kind_mxf4nvf4_t kind_mxf4nvf4{};
+[[maybe_unused]] static constexpr kind_mxf8f6f4_t kind_mxf8f6f4{};
+[[maybe_unused]] static constexpr kind_tf32_t kind_tf32{};
+
+template <int n>
+using n32_t = ::cuda::std::integral_constant<int, n>;
+
+_CCCL_END_NAMESPACE_CUDA_PTX
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_PTX_DOT_VARIANTS_H_

qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_helper_functions.h

@@ -0,0 +1,178 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_PTX_HELPER_FUNCTIONS_H_
+#define _CUDA_PTX_HELPER_FUNCTIONS_H_
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+
+#if _CCCL_CUDA_COMPILATION()
+
+#  include <cuda/std/__cccl/prologue.h>
+
+#  if defined(__CUDACC__) || defined(_NVHPC_CUDA) || defined(__CUDACC_RTC__)
+#    define _CUDA_PTX_CUDACC_MAJOR() __CUDACC_VER_MAJOR__
+#  elif defined(__CUDA__) && defined(__clang__)
+#    define _CUDA_PTX_CUDACC_MAJOR() (CUDA_VERSION / 1000)
+#  endif // ^^^ has cuda compiler ^^^
+
+#  if !defined(_LIBCUDA_PTX_ARCH_SPECIFIC)
+#    if defined(__CUDA_ARCH_SPECIFIC__)
+#      define _LIBCUDA_PTX_ARCH_SPECIFIC() __CUDA_ARCH_SPECIFIC__
+#    else
+#      if defined(__CUDA_ARCH_FEAT_SM90_ALL)
+#        define _LIBCUDA_PTX_ARCH_SPECIFIC() 900
+#      elif defined(__CUDA_ARCH_FEAT_SM100_ALL)
+#        define _LIBCUDA_PTX_ARCH_SPECIFIC() 1000
+#      elif defined(__CUDA_ARCH_FEAT_SM103_ALL)
+#        define _LIBCUDA_PTX_ARCH_SPECIFIC() 1030
+#      elif defined(__CUDA_ARCH_FEAT_SM120_ALL)
+#        define _LIBCUDA_PTX_ARCH_SPECIFIC() 1200
+#      else
+#        define _LIBCUDA_PTX_ARCH_SPECIFIC() 0
+#      endif
+#    endif // ^^^ !defined(__CUDA_ARCH_SPECIFIC__)
+#  endif // ^^^ !defined(_LIBCUDA_PTX_ARCH_SPECIFIC)
+
+#  if !defined(__CUDA_HAS_ARCH_FAMILY_SPECIFIC)
+
+#    define __CUDA_HAS_ARCH_FAMILY_SPECIFIC(N) false
+
+#  endif // !defined(__CUDA_HAS_ARCH_FAMILY_SPECIFIC)
+
+_CCCL_BEGIN_NAMESPACE_CUDA_PTX
+
+#  if _CUDA_PTX_CUDACC_MAJOR() < 13
+struct alignas(32) longlong4_32a
+{
+  long long x, y, z, w;
+};
+struct alignas(32) ulonglong4_32a
+{
+  unsigned long long x, y, z, w;
+};
+struct alignas(32) double4_32a
+{
+  double x, y, z, w;
+};
+#  else
+using ::double4_32a;
+using ::longlong4_32a;
+using ::ulonglong4_32a;
+#  endif // _CUDA_PTX_CUDACC_MAJOR() < 13
+
+/*************************************************************
+ *
+ * Conversion from generic pointer -> state space "pointer"
+ *
+ **************************************************************/
+_CCCL_DEVICE_API inline ::cuda::std::uint32_t __as_ptr_smem(const void* __ptr)
+{
+  // Consider adding debug asserts here.
+  return static_cast<::cuda::std::uint32_t>(::__cvta_generic_to_shared(__ptr));
+}
+
+_CCCL_DEVICE_API inline ::cuda::std::uint32_t __as_ptr_dsmem(const void* __ptr)
+{
+  // No difference in implementation to __as_ptr_smem.
+  return __as_ptr_smem(__ptr);
+}
+
+_CCCL_DEVICE_API inline ::cuda::std::uint32_t __as_ptr_remote_dsmem(const void* __ptr)
+{
+  // No difference in implementation to __as_ptr_smem.
+  // Consider adding debug asserts here.
+  return __as_ptr_smem(__ptr);
+}
+
+_CCCL_DEVICE_API inline ::cuda::std::uint64_t __as_ptr_gmem(const void* __ptr)
+{
+  // Consider adding debug asserts here.
+  return static_cast<::cuda::std::uint64_t>(::__cvta_generic_to_global(__ptr));
+}
+
+/*************************************************************
+ *
+ * Conversion from state space "pointer" -> generic pointer
+ *
+ **************************************************************/
+template <typename _Tp>
+_CCCL_DEVICE_API _Tp* __from_ptr_smem(::cuda::std::size_t __ptr)
+{
+  // Consider adding debug asserts here.
+  return reinterpret_cast<_Tp*>(::__cvta_shared_to_generic(__ptr));
+}
+
+template <typename _Tp>
+_CCCL_DEVICE_API _Tp* __from_ptr_dsmem(::cuda::std::size_t __ptr)
+{
+  // Consider adding debug asserts here.
+  return __from_ptr_smem<_Tp>(__ptr);
+}
+
+template <typename _Tp>
+_CCCL_DEVICE_API _Tp* __from_ptr_remote_dsmem(::cuda::std::size_t __ptr)
+{
+  // Consider adding debug asserts here.
+  return __from_ptr_smem<_Tp>(__ptr);
+}
+
+template <typename _Tp>
+_CCCL_DEVICE_API _Tp* __from_ptr_gmem(::cuda::std::size_t __ptr)
+{
+  // Consider adding debug asserts here.
+  return reinterpret_cast<_Tp*>(::__cvta_global_to_generic(__ptr));
+}
+
+/*************************************************************
+ *
+ * Conversion to and from b8 type
+ *
+ **************************************************************/
+
+template <typename _B8>
+_CCCL_DEVICE_API uint32_t __b8_as_u32(_B8 __val)
+{
+  static_assert(sizeof(_B8) == 1);
+  ::cuda::std::uint32_t __u32 = 0;
+  ::memcpy(&__u32, &__val, 1);
+  return __u32;
+}
+
+template <typename _B8>
+_CCCL_DEVICE_API _B8 __u32_as_b8(uint32_t __u32)
+{
+  static_assert(sizeof(_B8) == 1);
+  _B8 b8;
+  ::memcpy(&b8, &__u32, 1);
+  return b8;
+}
+
+_CCCL_END_NAMESPACE_CUDA_PTX
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+#endif // _CUDA_PTX_HELPER_FUNCTIONS_H_

qwen3_6_scripts/cccl_preload/include/cuda/__type_traits/is_trivially_copyable.h

@@ -0,0 +1,115 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CUDA__TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+#define __CUDA__TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__fwd/complex.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/array.h>
+#include <cuda/std/__fwd/complex.h>
+#include <cuda/std/__fwd/pair.h>
+#include <cuda/std/__fwd/tuple.h>
+#include <cuda/std/__type_traits/aggregate_members_all_of.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_aggregate.h>
+#include <cuda/std/__type_traits/is_trivially_copyable.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Tp, typename = void>
+inline constexpr bool __is_aggregate_trivially_copyable_v = false;
+
+template <typename _Tp>
+inline constexpr bool __is_trivially_copyable_v =
+  ::cuda::std::is_trivially_copyable_v<_Tp> || __is_aggregate_trivially_copyable_v<_Tp>;
+
+#if _CCCL_HAS_NVFP16()
+
+template <>
+inline constexpr bool __is_trivially_copyable_v<::__half> = true;
+
+template <>
+inline constexpr bool __is_trivially_copyable_v<::__half2> = true;
+
+#endif // _CCCL_HAS_NVFP16()
+
+#if _CCCL_HAS_NVBF16()
+
+template <>
+inline constexpr bool __is_trivially_copyable_v<::__nv_bfloat16> = true;
+template <>
+inline constexpr bool __is_trivially_copyable_v<::__nv_bfloat162> = true;
+
+#endif // _CCCL_HAS_NVBF16()
+
+template <typename _Tp>
+inline constexpr bool __is_trivially_copyable_v<_Tp[]> = __is_trivially_copyable_v<_Tp>;
+
+template <typename _Tp, ::cuda::std::size_t _Size>
+inline constexpr bool __is_trivially_copyable_v<_Tp[_Size]> = __is_trivially_copyable_v<_Tp>;
+
+template <typename _Tp, ::cuda::std::size_t _Size>
+inline constexpr bool __is_trivially_copyable_v<::cuda::std::array<_Tp, _Size>> = __is_trivially_copyable_v<_Tp>;
+
+template <typename _T1, typename _T2>
+inline constexpr bool __is_trivially_copyable_v<::cuda::std::pair<_T1, _T2>> =
+  __is_trivially_copyable_v<_T1> && __is_trivially_copyable_v<_T2>;
+
+template <typename... _Ts>
+inline constexpr bool __is_trivially_copyable_v<::cuda::std::tuple<_Ts...>> = (__is_trivially_copyable_v<_Ts> && ...);
+
+template <typename _Tp>
+inline constexpr bool __is_trivially_copyable_v<complex<_Tp>> = true;
+
+template <typename _Tp>
+inline constexpr bool __is_trivially_copyable_v<::cuda::std::complex<_Tp>> = true;
+
+// if all the previous conditions fail, check if the type is an aggregate and all its members are trivially copyable
+template <typename _Tp>
+using __is_trivially_copyable_callable = ::cuda::std::bool_constant<__is_trivially_copyable_v<_Tp>>;
+
+template <typename _Tp>
+inline constexpr bool __is_aggregate_trivially_copyable_v<
+  _Tp,
+  ::cuda::std::enable_if_t<::cuda::std::is_aggregate_v<_Tp> && !::cuda::std::is_trivially_copyable_v<_Tp>>> =
+  ::cuda::std::__aggregate_all_of_v<__is_trivially_copyable_callable, _Tp>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// public traits
+
+template <typename _Tp>
+inline constexpr bool is_trivially_copyable_v = __is_trivially_copyable_v<_Tp>;
+
+template <typename _Tp>
+inline constexpr bool is_trivially_copyable_v<const _Tp> = is_trivially_copyable_v<_Tp>;
+
+// defined as alias so users cannot specialize it (they should specialize the variable template instead)
+template <typename _Tp>
+using is_trivially_copyable = ::cuda::std::bool_constant<is_trivially_copyable_v<_Tp>>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // __CUDA__TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/clamp.h

@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_CLAMP_H
+#define _CUDA_STD___ALGORITHM_CLAMP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Tp, class _Compare>
+[[nodiscard]] _CCCL_API constexpr const _Tp&
+clamp(const _Tp& __v _CCCL_LIFETIMEBOUND,
+      const _Tp& __lo _CCCL_LIFETIMEBOUND,
+      const _Tp& __hi _CCCL_LIFETIMEBOUND,
+      _Compare __comp)
+{
+  _CCCL_ASSERT(!__comp(__hi, __lo), "Bad bounds passed to cuda::std::clamp");
+  return __comp(__v, __lo) ? __lo : __comp(__hi, __v) ? __hi : __v;
+}
+
+template <class _Tp>
+[[nodiscard]] _CCCL_API constexpr const _Tp&
+clamp(const _Tp& __v _CCCL_LIFETIMEBOUND, const _Tp& __lo _CCCL_LIFETIMEBOUND, const _Tp& __hi _CCCL_LIFETIMEBOUND)
+{
+  _CCCL_ASSERT(!(__hi < __lo), "Bad bounds passed to cuda::std::clamp");
+  return __v < __lo ? __lo : __hi < __v ? __hi : __v;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_CLAMP_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp.h

@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_COMP_H
+#define _CUDA_STD___ALGORITHM_COMP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__type_traits/integral_constant.h>
+#if defined(_LIBCUDACXX_HAS_STRING)
+#  include <cuda/std/__type_traits/predicate_traits.h>
+#endif // _LIBCUDACXX_HAS_STRING
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+struct __equal_to
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _T1, class _T2>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(const _T1& __lhs, const _T2& __rhs) const
+    noexcept(noexcept(__lhs == __rhs))
+  {
+    return __lhs == __rhs;
+  }
+};
+
+struct __less
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _Tp, class _Up>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(const _Tp& __lhs, const _Up& __rhs) const
+    noexcept(noexcept(__lhs < __rhs))
+  {
+    return __lhs < __rhs;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_COMP_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp_ref_type.h

@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_COMP_REF_TYPE_H
+#define _CUDA_STD___ALGORITHM_COMP_REF_TYPE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__utility/declval.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class _Compare>
+struct __debug_less
+{
+  _Compare& __comp_;
+  _CCCL_API constexpr __debug_less(_Compare& __c)
+      : __comp_(__c)
+  {}
+
+  template <class _Tp, class _Up>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(const _Tp& __x, const _Up& __y)
+  {
+    bool __r = __comp_(__x, __y);
+    if (__r)
+    {
+      __do_compare_assert(0, __y, __x);
+    }
+    return __r;
+  }
+
+  template <class _Tp, class _Up>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(_Tp& __x, _Up& __y)
+  {
+    bool __r = __comp_(__x, __y);
+    if (__r)
+    {
+      __do_compare_assert(0, __y, __x);
+    }
+    return __r;
+  }
+
+  template <class _LHS, class _RHS>
+  _CCCL_API constexpr decltype((void) declval<_Compare&>()(declval<_LHS&>(), declval<_RHS&>()))
+  __do_compare_assert(int, [[maybe_unused]] _LHS& __l, [[maybe_unused]] _RHS& __r)
+  {
+    _CCCL_ASSERT(!__comp_(__l, __r), "Comparator does not induce a strict weak ordering");
+  }
+
+  template <class _LHS, class _RHS>
+  _CCCL_API constexpr void __do_compare_assert(long, _LHS&, _RHS&)
+  {}
+};
+
+// Pass the comparator by lvalue reference. Or in debug mode, using a
+// debugging wrapper that stores a reference.
+#ifdef _CCCL_ENABLE_DEBUG_MODE
+template <class _Comp>
+using __comp_ref_type = __debug_less<_Comp>;
+#else // ^^^ _LIBCUDACXX_ENABLE_DEBUG_MODE ^^^ / vvv !_LIBCUDACXX_ENABLE_DEBUG_MODE vvv
+template <class _Comp>
+using __comp_ref_type = _Comp&;
+#endif // !_LIBCUDACXX_ENABLE_DEBUG_MODE
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_COMP_REF_TYPE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/equal.h

@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_EQUAL_H
+#define _CUDA_STD___ALGORITHM_EQUAL_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+#include <cuda/std/__iterator/distance.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__type_traits/add_lvalue_reference.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+[[nodiscard]] _CCCL_API constexpr bool
+equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred)
+{
+  bool __result = true;
+  for (; __first1 != __last1; ++__first1, (void) ++__first2)
+  {
+    if (!__pred(*__first1, *__first2))
+    {
+      __result = false;
+      break;
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator1, class _InputIterator2>
+[[nodiscard]] _CCCL_API constexpr bool equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2)
+{
+  return ::cuda::std::equal(__first1, __last1, __first2, __equal_to{});
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _BinaryPredicate, class _InputIterator1, class _InputIterator2>
+[[nodiscard]] _CCCL_API constexpr bool __equal(
+  _InputIterator1 __first1,
+  _InputIterator1 __last1,
+  _InputIterator2 __first2,
+  _InputIterator2 __last2,
+  _BinaryPredicate __pred,
+  input_iterator_tag,
+  input_iterator_tag)
+{
+  bool __result = true;
+  for (; __first1 != __last1 && __first2 != __last2; ++__first1, (void) ++__first2)
+  {
+    if (!__pred(*__first1, *__first2))
+    {
+      __result = false;
+      break;
+    }
+  }
+  return __result && __first1 == __last1 && __first2 == __last2;
+}
+
+template <class _BinaryPredicate, class _RandomAccessIterator1, class _RandomAccessIterator2>
+[[nodiscard]] _CCCL_API constexpr bool __equal(
+  _RandomAccessIterator1 __first1,
+  _RandomAccessIterator1 __last1,
+  _RandomAccessIterator2 __first2,
+  _RandomAccessIterator2 __last2,
+  _BinaryPredicate __pred,
+  random_access_iterator_tag,
+  random_access_iterator_tag)
+{
+  if (__last1 - __first1 != __last2 - __first2)
+  {
+    return false;
+  }
+  return ::cuda::std::equal<_RandomAccessIterator1, _RandomAccessIterator2, add_lvalue_reference_t<_BinaryPredicate>>(
+    __first1, __last1, __first2, __pred);
+}
+
+template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
+[[nodiscard]] _CCCL_API constexpr bool
+equal(_InputIterator1 __first1,
+      _InputIterator1 __last1,
+      _InputIterator2 __first2,
+      _InputIterator2 __last2,
+      _BinaryPredicate __pred)
+{
+  return ::cuda::std::__equal<add_lvalue_reference_t<_BinaryPredicate>>(
+    __first1,
+    __last1,
+    __first2,
+    __last2,
+    __pred,
+    __iterator_traits_category_or_concept_t<_InputIterator1>(),
+    __iterator_traits_category_or_concept_t<_InputIterator2>());
+}
+
+template <class _InputIterator1, class _InputIterator2>
+[[nodiscard]] _CCCL_API constexpr bool
+equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2)
+{
+  return ::cuda::std::__equal(
+    __first1,
+    __last1,
+    __first2,
+    __last2,
+    __equal_to{},
+    __iterator_traits_category_or_concept_t<_InputIterator1>(),
+    __iterator_traits_category_or_concept_t<_InputIterator2>());
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_EQUAL_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/fill_n.h

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_FILL_N_H
+#define _CUDA_STD___ALGORITHM_FILL_N_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__utility/convert_to_integral.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _OutputIterator, class _Size, class _Tp>
+_CCCL_API constexpr _OutputIterator __fill_n(_OutputIterator __first, _Size __n, const _Tp& __value_)
+{
+  for (; __n > 0; ++__first, (void) --__n)
+  {
+    *__first = __value_;
+  }
+  return __first;
+}
+
+template <class _OutputIterator, class _Size, class _Tp>
+_CCCL_API constexpr _OutputIterator fill_n(_OutputIterator __first, _Size __n, const _Tp& __value_)
+{
+  return ::cuda::std::__fill_n(__first, __convert_to_integral(__n), __value_);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_FILL_N_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iter_swap.h

@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_ITER_SWAP_H
+#define _CUDA_STD___ALGORITHM_ITER_SWAP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/swap.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+//! Intentionally not an algorithm to avoid breaking types that pull in `::std::iter_swap` via ADL
+_CCCL_BEGIN_NAMESPACE_CPO(__iter_swap)
+// "Poison pill" overload to intentionally create ambiguity with the unconstrained
+// `std::iter_swap` function.
+template <class _ForwardIterator1, class _ForwardIterator2>
+void iter_swap(_ForwardIterator1, _ForwardIterator2) = delete;
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+_CCCL_CONCEPT __unqualified_iter_swap =
+  _CCCL_REQUIRES_EXPR((_ForwardIterator1, _ForwardIterator2), _ForwardIterator1&& __a, _ForwardIterator2&& __b)(
+    iter_swap(::cuda::std::forward<_ForwardIterator1>(__a), ::cuda::std::forward<_ForwardIterator2>(__b)));
+
+template <class _ForwardIterator1, class _ForwardIterator2>
+_CCCL_CONCEPT __readable_swappable =
+  _CCCL_REQUIRES_EXPR((_ForwardIterator1, _ForwardIterator2), _ForwardIterator1 __a, _ForwardIterator2 __b)(
+    requires(!__unqualified_iter_swap<_ForwardIterator1, _ForwardIterator2>), swap(*__a, *__b));
+
+struct __fn
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  _CCCL_TEMPLATE(class _ForwardIterator1, class _ForwardIterator2)
+  _CCCL_REQUIRES(__unqualified_iter_swap<_ForwardIterator1, _ForwardIterator2>)
+  _CCCL_API constexpr void operator()(_ForwardIterator1&& __a, _ForwardIterator2&& __b) const
+    noexcept(noexcept(iter_swap(::cuda::std::declval<_ForwardIterator1>(), ::cuda::std::declval<_ForwardIterator2>())))
+  {
+    (void) iter_swap(::cuda::std::forward<_ForwardIterator1>(__a), ::cuda::std::forward<_ForwardIterator2>(__b));
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  _CCCL_TEMPLATE(class _ForwardIterator1, class _ForwardIterator2)
+  _CCCL_REQUIRES(__readable_swappable<_ForwardIterator1, _ForwardIterator2>)
+  _CCCL_API constexpr void operator()(_ForwardIterator1&& __a, _ForwardIterator2&& __b) const
+    noexcept(noexcept(swap(*::cuda::std::declval<_ForwardIterator1>(), *::cuda::std::declval<_ForwardIterator2>())))
+  {
+    swap(*__a, *__b);
+  }
+};
+
+_CCCL_END_NAMESPACE_CPO
+
+inline namespace __cpo
+{
+// This is a global constant to avoid breaking types that pull in `::std::iter_swap` via ADL
+_CCCL_GLOBAL_CONSTANT auto iter_swap = __iter_swap::__fn{};
+
+// We want to avoid using the CPO internally because of __tile__ access
+using __iter_swap_cpo = __iter_swap::__fn;
+} // namespace __cpo
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_ITER_SWAP_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iterator_operations.h

@@ -0,0 +1,179 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_ITERATOR_OPERATIONS_H
+#define _CUDA_STD___ALGORITHM_ITERATOR_OPERATIONS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/iter_swap.h>
+#include <cuda/std/__algorithm/ranges_iterator_concept.h>
+#include <cuda/std/__iterator/advance.h>
+#include <cuda/std/__iterator/distance.h>
+#include <cuda/std/__iterator/incrementable_traits.h>
+#include <cuda/std/__iterator/iter_move.h>
+#include <cuda/std/__iterator/iter_swap.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__iterator/next.h>
+#include <cuda/std/__iterator/prev.h>
+#include <cuda/std/__iterator/readable_traits.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_reference.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class _AlgPolicy>
+struct _IterOps;
+
+struct _RangeAlgPolicy
+{};
+
+template <>
+struct _IterOps<_RangeAlgPolicy>
+{
+  template <class _Iter>
+  using __value_type = iter_value_t<_Iter>;
+
+  template <class _Iter>
+  using __difference_type = iter_difference_t<_Iter>;
+
+  static constexpr auto advance      = ::cuda::std::ranges::__advance_cpo{};
+  static constexpr auto distance     = ::cuda::std::ranges::__distance_cpo{};
+  static constexpr auto __iter_move  = ::cuda::std::ranges::__iter_move_cpo{};
+  static constexpr auto iter_swap    = ::cuda::std::ranges::__iter_swap_cpo{};
+  static constexpr auto next         = ::cuda::std::ranges::__next_cpo{};
+  static constexpr auto prev         = ::cuda::std::ranges::__prev_cpo{};
+  static constexpr auto __advance_to = ::cuda::std::ranges::__advance_cpo{};
+};
+
+struct _ClassicAlgPolicy
+{};
+
+template <>
+struct _IterOps<_ClassicAlgPolicy>
+{
+  template <class _Iter>
+  using __value_type = typename iterator_traits<_Iter>::value_type;
+
+  template <class _Iter>
+  using __difference_type = typename iterator_traits<_Iter>::difference_type;
+
+  // advance
+  template <class _Iter, class _Distance>
+  _CCCL_API constexpr static void advance(_Iter& __iter, _Distance __count)
+  {
+    ::cuda::std::advance(__iter, __count);
+  }
+
+  // distance
+  template <class _Iter>
+  _CCCL_API constexpr static typename iterator_traits<_Iter>::difference_type distance(_Iter __first, _Iter __last)
+  {
+    return ::cuda::std::distance(__first, __last);
+  }
+
+  template <class _Iter>
+  using __deref_t = decltype(*::cuda::std::declval<_Iter&>());
+
+  template <class _Iter>
+  using __move_t = decltype(::cuda::std::move(*::cuda::std::declval<_Iter&>()));
+
+  template <class _Iter>
+  _CCCL_API constexpr static void __validate_iter_reference()
+  {
+    static_assert(
+      is_same_v<__deref_t<_Iter>, typename iterator_traits<remove_cvref_t<_Iter>>::reference>,
+      "It looks like your iterator's `iterator_traits<It>::reference` does not match the return type of "
+      "dereferencing the iterator, i.e., calling `*it`. This is undefined behavior according to [input.iterators] "
+      "and can lead to dangling reference issues at runtime, so we are flagging this.");
+  }
+
+  // iter_move
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _Iter, enable_if_t<is_reference_v<__deref_t<_Iter>>, int> = 0>
+  _CCCL_API constexpr static
+    // If the result of dereferencing `_Iter` is a reference type, deduce the result of calling `::cuda::std::move` on
+    // it. Note that the C++03 mode doesn't support `decltype(auto)` as the return type.
+    __move_t<_Iter>
+    __iter_move(_Iter&& __i)
+  {
+    __validate_iter_reference<_Iter>();
+
+    return ::cuda::std::move(*::cuda::std::forward<_Iter>(__i));
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _Iter, enable_if_t<!is_reference_v<__deref_t<_Iter>>, int> = 0>
+  _CCCL_API constexpr static
+    // If the result of dereferencing `_Iter` is a value type, deduce the return value of this function to also be a
+    // value -- otherwise, after `operator*` returns a temporary, this function would return a dangling reference to
+    // that temporary. Note that the C++03 mode doesn't support `auto` as the return type.
+    __deref_t<_Iter>
+    __iter_move(_Iter&& __i)
+  {
+    __validate_iter_reference<_Iter>();
+
+    return *::cuda::std::forward<_Iter>(__i);
+  }
+
+  // iter_swap
+  template <class _Iter1, class _Iter2>
+  _CCCL_API constexpr static void iter_swap(_Iter1&& __a, _Iter2&& __b)
+  {
+    ::cuda::std::__iter_swap_cpo{}(::cuda::std::forward<_Iter1>(__a), ::cuda::std::forward<_Iter2>(__b));
+  }
+
+  // next
+  template <class _Iterator>
+  _CCCL_API static constexpr _Iterator next(_Iterator, _Iterator __last)
+  {
+    return __last;
+  }
+
+  template <class _Iter>
+  _CCCL_API static constexpr remove_cvref_t<_Iter> next(_Iter&& __it, __difference_type<remove_cvref_t<_Iter>> __n = 1)
+  {
+    return ::cuda::std::next(::cuda::std::forward<_Iter>(__it), __n);
+  }
+
+  // prev
+  template <class _Iter>
+  _CCCL_API static constexpr remove_cvref_t<_Iter> prev(_Iter&& __iter, __difference_type<remove_cvref_t<_Iter>> __n = 1)
+  {
+    return ::cuda::std::prev(::cuda::std::forward<_Iter>(__iter), __n);
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _Iter>
+  _CCCL_API static constexpr void __advance_to(_Iter& __first, _Iter __last)
+  {
+    __first = __last;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_ITERATOR_OPERATIONS_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/lexicographical_compare.h

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+#define _CUDA_STD___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+#include <cuda/std/__algorithm/comp_ref_type.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Compare, class _InputIterator1, class _InputIterator2>
+[[nodiscard]] _CCCL_API constexpr bool __lexicographical_compare(
+  _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp)
+{
+  bool __result = false;
+  for (; __first2 != __last2; ++__first1, (void) ++__first2)
+  {
+    if (__first1 == __last1 || __comp(*__first1, *__first2))
+    {
+      __result = true;
+      break;
+    }
+    if (__comp(*__first2, *__first1))
+    {
+      break;
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Compare>
+[[nodiscard]] _CCCL_API constexpr bool lexicographical_compare(
+  _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp)
+{
+  return __lexicographical_compare<__comp_ref_type<_Compare>>(__first1, __last1, __first2, __last2, __comp);
+}
+
+template <class _InputIterator1, class _InputIterator2>
+[[nodiscard]] _CCCL_API constexpr bool lexicographical_compare(
+  _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2)
+{
+  return ::cuda::std::lexicographical_compare(__first1, __last1, __first2, __last2, __less{});
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max.h

@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_MAX_H
+#define _CUDA_STD___ALGORITHM_MAX_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+#include <cuda/std/__algorithm/comp_ref_type.h>
+#include <cuda/std/__algorithm/max_element.h>
+#include <cuda/std/initializer_list>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Tp, class _Compare>
+[[nodiscard]] _CCCL_API constexpr const _Tp&
+max(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND, _Compare __comp)
+{
+  return __comp(__a, __b) ? __b : __a;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Tp>
+[[nodiscard]] _CCCL_API constexpr const _Tp& max(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND)
+{
+  return __a < __b ? __b : __a;
+}
+
+template <class _Tp, class _Compare>
+[[nodiscard]] _CCCL_API constexpr _Tp max(initializer_list<_Tp> __t, _Compare __comp)
+{
+  return *::cuda::std::__max_element<__comp_ref_type<_Compare>>(__t.begin(), __t.end(), __comp);
+}
+
+template <class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp max(initializer_list<_Tp> __t)
+{
+  return *::cuda::std::max_element(__t.begin(), __t.end(), __less{});
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_MAX_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max_element.h

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_MAX_ELEMENT_H
+#define _CUDA_STD___ALGORITHM_MAX_ELEMENT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+#include <cuda/std/__algorithm/comp_ref_type.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Compare, class _ForwardIterator>
+_CCCL_API constexpr _ForwardIterator __max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
+{
+  static_assert(__has_forward_traversal<_ForwardIterator>, "::cuda::std::max_element requires a ForwardIterator");
+  if (__first != __last)
+  {
+    _ForwardIterator __i = __first;
+    while (++__i != __last)
+    {
+      if (__comp(*__first, *__i))
+      {
+        __first = __i;
+      }
+    }
+  }
+  return __first;
+}
+
+template <class _ForwardIterator, class _Compare>
+[[nodiscard]] _CCCL_API constexpr _ForwardIterator
+max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
+{
+  return ::cuda::std::__max_element<__comp_ref_type<_Compare>>(__first, __last, __comp);
+}
+
+template <class _ForwardIterator>
+[[nodiscard]] _CCCL_API constexpr _ForwardIterator max_element(_ForwardIterator __first, _ForwardIterator __last)
+{
+  return ::cuda::std::max_element(__first, __last, __less{});
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_MAX_ELEMENT_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min.h

@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_MIN_H
+#define _CUDA_STD___ALGORITHM_MIN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+#include <cuda/std/__algorithm/comp_ref_type.h>
+#include <cuda/std/__algorithm/min_element.h>
+#include <cuda/std/initializer_list>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Tp, class _Compare>
+[[nodiscard]] _CCCL_API constexpr const _Tp&
+min(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND, _Compare __comp)
+{
+  return __comp(__b, __a) ? __b : __a;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Tp>
+[[nodiscard]] _CCCL_API constexpr const _Tp& min(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND)
+{
+  return __b < __a ? __b : __a;
+}
+
+template <class _Tp, class _Compare>
+[[nodiscard]] _CCCL_API constexpr _Tp min(initializer_list<_Tp> __t, _Compare __comp)
+{
+  return *::cuda::std::__min_element<__comp_ref_type<_Compare>>(__t.begin(), __t.end(), __comp);
+}
+
+template <class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp min(initializer_list<_Tp> __t)
+{
+  return *::cuda::std::min_element(__t.begin(), __t.end(), __less{});
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_MIN_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min_element.h

@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_MIN_ELEMENT_H
+#define _CUDA_STD___ALGORITHM_MIN_ELEMENT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/comp.h>
+#include <cuda/std/__algorithm/comp_ref_type.h>
+#include <cuda/std/__functional/identity.h>
+#include <cuda/std/__functional/invoke.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__type_traits/is_callable.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Comp, class _Iter, class _Sent, class _Proj>
+_CCCL_API constexpr _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj)
+{
+  if (__first == __last)
+  {
+    return __first;
+  }
+
+  _Iter __i = __first;
+  while (++__i != __last)
+  {
+    if (::cuda::std::invoke(__comp, ::cuda::std::invoke(__proj, *__i), ::cuda::std::invoke(__proj, *__first)))
+    {
+      __first = __i;
+    }
+  }
+
+  return __first;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _Comp, class _Iter, class _Sent>
+_CCCL_API constexpr _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp)
+{
+  auto __proj = identity();
+  return ::cuda::std::__min_element<_Comp>(::cuda::std::move(__first), ::cuda::std::move(__last), __comp, __proj);
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _ForwardIterator, class _Compare>
+[[nodiscard]] _CCCL_API constexpr _ForwardIterator
+min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
+{
+  static_assert(__has_forward_traversal<_ForwardIterator>, "std::min_element requires a ForwardIterator");
+  static_assert(__is_callable<_Compare, decltype(*__first), decltype(*__first)>::value,
+                "The comparator has to be callable");
+
+  return ::cuda::std::__min_element<__comp_ref_type<_Compare>>(
+    ::cuda::std::move(__first), ::cuda::std::move(__last), __comp);
+}
+
+template <class _ForwardIterator>
+[[nodiscard]] _CCCL_API constexpr _ForwardIterator min_element(_ForwardIterator __first, _ForwardIterator __last)
+{
+  return ::cuda::std::min_element(__first, __last, __less{});
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_MIN_ELEMENT_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/ranges_iterator_concept.h

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+#define _CUDA_STD___ALGORITHM_RANGES_ITERATOR_CONCEPT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__iterator/concepts.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES
+
+template <class _IterMaybeQualified>
+_CCCL_API constexpr auto __get_iterator_concept()
+{
+  using _Iter = remove_cvref_t<_IterMaybeQualified>;
+
+  if constexpr (contiguous_iterator<_Iter>)
+  {
+    return contiguous_iterator_tag();
+  }
+  else if constexpr (random_access_iterator<_Iter>)
+  {
+    return random_access_iterator_tag();
+  }
+  else if constexpr (bidirectional_iterator<_Iter>)
+  {
+    return bidirectional_iterator_tag();
+  }
+  else if constexpr (forward_iterator<_Iter>)
+  {
+    return forward_iterator_tag();
+  }
+  else if constexpr (input_iterator<_Iter>)
+  {
+    return input_iterator_tag();
+  }
+}
+
+template <class _Iter>
+using __iterator_concept = decltype(__get_iterator_concept<_Iter>());
+
+_CCCL_END_NAMESPACE_CUDA_STD_RANGES
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_RANGES_ITERATOR_CONCEPT_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/swap_ranges.h

@@ -0,0 +1,78 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_SWAP_RANGES_H
+#define _CUDA_STD___ALGORITHM_SWAP_RANGES_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/iterator_operations.h>
+#include <cuda/std/__utility/move.h>
+#include <cuda/std/__utility/pair.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+// 2+2 iterators: the shorter size will be used.
+_CCCL_EXEC_CHECK_DISABLE
+template <class _AlgPolicy, class _ForwardIterator1, class _Sentinel1, class _ForwardIterator2, class _Sentinel2>
+_CCCL_API constexpr pair<_ForwardIterator1, _ForwardIterator2>
+__swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, _Sentinel2 __last2)
+{
+  while (__first1 != __last1 && __first2 != __last2)
+  {
+    _IterOps<_AlgPolicy>::iter_swap(__first1, __first2);
+    ++__first1;
+    ++__first2;
+  }
+
+  return pair<_ForwardIterator1, _ForwardIterator2>(::cuda::std::move(__first1), ::cuda::std::move(__first2));
+}
+
+// 2+1 iterators: size2 >= size1.
+_CCCL_EXEC_CHECK_DISABLE
+template <class _AlgPolicy, class _ForwardIterator1, class _Sentinel1, class _ForwardIterator2>
+_CCCL_API constexpr pair<_ForwardIterator1, _ForwardIterator2>
+__swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2)
+{
+  while (__first1 != __last1)
+  {
+    _IterOps<_AlgPolicy>::iter_swap(__first1, __first2);
+    ++__first1;
+    ++__first2;
+  }
+
+  return pair<_ForwardIterator1, _ForwardIterator2>(::cuda::std::move(__first1), ::cuda::std::move(__first2));
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _ForwardIterator1, class _ForwardIterator2>
+_CCCL_API constexpr _ForwardIterator2
+swap_ranges(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2)
+{
+  return ::cuda::std::__swap_ranges<_ClassicAlgPolicy>(
+           ::cuda::std::move(__first1), ::cuda::std::move(__last1), ::cuda::std::move(__first2))
+    .second;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_SWAP_RANGES_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/unwrap_iter.h

@@ -0,0 +1,95 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___ALGORITHM_UNWRAP_ITER_H
+#define _CUDA_STD___ALGORITHM_UNWRAP_ITER_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__memory/pointer_traits.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_copy_constructible.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+// TODO: Change the name of __unwrap_iter_impl to something more appropriate
+// The job of __unwrap_iter is to remove iterator wrappers (like reverse_iterator or __wrap_iter),
+// to reduce the number of template instantiations and to enable pointer-based optimizations e.g. in ::cuda::std::copy.
+// In debug mode, we don't do this.
+//
+// Some algorithms (e.g. ::cuda::std::copy, but not ::cuda::std::sort) need to convert an
+// "unwrapped" result back into the original iterator type. Doing that is the job of __rewrap_iter.
+
+// Default case - we can't unwrap anything
+template <class _Iter, bool = __has_contiguous_traversal<_Iter>>
+struct __unwrap_iter_impl
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  static _CCCL_API constexpr _Iter __rewrap(_Iter, _Iter __iter)
+  {
+    return __iter;
+  }
+  _CCCL_EXEC_CHECK_DISABLE
+  static _CCCL_API constexpr _Iter __unwrap(_Iter __i) noexcept
+  {
+    return __i;
+  }
+};
+
+// It's a contiguous iterator, so we can use a raw pointer instead
+template <class _Iter>
+struct __unwrap_iter_impl<_Iter, true>
+{
+  using _ToAddressT = decltype(::cuda::std::__to_address(::cuda::std::declval<_Iter>()));
+
+  _CCCL_EXEC_CHECK_DISABLE
+  static _CCCL_API constexpr _Iter __rewrap(_Iter __orig_iter, _ToAddressT __unwrapped_iter)
+  {
+    return __orig_iter + (__unwrapped_iter - ::cuda::std::__to_address(__orig_iter));
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  static _CCCL_API constexpr _ToAddressT __unwrap(_Iter __i) noexcept
+  {
+    return ::cuda::std::__to_address(__i);
+  }
+};
+
+template <class _Iter, class _Impl = __unwrap_iter_impl<_Iter>, enable_if_t<is_copy_constructible_v<_Iter>, int> = 0>
+_CCCL_API constexpr decltype(_Impl::__unwrap(::cuda::std::declval<_Iter>())) __unwrap_iter(_Iter __i) noexcept
+{
+  return _Impl::__unwrap(__i);
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _OrigIter, class _Iter, class _Impl = __unwrap_iter_impl<_OrigIter>>
+_CCCL_API constexpr _OrigIter __rewrap_iter(_OrigIter __orig_iter, _Iter __iter) noexcept
+{
+  return _Impl::__rewrap(::cuda::std::move(__orig_iter), ::cuda::std::move(__iter));
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ALGORITHM_UNWRAP_ITER_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__bit/bit_cast.h

@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024-26 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___BIT_BIT_CAST_H
+#define _CUDA_STD___BIT_BIT_CAST_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__type_traits/is_trivially_copyable.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstring/memcpy.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+// MSVC supports __builtin_bit_cast from 19.25 on
+#if _CCCL_CHECK_BUILTIN(builtin_bit_cast) || _CCCL_COMPILER(MSVC, >, 19, 25)
+#  define _CCCL_BUILTIN_BIT_CAST(...) __builtin_bit_cast(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_bit_cast)
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if defined(_CCCL_BUILTIN_BIT_CAST)
+#  define _CCCL_CONSTEXPR_BIT_CAST       constexpr
+#  define _CCCL_HAS_CONSTEXPR_BIT_CAST() 1
+#else // ^^^ _CCCL_BUILTIN_BIT_CAST ^^^ / vvv !_CCCL_BUILTIN_BIT_CAST vvv
+#  define _CCCL_CONSTEXPR_BIT_CAST
+#  define _CCCL_HAS_CONSTEXPR_BIT_CAST() 0
+#endif // !_CCCL_BUILTIN_BIT_CAST
+
+#if _CCCL_COMPILER(GCC, >=, 8)
+_CCCL_DIAG_PUSH
+_CCCL_DIAG_SUPPRESS_GCC("-Wclass-memaccess")
+#endif // _CCCL_COMPILER(GCC, >=, 8)
+
+template <class _To, class _From>
+[[nodiscard]] _CCCL_API inline _To __bit_cast_memcpy(const _From& __from) noexcept
+{
+  static_assert(::cuda::std::is_default_constructible_v<_To>,
+                "bit_cast memcpy fallback requires the destination type to be default constructible");
+  _To __temp;
+  ::cuda::std::memcpy(&__temp, &__from, sizeof(_To));
+  return __temp;
+}
+
+#if _CCCL_COMPILER(GCC, >=, 8)
+_CCCL_DIAG_POP
+#endif // _CCCL_COMPILER(GCC, >=, 8)
+
+_CCCL_TEMPLATE(class _To, class _From)
+_CCCL_REQUIRES((sizeof(_To) == sizeof(_From)) _CCCL_AND(::cuda::is_trivially_copyable_v<_To>)
+                 _CCCL_AND(::cuda::is_trivially_copyable_v<_From>))
+[[nodiscard]] _CCCL_API inline _CCCL_CONSTEXPR_BIT_CAST _To bit_cast(const _From& __from) noexcept
+{
+#if defined(_CCCL_BUILTIN_BIT_CAST)
+  if constexpr (::cuda::std::is_trivially_copyable_v<_To> && ::cuda::std::is_trivially_copyable_v<_From>)
+  {
+    return _CCCL_BUILTIN_BIT_CAST(_To, __from);
+  }
+  else
+#endif // _CCCL_BUILTIN_BIT_CAST
+  {
+    return ::cuda::std::__bit_cast_memcpy<_To>(__from);
+  }
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___BIT_BIT_CAST_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/architecture.h

@@ -0,0 +1,128 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_ARCH_H
+#define __CCCL_ARCH_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/preprocessor.h>
+
+// The header provides the following macros to determine the host architecture:
+//
+// _CCCL_HOST_ARCH(ARM64)  ARM64
+// _CCCL_HOST_ARCH(X86_64) X86 64 bit
+// CCCL_HOST_ARCH(ARM64)   ARM64
+// CCCL_HOST_ARCH(X86_64)  X86 64 bit
+
+// Determine the host architecture
+
+// Arm 64-bit
+#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) /*emulation*/)
+#  define _CCCL_HOST_ARCH_ARM64_() 1
+#else
+#  define _CCCL_HOST_ARCH_ARM64_() 0
+#endif
+
+// X86 64-bit
+
+// _M_X64 is defined even if we are compiling in Arm64 emulation mode
+#if (defined(_M_X64) && !defined(_M_ARM64EC)) || defined(__amd64__) || defined(__x86_64__)
+#  define _CCCL_HOST_ARCH_X86_64_() 1
+#else
+#  define _CCCL_HOST_ARCH_X86_64_() 0
+#endif
+
+#define _CCCL_HOST_ARCH(...) _CCCL_HOST_ARCH_##__VA_ARGS__##_()
+
+//! @def CCCL_HOST_ARCH(ARCH) /* implementation defined */
+//!
+//! @brief Detect the current host architecture.
+//!
+//! @param ARCH The name of the host architecture to test.
+//!
+//! @note This macro is made available when including any libcu++ header. Users that wish to
+//! include the smallest possible header for this macro should include `<cuda/std/version>`.
+//!
+//! For supported host architectures, the macro expands to an implementation-defined true value
+//! if the current host architecture matches, or false otherwise. These values may be used in
+//! boolean expressions (preprocessor or otherwise), but no other guarantees are made.
+//!
+//! Available values for `ARCH` include:
+//!
+//! - ``ARM64``: ARM 64-bit, including MSVC ARM64EC emulation.
+//! - ``X86_64``: X86 64-bit. This is false when compiling in MSVC ARM64EC emulation mode.
+//!
+//! Passing any other value will result in an undefined expansion, which may or may not be
+//! diagnosed by the compiler.
+//!
+//! @par Example
+//! @code
+//! #define MY_OTHER_MACRO 1
+//!
+//! // Expansion value can be used in ordinary macro conditionals
+//! #if CCCL_HOST_ARCH(X86_64) && MY_OTHER_MACRO
+//!   // ...
+//! #endif
+//!
+//! // Can be negated as usual
+//! #if !CCCL_HOST_ARCH(ARM64)
+//!   // ...
+//! #endif
+//! @endcode
+//!
+//! @return true if the specified host architecture is being compiled for, false otherwise.
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CCCL_HOST_ARCH(ARCH) /* implementation defined */
+#else
+#  define CCCL_HOST_ARCH(__arch__) _CCCL_HOST_ARCH_##__arch__##_()
+#endif
+
+// Note: the public API is single-arg to constrain the API and allow for future expansion. The
+// implementation is duplicated to guard against the architecture targets being accidentally
+// defined by the user.
+
+// Determine the endianness
+
+#define _CCCL_ENDIAN_LITTLE() 0xDEAD
+#define _CCCL_ENDIAN_BIG()    0xFACE
+#define _CCCL_ENDIAN_PDP()    0xBEEF
+
+#if _CCCL_COMPILER(NVRTC) || (_CCCL_COMPILER(MSVC) && (_CCCL_HOST_ARCH(X86_64) || _CCCL_HOST_ARCH(ARM64))) \
+  || __LITTLE_ENDIAN__
+#  define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
+#elif __BIG_ENDIAN__
+#  define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG()
+#elif defined(__BYTE_ORDER__)
+#  if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#    define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
+#  elif __BYTE_ORDER__ == __ORDER_PDP_ENDIAN__
+#    define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_PDP()
+#  elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#    define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG()
+#  endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#elif __has_include(<endian.h>)
+#  include <endian.h>
+#  if __BYTE_ORDER == __LITTLE_ENDIAN
+#    define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
+#  elif __BYTE_ORDER == __PDP_ENDIAN
+#    define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_PDP()
+#  elif __BYTE_ORDER == __BIG_ENDIAN
+#    define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG()
+#  endif // __BYTE_ORDER == __BIG_ENDIAN
+#endif // ^^^ has endian.h ^^^
+
+#if !defined(_CCCL_ENDIAN_NATIVE)
+_CCCL_WARNING("failed to determine the endianness of the host architecture, defaulting to little-endian")
+#  define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE()
+#endif // !_CCCL_ENDIAN_NATIVE
+
+#define _CCCL_ENDIAN(_NAME) (_CCCL_ENDIAN_NATIVE() == _CCCL_ENDIAN_##_NAME())
+
+#endif // __CCCL_ARCH_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/assert.h

@@ -0,0 +1,169 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_ASSERT_H
+#define __CCCL_ASSERT_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/attributes.h>
+#include <cuda/std/__cccl/builtin.h>
+#include <cuda/std/__cccl/execution_space.h>
+#include <cuda/std/__cccl/preprocessor.h>
+
+#if _CCCL_HOSTED()
+#  include <assert.h>
+#endif // _CCCL_HOSTED()
+
+#include <nv/target>
+
+#if defined(_DEBUG) || defined(DEBUG)
+#  ifndef _CCCL_ENABLE_DEBUG_MODE
+#    define _CCCL_ENABLE_DEBUG_MODE
+#  endif // !_CCCL_ENABLE_DEBUG_MODE
+#endif // _DEBUG || DEBUG
+
+// Automatically enable assertions when debug mode is enabled
+#ifdef _CCCL_ENABLE_DEBUG_MODE
+#  ifndef CCCL_ENABLE_ASSERTIONS
+#    define CCCL_ENABLE_ASSERTIONS
+#  endif // !CCCL_ENABLE_ASSERTIONS
+#endif // _CCCL_ENABLE_DEBUG_MODE
+
+//! Ensure that we switch on host assertions when all assertions are enabled
+#ifndef CCCL_ENABLE_HOST_ASSERTIONS
+#  ifdef CCCL_ENABLE_ASSERTIONS
+#    define CCCL_ENABLE_HOST_ASSERTIONS
+#  endif // CCCL_ENABLE_ASSERTIONS
+#endif // !CCCL_ENABLE_HOST_ASSERTIONS
+
+//! Ensure that we switch on device assertions when all assertions are enabled
+#ifndef CCCL_ENABLE_DEVICE_ASSERTIONS
+#  if defined(CCCL_ENABLE_ASSERTIONS) || defined(__CUDACC_DEBUG__)
+#    define CCCL_ENABLE_DEVICE_ASSERTIONS
+#  endif // CCCL_ENABLE_ASSERTIONS
+#endif // !CCCL_ENABLE_DEVICE_ASSERTIONS
+
+//! Use the different standard library implementations to implement host side asserts
+//! _CCCL_ASSERT_IMPL_HOST should never be used directly
+#if _CCCL_OS(QNX)
+#  define _CCCL_ASSERT_IMPL_HOST(expression, message) ((void) 0)
+#elif _CCCL_COMPILER(NVRTC) // There is no host standard library in nvrtc
+#  define _CCCL_ASSERT_IMPL_HOST(expression, message) ((void) 0)
+#elif __has_include(<yvals.h>) && _CCCL_OS(WINDOWS) // Windows uses _STL_VERIFY from <yvals.h>
+#  include <yvals.h>
+#  define _CCCL_ASSERT_IMPL_HOST(expression, message) _STL_VERIFY(expression, message)
+#else // ^^^ MSVC STL ^^^ / vvv !MSVC STL vvv
+#  ifdef NDEBUG
+// Reintroduce the __assert_fail / __assert_rtn declaration
+extern "C" {
+#    if !_CCCL_CUDA_COMPILER(CLANG)
+_CCCL_HOST_DEVICE
+#    endif // !_CCCL_CUDA_COMPILER(CLANG)
+#    if _CCCL_OS(APPLE)
+void __assert_rtn(const char* __function, const char* __assertion, const char* __file, unsigned int __line) noexcept
+  __attribute__((__noreturn__));
+#    else // ^^^ _CCCL_OS(APPLE) ^^^ / vvv !_CCCL_OS(APPLE) ^^^
+void __assert_fail(const char* __assertion, const char* __file, unsigned int __line, const char* __function) noexcept
+  __attribute__((__noreturn__));
+#    endif // !_CCCL_OS(APPLE)
+}
+#  endif // NDEBUG
+
+#  if _CCCL_OS(APPLE)
+#    define _CCCL_ASSERT_IMPL_HOST(expression, message)      \
+      _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+      ? (void) 0 : __assert_rtn(__func__, __FILE__, __LINE__, message)
+#  elif _CCCL_OS(ANDROID)
+#    define _CCCL_ASSERT_IMPL_HOST(expression, message)      \
+      _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+      ? (void) 0 : __assert2(__FILE__, __LINE__, __func__, message)
+#  else // ^^^ _CCCL_OS(APPLE) ^^^ / vvv !_CCCL_OS(APPLE) ^^^
+#    define _CCCL_ASSERT_IMPL_HOST(expression, message)      \
+      _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+      ? (void) 0 : __assert_fail(message, __FILE__, __LINE__, __func__)
+#  endif // !_CCCL_OS(APPLE)
+#endif // !MSVC STL
+
+//! Use custom implementations with nvcc on device and the host ones with clang-cuda and nvhpc
+//! _CCCL_ASSERT_IMPL_DEVICE should never be used directly
+#if _CCCL_OS(QNX) || _CCCL_OS(APPLE)
+#  define _CCCL_ASSERT_IMPL_DEVICE(expression, message) ((void) 0)
+#elif _CCCL_COMPILER(NVRTC)
+#  define _CCCL_ASSERT_IMPL_DEVICE(expression, message)    \
+    _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+    ? (void) 0 : __assertfail(message, __FILE__, __LINE__, __func__, sizeof(char))
+#elif _CCCL_CUDA_COMPILER(NVCC) //! Use __assert_fail to implement device side asserts
+#  if _CCCL_COMPILER(MSVC)
+#    define _CCCL_ASSERT_IMPL_DEVICE(expression, message)    \
+      _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+      ? (void) 0 : _wassert(_CRT_WIDE(#message), __FILEW__, __LINE__)
+#  elif _CCCL_OS(ANDROID)
+#    define _CCCL_ASSERT_IMPL_DEVICE(expression, message)    \
+      _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+      ? (void) 0 : __assert2(__FILE__, __LINE__, __func__, message)
+#  else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#    define _CCCL_ASSERT_IMPL_DEVICE(expression, message)    \
+      _CCCL_BUILTIN_EXPECT(static_cast<bool>(expression), 1) \
+      ? (void) 0 : __assert_fail(message, __FILE__, __LINE__, __func__)
+#  endif // !_CCCL_COMPILER(MSVC)
+#elif _CCCL_CUDA_COMPILATION()
+#  define _CCCL_ASSERT_IMPL_DEVICE(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
+#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv
+#  define _CCCL_ASSERT_IMPL_DEVICE(expression, message) ((void) 0)
+#endif // !_CCCL_CUDA_COMPILATION()
+
+//! _CCCL_ASSERT_HOST is enabled conditionally depending on CCCL_ENABLE_HOST_ASSERTIONS
+#ifdef CCCL_ENABLE_HOST_ASSERTIONS
+#  define _CCCL_ASSERT_HOST(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
+#else // ^^^ CCCL_ENABLE_HOST_ASSERTIONS ^^^ / vvv !CCCL_ENABLE_HOST_ASSERTIONS vvv
+#  define _CCCL_ASSERT_HOST(expression, message) ((void) 0)
+#endif // !CCCL_ENABLE_HOST_ASSERTIONS
+
+//! _CCCL_ASSERT_DEVICE is enabled conditionally depending on CCCL_ENABLE_DEVICE_ASSERTIONS
+#ifdef CCCL_ENABLE_DEVICE_ASSERTIONS
+#  define _CCCL_ASSERT_DEVICE(expression, message) _CCCL_ASSERT_IMPL_DEVICE(expression, message)
+#else // ^^^ CCCL_ENABLE_DEVICE_ASSERTIONS ^^^ / vvv !CCCL_ENABLE_DEVICE_ASSERTIONS vvv
+#  define _CCCL_ASSERT_DEVICE(expression, message) ((void) 0)
+#endif // !CCCL_ENABLE_DEVICE_ASSERTIONS
+
+//! _CCCL_VERIFY is enabled unconditionally and reserved for critical checks that are required to always be on
+//! _CCCL_ASSERT is enabled conditionally depending on CCCL_ENABLE_HOST_ASSERTIONS and CCCL_ENABLE_DEVICE_ASSERTIONS
+#if _CCCL_CUDA_COMPILER(NVHPC) // NVHPC can't have different behavior for host and device.
+                               // The host version of the assert will also work in device code.
+#  define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
+#  if defined(CCCL_ENABLE_HOST_ASSERTIONS) || defined(CCCL_ENABLE_DEVICE_ASSERTIONS)
+#    define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message)
+#  else
+#    define _CCCL_ASSERT(expression, message) ((void) 0)
+#  endif
+#elif _CCCL_CUDA_COMPILATION()
+#  if _CCCL_DEVICE_COMPILATION()
+#    define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_DEVICE(expression, message)
+#    define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_DEVICE(expression, message)
+#  else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
+#    define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
+#    define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message)
+#  endif // !_CCCL_DEVICE_COMPILATION()
+#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv
+#  define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message)
+#  define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message)
+#endif // !_CCCL_CUDA_COMPILATION()
+
+#endif // __CCCL_ASSERT_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/attributes.h

@@ -0,0 +1,221 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_ATTRIBUTES_H
+#define __CCCL_ATTRIBUTES_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/diagnostic.h>
+#include <cuda/std/__cccl/dialect.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#ifdef __has_attribute
+#  define _CCCL_HAS_ATTRIBUTE(__x) __has_attribute(__x)
+#else // ^^^ __has_attribute ^^^ / vvv !__has_attribute vvv
+#  define _CCCL_HAS_ATTRIBUTE(__x) 0
+#endif // !__has_attribute
+
+#ifdef __has_cpp_attribute
+#  define _CCCL_HAS_CPP_ATTRIBUTE(__x) __has_cpp_attribute(__x)
+#else // ^^^ __has_cpp_attribute ^^^ / vvv !__has_cpp_attribute vvv
+#  define _CCCL_HAS_CPP_ATTRIBUTE(__x) 0
+#endif // !__has_cpp_attribute
+
+#ifdef __has_declspec_attribute
+#  define _CCCL_HAS_DECLSPEC_ATTRIBUTE(__x) __has_declspec_attribute(__x)
+#else // ^^^ __has_declspec_attribute ^^^ / vvv !__has_declspec_attribute vvv
+#  define _CCCL_HAS_DECLSPEC_ATTRIBUTE(__x) 0
+#endif // !__has_declspec_attribute
+
+// MSVC needs extra help with empty base classes
+#if _CCCL_COMPILER(MSVC) || _CCCL_HAS_DECLSPEC_ATTRIBUTE(empty_bases)
+#  define _CCCL_DECLSPEC_EMPTY_BASES __declspec(empty_bases)
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_DECLSPEC_EMPTY_BASES
+#endif // !_CCCL_COMPILER(MSVC)
+
+#if _CCCL_HAS_ATTRIBUTE(__nodebug__)
+#  define _CCCL_NODEBUG __attribute__((__nodebug__))
+#else // ^^^ _CCCL_HAS_ATTRIBUTE(__nodebug__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__nodebug__) vvv
+#  define _CCCL_NODEBUG
+#endif // !_CCCL_HAS_ATTRIBUTE(__nodebug__)
+
+// Debuggers do not step into functions marked with __attribute__((__artificial__)). This
+// is useful for small wrapper functions that just dispatch to other functions and that
+// are inlined into the caller.
+#if _CCCL_HAS_ATTRIBUTE(__artificial__) && !_CCCL_CUDA_COMPILER(NVCC)
+#  define _CCCL_ARTIFICIAL __attribute__((__artificial__))
+#else // ^^^ _CCCL_HAS_ATTRIBUTE(__artificial__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__artificial__) vvv
+#  define _CCCL_ARTIFICIAL
+#endif // !_CCCL_HAS_ATTRIBUTE(__artificial__)
+
+// The nodebug attribute flattens aliases down to the actual type rather typename meow<T>::type
+#if _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_NODEBUG_ALIAS _CCCL_NODEBUG
+#else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv
+#  define _CCCL_NODEBUG_ALIAS
+#endif // !_CCCL_CUDA_COMPILER(CLANG)
+
+// _CCCL_ASSUME
+// NVCC does not properly respect [[assume()]], so use __builtin_assume, see nvbug5458663
+#if _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION()
+#  define _CCCL_ASSUME(...) __builtin_assume(__VA_ARGS__)
+#elif _CCCL_HAS_CPP_ATTRIBUTE(assume)
+#  define _CCCL_ASSUME(...) [[assume(__VA_ARGS__)]]
+#else
+#  define _CCCL_ASSUME(...) _CCCL_BUILTIN_ASSUME(__VA_ARGS__)
+#endif
+
+#if _CCCL_TILE_COMPILATION() // nvbug6100910: __builtin_assume is not supported in tile mode
+#  undef _CCCL_ASSUME
+#  define _CCCL_ASSUME(...)
+#endif // _CCCL_TILE_COMPILATION()
+
+// _CCCL_CONST
+
+#if _CCCL_HAS_CPP_ATTRIBUTE(__gnu__::__const__)
+#  define _CCCL_CONST [[__gnu__::__const__]]
+#else // ^^^ has gnu::const ^^^ / vvv no gnu::const vvv
+#  define _CCCL_CONST _CCCL_PURE
+#endif // ^^^ no gnu::const ^^^
+
+// _CCCL_DIAGNOSE_IF
+
+#if _CCCL_HAS_ATTRIBUTE(__diagnose_if__)
+#  define _CCCL_DIAGNOSE_IF(_COND, _MSG, _TYPE) __attribute__((__diagnose_if__(_COND, _MSG, _TYPE)))
+#else // ^^^ _CCCL_HAS_ATTRIBUTE(diagnose_if) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(diagnose_if) vvv
+#  define _CCCL_DIAGNOSE_IF(_COND, _MSG, _TYPE)
+#endif // !_CCCL_HAS_ATTRIBUTE(diagnose_if)
+
+// _CCCL_INTRINSIC
+
+// MSVC provides a way to mark functions as intrinsic provided the function's body consists of a single
+// return statement of a cast expression (e.g., move(x) or forward<T>(u)).
+#if _CCCL_COMPILER(MSVC) && _CCCL_HAS_CPP_ATTRIBUTE(msvc::intrinsic)
+#  define _CCCL_INTRINSIC [[msvc::intrinsic]]
+#else
+#  define _CCCL_INTRINSIC
+#endif
+
+// _CCCL_PURE
+
+#if _CCCL_CUDA_COMPILER(NVCC, >=, 12, 5)
+#  define _CCCL_PURE __nv_pure__
+#elif _CCCL_HAS_CPP_ATTRIBUTE(__gnu__::__pure__)
+#  define _CCCL_PURE [[__gnu__::__pure__]]
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_PURE __declspec(noalias)
+#else
+#  define _CCCL_PURE
+#endif
+
+// _CCCL_NO_CFI
+
+#if !_CCCL_COMPILER(GCC)
+#  define _CCCL_NO_CFI _CCCL_NO_SANITIZE("cfi")
+#else
+#  define _CCCL_NO_CFI
+#endif
+
+// _CCCL_NO_SANITIZE
+
+#if _CCCL_HAS_ATTRIBUTE(__no_sanitize__)
+#  define _CCCL_NO_SANITIZE(_STR) __attribute__((__no_sanitize__(_STR)))
+#else // ^^^ _CCCL_HAS_ATTRIBUTE(no_sanitize) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(no_sanitize) vvv
+#  define _CCCL_NO_SANITIZE(_STR)
+#endif // !_CCCL_HAS_ATTRIBUTE(no_sanitize)
+
+// _CCCL_NO_SPECIALIZATIONS
+
+#if _CCCL_HAS_CPP_ATTRIBUTE(clang::__no_specializations__)
+#  define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG)   [[clang::__no_specializations__(_MSG)]]
+#  define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 1
+#elif _CCCL_HAS_CPP_ATTRIBUTE(msvc::no_specializations)
+#  define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG)   [[msvc::no_specializations(_MSG)]]
+#  define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 1
+#else // ^^^ has attribute no_specializations ^^^ / vvv hasn't attribute no_specializations vvv
+#  define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG)
+#  define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 0
+#endif // ^^^ hasn't attribute no_specializations ^^^
+
+#define _CCCL_NO_SPECIALIZATIONS \
+  _CCCL_NO_SPECIALIZATIONS_BECAUSE("Users are not allowed to specialize this cccl entity")
+
+// _CCCL_LIFETIMEBOUND
+
+#if _CCCL_HAS_CPP_ATTRIBUTE(clang::lifetimebound) || _CCCL_COMPILER(CLANG)
+#  define _CCCL_LIFETIMEBOUND [[clang::lifetimebound]]
+#elif _CCCL_HAS_CPP_ATTRIBUTE(msvc::lifetimebound) || _CCCL_COMPILER(MSVC, >=, 19, 37)
+#  define _CCCL_LIFETIMEBOUND [[msvc::lifetimebound]]
+#else
+#  define _CCCL_LIFETIMEBOUND
+#endif
+
+// _CCCL_NO_UNIQUE_ADDRESS
+
+#if _CCCL_COMPILER(MSVC) || _CCCL_HAS_CPP_ATTRIBUTE(no_unique_address) < 201803L
+// MSVC implementation has lead to multiple issues with silent runtime corruption when passing data into kernels
+#  define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0
+#  define _CCCL_NO_UNIQUE_ADDRESS
+#elif _CCCL_HAS_CPP_ATTRIBUTE(no_unique_address)
+#  define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 1
+#  define _CCCL_NO_UNIQUE_ADDRESS                 [[no_unique_address]]
+#else
+#  define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0
+#  define _CCCL_NO_UNIQUE_ADDRESS
+#endif
+
+// Passing objects with nested [[no_unique_address]] to kernels leads to data corruption.
+// This is caused by cudafe++ not honoring [[no_unique_address]] when compiling for C++17
+// with clang as the host compiler. See nvbug 5265027 for more details.
+#if _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() && _CCCL_COMPILER(CLANG) && _CCCL_STD_VER < 2020 \
+  && _CCCL_CUDA_COMPILER(NVCC)
+#  undef _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS
+#  undef _CCCL_NO_UNIQUE_ADDRESS
+#  define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0
+#  define _CCCL_NO_UNIQUE_ADDRESS
+#endif // _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() && _CCCL_COMPILER(CLANG)
+
+// _CCCL_PREFERRED_NAME
+
+#if _CCCL_HAS_ATTRIBUTE(__preferred_name__)
+#  define _CCCL_PREFERRED_NAME(x) __attribute__((__preferred_name__(x)))
+#else
+#  define _CCCL_PREFERRED_NAME(x)
+#endif
+
+#if _CCCL_HAS_ATTRIBUTE(__require_constant_initialization__)
+#  define _CCCL_REQUIRE_CONSTANT_INITIALIZATION __attribute__((__require_constant_initialization__))
+#else
+#  define _CCCL_REQUIRE_CONSTANT_INITIALIZATION
+#endif
+
+// _CCCL_RESTRICT
+
+#if _CCCL_COMPILER(MSVC) // vvv _CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_RESTRICT __restrict
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_RESTRICT __restrict__
+#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // __CCCL_ATTRIBUTES_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/builtin.h

@@ -0,0 +1,474 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_BUILTIN_H
+#define __CCCL_BUILTIN_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/preprocessor.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/cuda_capabilities.h>
+#include <cuda/std/__cccl/extended_data_types.h>
+#include <cuda/std/__cccl/host_std_lib.h>
+
+//! This file consolidates all compiler builtin detection for CCCL.
+//!
+//! To work around older compilers not supporting `__has_builtin` we use `_CCCL_CHECK_BUILTIN` that detects more
+//! cases
+//!
+//! * We work around old clang versions (before clang-10) not supporting __has_builtin via _CCCL_CHECK_BUILTIN
+//! * We work around old intel versions (before 2021.3)   not supporting __has_builtin via _CCCL_CHECK_BUILTIN
+//! * We work around old nvhpc versions (before 2022.11)  not supporting __has_builtin via _CCCL_CHECK_BUILTIN
+//! * MSVC needs manual handling, has no real way of checking builtins so all is manual
+//! * GCC  needs manual handling, before gcc-10 as that finally supports __has_builtin
+//!
+//! In case compiler support for a builtin is advertised but leads to regressions we explicitly undef the macro
+//!
+//! Finally, because `_CCCL_CHECK_BUILTIN` may lead to false positives, we move detection of new builtins over towards
+//! just using _CCCL_HAS_BUILTIN
+
+#ifdef __has_builtin
+#  define _CCCL_HAS_BUILTIN(__x) __has_builtin(__x)
+#else // ^^^ __has_builtin ^^^ / vvv !__has_builtin vvv
+#  define _CCCL_HAS_BUILTIN(__x) 0
+#endif // !__has_builtin
+
+#ifdef __has_feature
+#  define _CCCL_HAS_FEATURE(__x) __has_feature(__x)
+#else // ^^^ __has_feature ^^^ / vvv !__has_feature vvv
+#  define _CCCL_HAS_FEATURE(__x) 0
+#endif // !__has_feature
+
+// '__is_identifier' returns '0' if '__x' is a reserved identifier provided by the compiler and '1' otherwise.
+#ifdef __is_identifier
+#  define _CCCL_IS_IDENTIFIER(__x) __is_identifier(__x)
+#else // ^^^ __is_identifier ^^^ / vvv !__is_identifier vvv
+#  define _CCCL_IS_IDENTIFIER(__x) 1
+#endif // !__is_identifier
+
+#define _CCCL_HAS_KEYWORD(__x) !(_CCCL_IS_IDENTIFIER(__x))
+
+// https://bugs.llvm.org/show_bug.cgi?id=44517
+#define _CCCL_CHECK_BUILTIN(__x) (_CCCL_HAS_BUILTIN(__##__x) || _CCCL_HAS_KEYWORD(__##__x) || _CCCL_HAS_FEATURE(__x))
+
+// NVCC has issues with function pointers
+#if _CCCL_HAS_BUILTIN(__add_lvalue_reference) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_ADD_LVALUE_REFERENCE(...) __add_lvalue_reference(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__add_lvalue_reference)
+
+// NVCC has issues with function pointers
+#if _CCCL_HAS_BUILTIN(__add_pointer) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_ADD_POINTER(...) __add_pointer(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__add_pointer)
+
+// NVCC has issues with function pointers
+#if _CCCL_HAS_BUILTIN(__add_rvalue_reference) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_ADD_RVALUE_REFERENCE(...) __add_rvalue_reference(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__add_rvalue_reference)
+
+// TODO: Enable using the builtin __array_rank when https://llvm.org/PR57133 is resolved
+#if 0 // _CCCL_CHECK_BUILTIN(array_rank)
+#  define _CCCL_BUILTIN_ARRAY_RANK(...) __array_rank(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(array_rank)
+
+// nvhpc has a bug where it supports __builtin_addressof but does not mark it via _CCCL_CHECK_BUILTIN
+#if _CCCL_CHECK_BUILTIN(builtin_addressof) || _CCCL_COMPILER(GCC, >=, 7) || _CCCL_COMPILER(MSVC) \
+  || _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC, >=, 12, 3)
+#  define _CCCL_BUILTIN_ADDRESSOF(...) __builtin_addressof(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_addressof)
+
+#if _CCCL_CHECK_BUILTIN(builtin_assume) || _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(NVHPC)
+#  define _CCCL_BUILTIN_ASSUME(...) __builtin_assume(__VA_ARGS__)
+#elif _CCCL_COMPILER(GCC, >=, 13)
+#  define _CCCL_BUILTIN_ASSUME(...) __attribute__((__assume__(__VA_ARGS__)))
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_BUILTIN_ASSUME(...) __assume(__VA_ARGS__)
+#else
+#  define _CCCL_BUILTIN_ASSUME(...)
+#endif // _CCCL_CHECK_BUILTIN(builtin_assume)
+
+#if _CCCL_TILE_COMPILATION() // nvbug6100910: __builtin_assume is not supported in tile mode
+#  undef _CCCL_BUILTIN_ASSUME
+#  define _CCCL_BUILTIN_ASSUME(...)
+#endif // _CCCL_TILE_COMPILATION()
+
+#if _CCCL_HAS_BUILTIN(__builtin_assume_aligned) || _CCCL_COMPILER(MSVC, >=, 19, 23) || _CCCL_COMPILER(GCC)
+#  define _CCCL_BUILTIN_ASSUME_ALIGNED(...) __builtin_assume_aligned(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_assume_aligned)
+
+#if _CCCL_CHECK_BUILTIN(builtin_constant_p) || _CCCL_COMPILER(GCC)
+#  define _CCCL_BUILTIN_CONSTANT_P(...) __builtin_constant_p(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_constant_p)
+
+#if _CCCL_CHECK_BUILTIN(builtin_expect) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC)
+#  define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) __builtin_expect(_EXPR, _VAL)
+#else // ^^^ has __builtin_expect ^^^ / vvv no __builtin_expect vvv
+#  define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) (_EXPR)
+#endif // ^^^ no __builtin_expect ^^^
+
+#if _CCCL_TILE_COMPILATION() // nvbug6100927:  __builtin_expect is unsupported in tile mode
+#  undef _CCCL_BUILTIN_EXPECT
+#  define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) (_EXPR)
+#endif // _CCCL_TILE_COMPILATION()
+
+#if _CCCL_CHECK_BUILTIN(builtin_huge_valf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_HUGE_VALF() __builtin_huge_valf()
+#endif // _CCCL_CHECK_BUILTIN(builtin_huge_valf)
+
+#if _CCCL_CHECK_BUILTIN(builtin_huge_val) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_HUGE_VAL() __builtin_huge_val()
+#endif // _CCCL_CHECK_BUILTIN(builtin_huge_val)
+
+#if _CCCL_CHECK_BUILTIN(builtin_huge_vall) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_HUGE_VALL() __builtin_huge_vall()
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_BUILTIN_HUGE_VALL() static_cast<long double>(__builtin_huge_val())
+#endif // _CCCL_CHECK_BUILTIN(builtin_huge_vall)
+
+#if _CCCL_HAS_FLOAT128()
+#  if _CCCL_CHECK_BUILTIN(builtin_huge_valf128) || _CCCL_COMPILER(GCC, >=, 7)
+#    define _CCCL_BUILTIN_HUGE_VALF128() __builtin_huge_valf128()
+#  endif // _CCCL_CHECK_BUILTIN(builtin_huge_valf128) || _CCCL_COMPILER(GCC, >=, 7)
+
+// nvcc does not implement __builtin_huge_valf128
+#  if _CCCL_CUDA_COMPILER(NVCC)
+#    undef _CCCL_BUILTIN_HUGE_VALF128
+#  endif // _CCCL_CUDA_COMPILER(NVCC)
+#endif // _CCCL_HAS_FLOAT128()
+
+#if _CCCL_CHECK_BUILTIN(builtin_is_constant_evaluated) || _CCCL_COMPILER(GCC, >=, 9) || _CCCL_COMPILER(MSVC, >, 19, 24)
+#  define _CCCL_BUILTIN_IS_CONSTANT_EVALUATED(...) __builtin_is_constant_evaluated(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_is_constant_evaluated)
+
+#if _CCCL_TILE_COMPILATION() // nvbug6067464: __builtin_is_constant_evaluated is unsupported in tile mode
+#  undef _CCCL_BUILTIN_IS_CONSTANT_EVALUATED
+#endif // _CCCL_TILE_COMPILATION()
+
+#if _CCCL_CHECK_BUILTIN(builtin_is_corresponding_member)
+#  define _CCCL_BUILTIN_IS_CORRESPONDING_MEMBER(_C1, _C2, _MPtr1, _MPtr2) \
+    __builtin_is_corresponding_member(_MPtr1, _MPtr2)
+#elif _CCCL_COMPILER(MSVC, >=, 19, 29)
+// using __is_corresponding_member with msvc outside of constexpr context causes linker errors, see
+// https://developercommunity.visualstudio.com/t/Using-compiler-builtins-causes-linking-n/10888080
+// #  define _CCCL_BUILTIN_IS_CORRESPONDING_MEMBER(_C1, _C2, _MPtr1, _MPtr2) __is_corresponding_member(_C1, _C2, _MPtr1,
+// _MPtr2)
+#endif // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 29) ^^^
+
+#if _CCCL_CHECK_BUILTIN(builtin_is_pointer_interconvertible_with_class)
+#  define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS(_S, _MPtr) \
+    __builtin_is_pointer_interconvertible_with_class(_MPtr)
+#elif _CCCL_COMPILER(MSVC, >=, 19, 29)
+// using __is_pointer_interconvertible_with_class with msvc outside of constexpr context causes linker errors, see
+// https://developercommunity.visualstudio.com/t/Using-compiler-builtins-causes-linking-n/10888080
+// #  define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS(_S, _MPtr)
+// __is_pointer_interconvertible_with_class(_S, _MPtr)
+#endif // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 29) ^^^
+
+#if _CCCL_CHECK_BUILTIN(builtin_is_virtual_base_of)
+#  define _CCCL_BUILTIN_IS_VIRTUAL_BASE_OF(...) __builtin_is_virtual_base_of(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_is_virtual_base_of)
+
+// nvcc < 13.3 doesn't implement __builtin_is_virtual_base_of
+#if _CCCL_CUDA_COMPILER(NVCC, <, 13, 3)
+#  undef _CCCL_BUILTIN_IS_VIRTUAL_BASE_OF
+#endif // _CCCL_CUDA_COMPILER(NVCC, <, 13, 3)
+
+#if _CCCL_CHECK_BUILTIN(builtin_nanf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_NANF(...) __builtin_nanf(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_nanf)
+
+#if _CCCL_CHECK_BUILTIN(builtin_nan) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_NAN(...) __builtin_nan(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_nan)
+
+#if _CCCL_CHECK_BUILTIN(builtin_nanl) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_NANL(...) __builtin_nanl(__VA_ARGS__)
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_BUILTIN_NANL(...) static_cast<long double>(__builtin_nan(__VA_ARGS__))
+#endif // _CCCL_CHECK_BUILTIN(builtin_nanl)
+
+#if _CCCL_HAS_FLOAT128()
+#  if _CCCL_CHECK_BUILTIN(builtin_nanf128) || _CCCL_COMPILER(GCC, >=, 7)
+#    define _CCCL_BUILTIN_NANF128(...) __builtin_nanf128(__VA_ARGS__)
+#  endif // _CCCL_CHECK_BUILTIN(builtin_nanf128) || _CCCL_COMPILER(GCC, >=, 7)
+
+// nvcc does not implement __builtin_nanf128
+#  if _CCCL_CUDA_COMPILER(NVCC)
+#    undef _CCCL_BUILTIN_NANF128
+#  endif // _CCCL_CUDA_COMPILER(NVCC)
+#endif // _CCCL_HAS_FLOAT128()
+
+#if _CCCL_CHECK_BUILTIN(builtin_nansf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_NANSF(...) __builtin_nansf(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_nansf)
+
+#if _CCCL_CHECK_BUILTIN(builtin_nans) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_NANS(...) __builtin_nans(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_nans)
+
+#if _CCCL_CHECK_BUILTIN(builtin_nansl) || _CCCL_COMPILER(GCC, <, 10)
+#  define _CCCL_BUILTIN_NANSL(...) __builtin_nansl(__VA_ARGS__)
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_BUILTIN_NANSL(...) static_cast<long double>(__builtin_nans(__VA_ARGS__))
+#endif // _CCCL_CHECK_BUILTIN(builtin_nansl)
+
+#if _CCCL_HAS_FLOAT128()
+#  if _CCCL_CHECK_BUILTIN(builtin_nansf128) || _CCCL_COMPILER(GCC, >=, 7)
+#    define _CCCL_BUILTIN_NANSF128(...) __builtin_nansf128(__VA_ARGS__)
+#  endif // _CCCL_CHECK_BUILTIN(builtin_nansf128) || _CCCL_COMPILER(GCC, >=, 7)
+
+// nvcc does not implement __builtin_nansf128
+#  if _CCCL_CUDA_COMPILER(NVCC)
+#    undef _CCCL_BUILTIN_NANSF128
+#  endif // _CCCL_CUDA_COMPILER(NVCC)
+#endif // _CCCL_HAS_FLOAT128()
+
+#if _CCCL_CHECK_BUILTIN(builtin_memcmp) || _CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC, >=, 19, 28)
+#  define _CCCL_BUILTIN_MEMCMP(...) __builtin_memcmp(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_memcmp) || _CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC, >=, 19, 28)
+
+#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)
+#  undef _CCCL_BUILTIN_MEMCMP
+#endif // _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)
+
+#if _CCCL_CHECK_BUILTIN(builtin_memmove) || _CCCL_COMPILER(GCC)
+#  define _CCCL_BUILTIN_MEMMOVE(...) __builtin_memmove(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_memmove) || _CCCL_COMPILER(GCC)
+
+#if _CCCL_CUDA_COMPILER(NVCC)
+#  undef _CCCL_BUILTIN_MEMMOVE
+#endif // _CCCL_CUDA_COMPILER(NVCC)
+
+#if _CCCL_CHECK_BUILTIN(builtin_operator_new) && _CCCL_CHECK_BUILTIN(builtin_operator_delete) \
+  && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_OPERATOR_DELETE(...) __builtin_operator_delete(__VA_ARGS__)
+#  define _CCCL_BUILTIN_OPERATOR_NEW(...)    __builtin_operator_new(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(builtin_operator_new) && _CCCL_CHECK_BUILTIN(builtin_operator_delete)
+
+#if _CCCL_CHECK_BUILTIN(builtin_prefetch) || _CCCL_COMPILER(GCC)
+#  define _CCCL_BUILTIN_PREFETCH(...) NV_IF_TARGET(NV_IS_HOST, __builtin_prefetch(__VA_ARGS__);)
+#else
+#  define _CCCL_BUILTIN_PREFETCH(...)
+#endif // _CCCL_CHECK_BUILTIN(builtin_prefetch)
+
+#if _CCCL_HAS_BUILTIN(__decay) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_DECAY(...) __decay(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__decay) && clang-cuda
+
+#if _CCCL_CHECK_BUILTIN(has_nothrow_assign) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \
+  || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_HAS_NOTHROW_ASSIGN(...) __has_nothrow_assign(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(has_nothrow_assign) && gcc >= 4.3
+
+#if _CCCL_CHECK_BUILTIN(has_nothrow_constructor) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \
+  || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_HAS_NOTHROW_CONSTRUCTOR(...) __has_nothrow_constructor(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(has_nothrow_constructor) && gcc >= 4.3
+
+#if _CCCL_CHECK_BUILTIN(has_nothrow_copy) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \
+  || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_HAS_NOTHROW_COPY(...) __has_nothrow_copy(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(has_nothrow_copy) && gcc >= 4.3
+
+#if _CCCL_HAS_BUILTIN(__integer_pack)
+#  define _CCCL_BUILTIN_INTEGER_PACK(...) __integer_pack(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__integer_pack)
+
+#if _CCCL_CHECK_BUILTIN(is_array)
+#  define _CCCL_BUILTIN_IS_ARRAY(...) __is_array(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_array)
+
+// clang prior to clang-19 gives wrong results for __is_array of _Tp[0]
+#if _CCCL_COMPILER(CLANG, <, 19)
+#  undef _CCCL_BUILTIN_IS_ARRAY
+#endif // clang < 19
+
+#if _CCCL_CHECK_BUILTIN(is_assignable) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, >=, 9)
+#  define _CCCL_BUILTIN_IS_ASSIGNABLE(...) __is_assignable(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_assignable) && gcc >= 9.0
+
+#if _CCCL_CHECK_BUILTIN(is_constructible) || _CCCL_COMPILER(GCC, >=, 8) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_IS_CONSTRUCTIBLE(...) __is_constructible(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_constructible) && gcc >= 8.0
+
+#if _CCCL_CHECK_BUILTIN(is_convertible_to) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_IS_CONVERTIBLE_TO(...) __is_convertible_to(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_convertible_to)
+
+#if _CCCL_CHECK_BUILTIN(is_destructible) || _CCCL_COMPILER(MSVC)
+#  define _CCCL_BUILTIN_IS_DESTRUCTIBLE(...) __is_destructible(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_destructible)
+
+#if _CCCL_CHECK_BUILTIN(is_layout_compatible) || _CCCL_COMPILER(MSVC, >=, 19, 29)
+#  define _CCCL_BUILTIN_IS_LAYOUT_COMPATIBLE(...) __is_layout_compatible(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_layout_compatible) || _CCCL_COMPILER(MSVC, >=, 19, 29)
+
+#if _CCCL_CHECK_BUILTIN(is_lvalue_reference)
+#  define _CCCL_BUILTIN_IS_LVALUE_REFERENCE(...) __is_lvalue_reference(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_lvalue_reference)
+
+#if _CCCL_HAS_BUILTIN(__is_member_function_pointer)
+#  define _CCCL_BUILTIN_IS_MEMBER_FUNCTION_POINTER(...) __is_member_function_pointer(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_member_function_pointer)
+
+#if _CCCL_HAS_BUILTIN(__is_member_object_pointer)
+#  define _CCCL_BUILTIN_IS_MEMBER_OBJECT_POINTER(...) __is_member_object_pointer(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_member_object_pointer)
+
+#if _CCCL_HAS_BUILTIN(__is_member_pointer)
+#  define _CCCL_BUILTIN_IS_MEMBER_POINTER(...) __is_member_pointer(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_member_pointer)
+
+#if _CCCL_CHECK_BUILTIN(is_nothrow_assignable) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_IS_NOTHROW_ASSIGNABLE(...) __is_nothrow_assignable(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_nothrow_assignable)
+
+#if _CCCL_CHECK_BUILTIN(is_nothrow_constructible) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_IS_NOTHROW_CONSTRUCTIBLE(...) __is_nothrow_constructible(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_nothrow_constructible)
+
+#if _CCCL_CHECK_BUILTIN(is_nothrow_destructible) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_IS_NOTHROW_DESTRUCTIBLE(...) __is_nothrow_destructible(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_nothrow_destructible)
+
+#if _CCCL_CHECK_BUILTIN(is_object)
+#  define _CCCL_BUILTIN_IS_OBJECT(...) __is_object(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_object)
+
+// Disabled due to libstdc++ conflict
+#if 0 // _CCCL_HAS_BUILTIN(__is_pointer)
+#  define _CCCL_BUILTIN_IS_POINTER(...) __is_pointer(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_pointer)
+
+#if _CCCL_CHECK_BUILTIN(is_pointer_interconvertible_base_of) || _CCCL_COMPILER(MSVC, >=, 19, 29)
+#  define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_BASE_OF(...) __is_pointer_interconvertible_base_of(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(is_pointer_interconvertible_base_of) || _CCCL_COMPILER(MSVC, >=, 19, 29)
+
+#if _CCCL_HAS_BUILTIN(__is_reference)
+#  define _CCCL_BUILTIN_IS_REFERENCE(...) __is_reference(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_reference)
+
+// Disabled due to libstdc++ conflict
+#if 0 // _CCCL_HAS_BUILTIN(__is_referenceable)
+#  define _CCCL_BUILTIN_IS_REFERENCEABLE(...) __is_referenceable(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_referenceable)
+
+#if _CCCL_HAS_BUILTIN(__is_rvalue_reference)
+#  define _CCCL_BUILTIN_IS_RVALUE_REFERENCE(...) __is_rvalue_reference(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_rvalue_reference)
+
+// Disabled due to libstdc++ conflict
+#if 0 // _CCCL_HAS_BUILTIN(__is_scalar)
+#  define _CCCL_BUILTIN_IS_SCALAR(...) __is_scalar(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_scalar)
+
+#if _CCCL_CHECK_BUILTIN(make_integer_seq) || _CCCL_COMPILER(MSVC, >=, 19, 23)
+#  define _CCCL_BUILTIN_MAKE_INTEGER_SEQ(...) __make_integer_seq<__VA_ARGS__>
+#endif // _CCCL_CHECK_BUILTIN(make_integer_seq)
+
+#if _CCCL_HAS_BUILTIN(__reference_constructs_from_temporary)
+#  define _CCCL_BUILTIN_REFERENCE_CONSTRUCTS_FROM_TEMPORARY(...) __reference_constructs_from_temporary(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__reference_constructs_from_temporary)
+
+#if _CCCL_HAS_BUILTIN(__reference_converts_from_temporary)
+#  define _CCCL_BUILTIN_REFERENCE_CONVERTS_FROM_TEMPORARY(...) __reference_converts_from_temporary(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__reference_converts_from_temporary)
+
+#if _CCCL_HAS_BUILTIN(__remove_const) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_CONST(...) __remove_const(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_const)
+
+#if _CCCL_HAS_BUILTIN(__remove_cv) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_CV(...) __remove_cv(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_cv)
+
+#if _CCCL_HAS_BUILTIN(__remove_cvref) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_CVREF(...) __remove_cvref(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_cvref)
+
+#if _CCCL_COMPILER(NVRTC, <, 12, 4) // NVRTC below 12.4 fails to properly compile that builtin
+#  undef _CCCL_BUILTIN_REMOVE_CVREF
+#endif // _CCCL_COMPILER(NVRTC, <, 12, 4)
+
+#if _CCCL_HAS_BUILTIN(__remove_extent) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_EXTENT(...) __remove_extent(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_extent)
+
+#if _CCCL_HAS_BUILTIN(__remove_pointer) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_POINTER(...) __remove_pointer(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_pointer)
+
+#if _CCCL_HAS_BUILTIN(__remove_reference)
+#  define _CCCL_BUILTIN_REMOVE_REFERENCE_T(...) __remove_reference(__VA_ARGS__)
+#elif _CCCL_HAS_BUILTIN(__remove_reference_t) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_REFERENCE_T(...) __remove_reference_t(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_reference_t)
+
+#if _CCCL_COMPILER(NVRTC, <, 12, 4) // NVRTC below 12.4 fails to properly compile cuda::std::move with that
+#  undef _CCCL_BUILTIN_REMOVE_REFERENCE_T
+#endif // _CCCL_COMPILER(NVRTC, <, 12, 4)
+
+#if _CCCL_HAS_BUILTIN(__remove_volatile) && _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_BUILTIN_REMOVE_VOLATILE(...) __remove_volatile(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__remove_volatile)
+
+#if _CCCL_HAS_BUILTIN(__type_pack_element)
+#  define _CCCL_BUILTIN_TYPE_PACK_ELEMENT(...) __type_pack_element<__VA_ARGS__>
+#endif // _CCCL_HAS_BUILTIN(__type_pack_element)
+
+#if _CCCL_HAS_BUILTIN(__is_complete_type)
+#  define _CCCL_BUILTIN_IS_COMPLETE_TYPE(...) __is_complete_type(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__is_complete_type)
+
+#if _CCCL_HAS_BUILTIN(__builtin_clear_padding) \
+  && (_CCCL_HOST_COMPILATION() || !(_CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC)))
+#  define _CCCL_BUILTIN_CLEAR_PADDING(...) __builtin_clear_padding(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_clear_padding) && (_CCCL_HOST_COMPILATION() || !(_CCCL_COMPILER(GCC) ||
+       // _CCCL_COMPILER(NVHPC)))
+
+// NVCC prior to 12.2 have trouble with pack expansion into __type_pack_element in an alias template
+#if _CCCL_CUDACC_BELOW(12, 2)
+#  undef _CCCL_BUILTIN_TYPE_PACK_ELEMENT
+#endif // _CCCL_CUDACC_BELOW(12, 2)
+
+#if _CCCL_COMPILER(MSVC) // To use __builtin_FUNCSIG(), both MSVC and nvcc need to support it
+#  if _CCCL_COMPILER(MSVC, >=, 19, 35) && _CCCL_CUDACC_AT_LEAST(12, 3)
+#    define _CCCL_BUILTIN_PRETTY_FUNCTION() __builtin_FUNCSIG()
+#  else // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 35) ^^^ / vvv _CCCL_COMPILER(MSVC, <, 19, 35) vvv
+#    define _CCCL_BUILTIN_PRETTY_FUNCTION() __FUNCSIG__
+#    define _CCCL_BROKEN_MSVC_FUNCSIG
+#  endif // _CCCL_COMPILER(MSVC, <, 19, 35)
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_BUILTIN_PRETTY_FUNCTION() __PRETTY_FUNCTION__
+#endif // !_CCCL_COMPILER(MSVC)
+
+// GCC's builtin_strlen isn't reliable at constexpr time
+// NVRTC does not expose builtin_strlen
+#if !_CCCL_COMPILER(GCC) && !_CCCL_COMPILER(NVRTC)
+#  define _CCCL_BUILTIN_STRLEN(...) __builtin_strlen(__VA_ARGS__)
+#endif
+
+// The new __nv_atomic builtins are available when __CUDACC_DEVICE_ATOMIC_BUILTINS__ is defined
+#if defined(__CUDACC_DEVICE_ATOMIC_BUILTINS__) && _CCCL_PTX_ARCH() >= 600 && !_CCCL_COMPILER(MSVC)
+#  define _CCCL_HAS_NV_ATOMIC_BUILTINS() 1
+#else // ^^^ has intrinsics ^^^ / vvv no intrinsics
+#  define _CCCL_HAS_NV_ATOMIC_BUILTINS() 0
+#endif // no intrinsics
+
+#endif // __CCCL_BUILTIN_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/compiler.h

@@ -0,0 +1,238 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_COMPILER_H
+#define __CCCL_COMPILER_H
+
+#include <cuda/std/__cccl/preprocessor.h>
+
+// Utility to compare version numbers. To use:
+// 1) Define a macro that makes a pair of (major, minor) numbers:
+//    #define MYPRODUCT_MAKE_VERSION(_MAJOR, _MINOR) (_MAJOR * 100 + _MINOR)
+// 2) Define a macro that you will use to compare versions, e.g.:
+//    #define MYPRODUCT(...) _CCCL_VERSION_COMPARE(MYPRODUCT, MYPRODUCT_##__VA_ARGS__)
+//    Signatures:
+//       MYPRODUCT(_PROD)                      - is the product _PROD version non-zero?
+//       MYPRODUCT(_PROD, _OP, _MAJOR)         - compare the product _PROD major version to _MAJOR using operator _OP
+//       MYPRODUCT(_PROD, _OP, _MAJOR, _MINOR) - compare the product _PROD version to _MAJOR._MINOR using operator _OP
+// 3) Define the product version macros as a function-like macro that returns the version number or
+//    _CCCL_VERSION_INVALID() if the version cannot be determined, e. g.:
+//    #define MYPRODUCT_<_PROD>() (1, 2)
+//      or
+//    #define MYPRODUCT_<_PROD>() _CCCL_VERSION_INVALID()
+#define _CCCL_VERSION_MAJOR_(_MAJOR, _MINOR)   _MAJOR
+#define _CCCL_VERSION_MAJOR(_PAIR)             _CCCL_VERSION_MAJOR_ _PAIR
+#define _CCCL_VERSION_INVALID()                (-1, -1)
+#define _CCCL_MAKE_VERSION(_PREFIX, _PAIR)     (_CCCL_PP_EVAL(_CCCL_PP_CAT(_PREFIX, MAKE_VERSION), _CCCL_PP_EXPAND _PAIR))
+#define _CCCL_VERSION_IS_INVALID(_PAIR)        (_CCCL_VERSION_MAJOR(_PAIR) == _CCCL_VERSION_MAJOR(_CCCL_VERSION_INVALID()))
+#define _CCCL_VERSION_COMPARE_1(_PREFIX, _VER) (!_CCCL_VERSION_IS_INVALID(_VER()))
+#define _CCCL_VERSION_COMPARE_3(_PREFIX, _VER, _OP, _MAJOR) \
+  (!_CCCL_VERSION_IS_INVALID(_VER()) && (_CCCL_VERSION_MAJOR(_VER()) _OP _MAJOR))
+#define _CCCL_VERSION_COMPARE_4(_PREFIX, _VER, _OP, _MAJOR, _MINOR) \
+  (!_CCCL_VERSION_IS_INVALID(_VER())                                \
+   && (_CCCL_MAKE_VERSION(_PREFIX, _VER()) _OP _CCCL_MAKE_VERSION(_PREFIX, (_MAJOR, _MINOR))))
+#define _CCCL_VERSION_SELECT_COUNT(_ARG1, _ARG2, _ARG3, _ARG4, _ARG5, ...) _ARG5
+#define _CCCL_VERSION_SELECT2(_ARGS)                                       _CCCL_VERSION_SELECT_COUNT _ARGS
+// MSVC traditonal preprocessor requires an extra level of indirection
+#define _CCCL_VERSION_SELECT(...)         \
+  _CCCL_VERSION_SELECT2(                  \
+    (__VA_ARGS__,                         \
+     _CCCL_VERSION_COMPARE_4,             \
+     _CCCL_VERSION_COMPARE_3,             \
+     _CCCL_VERSION_COMPARE_BAD_ARG_COUNT, \
+     _CCCL_VERSION_COMPARE_1,             \
+     _CCCL_VERSION_COMPARE_BAD_ARG_COUNT))
+#define _CCCL_VERSION_COMPARE(_PREFIX, ...) _CCCL_VERSION_SELECT(__VA_ARGS__)(_PREFIX, __VA_ARGS__)
+
+#define _CCCL_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 100 + (_MINOR))
+#define _CCCL_COMPILER(...)                         _CCCL_VERSION_COMPARE(_CCCL_COMPILER_, _CCCL_COMPILER_##__VA_ARGS__)
+
+#define _CCCL_COMPILER_NVHPC()    _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_CLANG()    _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_GCC()      _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_MSVC()     _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_MSVC2019() _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_MSVC2022() _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_MSVC2026() _CCCL_VERSION_INVALID()
+#define _CCCL_COMPILER_NVRTC()    _CCCL_VERSION_INVALID()
+
+// Determine the host compiler and its version
+#if defined(__INTEL_COMPILER)
+#  ifndef CCCL_IGNORE_DEPRECATED_COMPILER
+#    warning \
+      "The Intel C++ Compiler Classic (icc/icpc) is not supported by CCCL. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message."
+#  endif // !CCCL_IGNORE_DEPRECATED_COMPILER
+#elif defined(__NVCOMPILER)
+#  undef _CCCL_COMPILER_NVHPC
+#  define _CCCL_COMPILER_NVHPC() (__NVCOMPILER_MAJOR__, __NVCOMPILER_MINOR__)
+#elif defined(__clang__)
+#  undef _CCCL_COMPILER_CLANG
+#  define _CCCL_COMPILER_CLANG() (__clang_major__, __clang_minor__)
+#elif defined(__GNUC__)
+#  undef _CCCL_COMPILER_GCC
+#  define _CCCL_COMPILER_GCC() (__GNUC__, __GNUC_MINOR__)
+#elif defined(_MSC_VER)
+// see https://learn.microsoft.com/en-us/cpp/overview/compiler-versions?view=msvc-180#version-macros
+#  undef _CCCL_COMPILER_MSVC
+#  define _CCCL_COMPILER_MSVC() (_MSC_VER / 100, _MSC_VER % 100)
+#  if _CCCL_COMPILER(MSVC, <, 19, 20)
+#    ifndef CCCL_IGNORE_DEPRECATED_COMPILER
+#      error \
+        "Visual Studio 2017 (MSC_VER < 1920) and older are not supported by CCCL. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this error."
+#    endif
+#  endif // _CCCL_COMPILER(MSVC, <, 19, 20)
+#  if _CCCL_COMPILER(MSVC, >=, 19, 20) && _CCCL_COMPILER(MSVC, <, 19, 30)
+#    undef _CCCL_COMPILER_MSVC2019
+#    define _CCCL_COMPILER_MSVC2019() _CCCL_COMPILER_MSVC()
+#  endif // _CCCL_COMPILER(MSVC, >=, 19, 20) && _CCCL_COMPILER(MSVC, <, 19, 30)
+#  if _CCCL_COMPILER(MSVC, >=, 19, 30) && _CCCL_COMPILER(MSVC, <, 19, 50)
+#    undef _CCCL_COMPILER_MSVC2022
+#    define _CCCL_COMPILER_MSVC2022() _CCCL_COMPILER_MSVC()
+#  endif // _CCCL_COMPILER(MSVC, >=, 19, 30) && _CCCL_COMPILER(MSVC, <, 19, 50)
+#  if _CCCL_COMPILER(MSVC, >=, 19, 50)
+#    undef _CCCL_COMPILER_MSVC2026
+#    define _CCCL_COMPILER_MSVC2026() _CCCL_COMPILER_MSVC()
+#  endif // _CCCL_COMPILER(MSVC, >=, 19, 45)
+#elif defined(__CUDACC_RTC__)
+#  undef _CCCL_COMPILER_NVRTC
+#  define _CCCL_COMPILER_NVRTC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__)
+#endif
+
+#if defined(__CUDACC__) || defined(_NVHPC_CUDA)
+#  define _CCCL_CUDA_COMPILATION() 1
+#else // ^^^ compiling .cu file ^^^ / vvv not compiling .cu file vvv
+#  define _CCCL_CUDA_COMPILATION() 0
+#endif // ^^^ not compiling .cu file ^^^
+
+// The CUDA compiler version shares the implementation with the C++ compiler
+#define _CCCL_CUDA_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) _CCCL_COMPILER_MAKE_VERSION(_MAJOR, _MINOR)
+#define _CCCL_CUDA_COMPILER(...)                         _CCCL_VERSION_COMPARE(_CCCL_CUDA_COMPILER_, _CCCL_CUDA_COMPILER_##__VA_ARGS__)
+
+#define _CCCL_CUDA_COMPILER_NVCC()  _CCCL_VERSION_INVALID()
+#define _CCCL_CUDA_COMPILER_NVHPC() _CCCL_VERSION_INVALID()
+#define _CCCL_CUDA_COMPILER_CLANG() _CCCL_VERSION_INVALID()
+#define _CCCL_CUDA_COMPILER_NVRTC() _CCCL_VERSION_INVALID()
+
+// Determine the cuda compiler
+#if _CCCL_CUDA_COMPILATION()
+#  if defined(__NVCC__)
+#    undef _CCCL_CUDA_COMPILER_NVCC
+#    define _CCCL_CUDA_COMPILER_NVCC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__)
+#  elif defined(_NVHPC_CUDA)
+#    undef _CCCL_CUDA_COMPILER_NVHPC
+#    define _CCCL_CUDA_COMPILER_NVHPC() _CCCL_COMPILER_NVHPC()
+#  elif defined(__CUDA__) && _CCCL_COMPILER(CLANG)
+#    undef _CCCL_CUDA_COMPILER_CLANG
+#    define _CCCL_CUDA_COMPILER_CLANG() _CCCL_COMPILER_CLANG()
+#  elif _CCCL_COMPILER(NVRTC)
+#    undef _CCCL_CUDA_COMPILER_NVRTC
+#    define _CCCL_CUDA_COMPILER_NVRTC() _CCCL_COMPILER_NVRTC()
+#  endif // ^^^ _CCCL_COMPILER(NVRTC) ^^^
+#endif // _CCCL_CUDA_COMPILATION()
+
+// Determine if we are compiling host code, this includes both CUDA and C++ compilation
+// nvc++ does not define __CUDA_ARCH__, but it compiles both host and device code at the same time
+#if !defined(__CUDA_ARCH__)
+#  define _CCCL_HOST_COMPILATION() 1
+#else // ^^^ compiling host code ^^^ / vvv not compiling host code vvv
+#  define _CCCL_HOST_COMPILATION() 0
+#endif // ^^^ not compiling host code ^^^
+
+#if (_CCCL_CUDA_COMPILATION() && defined(__CUDA_ARCH__)) || _CCCL_CUDA_COMPILER(NVHPC)
+#  define _CCCL_DEVICE_COMPILATION() 1
+#else // ^^^ compiling device code ^^^ / vvv not compiling device code vvv
+#  define _CCCL_DEVICE_COMPILATION() 0
+#endif // ^^^ not compiling device code ^^^
+
+#if defined(__CUDACC_TILE__) && _CCCL_CUDA_COMPILER(NVCC, >, 13, 3)
+#  define _CCCL_TILE_COMPILATION() 1
+#else // ^^^ compiling .cu file in tile mode ^^^ / vvv not compiling in tile mode vvv
+#  define _CCCL_TILE_COMPILATION() 0
+#endif // ^^^ not compiling .cu file ^^^
+
+#define _CCCL_CUDACC_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 1000 + (_MINOR) * 10)
+
+// clang-cuda does not define __CUDACC_VER_MAJOR__ and friends. They are instead retrieved from the CUDA_VERSION macro
+// defined in "cuda.h". clang-cuda automatically pre-includes "__clang_cuda_runtime_wrapper.h" which includes "cuda.h"
+#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(NVHPC) || _CCCL_CUDA_COMPILER(NVRTC)
+#  define _CCCL_CUDACC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__)
+#elif _CCCL_CUDA_COMPILER(CLANG)
+#  define _CCCL_CUDACC() (CUDA_VERSION / 1000, (CUDA_VERSION % 1000) / 10)
+#endif // ^^^ has cuda compiler ^^^
+
+#if !defined(_CCCL_CUDACC) || !_CCCL_CUDA_COMPILATION()
+#  undef _CCCL_CUDACC
+#  define _CCCL_CUDACC() _CCCL_VERSION_INVALID()
+#endif // !_CCCL_CUDACC || !_CCCL_CUDA_COMPILATION()
+
+#define _CCCL_CUDACC_EQUAL(...)    _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, ==, __VA_ARGS__)
+#define _CCCL_CUDACC_BELOW(...)    _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, <, __VA_ARGS__)
+#define _CCCL_CUDACC_AT_LEAST(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, >=, __VA_ARGS__)
+
+#if _CCCL_CUDA_COMPILATION() && _CCCL_CUDACC_BELOW(12) && !defined(CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12)
+#  error "CUDA versions below 12 are not supported." \
+"Define CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12 to suppress this message."
+#endif
+
+// Define the pragma for the host compiler
+#if _CCCL_COMPILER(MSVC)
+#  define _CCCL_PRAGMA(_ARG) __pragma(_ARG)
+#else
+#  define _CCCL_PRAGMA(_ARG) _Pragma(_CCCL_TO_STRING(_ARG))
+#endif // _CCCL_COMPILER(MSVC)
+
+// Define the proper object format for NVHPC and NVRTC
+#if (_CCCL_COMPILER(NVHPC) && defined(__linux__)) || _CCCL_COMPILER(NVRTC)
+#  ifndef __ELF__
+#    define __ELF__
+#  endif // !__ELF__
+#endif // _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC)
+
+#if _CCCL_DEVICE_COMPILATION()
+#  define _CCCL_PRAGMA_UNROLL(_N)    _CCCL_PRAGMA(unroll _N)
+#  define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA(unroll)
+#elif _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC) || _CCCL_COMPILER(CLANG)
+#  define _CCCL_PRAGMA_UNROLL(_N)    _CCCL_PRAGMA(unroll _N)
+#  define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA(unroll)
+#elif _CCCL_COMPILER(GCC, >=, 8)
+// gcc supports only #pragma GCC unroll, but that causes problems when compiling with nvcc. So, we use #pragma unroll
+// when compiling device code, and #pragma GCC unroll when compiling host code, but we need to suppress the warning
+// about the unknown pragma for nvcc.
+// #pragma GCC unroll does not support full unrolling, so we use the maximum value that it supports.
+#  define _CCCL_PRAGMA_UNROLL(_N) \
+    _CCCL_BEGIN_NV_DIAG_SUPPRESS(1675) _CCCL_PRAGMA(GCC unroll _N) _CCCL_END_NV_DIAG_SUPPRESS()
+#  define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA_UNROLL(65534)
+#else // ^^^ has pragma unroll support ^^^ / vvv no pragma unroll support vvv
+#  define _CCCL_PRAGMA_UNROLL(_N)
+#  define _CCCL_PRAGMA_UNROLL_FULL()
+#endif // ^^^ no pragma unroll support ^^^
+
+#define _CCCL_PRAGMA_NOUNROLL() _CCCL_PRAGMA_UNROLL(1)
+
+#if _CCCL_COMPILER(MSVC)
+#  define _CCCL_WARNING(_MSG) _CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " _MSG))
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_WARNING(_MSG) _CCCL_PRAGMA(GCC warning _MSG)
+#endif // !_CCCL_COMPILER(MSVC)
+
+// Freestanding environment detection
+// NVRTC is treated as freestanding since it has no access to the host standard library
+#if defined(_CCCL_ENABLE_FREESTANDING) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_FREESTANDING() 1
+#  define _CCCL_HOSTED()       0
+#  define _CCCL_HOSTJIT()      (!_CCCL_COMPILER(NVRTC))
+#  define _CCCL_NO_TYPEID
+#else // ^^^ _CCCL_ENABLE_FREESTANDING ||  _CCCL_COMPILER(NVRTC) ^^^ / vvv Hosted environment vvv
+#  define _CCCL_FREESTANDING() 0
+#  define _CCCL_HOSTED()       1
+#  define _CCCL_HOSTJIT()      0
+#endif // Hosted environment
+
+#endif // __CCCL_COMPILER_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_capabilities.h

@@ -0,0 +1,118 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_CUDA_CAPABILITIES
+#define __CCCL_CUDA_CAPABILITIES
+
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/cuda_toolkit.h>
+
+#include <nv/target>
+
+/// In device code, _CCCL_PTX_ARCH() expands to the PTX version for which we are compiling.
+/// In host code, _CCCL_PTX_ARCH()'s value is implementation defined.
+#if !defined(__CUDA_ARCH__)
+#  define _CCCL_PTX_ARCH() 0
+#else
+#  define _CCCL_PTX_ARCH() __CUDA_ARCH__
+#endif
+
+#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes:
+//! When this macro is defined, Programmatic Dependent Launch (PDL) is disabled across CCCL
+#  define CCCL_DISABLE_PDL
+#endif // _CCCL_DOXYGEN_INVOKED
+
+#ifdef CCCL_DISABLE_PDL
+#  define _CCCL_HAS_PDL() 0
+#else // CCCL_DISABLE_PDL
+#  define _CCCL_HAS_PDL() 1
+#endif // CCCL_DISABLE_PDL
+
+#if _CCCL_HAS_PDL()
+// Waits for the previous kernel to complete (when it reaches its final membar). Should be put before the first global
+// memory access in a kernel.
+#  define _CCCL_PDL_GRID_DEPENDENCY_SYNC() NV_IF_TARGET(NV_PROVIDES_SM_90, ::cudaGridDependencySynchronize();)
+// Allows the subsequent kernel in the same stream to launch. Can be put anywhere in a kernel.
+// Heuristic(ahendriksen): put it after the last load.
+#  define _CCCL_PDL_TRIGGER_NEXT_LAUNCH() NV_IF_TARGET(NV_PROVIDES_SM_90, ::cudaTriggerProgrammaticLaunchCompletion();)
+#else // _CCCL_HAS_PDL()
+#  define _CCCL_PDL_GRID_DEPENDENCY_SYNC()
+#  define _CCCL_PDL_TRIGGER_NEXT_LAUNCH()
+#endif // _CCCL_HAS_PDL()
+
+// Check whether the relocatable device code (RDC) is being generated.
+#if defined(__CUDACC_RDC__) || defined(__CLANG_RDC__) || defined(_NVHPC_RDC)
+#  define _CCCL_HAS_RDC() 1
+#else // ^^^ has RDC ^^^ / vvv no RDC vvv
+#  define _CCCL_HAS_RDC() 0
+#endif // ^^^ no RDC ^^^
+
+// Check whether extensible whole program is being compiled.
+#if defined(__CUDACC_EWP__)
+#  define _CCCL_HAS_EWP() 1
+#else // ^^^ has EWP ^^^ / vvv no EWP vvv
+#  define _CCCL_HAS_EWP() 0
+#endif // ^^^ no EWP ^^^
+
+// Control whether device runtime APIs can be used, because they require libcudadevrt to be linked. Defaults to true
+// when RDC or EWP are enabled. Can be disabled by defining CCCL_DISABLE_DEVICE_RUNTIME.
+#if (_CCCL_HAS_RDC() || _CCCL_HAS_EWP()) && !defined(CCCL_DISABLE_DEVICE_RUNTIME)
+#  define _CCCL_HAS_DEVICE_RUNTIME() 1
+#else // ^^^ has device runtime ^^^ / vvv no device runtime vvv
+#  define _CCCL_HAS_DEVICE_RUNTIME() 0
+#endif // ^^^ no device runtime ^^^
+
+// Some functions can be called from host or device code and launch kernels inside. Thus, they use CUDA Dynamic
+// Parallelism (CDP) and require compiling with Relocatable Device Code (RDC) or extensible whole program (EWP) and link
+// with device runtime library. CDP is unsupported with clang-cuda below 22.
+// TODO(bgruber): remove CUB_DISABLE_CDP in CCCL 4.0
+#if _CCCL_HAS_DEVICE_RUNTIME() && !defined(CCCL_DISABLE_CDP) && !defined(CUB_DISABLE_CDP) \
+  && !_CCCL_CUDA_COMPILER(CLANG, <, 22)
+// We have CDP, so host and device APIs can call kernels
+#  define _CCCL_HAS_CDP() 1
+#else // ^^^ has CDP ^^^ / vvv no CDP vvv
+// We don't have CDP, only host APIs can call kernels
+#  define _CCCL_HAS_CDP() 0
+#endif // ^^^ no CDP ^^^
+
+// When RDC is enabled, __launch_bounds__ cannot be used reliably. See #902.
+#if !_CCCL_HAS_RDC() && !defined(CCCL_DISABLE_LAUNCH_BOUNDS)
+#  define _CCCL_LAUNCH_BOUNDS(...) __launch_bounds__(__VA_ARGS__)
+#else // ^^^ has launch bounds attribute ^^^ / vvv no launch bounds attribute vvv
+#  define _CCCL_LAUNCH_BOUNDS(...)
+#endif // ^^^ no launch bounds attribute ^^^
+
+// __block_size__ attribute is available for nvcc and nvrtc 12.9+ for hopper+ architectures. For older nvcc and nvrtc,
+// we can fallback to __cluster_dims__ attribute only specifying the ncta per cluster.
+// This attribute should be used only for cluster launches.
+#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 9) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 9)) && _CCCL_PTX_ARCH() >= 900
+#  define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) __block_size__(_NTID, _NCTA_PER_CLUSTER)
+#elif (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(NVRTC)) && _CCCL_PTX_ARCH() >= 900
+#  define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) __cluster_dims__ _NCTA_PER_CLUSTER
+#else // ^^ has __block_size__ attribute ^^^ / vvv no __block_size__ attribute vvv
+#  define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER)
+#endif // ^^^ no __block_size__ attribute ^^^
+
+#if _CCCL_HAS_CDP()
+#  ifdef CUDA_FORCE_CDP1_IF_SUPPORTED
+#    error "CUDA Dynamic Parallelism 1 is no longer supported. Please undefine CUDA_FORCE_CDP1_IF_SUPPORTED."
+#  endif // CUDA_FORCE_CDP1_IF_SUPPORTED
+#endif // _CCCL_HAS_CDP()
+
+#endif // __CCCL_CUDA_CAPABILITIES

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_toolkit.h

@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_CUDA_TOOLKIT_H
+#define __CCCL_CUDA_TOOLKIT_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_CUDA_COMPILATION() || __has_include(<cuda_runtime_api.h>)
+#  define _CCCL_HAS_CTK() 1
+#else // ^^^ has cuda toolkit ^^^ / vvv no cuda toolkit vvv
+#  define _CCCL_HAS_CTK() 0
+#endif // ^^^ no cuda toolkit ^^^
+
+// CUDA compilers preinclude cuda_runtime.h, so we need to include it here to get the CUDART_VERSION macro
+#if _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILATION()
+#  include <cuda_runtime_api.h>
+#endif // _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILATION()
+
+// Check compatibility of the CUDA compiler and CUDA toolkit headers
+// Some users might want to use a newer version of the CTK than the compiler ships. Enable that on their own peril
+#ifndef CCCL_DISABLE_CTK_COMPATIBILITY_CHECK
+#  if _CCCL_CUDA_COMPILATION()
+#    if !_CCCL_CUDACC_EQUAL((CUDART_VERSION / 1000), (CUDART_VERSION % 1000) / 10)
+#      error "CUDA compiler and CUDA toolkit headers are incompatible, please check your include paths"
+#    endif // !_CCCL_CUDACC_EQUAL((CUDART_VERSION / 1000), (CUDART_VERSION % 1000) / 10)
+#  endif // _CCCL_CUDA_COMPILATION()
+#endif // CCCL_DISABLE_CTK_COMPATIBILITY_CHECK
+
+#if _CCCL_HAS_CTK()
+#  define _CCCL_CTK() (CUDART_VERSION / 1000, (CUDART_VERSION % 1000) / 10)
+#else // ^^^ has cuda toolkit ^^^ / vvv no cuda toolkit vvv
+#  define _CCCL_CTK() _CCCL_VERSION_INVALID()
+#endif // ^^^ no cuda toolkit ^^^
+
+#define _CCCL_CTK_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 1000 + (_MINOR) * 10)
+#define _CCCL_CTK_BELOW(...)                   _CCCL_VERSION_COMPARE(_CCCL_CTK_, _CCCL_CTK, <, __VA_ARGS__)
+#define _CCCL_CTK_AT_LEAST(...)                _CCCL_VERSION_COMPARE(_CCCL_CTK_, _CCCL_CTK, >=, __VA_ARGS__)
+
+#endif // __CCCL_CUDA_TOOLKIT_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/deprecated.h

@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_DEPRECATED_H
+#define __CCCL_DEPRECATED_H
+
+#include <cuda/std/__cccl/attributes.h>
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/dialect.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+// Check for deprecation opt outs
+#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_DIALECT)
+#  if !defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT)
+#    define CCCL_IGNORE_DEPRECATED_CPP_DIALECT
+#  endif
+#endif // suppress all dialect deprecation warnings
+#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_14) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT)
+#  if !defined(CCCL_IGNORE_DEPRECATED_CPP_14)
+#    define CCCL_IGNORE_DEPRECATED_CPP_14
+#  endif
+#endif // suppress all c++14 dialect deprecation warnings
+#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_11) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) \
+  || defined(CCCL_IGNORE_DEPRECATED_CPP_14)
+#  if !defined(CCCL_IGNORE_DEPRECATED_CPP_11)
+#    define CCCL_IGNORE_DEPRECATED_CPP_11
+#  endif
+#endif // suppress all c++11 dialect deprecation warnings
+#if defined(LIBCUDACXX_IGNORE_DEPRECATED_COMPILER) || defined(THRUST_IGNORE_DEPRECATED_COMPILER) \
+  || defined(CUB_IGNORE_DEPRECATED_COMPILER) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT)      \
+  || defined(CCCL_IGNORE_DEPRECATED_CPP_14) || defined(CCCL_IGNORE_DEPRECATED_CPP_11)
+#  if !defined(CCCL_IGNORE_DEPRECATED_COMPILER)
+#    define CCCL_IGNORE_DEPRECATED_COMPILER
+#  endif
+#endif // suppress all compiler deprecation warnings
+#if defined(LIBCUDACXX_IGNORE_DEPRECATED_API) || defined(THRUST_IGNORE_DEPRECATED_API) \
+  || defined(CUB_IGNORE_DEPRECATED_API)
+#  if !defined(CCCL_IGNORE_DEPRECATED_API)
+#    define CCCL_IGNORE_DEPRECATED_API
+#  endif
+#endif // suppress all API deprecation warnings
+
+#if defined(CCCL_IGNORE_DEPRECATED_API) || defined(_LIBCUDACXX_DISABLE_DEPRECATION_WARNINGS)
+//! deprecated [Since 2.8]
+#  define CCCL_DEPRECATED
+//! deprecated [Since 2.8]
+#  define CCCL_DEPRECATED_BECAUSE(MSG)
+#elif _CCCL_HAS_ATTRIBUTE(deprecated)
+//! deprecated [Since 2.8]
+#  define CCCL_DEPRECATED              __attribute__((deprecated))
+//! deprecated [Since 2.8]
+#  define CCCL_DEPRECATED_BECAUSE(MSG) __attribute__((deprecated(MSG)))
+#else // ^^^ attribute deprecated ^^^ / vvv standard deprecated attribute vvv
+//! deprecated [Since 2.8]
+#  define CCCL_DEPRECATED              [[deprecated]]
+//! deprecated [Since 2.8]
+#  define CCCL_DEPRECATED_BECAUSE(MSG) [[deprecated(MSG)]]
+#endif // ^^^ standard deprecated attribute ^^^
+
+#if _CCCL_STD_VER >= 2020
+#  define _CCCL_DEPRECATED_IN_CXX20 CCCL_DEPRECATED
+#else // ^^^ _CCCL_STD_VER >= 2020 ^^^ / vvv _CCCL_STD_VER < 2020 vvv
+#  define _CCCL_DEPRECATED_IN_CXX20
+#endif // ^^^ _CCCL_STD_VER < 2020 ^^^
+
+#if _CCCL_STD_VER >= 2023
+#  define _CCCL_DEPRECATED_IN_CXX23 CCCL_DEPRECATED
+#else // ^^^ _CCCL_STD_VER >= 2023 ^^^ / vvv _CCCL_STD_VER < 2023 vvv
+#  define _CCCL_DEPRECATED_IN_CXX23
+#endif // ^^^ _CCCL_STD_VER < 2023 ^^^
+
+#endif // __CCCL_DEPRECATED_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/diagnostic.h

@@ -0,0 +1,145 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_DIAGNOSTIC_H
+#define __CCCL_DIAGNOSTIC_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+// Enable us to selectively silence host compiler warnings
+#if _CCCL_COMPILER(CLANG)
+#  define _CCCL_DIAG_PUSH                     _CCCL_PRAGMA(clang diagnostic push)
+#  define _CCCL_DIAG_POP                      _CCCL_PRAGMA(clang diagnostic pop)
+#  define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) _CCCL_PRAGMA(clang diagnostic ignored _WARNING)
+#  define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
+#elif _CCCL_COMPILER(GCC)
+#  define _CCCL_DIAG_PUSH _CCCL_PRAGMA(GCC diagnostic push)
+#  define _CCCL_DIAG_POP  _CCCL_PRAGMA(GCC diagnostic pop)
+#  define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) _CCCL_PRAGMA(GCC diagnostic ignored _WARNING)
+#  define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
+#elif _CCCL_COMPILER(NVHPC)
+#  define _CCCL_DIAG_PUSH _CCCL_PRAGMA(diagnostic push)
+#  define _CCCL_DIAG_POP  _CCCL_PRAGMA(diagnostic pop)
+#  define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) _CCCL_PRAGMA(diag_suppress _WARNING)
+#  define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_DIAG_PUSH _CCCL_PRAGMA(warning(push))
+#  define _CCCL_DIAG_POP  _CCCL_PRAGMA(warning(pop))
+#  define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) _CCCL_PRAGMA(warning(disable : _WARNING))
+#else
+#  define _CCCL_DIAG_PUSH
+#  define _CCCL_DIAG_POP
+#  define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_GCC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING)
+#  define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING)
+#endif
+
+// Enable us to selectively silence cuda compiler warnings
+#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_COMPILER(NVRTC)
+#  if defined(__NVCC_DIAG_PRAGMA_SUPPORT__)
+#    define _CCCL_NV_DIAG_PUSH()               _CCCL_PRAGMA(nv_diagnostic push)
+#    define _CCCL_NV_DIAG_POP()                _CCCL_PRAGMA(nv_diagnostic pop)
+#    define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) _CCCL_PRAGMA(nv_diag_suppress _WARNING)
+#    define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) \
+      _CCCL_NV_DIAG_PUSH() _CCCL_PP_FOR_EACH(_CCCL_DIAG_SUPPRESS_NVCC, __VA_ARGS__)
+#    define _CCCL_END_NV_DIAG_SUPPRESS() _CCCL_NV_DIAG_POP()
+#  else // ^^^ __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^ / vvv !__NVCC_DIAG_PRAGMA_SUPPORT__ vvv
+#    define _CCCL_NV_DIAG_PUSH()               _CCCL_PRAGMA(diagnostic push)
+#    define _CCCL_NV_DIAG_POP()                _CCCL_PRAGMA(diagnostic pop)
+#    define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) _CCCL_PRAGMA(diag_suppress _WARNING)
+#    define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) \
+      _CCCL_NV_DIAG_PUSH() _CCCL_PP_FOR_EACH(_CCCL_DIAG_SUPPRESS_NVCC, __VA_ARGS__)
+#    define _CCCL_END_NV_DIAG_SUPPRESS() _CCCL_NV_DIAG_POP()
+#  endif // !__NVCC_DIAG_PRAGMA_SUPPORT__
+#else // ^^^ _CCCL_CUDA_COMPILER(NVCC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVCC) vvv
+#  define _CCCL_NV_DIAG_PUSH()
+#  define _CCCL_NV_DIAG_POP()
+#  define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING)
+#  define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...)
+#  define _CCCL_END_NV_DIAG_SUPPRESS()
+#endif // !_CCCL_CUDA_COMPILER(NVCC)
+
+// Convenient shortcuts to silence common warnings
+#if _CCCL_COMPILER(CLANG)
+#  define _CCCL_SUPPRESS_DEPRECATED_PUSH                   \
+    _CCCL_DIAG_PUSH                                        \
+    _CCCL_DIAG_SUPPRESS_CLANG("-Wdeprecated")              \
+    _CCCL_DIAG_SUPPRESS_CLANG("-Wdeprecated-declarations") \
+    _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
+#  define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
+#  define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
+#elif _CCCL_COMPILER(GCC)
+#  define _CCCL_SUPPRESS_DEPRECATED_PUSH                 \
+    _CCCL_DIAG_PUSH                                      \
+    _CCCL_DIAG_SUPPRESS_GCC("-Wdeprecated")              \
+    _CCCL_DIAG_SUPPRESS_GCC("-Wdeprecated-declarations") \
+    _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
+#  define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
+#  define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
+#elif _CCCL_COMPILER(NVHPC)
+#  define _CCCL_SUPPRESS_DEPRECATED_PUSH                             \
+    _CCCL_DIAG_PUSH                                                  \
+    _CCCL_DIAG_SUPPRESS_NVHPC(deprecated_entity)                     \
+    _CCCL_DIAG_SUPPRESS_NVHPC(deprecated_entity_with_custom_message) \
+    _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
+#  define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
+#  define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_SUPPRESS_DEPRECATED_PUSH \
+    _CCCL_DIAG_PUSH                      \
+    _CCCL_DIAG_SUPPRESS_MSVC(4996)       \
+    _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444)
+#  define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
+#  define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP
+#elif _CCCL_COMPILER(NVRTC)
+#  if _CCCL_COMPILER(NVRTC, >=, 13, 3) && defined(__NVCC_DIAG_PRAGMA_SUPPORT__)
+#    define _CCCL_SUPPRESS_DEPRECATED_PUSH       _CCCL_NV_DIAG_PUSH()
+// NVRTC 13.3 does not honor nv_diag_suppress when it is emitted in the same macro expansion as
+// nv_diagnostic push. Keep the suppression in a separate source-level macro invocation.
+// See https://github.com/NVIDIA/cccl/issues/9170 and nvbug 6239043.
+#    define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG _Pragma("nv_diag_suppress 1444,20199")
+#  else // ^^^ NVRTC >= 13.3 with __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^
+#    define _CCCL_SUPPRESS_DEPRECATED_PUSH _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199)
+#    define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
+#  endif // ^^^ NVRTC >= 13.3 with __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^
+#  define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP()
+#else // unknown compiler
+#  define _CCCL_SUPPRESS_DEPRECATED_PUSH
+#  define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG
+#  define _CCCL_SUPPRESS_DEPRECATED_POP
+#endif // unknown compiler
+
+#if _CCCL_COMPILER(MSVC)
+#  define _CCCL_HAS_PRAGMA_MSVC_WARNING
+#  if !defined(_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING)
+#    define _CCCL_USE_PRAGMA_MSVC_WARNING
+#  endif // !_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING
+#endif // !_CCCL_COMPILER(MSVC)
+
+#endif // __CCCL_DIAGNOSTIC_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/dialect.h

@@ -0,0 +1,230 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_DIALECT_H
+#define __CCCL_DIALECT_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/builtin.h>
+#include <cuda/std/__cccl/host_std_lib.h>
+
+///////////////////////////////////////////////////////////////////////////////
+// Determine the C++ standard dialect
+///////////////////////////////////////////////////////////////////////////////
+#if _CCCL_COMPILER(MSVC)
+#  if _MSVC_LANG <= 201103L
+#    define _CCCL_STD_VER 2011
+#  elif _MSVC_LANG <= 201402L
+#    define _CCCL_STD_VER 2014
+#  elif _MSVC_LANG <= 201703L
+#    define _CCCL_STD_VER 2017
+#  elif _MSVC_LANG <= 202002L
+#    define _CCCL_STD_VER 2020
+#  else
+#    define _CCCL_STD_VER 2023 // current year, or date of c++2b ratification
+#  endif
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  if __cplusplus <= 199711L
+#    define _CCCL_STD_VER 2003
+#  elif __cplusplus <= 201103L
+#    define _CCCL_STD_VER 2011
+#  elif __cplusplus <= 201402L
+#    define _CCCL_STD_VER 2014
+#  elif __cplusplus <= 201703L
+#    define _CCCL_STD_VER 2017
+#  elif __cplusplus <= 202002L
+#    define _CCCL_STD_VER 2020
+#  elif __cplusplus <= 202302L
+#    define _CCCL_STD_VER 2023
+#  else
+#    define _CCCL_STD_VER 2024 // current year, or date of c++2c ratification
+#  endif
+#endif // !_CCCL_COMPILER(MSVC)
+
+///////////////////////////////////////////////////////////////////////////////
+// Conditionally enable constexpr per standard dialect
+///////////////////////////////////////////////////////////////////////////////
+
+#if _CCCL_STD_VER >= 2020
+#  define _CCCL_CONSTEXPR_CXX20 constexpr
+#else // ^^^ C++20 ^^^ / vvv C++17 vvv
+#  define _CCCL_CONSTEXPR_CXX20
+#endif // _CCCL_STD_VER <= 2017
+
+#if _CCCL_STD_VER >= 2023
+#  define _CCCL_CONSTEXPR_CXX23 constexpr
+#else // ^^^ C++23 ^^^ / vvv C++20 vvv
+#  define _CCCL_CONSTEXPR_CXX23
+#endif // _CCCL_STD_VER <= 2020
+
+///////////////////////////////////////////////////////////////////////////////
+// Detect whether we can use some language features based on standard dialect
+///////////////////////////////////////////////////////////////////////////////
+
+// concepts are only available from C++20 onwards
+#if _CCCL_STD_VER <= 2017 || __cpp_concepts < 201907L
+#  define _CCCL_HAS_CONCEPTS() 0
+#else // ^^^ no concepts ^^^ / vvv has concepts vvv
+#  define _CCCL_HAS_CONCEPTS() 1
+#endif // ^^^ has concepts ^^^
+
+// Three way comparison is only available from C++20 onwards
+#if _CCCL_STD_VER <= 2017 || __cpp_impl_three_way_comparison < 201907L
+#  define _CCCL_NO_THREE_WAY_COMPARISON
+#endif // _CCCL_STD_VER <= 2017 || __cpp_impl_three_way_comparison < 201907L
+
+// Some compilers turn on pack indexing in pre-C++26 code. We want to use it if it is
+// available.
+#if __cpp_pack_indexing >= 202311L && !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_COMPILER(CLANG, <, 20)
+#  define _CCCL_HAS_PACK_INDEXING() 1
+#else // ^^^ has pack indexing ^^^ / vvv no pack indexing vvv
+#  define _CCCL_HAS_PACK_INDEXING() 0
+#endif // no pack indexing
+
+#if _CCCL_STD_VER <= 2017 || __cpp_consteval < 201811L
+#  define _CCCL_NO_CONSTEVAL
+#  define _CCCL_CONSTEVAL constexpr
+#else
+#  define _CCCL_CONSTEVAL consteval
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+// Conditionally use certain language features depending on availability
+///////////////////////////////////////////////////////////////////////////////
+
+// We need to treat host and device separately
+#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)
+#  define _CCCL_GLOBAL_CONSTANT _CCCL_DEVICE constexpr
+#else // ^^^ _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) ^^^ /
+      // vvv !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) vvv
+#  define _CCCL_GLOBAL_CONSTANT inline constexpr
+#endif // !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC)
+
+#if _CCCL_STD_VER >= 2020 && __cpp_constinit >= 201907L
+#  define _CCCL_CONSTINIT constinit
+#else // ^^^ has constinit ^^^ / vvv no constinit vvv
+#  define _CCCL_CONSTINIT _CCCL_REQUIRE_CONSTANT_INITIALIZATION
+#endif // ^^^ no constinit ^^^
+
+// nvcc and nvrtc don't implement multiarg operator[] even in C++23 mode
+#if __cpp_multidimensional_subscript >= 202110L && !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC)
+#  define _CCCL_HAS_MULTIARG_OPERATOR_BRACKETS() 1
+#else // ^^^ has multiarg operator[] ^^^ / vvv no multiarg operator[] vvv
+#  define _CCCL_HAS_MULTIARG_OPERATOR_BRACKETS() 0
+#endif // ^^^ no mutiarg operator[] ^^^
+
+// clang 16+, gcc 13+ and nvc++ 25.9+ backport the static subscript operator back to c++17.
+#if __cpp_multidimensional_subscript >= 202211L                                        \
+  || ((_CCCL_COMPILER(CLANG, >=, 16) || _CCCL_COMPILER(GCC, >=, 13)                    \
+       || (_CCCL_COMPILER(NVHPC, >=, 25, 9) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 12))) \
+      && (!_CCCL_CUDA_COMPILATION() || _CCCL_CUDA_COMPILER(CLANG)))
+#  define _CCCL_HAS_STATIC_SUBSCRIPT_OPERATOR() 1
+#else // ^^^ has static operator[] ^^^ / vvv no static operator[] vvv
+#  define _CCCL_HAS_STATIC_SUBSCRIPT_OPERATOR() 0
+#endif // ^^^ no static operator[] ^^^
+
+// nvcc 13+, clang 16+ and gcc 13+ backport the static call operator back to c++17.
+#if __cpp_static_call_operator >= 202207L                                              \
+  || ((_CCCL_COMPILER(CLANG, >=, 16) || _CCCL_COMPILER(GCC, >=, 13)                    \
+       || (_CCCL_COMPILER(NVHPC, >=, 26, 1) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 13))) \
+      && (!_CCCL_CUDA_COMPILATION() || _CCCL_CUDA_COMPILER(NVCC, >=, 13, 0) || _CCCL_CUDA_COMPILER(CLANG)))
+#  define _CCCL_HAS_STATIC_CALL_OPERATOR() 1
+#else // ^^^ has static operator() ^^^ / vvv no static operator() vvv
+#  define _CCCL_HAS_STATIC_CALL_OPERATOR() 0
+#endif // ^^^ no static operator() ^^^
+
+// if consteval requires C++23, but most compilers support it even in C++20 mode while emitting some warnings. Those are
+// silenced in prologue/epilogue. nvcc is happy about using it in C++20 since 13.0, but only when compiling host code.
+// nvc++ requires libstdc++ at least 12 to support if consteval.
+#if _CCCL_STD_VER == 2020                                  \
+  && (_CCCL_COMPILER(GCC, >=, 12) || _CCCL_COMPILER(CLANG) \
+      || (_CCCL_COMPILER(NVHPC) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 12)))
+#  define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 1
+#else
+#  define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 0
+#endif
+
+// nvcc before 13 doesn't support if consteval at all. Since 13, it accepts if consteval in host code (clang doesn't
+// work) and since 13.1 it works in device code, too.
+#if _CCCL_CUDA_COMPILER(NVCC, <, 13) || (_CCCL_CUDA_COMPILER(NVCC, <, 13, 1) && _CCCL_DEVICE_COMPILATION()) \
+  || (_CCCL_CUDA_COMPILER(NVCC) && _CCCL_COMPILER(CLANG))
+#  undef _CCCL_HAS_IF_CONSTEVAL_IN_CXX20
+#  define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 0
+#endif // ^^^ disable if consteval in c++20 for nvcc ^^^
+
+#if __cpp_if_consteval >= 202106L || _CCCL_HAS_IF_CONSTEVAL_IN_CXX20()
+#  define _CCCL_IF_CONSTEVAL             if consteval
+#  define _CCCL_IF_CONSTEVAL_DEFAULT     _CCCL_IF_CONSTEVAL
+#  define _CCCL_IF_NOT_CONSTEVAL         if !consteval
+#  define _CCCL_IF_NOT_CONSTEVAL_DEFAULT _CCCL_IF_NOT_CONSTEVAL
+#elif defined(_CCCL_BUILTIN_IS_CONSTANT_EVALUATED)
+#  if _CCCL_HOST_COMPILATION() && _CCCL_COMPILER(GCC)
+#    define _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() _CCCL_DIAG_PUSH _CCCL_DIAG_SUPPRESS_GCC("-Wtautological-compare")
+#    define _CCCL_END_IF_CONSTEVAL_SUPPRESS()   _CCCL_DIAG_POP
+#  else // ^^^ _CCCL_HOST_COMPILATION() && _CCCL_COMPILER(GCC) ^^^ /
+        // vvv !_CCCL_HOST_COMPILATION() || ! _CCCL_COMPILER(GCC) vvv
+#    define _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS()
+#    define _CCCL_END_IF_CONSTEVAL_SUPPRESS()
+#  endif // ^^^ !_CCCL_HOST_COMPILATION() || ! _CCCL_COMPILER(GCC) ^^^
+
+#  define _CCCL_IF_CONSTEVAL \
+    _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() if (_CCCL_BUILTIN_IS_CONSTANT_EVALUATED()) _CCCL_END_IF_CONSTEVAL_SUPPRESS()
+#  define _CCCL_IF_CONSTEVAL_DEFAULT _CCCL_IF_CONSTEVAL
+#  define _CCCL_IF_NOT_CONSTEVAL \
+    _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() if (!_CCCL_BUILTIN_IS_CONSTANT_EVALUATED()) _CCCL_END_IF_CONSTEVAL_SUPPRESS()
+#  define _CCCL_IF_NOT_CONSTEVAL_DEFAULT _CCCL_IF_NOT_CONSTEVAL
+#else // ^^^ has is constant evaluated ^^^ / vvv no is constant evaluated vvv
+#  define _CCCL_IF_CONSTEVAL             if constexpr (false)
+#  define _CCCL_IF_CONSTEVAL_DEFAULT     if constexpr (true)
+#  define _CCCL_IF_NOT_CONSTEVAL         if constexpr (true)
+#  define _CCCL_IF_NOT_CONSTEVAL_DEFAULT if constexpr (false)
+#endif // ^^^ no is constant evaluated ^^^
+
+#if _CCCL_STD_VER >= 2020 && __cpp_char8_t >= 201811L
+#  define _CCCL_HAS_CHAR8_T() 1
+#else // ^^^ has char8_t ^^^ / vvv no char8_t vvv
+#  define _CCCL_HAS_CHAR8_T() 0
+#endif // ^^^ no char8_t ^^^
+
+// We currently do not support any of the STL wchar facilities
+#define _CCCL_HAS_WCHAR_T() 0
+
+// Fixme: replace the condition with (!_CCCL_DEVICE_COMPILATION())
+// FIXME: Enable this for clang-cuda in a followup
+#if !_CCCL_CUDA_COMPILATION() && !defined(CCCL_DISABLE_LONG_DOUBLE_SUPPORT)
+#  define _CCCL_HAS_LONG_DOUBLE() 1
+#else // ^^^ has long double ^^^ / vvv no long double vvv
+#  define _CCCL_HAS_LONG_DOUBLE() 0
+#endif // ^^^ no long double ^^^
+
+// clang-21+ and gcc-16+ allow structured bindings to introduce a pack since C++17.
+#if __cpp_structured_bindings >= 202411L || _CCCL_COMPILER(CLANG, >=, 21) || _CCCL_COMPILER(GCC, >=, 16)
+#  define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 1
+#else // ^^^ has structured bindings with pack ^^^ / vvv no structured bindings with pack vvv
+#  define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 0
+#endif // ^^^ no structured bindings with pack ^^^
+
+// nvcc doesn't implement structured bindings pack yet.
+#if _CCCL_CUDA_COMPILER(NVCC)
+#  undef _CCCL_HAS_STRUCTURED_BINDINGS_PACK
+#  define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 0
+#endif // _CCCL_CUDA_COMPILER(NVCC)
+
+#endif // __CCCL_DIALECT_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/epilogue.h

@@ -0,0 +1,390 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py.
+
+// NO include guards here (this file is included multiple times)
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/diagnostic.h>
+
+#if !defined(_CCCL_PROLOGUE_INCLUDED)
+#  error "cccl internal error: <cuda/std/__cccl/prologue.h> must be included before <cuda/std/__cccl/epilogue.h>"
+#endif
+#undef _CCCL_PROLOGUE_INCLUDED
+
+_CCCL_NV_DIAG_POP()
+_CCCL_DIAG_POP
+
+// __declspec modifiers
+
+#if defined(align)
+#  error \
+    "cccl internal error: macro `align` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_align)
+#  pragma pop_macro("align")
+#  undef _CCCL_POP_MACRO_align
+#endif
+
+#if defined(allocate)
+#  error \
+    "cccl internal error: macro `allocate` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_allocate)
+#  pragma pop_macro("allocate")
+#  undef _CCCL_POP_MACRO_allocate
+#endif
+
+#if defined(allocator)
+#  error \
+    "cccl internal error: macro `allocator` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_allocator)
+#  pragma pop_macro("allocator")
+#  undef _CCCL_POP_MACRO_allocator
+#endif
+
+#if defined(appdomain)
+#  error \
+    "cccl internal error: macro `appdomain` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_appdomain)
+#  pragma pop_macro("appdomain")
+#  undef _CCCL_POP_MACRO_appdomain
+#endif
+
+#if defined(code_seg)
+#  error \
+    "cccl internal error: macro `code_seg` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_code_seg)
+#  pragma pop_macro("code_seg")
+#  undef _CCCL_POP_MACRO_code_seg
+#endif
+
+#if defined(deprecated)
+#  error \
+    "cccl internal error: macro `deprecated` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_deprecated)
+#  pragma pop_macro("deprecated")
+#  undef _CCCL_POP_MACRO_deprecated
+#endif
+
+#if defined(dllimport)
+#  error \
+    "cccl internal error: macro `dllimport` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_dllimport)
+#  pragma pop_macro("dllimport")
+#  undef _CCCL_POP_MACRO_dllimport
+#endif
+
+#if defined(dllexport)
+#  error \
+    "cccl internal error: macro `dllexport` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_dllexport)
+#  pragma pop_macro("dllexport")
+#  undef _CCCL_POP_MACRO_dllexport
+#endif
+
+#if defined(empty_bases)
+#  error \
+    "cccl internal error: macro `empty_bases` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_empty_bases)
+#  pragma pop_macro("empty_bases")
+#  undef _CCCL_POP_MACRO_empty_bases
+#endif
+
+#if defined(hybrid_patchable)
+#  error \
+    "cccl internal error: macro `hybrid_patchable` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_hybrid_patchable)
+#  pragma pop_macro("hybrid_patchable")
+#  undef _CCCL_POP_MACRO_hybrid_patchable
+#endif
+
+#if defined(jitintrinsic)
+#  error \
+    "cccl internal error: macro `jitintrinsic` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_jitintrinsic)
+#  pragma pop_macro("jitintrinsic")
+#  undef _CCCL_POP_MACRO_jitintrinsic
+#endif
+
+#if defined(lifetimebound)
+#  error \
+    "cccl internal error: macro `lifetimebound` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_lifetimebound)
+#  pragma pop_macro("lifetimebound")
+#  undef _CCCL_POP_MACRO_lifetimebound
+#endif
+
+#if defined(naked)
+#  error \
+    "cccl internal error: macro `naked` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_naked)
+#  pragma pop_macro("naked")
+#  undef _CCCL_POP_MACRO_naked
+#endif
+
+#if defined(noalias)
+#  error \
+    "cccl internal error: macro `noalias` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_noalias)
+#  pragma pop_macro("noalias")
+#  undef _CCCL_POP_MACRO_noalias
+#endif
+
+#if defined(noinline)
+#  error \
+    "cccl internal error: macro `noinline` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_noinline)
+#  pragma pop_macro("noinline")
+#  undef _CCCL_POP_MACRO_noinline
+#endif
+
+#if defined(noreturn)
+#  error \
+    "cccl internal error: macro `noreturn` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_noreturn)
+#  pragma pop_macro("noreturn")
+#  undef _CCCL_POP_MACRO_noreturn
+#endif
+
+#if defined(nothrow)
+#  error \
+    "cccl internal error: macro `nothrow` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_nothrow)
+#  pragma pop_macro("nothrow")
+#  undef _CCCL_POP_MACRO_nothrow
+#endif
+
+#if defined(novtable)
+#  error \
+    "cccl internal error: macro `novtable` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_novtable)
+#  pragma pop_macro("novtable")
+#  undef _CCCL_POP_MACRO_novtable
+#endif
+
+#if defined(no_sanitize_address)
+#  error \
+    "cccl internal error: macro `no_sanitize_address` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_no_sanitize_address)
+#  pragma pop_macro("no_sanitize_address")
+#  undef _CCCL_POP_MACRO_no_sanitize_address
+#endif
+
+#if defined(process)
+#  error \
+    "cccl internal error: macro `process` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_process)
+#  pragma pop_macro("process")
+#  undef _CCCL_POP_MACRO_process
+#endif
+
+#if defined(property)
+#  error \
+    "cccl internal error: macro `property` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_property)
+#  pragma pop_macro("property")
+#  undef _CCCL_POP_MACRO_property
+#endif
+
+#if defined(restrict)
+#  error \
+    "cccl internal error: macro `restrict` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_restrict)
+#  pragma pop_macro("restrict")
+#  undef _CCCL_POP_MACRO_restrict
+#endif
+
+#if defined(safebuffers)
+#  error \
+    "cccl internal error: macro `safebuffers` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_safebuffers)
+#  pragma pop_macro("safebuffers")
+#  undef _CCCL_POP_MACRO_safebuffers
+#endif
+
+#if defined(selectany)
+#  error \
+    "cccl internal error: macro `selectany` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_selectany)
+#  pragma pop_macro("selectany")
+#  undef _CCCL_POP_MACRO_selectany
+#endif
+
+#if defined(spectre)
+#  error \
+    "cccl internal error: macro `spectre` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_spectre)
+#  pragma pop_macro("spectre")
+#  undef _CCCL_POP_MACRO_spectre
+#endif
+
+#if defined(thread)
+#  error \
+    "cccl internal error: macro `thread` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_thread)
+#  pragma pop_macro("thread")
+#  undef _CCCL_POP_MACRO_thread
+#endif
+
+#if defined(uuid)
+#  error \
+    "cccl internal error: macro `uuid` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_uuid)
+#  pragma pop_macro("uuid")
+#  undef _CCCL_POP_MACRO_uuid
+#endif
+
+// [[msvc::attribute]] attributes
+
+#if defined(msvc)
+#  error \
+    "cccl internal error: macro `msvc` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_msvc)
+#  pragma pop_macro("msvc")
+#  undef _CCCL_POP_MACRO_msvc
+#endif
+
+#if defined(flatten)
+#  error \
+    "cccl internal error: macro `flatten` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_flatten)
+#  pragma pop_macro("flatten")
+#  undef _CCCL_POP_MACRO_flatten
+#endif
+
+#if defined(forceinline)
+#  error \
+    "cccl internal error: macro `forceinline` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_forceinline)
+#  pragma pop_macro("forceinline")
+#  undef _CCCL_POP_MACRO_forceinline
+#endif
+
+#if defined(forceinline_calls)
+#  error \
+    "cccl internal error: macro `forceinline_calls` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_forceinline_calls)
+#  pragma pop_macro("forceinline_calls")
+#  undef _CCCL_POP_MACRO_forceinline_calls
+#endif
+
+#if defined(intrinsic)
+#  error \
+    "cccl internal error: macro `intrinsic` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_intrinsic)
+#  pragma pop_macro("intrinsic")
+#  undef _CCCL_POP_MACRO_intrinsic
+#endif
+
+#if defined(noinline)
+#  error \
+    "cccl internal error: macro `noinline` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_noinline)
+#  pragma pop_macro("noinline")
+#  undef _CCCL_POP_MACRO_noinline
+#endif
+
+#if defined(noinline_calls)
+#  error \
+    "cccl internal error: macro `noinline_calls` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_noinline_calls)
+#  pragma pop_macro("noinline_calls")
+#  undef _CCCL_POP_MACRO_noinline_calls
+#endif
+
+#if defined(no_tls_guard)
+#  error \
+    "cccl internal error: macro `no_tls_guard` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_no_tls_guard)
+#  pragma pop_macro("no_tls_guard")
+#  undef _CCCL_POP_MACRO_no_tls_guard
+#endif
+
+// Windows nasty macros
+
+#if defined(min)
+#  error \
+    "cccl internal error: macro `min` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_min)
+#  pragma pop_macro("min")
+#  undef _CCCL_POP_MACRO_min
+#endif
+
+#if defined(max)
+#  error \
+    "cccl internal error: macro `max` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_max)
+#  pragma pop_macro("max")
+#  undef _CCCL_POP_MACRO_max
+#endif
+
+#if defined(interface)
+#  error \
+    "cccl internal error: macro `interface` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_interface)
+#  pragma pop_macro("interface")
+#  undef _CCCL_POP_MACRO_interface
+#endif
+
+// sal.h on Windows
+
+#if defined(__valid)
+#  error \
+    "cccl internal error: macro `__valid` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO___valid)
+#  pragma pop_macro("__valid")
+#  undef _CCCL_POP_MACRO___valid
+#endif
+
+#if defined(__callback)
+#  error \
+    "cccl internal error: macro `__callback` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO___callback)
+#  pragma pop_macro("__callback")
+#  undef _CCCL_POP_MACRO___callback
+#endif
+
+// other macros
+
+#if defined(clang)
+#  error \
+    "cccl internal error: macro `clang` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_clang)
+#  pragma pop_macro("clang")
+#  undef _CCCL_POP_MACRO_clang
+#endif
+
+// sys/sysmacros.h on linux
+
+#if defined(major)
+#  error \
+    "cccl internal error: macro `major` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_major)
+#  pragma pop_macro("major")
+#  undef _CCCL_POP_MACRO_major
+#endif
+
+#if defined(minor)
+#  error \
+    "cccl internal error: macro `minor` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_minor)
+#  pragma pop_macro("minor")
+#  undef _CCCL_POP_MACRO_minor
+#endif
+
+#if defined(makedev)
+#  error \
+    "cccl internal error: macro `makedev` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_makedev)
+#  pragma pop_macro("makedev")
+#  undef _CCCL_POP_MACRO_makedev
+#endif
+
+// NO include guards here (this file is included multiple times)

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/exceptions.h

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_EXCEPTIONS_H
+#define __CCCL_EXCEPTIONS_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/execution_space.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if defined(CCCL_DISABLE_EXCEPTIONS) // Escape hatch for users to manually disable exceptions
+#  define _CCCL_HAS_EXCEPTIONS() 0
+#elif _CCCL_COMPILER(NVRTC) // NVRTC has no exceptions
+#  define _CCCL_HAS_EXCEPTIONS() 0
+#elif _CCCL_COMPILER(MSVC) // MSVC needs special checks for `_HAS_EXCEPTIONS` and `_CPPUNWIND`
+#  define _CCCL_HAS_EXCEPTIONS() ((_HAS_EXCEPTIONS != 0) && (_CPPUNWIND != 0)) // disabled with /EH
+#else // other compilers use `__EXCEPTIONS`
+#  define _CCCL_HAS_EXCEPTIONS() (__EXCEPTIONS) // disabled with -fno-exceptions
+#endif // has exceptions
+
+#if _CCCL_HAS_EXCEPTIONS() && __cpp_constexpr_exceptions >= 202411L
+#  define _CCCL_HAS_CONSTEXPR_EXCEPTIONS() 1
+#else // ^^^ has constexpr exceptions ^^^ / vvv no constexpr exceptions vvv
+#  define _CCCL_HAS_CONSTEXPR_EXCEPTIONS() 0
+#endif // ^^^ no constexpr exceptions ^^^
+
+#endif // __CCCL_EXCEPTIONS_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/execution_space.h

@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_EXECUTION_SPACE_H
+#define __CCCL_EXECUTION_SPACE_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/cuda_capabilities.h>
+
+#if _CCCL_CUDA_COMPILATION()
+#  define _CCCL_HOST        __host__
+#  define _CCCL_DEVICE      __device__
+#  define _CCCL_HOST_DEVICE __host__ __device__
+#else // ^^^ _CCCL_CUDA_COMPILATION ^^^ / vvv !_CCCL_CUDA_COMPILATION vvv
+#  define _CCCL_HOST
+#  define _CCCL_DEVICE
+#  define _CCCL_HOST_DEVICE
+#endif // !_CCCL_CUDA_COMPILATION
+
+#if _CCCL_TILE_COMPILATION()
+#  define _CCCL_TILE __tile__
+#else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() vvv
+#  define _CCCL_TILE
+#endif // ^^^ !_CCCL_TILE_COMPILATION() ^^^
+
+// clang-cuda before version 22 requires __host__ __device__ annotations on deduction guides
+#if _CCCL_CUDA_COMPILER(CLANG, <, 22)
+#  define _CCCL_DEDUCTION_GUIDE_ATTRIBUTES _CCCL_HOST_DEVICE
+#else // ^^^ _CCCL_CUDA_COMPILER(CLANG, <, 22) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG, <, 22) vvv
+#  define _CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+#endif // ^^ !_CCCL_CUDA_COMPILER(CLANG, <, 22) ^^^
+
+// Global variables of non builtin types are only device accessible if they are marked as `__device__`
+#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)
+#  define _CCCL_GLOBAL_VARIABLE _CCCL_DEVICE
+#else // ^^^ _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) ^^^ /
+      // vvv !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) vvv
+#  define _CCCL_GLOBAL_VARIABLE
+#endif // ^^^ !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) ^^^
+
+#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) || _CCCL_CUDA_COMPILER(NVRTC) || _CCCL_CUDA_COMPILER(CLANG, >=, 20)) \
+  && _CCCL_PTX_ARCH() >= 700
+#  define _CCCL_HAS_GRID_CONSTANT() 1
+#  define _CCCL_GRID_CONSTANT       __grid_constant__
+#else // ^^^ has __grid_constant__ ^^^ / vvv no __grid_constant__ vvv
+#  define _CCCL_HAS_GRID_CONSTANT() 0
+#  define _CCCL_GRID_CONSTANT
+#endif // ^^^ no __grid_constant__ ^^^
+
+#if !defined(_CCCL_EXEC_CHECK_DISABLE)
+#  if _CCCL_CUDA_COMPILER(NVCC)
+#    define _CCCL_EXEC_CHECK_DISABLE _CCCL_PRAGMA(nv_exec_check_disable)
+#  else
+#    define _CCCL_EXEC_CHECK_DISABLE
+#  endif // _CCCL_CUDA_COMPILER(NVCC)
+#endif // !_CCCL_EXEC_CHECK_DISABLE
+
+#if _CCCL_CUDA_COMPILER(NVHPC)
+#  define _CCCL_TARGET_CONSTEXPR
+#else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv
+#  define _CCCL_TARGET_CONSTEXPR constexpr
+#endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^
+
+//! @brief List of all known PTX architectures supported by this CCCL version.
+#define _CCCL_KNOWN_CUDA_ARCH_LIST 50, 52, 53, 60, 61, 62, 70, 75, 80, 86, 87, 88, 89, 90, 100, 103, 110, 120, 121
+
+//! @brief List of all known architecture specific architectures supported by this CCCL version.
+#define _CCCL_KNOWN_CUDA_ARCH_SPECIFIC_LIST 90, 100, 103, 110, 120, 121
+
+#endif // __CCCL_EXECUTION_SPACE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/extended_data_types.h

@@ -0,0 +1,148 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_EXTENDED_DATA_TYPES_H
+#define __CCCL_EXTENDED_DATA_TYPES_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/architecture.h>
+#include <cuda/std/__cccl/cuda_capabilities.h>
+#include <cuda/std/__cccl/cuda_toolkit.h>
+#include <cuda/std/__cccl/diagnostic.h>
+#include <cuda/std/__cccl/os.h>
+#include <cuda/std/__cccl/preprocessor.h>
+
+#define _CCCL_HAS_INT128()   0
+#define _CCCL_HAS_NVFP4()    0
+#define _CCCL_HAS_NVFP6()    0
+#define _CCCL_HAS_NVFP8()    0
+#define _CCCL_HAS_NVFP16()   0
+#define _CCCL_HAS_NVBF16()   0
+#define _CCCL_HAS_FLOAT128() 0
+
+#if _CCCL_TILE_COMPILATION() // TODO(miscco): Fix access to extended floating point types
+#  define CCCL_DISABLE_NVFP4_SUPPORT
+#  define CCCL_DISABLE_NVFP6_SUPPORT
+#  define CCCL_DISABLE_NVFP8_SUPPORT
+#  define CCCL_DISABLE_INT128_SUPPORT
+#  define CCCL_DISABLE_FLOAT128_SUPPORT
+#endif // _CCCL_TILE_COMPILATION()
+
+#if !defined(CCCL_DISABLE_INT128_SUPPORT) && _CCCL_OS(LINUX) \
+  && ((_CCCL_COMPILER(NVRTC) && defined(__CUDACC_RTC_INT128__)) || defined(__SIZEOF_INT128__))
+#  undef _CCCL_HAS_INT128
+#  define _CCCL_HAS_INT128() 1
+#endif
+
+#if __has_include(<cuda_fp16.h>) && (_CCCL_HAS_CTK() || defined(LIBCUDACXX_ENABLE_HOST_NVFP16)) \
+                      && !defined(CCCL_DISABLE_FP16_SUPPORT)
+#  undef _CCCL_HAS_NVFP16
+#  define _CCCL_HAS_NVFP16() 1
+struct __half;
+struct __half2;
+#endif
+
+#if __has_include(<cuda_bf16.h>) && _CCCL_HAS_NVFP16() && !defined(CCCL_DISABLE_BF16_SUPPORT)
+#  undef _CCCL_HAS_NVBF16
+#  define _CCCL_HAS_NVBF16() 1
+struct __nv_bfloat16;
+struct __nv_bfloat162;
+#endif
+
+#if __has_include(<cuda_fp8.h>) && _CCCL_HAS_NVFP16() && _CCCL_HAS_NVBF16() && !defined(CCCL_DISABLE_NVFP8_SUPPORT)
+#  undef _CCCL_HAS_NVFP8
+#  define _CCCL_HAS_NVFP8() 1
+struct __nv_fp8_e5m2;
+struct __nv_fp8x2_e5m2;
+struct __nv_fp8x4_e5m2;
+
+struct __nv_fp8_e4m3;
+struct __nv_fp8x2_e4m3;
+struct __nv_fp8x4_e4m3;
+
+#  if _CCCL_CTK_AT_LEAST(12, 8)
+struct __nv_fp8_e8m0;
+struct __nv_fp8x2_e8m0;
+struct __nv_fp8x4_e8m0;
+#  endif // _CCCL_CTK_AT_LEAST(12, 8)
+#endif
+
+#if __has_include(<cuda_fp6.h>) && _CCCL_HAS_NVFP8() && !_CCCL_CUDA_COMPILER(NVHPC) \
+                      && !defined(CCCL_DISABLE_NVFP6_SUPPORT)
+#  undef _CCCL_HAS_NVFP6
+#  define _CCCL_HAS_NVFP6() 1
+struct __nv_fp6_e3m2;
+struct __nv_fp6x2_e3m2;
+struct __nv_fp6x4_e3m2;
+
+struct __nv_fp6_e2m3;
+struct __nv_fp6x2_e2m3;
+struct __nv_fp6x4_e2m3;
+#endif
+
+#if __has_include(<cuda_fp4.h>) && _CCCL_HAS_NVFP6() && !defined(CCCL_DISABLE_NVFP4_SUPPORT)
+#  undef _CCCL_HAS_NVFP4
+#  define _CCCL_HAS_NVFP4() 1
+struct __nv_fp4_e2m1;
+struct __nv_fp4x2_e2m1;
+struct __nv_fp4x4_e2m1;
+#endif
+
+#define _CCCL_HAS_NVFP4_E2M1() _CCCL_HAS_NVFP4()
+#define _CCCL_HAS_NVFP6_E2M3() _CCCL_HAS_NVFP6()
+#define _CCCL_HAS_NVFP6_E3M2() _CCCL_HAS_NVFP6()
+#define _CCCL_HAS_NVFP8_E4M3() _CCCL_HAS_NVFP8()
+#define _CCCL_HAS_NVFP8_E5M2() _CCCL_HAS_NVFP8()
+#define _CCCL_HAS_NVFP8_E8M0() (_CCCL_HAS_NVFP8() && _CCCL_CTK_AT_LEAST(12, 8))
+
+/***********************************************************************************************************************
+ * __float128
+ **********************************************************************************************************************/
+
+#if !defined(CCCL_DISABLE_FLOAT128_SUPPORT) && _CCCL_HAS_INT128() && _CCCL_OS(LINUX) && !_CCCL_HOST_ARCH(ARM64) \
+  && !_CCCL_TILE_COMPILATION()
+// Detect host compiler support
+#  if (defined(__CUDACC_RTC_FLOAT128__) || defined(__SIZEOF_FLOAT128__) || defined(__FLOAT128__))
+#    if _CCCL_DEVICE_COMPILATION()
+// Only NVCC and NVRTC 12.8+ on architectures at least SM100 supports __float128 on device
+#      if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 8)) && _CCCL_PTX_ARCH() >= 1000
+#        undef _CCCL_HAS_FLOAT128
+#        define _CCCL_HAS_FLOAT128() 1
+#      endif // _CCCL_CUDA_COMPILER(NVCC) && _CCCL_PTX_ARCH() >= 1000
+#    else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
+#      undef _CCCL_HAS_FLOAT128
+#      define _CCCL_HAS_FLOAT128() 1
+#    endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^
+#  endif // Host compiler support
+#endif // !defined(CCCL_DISABLE_FLOAT128_SUPPORT) && _CCCL_HAS_INT128() && _CCCL_OS(LINUX) && !_CCCL_HOST_ARCH(ARM64)
+
+// gcc does not allow to use q/Q floating point literals when __STRICT_ANSI__ is defined. They may be allowed by
+// -fext-numeric-literals, but there is no way to detect it in the preprocessor. The user is required to define
+// CCCL_GCC_HAS_EXTENDED_NUMERIC_LITERALS in this case. Otherwise, we disable the __float128 support.
+//
+// Note: since GCC 13, we could use f128/F128 literals, but for values > DBL_MAX, the compilation with nvcc fails due to
+//       "floating constant is out of range".
+#if _CCCL_HAS_FLOAT128() && _CCCL_COMPILER(GCC) && defined(__STRICT_ANSI__) \
+  && !defined(CCCL_GCC_HAS_EXTENDED_NUMERIC_LITERALS)
+#  undef _CCCL_HAS_FLOAT128
+#  define _CCCL_HAS_FLOAT128() 0
+#endif // _CCCL_HAS_FLOAT128()
+
+#endif // __CCCL_EXTENDED_DATA_TYPES_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/host_std_lib.h

@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_HOST_STD_LIB_H
+#define __CCCL_HOST_STD_LIB_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/preprocessor.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#define _CCCL_HOST_STD_LIB_LIBSTDCXX() _CCCL_VERSION_INVALID()
+#define _CCCL_HOST_STD_LIB_LIBCXX()    _CCCL_VERSION_INVALID()
+#define _CCCL_HOST_STD_LIB_STL()       _CCCL_VERSION_INVALID()
+
+// include a minimal header
+#if __has_include(<version>)
+#  include <version>
+#elif __has_include(<ciso646>)
+#  include <ciso646>
+#endif // ^^^ __has_include(<ciso646>) ^^^
+
+#define _CCCL_HOST_STD_LIB_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 100 + (_MINOR))
+#define _CCCL_HOST_STD_LIB(...)                         _CCCL_VERSION_COMPARE(_CCCL_HOST_STD_LIB_, _CCCL_HOST_STD_LIB_##__VA_ARGS__)
+
+#if _CCCL_HOSTED()
+#  if defined(_MSVC_STL_VERSION)
+#    undef _CCCL_HOST_STD_LIB_STL
+#    define _CCCL_HOST_STD_LIB_STL() (_MSVC_STL_VERSION, 0)
+#  elif defined(__GLIBCXX__)
+#    undef _CCCL_HOST_STD_LIB_LIBSTDCXX
+#    define _CCCL_HOST_STD_LIB_LIBSTDCXX() (_GLIBCXX_RELEASE, 0)
+#  elif defined(_LIBCPP_VERSION)
+#    undef _CCCL_HOST_STD_LIB_LIBCXX
+// since llvm-16, the version scheme has been changed from MMppp to MMmmpp
+#    if _LIBCPP_VERSION / 10000 < 2
+#      define _CCCL_HOST_STD_LIB_LIBCXX() (_LIBCPP_VERSION / 1000, 0)
+#    else
+#      define _CCCL_HOST_STD_LIB_LIBCXX() (_LIBCPP_VERSION / 10000, (_LIBCPP_VERSION / 100) % 100)
+#    endif
+#  endif // ^^^ _LIBCPP_VERSION ^^^
+#endif // _CCCL_HOSTED()
+
+#define _CCCL_HAS_HOST_STD_LIB() \
+  (_CCCL_HOST_STD_LIB(LIBSTDCXX) || _CCCL_HOST_STD_LIB(LIBCXX) || _CCCL_HOST_STD_LIB(STL))
+
+#endif // __CCCL_HOST_STD_LIB_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/is_non_narrowing_convertible.h

@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_IS_NON_NARROWING_CONVERTIBLE_H
+#define __CCCL_IS_NON_NARROWING_CONVERTIBLE_H
+
+#include <cuda/std/__cccl/compiler.h>
+
+//! There is compiler bug that results in incorrect results for the below `__is_non_narrowing_convertible` check.
+//! This breaks some common functionality, so this *must* be included outside of a system header. See nvbug4867473.
+#if defined(_CCCL_FORCE_SYSTEM_HEADER_GCC) || defined(_CCCL_FORCE_SYSTEM_HEADER_CLANG) \
+  || defined(_CCCL_FORCE_SYSTEM_HEADER_MSVC)
+#  error \
+    "This header must be included only within the <cuda/std/__cccl/system_header>. This most likely means a mix and match of different versions of CCCL."
+#endif // system header detected
+
+namespace __cccl_internal
+{
+#if _CCCL_CUDA_COMPILATION()
+template <class _Tp>
+__host__ __device__ _Tp&& __cccl_declval(int);
+template <class _Tp>
+__host__ __device__ _Tp __cccl_declval(long);
+template <class _Tp>
+__host__ __device__ decltype(__cccl_internal::__cccl_declval<_Tp>(0)) __cccl_declval() noexcept;
+
+// This requires a type to be implicitly convertible (also non-arithmetic)
+template <class _Tp>
+__host__ __device__ void __cccl_accepts_implicit_conversion(_Tp) noexcept;
+#else // ^^^ CUDA compilation ^^^ / vvv no CUDA compilation
+template <class _Tp>
+_Tp&& __cccl_declval(int);
+template <class _Tp>
+_Tp __cccl_declval(long);
+template <class _Tp>
+decltype(__cccl_internal::__cccl_declval<_Tp>(0)) __cccl_declval() noexcept;
+
+// This requires a type to be implicitly convertible (also non-arithmetic)
+template <class _Tp>
+void __cccl_accepts_implicit_conversion(_Tp) noexcept;
+#endif // no CUDA compilation
+
+template <class...>
+using __cccl_void_t = void;
+
+template <class _Dest, class _Source, class = void>
+struct __is_non_narrowing_convertible
+{
+  static constexpr bool value = false;
+};
+
+// This also prohibits narrowing conversion in case of arithmetic types
+template <class _Dest, class _Source>
+struct __is_non_narrowing_convertible<_Dest,
+                                      _Source,
+                                      __cccl_void_t<decltype(__cccl_internal::__cccl_accepts_implicit_conversion<_Dest>(
+                                                      __cccl_internal::__cccl_declval<_Source>())),
+                                                    decltype(_Dest{__cccl_internal::__cccl_declval<_Source>()})>>
+{
+  static constexpr bool value = true;
+};
+} // namespace __cccl_internal
+
+#endif // __CCCL_IS_NON_NARROWING_CONVERTIBLE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/os.h

@@ -0,0 +1,120 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_OS_H
+#define __CCCL_OS_H
+
+// The header provides the following macros to determine the host architecture:
+//
+// _CCCL_OS(WINDOWS)
+// _CCCL_OS(LINUX)
+// _CCCL_OS(ANDROID)
+// _CCCL_OS(QNX)
+
+// Determine the host compiler and its version
+#if defined(_WIN32) || defined(_WIN64) /* _WIN64 for NVRTC */
+#  define _CCCL_OS_WINDOWS_() 1
+#else
+#  define _CCCL_OS_WINDOWS_() 0
+#endif
+
+#if defined(__linux__) || defined(__LP64__) /* __LP64__ for NVRTC */
+#  define _CCCL_OS_LINUX_() 1
+#else
+#  define _CCCL_OS_LINUX_() 0
+#endif
+
+#if defined(__ANDROID__)
+#  define _CCCL_OS_ANDROID_() 1
+#else
+#  define _CCCL_OS_ANDROID_() 0
+#endif
+
+#if defined(__QNX__) || defined(__QNXNTO__)
+#  define _CCCL_OS_QNX_() 1
+#else
+#  define _CCCL_OS_QNX_() 0
+#endif
+
+#if defined(__APPLE__) || defined(__APPLE_CC__)
+#  define _CCCL_OS_APPLE_() 1
+#else
+#  define _CCCL_OS_APPLE_() 0
+#endif
+
+#define _CCCL_OS(...) _CCCL_OS_##__VA_ARGS__##_()
+
+//! @def CCCL_OS(os) /* implementation defined */
+//!
+//! @brief Detect the current operating system.
+//!
+//! @param os The name of the operating system to test.
+//!
+//! @note This macro is made available when including any libcu++ header. Users that wish to
+//! include the smallest possible header for this macro should include `<cuda/std/version>`.
+//!
+//! For supported operating systems, the macro expands to an implementation-defined true value
+//! if the current operating system matches, or false otherwise. These values may be used in
+//! boolean expressions (preprocessor or otherwise), but no other guarantees are made.
+//!
+//! Available values for `os` include:
+//!
+//! - ``WINDOWS``: Windows, either in 32-bit or 64-bit mode.
+//! - ``LINUX``: Any kind of Linux installation. Note that other unix-based operating systems will
+//!              also match against this.
+//! - ``ANDROID``: Android operating system.
+//! - ``QNX``: QNX real-time operating system.
+//! - ``APPLE``: macOS (Intel or Apple Silicon).
+//!
+//! Passing any other value will result in an undefined expansion, which may or may not be
+//! diagnosed by the compiler.
+//!
+//! @note Some operating systems may satisfy multiple conditions. For example macOS and Android
+//! satisfy both `APPLE`/`ANDROID` and `LINUX`.
+//!
+//! @par Example
+//! @code
+//! #define MY_OTHER_MACRO 1
+//!
+//! // Expansion value can be used in ordinary macro conditionals
+//! #if CCCL_OS(WINDOWS) && MY_OTHER_MACRO
+//!   // ...
+//! #endif
+//!
+//! // Can be negated as usual
+//! #if !CCCL_OS(QNX)
+//!   // ...
+//! #endif
+//!
+//! #if CCCL_OS(APPLE)
+//!   // Will be visible only on macOS
+//! #endif
+//!
+//! #if CCCL_OS(ANDROID)
+//!   // Will be visible only on Android
+//! #endif
+//!
+//! #if CCCL_OS(LINUX) && !CCCL_OS(APPLE) && !CCCL_OS(ANDROID)
+//!   // Only visible on Linux
+//! #endif
+//! @endcode
+//!
+//! @return true if the specified OS is begin compiled for, false otherwise.
+#ifdef _CCCL_DOXYGEN_INVOKED
+#  define CCCL_OS(os) /* implementation defined */
+#else
+#  define CCCL_OS(__os__) _CCCL_OS_##__os__##_()
+#endif
+
+// Note: the public API is single-arg to constrain the API and allow for future expansion. The
+// implementation is duplicated to guard against the OS targets being accidentally defined by
+// the user.
+
+#endif // __CCCL_OS_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/prologue.h

@@ -0,0 +1,348 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py.
+
+// NO include guards here (this file is included multiple times)
+
+#if defined(_CCCL_PROLOGUE_INCLUDED)
+#  error \
+    "cccl internal error: <cuda/std/__cccl/epilogue.h> must be included before next <cuda/std/__cccl/prologue.h> is reincluded"
+#endif
+#define _CCCL_PROLOGUE_INCLUDED() 1
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/diagnostic.h>
+#include <cuda/std/__cccl/dialect.h>
+
+// __declspec modifiers
+
+#if defined(align)
+#  pragma push_macro("align")
+#  undef align
+#  define _CCCL_POP_MACRO_align
+#endif // defined(align)
+
+#if defined(allocate)
+#  pragma push_macro("allocate")
+#  undef allocate
+#  define _CCCL_POP_MACRO_allocate
+#endif // defined(allocate)
+
+#if defined(allocator)
+#  pragma push_macro("allocator")
+#  undef allocator
+#  define _CCCL_POP_MACRO_allocator
+#endif // defined(allocator)
+
+#if defined(appdomain)
+#  pragma push_macro("appdomain")
+#  undef appdomain
+#  define _CCCL_POP_MACRO_appdomain
+#endif // defined(appdomain)
+
+#if defined(code_seg)
+#  pragma push_macro("code_seg")
+#  undef code_seg
+#  define _CCCL_POP_MACRO_code_seg
+#endif // defined(code_seg)
+
+#if defined(deprecated)
+#  pragma push_macro("deprecated")
+#  undef deprecated
+#  define _CCCL_POP_MACRO_deprecated
+#endif // defined(deprecated)
+
+#if defined(dllimport)
+#  pragma push_macro("dllimport")
+#  undef dllimport
+#  define _CCCL_POP_MACRO_dllimport
+#endif // defined(dllimport)
+
+#if defined(dllexport)
+#  pragma push_macro("dllexport")
+#  undef dllexport
+#  define _CCCL_POP_MACRO_dllexport
+#endif // defined(dllexport)
+
+#if defined(empty_bases)
+#  pragma push_macro("empty_bases")
+#  undef empty_bases
+#  define _CCCL_POP_MACRO_empty_bases
+#endif // defined(empty_bases)
+
+#if defined(hybrid_patchable)
+#  pragma push_macro("hybrid_patchable")
+#  undef hybrid_patchable
+#  define _CCCL_POP_MACRO_hybrid_patchable
+#endif // defined(hybrid_patchable)
+
+#if defined(jitintrinsic)
+#  pragma push_macro("jitintrinsic")
+#  undef jitintrinsic
+#  define _CCCL_POP_MACRO_jitintrinsic
+#endif // defined(jitintrinsic)
+
+#if defined(lifetimebound)
+#  pragma push_macro("lifetimebound")
+#  undef lifetimebound
+#  define _CCCL_POP_MACRO_lifetimebound
+#endif // defined(lifetimebound)
+
+#if defined(naked)
+#  pragma push_macro("naked")
+#  undef naked
+#  define _CCCL_POP_MACRO_naked
+#endif // defined(naked)
+
+#if defined(noalias)
+#  pragma push_macro("noalias")
+#  undef noalias
+#  define _CCCL_POP_MACRO_noalias
+#endif // defined(noalias)
+
+#if defined(noinline)
+#  pragma push_macro("noinline")
+#  undef noinline
+#  define _CCCL_POP_MACRO_noinline
+#endif // defined(noinline)
+
+#if defined(noreturn)
+#  pragma push_macro("noreturn")
+#  undef noreturn
+#  define _CCCL_POP_MACRO_noreturn
+#endif // defined(noreturn)
+
+#if defined(nothrow)
+#  pragma push_macro("nothrow")
+#  undef nothrow
+#  define _CCCL_POP_MACRO_nothrow
+#endif // defined(nothrow)
+
+#if defined(novtable)
+#  pragma push_macro("novtable")
+#  undef novtable
+#  define _CCCL_POP_MACRO_novtable
+#endif // defined(novtable)
+
+#if defined(no_sanitize_address)
+#  pragma push_macro("no_sanitize_address")
+#  undef no_sanitize_address
+#  define _CCCL_POP_MACRO_no_sanitize_address
+#endif // defined(no_sanitize_address)
+
+#if defined(process)
+#  pragma push_macro("process")
+#  undef process
+#  define _CCCL_POP_MACRO_process
+#endif // defined(process)
+
+#if defined(property)
+#  pragma push_macro("property")
+#  undef property
+#  define _CCCL_POP_MACRO_property
+#endif // defined(property)
+
+#if defined(restrict)
+#  pragma push_macro("restrict")
+#  undef restrict
+#  define _CCCL_POP_MACRO_restrict
+#endif // defined(restrict)
+
+#if defined(safebuffers)
+#  pragma push_macro("safebuffers")
+#  undef safebuffers
+#  define _CCCL_POP_MACRO_safebuffers
+#endif // defined(safebuffers)
+
+#if defined(selectany)
+#  pragma push_macro("selectany")
+#  undef selectany
+#  define _CCCL_POP_MACRO_selectany
+#endif // defined(selectany)
+
+#if defined(spectre)
+#  pragma push_macro("spectre")
+#  undef spectre
+#  define _CCCL_POP_MACRO_spectre
+#endif // defined(spectre)
+
+#if defined(thread)
+#  pragma push_macro("thread")
+#  undef thread
+#  define _CCCL_POP_MACRO_thread
+#endif // defined(thread)
+
+#if defined(uuid)
+#  pragma push_macro("uuid")
+#  undef uuid
+#  define _CCCL_POP_MACRO_uuid
+#endif // defined(uuid)
+
+// [[msvc::attribute]] attributes
+
+#if defined(msvc)
+#  pragma push_macro("msvc")
+#  undef msvc
+#  define _CCCL_POP_MACRO_msvc
+#endif // defined(msvc)
+
+#if defined(flatten)
+#  pragma push_macro("flatten")
+#  undef flatten
+#  define _CCCL_POP_MACRO_flatten
+#endif // defined(flatten)
+
+#if defined(forceinline)
+#  pragma push_macro("forceinline")
+#  undef forceinline
+#  define _CCCL_POP_MACRO_forceinline
+#endif // defined(forceinline)
+
+#if defined(forceinline_calls)
+#  pragma push_macro("forceinline_calls")
+#  undef forceinline_calls
+#  define _CCCL_POP_MACRO_forceinline_calls
+#endif // defined(forceinline_calls)
+
+#if defined(intrinsic)
+#  pragma push_macro("intrinsic")
+#  undef intrinsic
+#  define _CCCL_POP_MACRO_intrinsic
+#endif // defined(intrinsic)
+
+#if defined(noinline)
+#  pragma push_macro("noinline")
+#  undef noinline
+#  define _CCCL_POP_MACRO_noinline
+#endif // defined(noinline)
+
+#if defined(noinline_calls)
+#  pragma push_macro("noinline_calls")
+#  undef noinline_calls
+#  define _CCCL_POP_MACRO_noinline_calls
+#endif // defined(noinline_calls)
+
+#if defined(no_tls_guard)
+#  pragma push_macro("no_tls_guard")
+#  undef no_tls_guard
+#  define _CCCL_POP_MACRO_no_tls_guard
+#endif // defined(no_tls_guard)
+
+// Windows nasty macros
+
+#if defined(min)
+#  pragma push_macro("min")
+#  undef min
+#  define _CCCL_POP_MACRO_min
+#endif // defined(min)
+
+#if defined(max)
+#  pragma push_macro("max")
+#  undef max
+#  define _CCCL_POP_MACRO_max
+#endif // defined(max)
+
+#if defined(interface)
+#  pragma push_macro("interface")
+#  undef interface
+#  define _CCCL_POP_MACRO_interface
+#endif // defined(interface)
+
+// sal.h on Windows
+
+#if defined(__valid)
+#  pragma push_macro("__valid")
+#  undef __valid
+#  define _CCCL_POP_MACRO___valid
+#endif // defined(__valid)
+
+#if defined(__callback)
+#  pragma push_macro("__callback")
+#  undef __callback
+#  define _CCCL_POP_MACRO___callback
+#endif // defined(__callback)
+
+// other macros
+
+#if defined(clang)
+#  pragma push_macro("clang")
+#  undef clang
+#  define _CCCL_POP_MACRO_clang
+#endif // defined(clang)
+
+// sys/sysmacros.h on linux
+
+#if defined(major)
+#  pragma push_macro("major")
+#  undef major
+#  define _CCCL_POP_MACRO_major
+#endif // defined(major)
+
+#if defined(minor)
+#  pragma push_macro("minor")
+#  undef minor
+#  define _CCCL_POP_MACRO_minor
+#endif // defined(minor)
+
+#if defined(makedev)
+#  pragma push_macro("makedev")
+#  undef makedev
+#  define _CCCL_POP_MACRO_makedev
+#endif // defined(makedev)
+
+_CCCL_DIAG_PUSH
+_CCCL_NV_DIAG_PUSH()
+
+// disable some msvc warnings
+// https://github.com/microsoft/STL/blob/master/stl/inc/yvals_core.h#L353
+// warning C4100: 'quack': unreferenced formal parameter
+// warning C4127: conditional expression is constant
+// warning C4180: qualifier applied to function type has no meaning; ignored
+// warning C4197: 'purr': top-level volatile in cast is ignored
+// warning C4324: 'roar': structure was padded due to alignment specifier
+// warning C4455: literal suffix identifiers that do not start with an underscore are reserved
+// warning C4503: 'hum': decorated name length exceeded, name was truncated
+// warning C4522: 'woof' : multiple assignment operators specified
+// warning C4668: 'meow' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
+// warning C4800: 'boo': forcing value to bool 'true' or 'false' (performance warning)
+// warning C4996: 'meow': was declared deprecated
+_CCCL_DIAG_SUPPRESS_MSVC(4100 4127 4180 4197 4296 4324 4455 4503 4522 4668 4800 4996)
+
+// Suppress compiler warnings about C++ extensions.
+
+#if _CCCL_COMPILER(GCC, >=, 12)
+_CCCL_DIAG_SUPPRESS_GCC("-Wc++20-extensions")
+_CCCL_DIAG_SUPPRESS_GCC("-Wc++23-extensions")
+#endif // _CCCL_COMPILER(GCC, >=, 12)
+#if _CCCL_COMPILER(GCC, >=, 14)
+_CCCL_DIAG_SUPPRESS_GCC("-Wc++26-extensions")
+#endif // _CCCL_COMPILER(GCC, >=, 14)
+
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++20-extensions")
+#if _CCCL_COMPILER(CLANG, >=, 17)
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++23-extensions")
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++26-extensions")
+#else // ^^^ _CCCL_COMPILER(CLANG, >=, 17) ^^^ / vvv _CCCL_COMPILER(CLANG, <, 17) vvv
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++2b-extensions")
+#endif // ^^^ _CCCL_COMPILER(CLANG, <, 17) ^^^
+
+// Suppress `if consteval`-related warnings.
+
+_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_nonstandard)
+_CCCL_DIAG_SUPPRESS_NVHPC(is_constant_evaluated_in_nonconstexpr_context)
+_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_in_nonconstexpr_function)
+
+_CCCL_DIAG_SUPPRESS_NVCC(3215) // "if consteval" and "if not consteval" are not standard in this mode
+_CCCL_DIAG_SUPPRESS_NVCC(3206) // "if consteval" and "if not consteval" are meaningless in a non-constexpr function
+_CCCL_DIAG_SUPPRESS_NVCC(3060) // call to __builtin_is_constant_evaluated appearing in a non-constexpr function always
+                               // produces "false"
+
+// NO include guards here (this file is included multiple times)

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/ptx_isa.h

@@ -0,0 +1,369 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_PTX_ISA_H_
+#define __CCCL_PTX_ISA_H_
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
+
+/*
+ * Targeting macros
+ *
+ * Information from:
+ * https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#release-notes
+ */
+
+// The first define is for future major versions of CUDACC.
+// We make sure that these get the highest known PTX ISA version.
+// For clang cuda check
+// https://github.com/llvm/llvm-project/blob/release/<VER>.x/clang/lib/Driver/ToolChains/Cuda.cpp getNVPTXTargetFeatures
+#if _CCCL_CUDACC_AT_LEAST(14, 0) && !_CCCL_CUDA_COMPILER(CLANG)
+#  define __cccl_ptx_isa 940ULL
+// PTX ISA 9.4 is available from CUDA 13.4
+#elif _CCCL_CUDACC_AT_LEAST(13, 4) && !_CCCL_CUDA_COMPILER(CLANG)
+#  define __cccl_ptx_isa 940ULL
+// PTX ISA 9.3 is available from CUDA 13.3
+#elif _CCCL_CUDACC_AT_LEAST(13, 3) && !_CCCL_CUDA_COMPILER(CLANG)
+#  define __cccl_ptx_isa 930ULL
+// PTX ISA 9.2 is available from CUDA 13.2
+#elif _CCCL_CUDACC_AT_LEAST(13, 2) && !_CCCL_CUDA_COMPILER(CLANG)
+#  define __cccl_ptx_isa 920ULL
+// PTX ISA 9.1 is available from CUDA 13.1
+#elif _CCCL_CUDACC_AT_LEAST(13, 1) && !_CCCL_CUDA_COMPILER(CLANG)
+#  define __cccl_ptx_isa 910ULL
+// PTX ISA 9.0 is available from CUDA 13.0, driver r580
+#elif _CCCL_CUDACC_AT_LEAST(13, 0) && !_CCCL_CUDA_COMPILER(CLANG)
+#  define __cccl_ptx_isa 900ULL
+// PTX ISA 8.8 is available from CUDA 12.9, driver r575
+#elif _CCCL_CUDACC_AT_LEAST(12, 9) && !_CCCL_CUDA_COMPILER(CLANG, <, 22)
+#  define __cccl_ptx_isa 880ULL
+// PTX ISA 8.7 is available from CUDA 12.8, driver r570
+#elif _CCCL_CUDACC_AT_LEAST(12, 8) && !_CCCL_CUDA_COMPILER(CLANG, <, 20)
+#  define __cccl_ptx_isa 870ULL
+// PTX ISA 8.5 is available from CUDA 12.5, driver r555
+#elif _CCCL_CUDACC_AT_LEAST(12, 5) && !_CCCL_CUDA_COMPILER(CLANG, <, 19)
+#  define __cccl_ptx_isa 850ULL
+// PTX ISA 8.4 is available from CUDA 12.4, driver r550
+#elif _CCCL_CUDACC_AT_LEAST(12, 4) && !_CCCL_CUDA_COMPILER(CLANG, <, 19)
+#  define __cccl_ptx_isa 840ULL
+// PTX ISA 8.3 is available from CUDA 12.3, driver r545
+#elif _CCCL_CUDACC_AT_LEAST(12, 3) && !_CCCL_CUDA_COMPILER(CLANG, <, 18)
+#  define __cccl_ptx_isa 830ULL
+// PTX ISA 8.2 is available from CUDA 12.2, driver r535
+#elif _CCCL_CUDACC_AT_LEAST(12, 2) && !_CCCL_CUDA_COMPILER(CLANG, <, 18)
+#  define __cccl_ptx_isa 820ULL
+// PTX ISA 8.1 is available from CUDA 12.1, driver r530
+#elif _CCCL_CUDACC_AT_LEAST(12, 1) && !_CCCL_CUDA_COMPILER(CLANG, <, 17)
+#  define __cccl_ptx_isa 810ULL
+// PTX ISA 8.0 is available from CUDA 12.0, driver r525
+#elif _CCCL_CUDACC_AT_LEAST(12, 0) && !_CCCL_CUDA_COMPILER(CLANG, <, 17)
+#  define __cccl_ptx_isa 800ULL
+// PTX ISA 7.8 is available from CUDA 11.8, driver r520
+#elif _CCCL_CUDACC_AT_LEAST(11, 8) && !_CCCL_CUDA_COMPILER(CLANG, <, 16)
+#  define __cccl_ptx_isa 780ULL
+// PTX ISA 7.7 is available from CUDA 11.7, driver r515
+#elif _CCCL_CUDACC_AT_LEAST(11, 7) && !_CCCL_CUDA_COMPILER(CLANG, <, 16)
+#  define __cccl_ptx_isa 770ULL
+// PTX ISA 7.6 is available from CUDA 11.6, driver r510
+#elif _CCCL_CUDACC_AT_LEAST(11, 6) && !_CCCL_CUDA_COMPILER(CLANG, <, 16)
+#  define __cccl_ptx_isa 760ULL
+// PTX ISA 7.5 is available from CUDA 11.5, driver r495
+#elif _CCCL_CUDACC_AT_LEAST(11, 5) && !_CCCL_CUDA_COMPILER(CLANG, <, 14)
+#  define __cccl_ptx_isa 750ULL
+// PTX ISA 7.4 is available from CUDA 11.4, driver r470
+#elif _CCCL_CUDACC_AT_LEAST(11, 4) && !_CCCL_CUDA_COMPILER(CLANG, <, 14)
+#  define __cccl_ptx_isa 740ULL
+// PTX ISA 7.3 is available from CUDA 11.3, driver r465
+#elif _CCCL_CUDACC_AT_LEAST(11, 3) && !_CCCL_CUDA_COMPILER(CLANG, <, 14)
+#  define __cccl_ptx_isa 730ULL
+// PTX ISA 7.2 is available from CUDA 11.2, driver r460
+#elif _CCCL_CUDACC_AT_LEAST(11, 2) && !_CCCL_CUDA_COMPILER(CLANG, <, 13)
+#  define __cccl_ptx_isa 720ULL
+// PTX ISA 7.1 is available from CUDA 11.1, driver r455
+#elif _CCCL_CUDACC_AT_LEAST(11, 1) && !_CCCL_CUDA_COMPILER(CLANG, <, 13)
+#  define __cccl_ptx_isa 710ULL
+// PTX ISA 7.0 is available from CUDA 11.0, driver r445
+#elif _CCCL_CUDACC_AT_LEAST(11, 0) && !_CCCL_CUDA_COMPILER(CLANG, <, 12)
+#  define __cccl_ptx_isa 700ULL
+// Fallback case. Define the ISA version to be zero. This ensures that the macro is always defined.
+#else
+#  define __cccl_ptx_isa 0ULL
+#endif
+
+// We define certain feature test macros depending on availability. When
+// __CUDA_MINIMUM_ARCH__ is not available, we define the following features
+// depending on PTX ISA. This permits checking for the feature in host code.
+// When __CUDA_MINIMUM_ARCH__ is available, we only enable the feature when the
+// hardware supports it.
+#if __cccl_ptx_isa >= 800
+#  if (!defined(__CUDA_MINIMUM_ARCH__)) || (defined(__CUDA_MINIMUM_ARCH__) && 900 <= __CUDA_MINIMUM_ARCH__)
+#    define __cccl_lib_local_barrier_arrive_tx
+#    define __cccl_lib_experimental_ctk12_cp_async_exposure
+#  endif
+#endif // __cccl_ptx_isa >= 800
+
+// NVRTC ships a built-in copy of <nv/detail/__target_macros>, so including CCCL's version of this header will omit the
+// content since the header guards are already defined. To make older NVRTC versions have a few newer feature macros
+// required for the PTX tests, we define them here outside the header guards.
+// TODO(bgruber): limit this workaround to NVRTC versions older than the first one shipping those macros
+#if _CCCL_COMPILER(NVRTC)
+
+// missing SM_88
+#  if !defined(NV_PROVIDES_SM_88)
+#    define _NV_TARGET_VAL_SM_88 880
+#    define NV_PROVIDES_SM_88    __NV_PROVIDES_SM_88
+#    define NV_IS_EXACTLY_SM_88  __NV_IS_EXACTLY_SM_88
+#    if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_88)
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_88 1
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_88      1
+#    else
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_88 0
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_88      0
+#    endif
+#    if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_88)
+#      define _NV_TARGET___NV_PROVIDES_SM_88      1
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_88 1
+#    else
+#      define _NV_TARGET___NV_PROVIDES_SM_88      0
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_88 0
+#    endif
+#  endif // !NV_PROVIDES_SM_88
+
+// missing SM_90a
+#  ifndef NV_HAS_FEATURE_SM_90a
+#    define NV_HAS_FEATURE_SM_90a __NV_HAS_FEATURE_SM_90a
+#    if defined(__CUDA_ARCH_FEAT_SM90_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 900))
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_90a 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_90a 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_90a
+
+// missing SM_100
+#  ifndef NV_PROVIDES_SM_100
+#    define _NV_TARGET_VAL_SM_100 1000
+#    define NV_PROVIDES_SM_100    __NV_PROVIDES_SM_100
+#    define NV_IS_EXACTLY_SM_100  __NV_IS_EXACTLY_SM_100
+#    if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_100)
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_100 1
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_100      1
+#    else
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_100 0
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_100      0
+#    endif
+#    if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_100)
+#      define _NV_TARGET___NV_PROVIDES_SM_100      1
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_100 1
+#    else
+#      define _NV_TARGET___NV_PROVIDES_SM_100      0
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_100 0
+#    endif
+#  endif // !NV_PROVIDES_SM_100
+
+// missing SM_100a
+#  ifndef NV_HAS_FEATURE_SM_100a
+#    define NV_HAS_FEATURE_SM_100a __NV_HAS_FEATURE_SM_100a
+#    if defined(__CUDA_ARCH_FEAT_SM100_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1000))
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100a 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100a 0
+#    endif
+#  endif // !NV_HAS_FEATURE_SM_100a
+
+// missing SM_103
+#  ifndef NV_PROVIDES_SM_103
+#    define _NV_TARGET_VAL_SM_103 1030
+#    define NV_PROVIDES_SM_103    __NV_PROVIDES_SM_103
+#    define NV_IS_EXACTLY_SM_103  __NV_IS_EXACTLY_SM_103
+#    if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_103)
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_103 1
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_103      1
+#    else
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_103 0
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_103      0
+#    endif
+#    if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_103)
+#      define _NV_TARGET___NV_PROVIDES_SM_103      1
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_103 1
+#    else
+#      define _NV_TARGET___NV_PROVIDES_SM_103      0
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_103 0
+#    endif
+#  endif // !NV_PROVIDES_SM_103
+
+// missing SM_103
+#  ifndef NV_HAS_FEATURE_SM_103a
+#    define NV_HAS_FEATURE_SM_103a __NV_HAS_FEATURE_SM_103a
+#    if defined(__CUDA_ARCH_FEAT_SM103_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1030))
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103a 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103a 0
+#    endif
+#  endif // !NV_HAS_FEATURE_SM_103a
+
+// missing SM_110
+#  ifndef NV_PROVIDES_SM_110
+#    define _NV_TARGET_VAL_SM_110 1100
+#    define NV_PROVIDES_SM_110    __NV_PROVIDES_SM_110
+#    define NV_IS_EXACTLY_SM_110  __NV_IS_EXACTLY_SM_110
+#    if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_110)
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_110 1
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_110      1
+#    else
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_110 0
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_110      0
+#    endif
+#    if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_110)
+#      define _NV_TARGET___NV_PROVIDES_SM_110      1
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_110 1
+#    else
+#      define _NV_TARGET___NV_PROVIDES_SM_110      0
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_110 0
+#    endif
+#  endif // !NV_PROVIDES_SM_110
+
+// missing SM_110a
+#  ifndef NV_HAS_FEATURE_SM_110a
+#    define NV_HAS_FEATURE_SM_110a __NV_HAS_FEATURE_SM_110a
+#    if defined(__CUDA_ARCH_FEAT_SM110_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1100))
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110a 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110a 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_110a
+
+// missing SM_120
+#  ifndef NV_PROVIDES_SM_120
+#    define _NV_TARGET_VAL_SM_120 1200
+#    define NV_PROVIDES_SM_120    __NV_PROVIDES_SM_120
+#    define NV_IS_EXACTLY_SM_120  __NV_IS_EXACTLY_SM_120
+#    if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_120)
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_120 1
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_120      1
+#    else
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_120 0
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_120      0
+#    endif
+#    if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_120)
+#      define _NV_TARGET___NV_PROVIDES_SM_120      1
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_120 1
+#    else
+#      define _NV_TARGET___NV_PROVIDES_SM_120      0
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_120 0
+#    endif
+#  endif // !NV_PROVIDES_SM_120
+
+// missing SM_120a
+#  ifndef NV_HAS_FEATURE_SM_120a
+#    define NV_HAS_FEATURE_SM_120a __NV_HAS_FEATURE_SM_120a
+#    if defined(__CUDA_ARCH_FEAT_SM120_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1200))
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120a 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120a 0
+#    endif
+#  endif // _CCCL_COMPILER(NVRTC)
+
+// missing SM_121
+#  if !defined(NV_PROVIDES_SM_121)
+#    define _NV_TARGET_VAL_SM_121 1210
+#    define NV_PROVIDES_SM_121    __NV_PROVIDES_SM_121
+#    define NV_IS_EXACTLY_SM_121  __NV_IS_EXACTLY_SM_121
+#    if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_121)
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_121 1
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_121      1
+#    else
+#      define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_121 0
+#      define _NV_TARGET___NV_IS_EXACTLY_SM_121      0
+#    endif
+#    if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_121)
+#      define _NV_TARGET___NV_PROVIDES_SM_121      1
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_121 1
+#    else
+#      define _NV_TARGET___NV_PROVIDES_SM_121      0
+#      define _NV_TARGET_BOOL___NV_PROVIDES_SM_121 0
+#    endif
+#  endif // !NV_PROVIDES_SM_121
+
+// missing SM_121a
+#  ifndef NV_HAS_FEATURE_SM_121a
+#    define NV_HAS_FEATURE_SM_121a __NV_HAS_FEATURE_SM_121a
+#    if defined(__CUDA_ARCH_FEAT_SM121_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1210))
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121a 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121a 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_121a
+
+//----------------------------------------------------------------------------------------------------------------------
+// family-specific SM versions
+
+// missing SM_100f
+#  ifndef NV_HAS_FEATURE_SM_100f
+#    define NV_HAS_FEATURE_SM_100f __NV_HAS_FEATURE_SM_100f
+#    if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1000)
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100f 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100f 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_100
+
+// missing SM_103f
+#  ifndef NV_HAS_FEATURE_SM_103f
+#    define NV_HAS_FEATURE_SM_103f __NV_HAS_FEATURE_SM_103f
+#    if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1030)
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103f 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103f 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_103f
+
+// missing SM_110f
+#  ifndef NV_HAS_FEATURE_SM_110f
+#    define NV_HAS_FEATURE_SM_110f __NV_HAS_FEATURE_SM_110f
+#    if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1100)
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110f 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110f 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_110f
+
+// missing SM_120f
+#  ifndef NV_HAS_FEATURE_SM_120f
+#    define NV_HAS_FEATURE_SM_120f __NV_HAS_FEATURE_SM_120f
+#    if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1200)
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120f 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120f 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_120f
+
+// missing SM_121f
+#  ifndef NV_HAS_FEATURE_SM_121f
+#    define NV_HAS_FEATURE_SM_121f __NV_HAS_FEATURE_SM_121f
+#    if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1210)
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121f 1
+#    else
+#      define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121f 0
+#    endif
+#  endif // NV_HAS_FEATURE_SM_121f
+
+#endif // _CCCL_COMPILER(NVRTC)
+#endif // __CCCL_PTX_ISA_H_

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/rtti.h

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_RTTI_H
+#define __CCCL_RTTI_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/builtin.h>
+
+// NOTE: some compilers support the `typeid` feature but not the `dynamic_cast`
+// feature. This is why we have separate macros for each.
+
+#ifndef _CCCL_NO_RTTI
+#  if defined(CCCL_DISABLE_RTTI) // Escape hatch for users to manually disable RTTI
+#    define _CCCL_NO_RTTI
+#  elif defined(__CUDA_ARCH__)
+#    define _CCCL_NO_RTTI // No RTTI in CUDA device code
+#  elif _CCCL_COMPILER(NVRTC)
+#    define _CCCL_NO_RTTI
+#  elif _CCCL_COMPILER(MSVC)
+#    if _CPPRTTI == 0
+#      define _CCCL_NO_RTTI
+#    endif
+#  elif _CCCL_COMPILER(CLANG)
+#    if !_CCCL_HAS_FEATURE(cxx_rtti)
+#      define _CCCL_NO_RTTI
+#    endif
+#  else
+#    if __GXX_RTTI == 0 && __cpp_rtti == 0
+#      define _CCCL_NO_RTTI
+#    endif
+#  endif
+#endif // !_CCCL_NO_RTTI
+
+#ifndef _CCCL_NO_TYPEID
+#  if defined(CCCL_DISABLE_RTTI) // CCCL_DISABLE_RTTI disables typeid also
+#    define _CCCL_NO_TYPEID
+#  elif defined(__CUDA_ARCH__)
+#    define _CCCL_NO_TYPEID // No typeid in CUDA device code
+#  elif _CCCL_COMPILER(NVRTC)
+#    define _CCCL_NO_TYPEID
+#  elif _CCCL_COMPILER(MSVC)
+// No-op, MSVC always supports typeid even when RTTI is disabled
+#  elif _CCCL_COMPILER(CLANG)
+#    if !_CCCL_HAS_FEATURE(cxx_rtti)
+#      define _CCCL_NO_TYPEID
+#    endif
+#  else
+#    if __GXX_RTTI == 0 && __cpp_rtti == 0
+#      define _CCCL_NO_TYPEID
+#    endif
+#  endif
+#endif // !_CCCL_NO_TYPEID
+
+#endif // __CCCL_RTTI_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/sequence_access.h

@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_SEQUENCE_ACCESS_H
+#define __CCCL_SEQUENCE_ACCESS_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+// We need to define hidden friends for {cr,r,}{begin,end} of our containers as we will otherwise encounter ambigouities
+#define _CCCL_SYNTHESIZE_SEQUENCE_ACCESS(_ClassName, _ConstIter)                                                       \
+  [[nodiscard]] _CCCL_API friend iterator begin(_ClassName& __sequence) noexcept(noexcept(__sequence.begin()))         \
+  {                                                                                                                    \
+    return __sequence.begin();                                                                                         \
+  }                                                                                                                    \
+  [[nodiscard]] _CCCL_API friend _ConstIter begin(const _ClassName& __sequence) noexcept(noexcept(__sequence.begin())) \
+  {                                                                                                                    \
+    return __sequence.begin();                                                                                         \
+  }                                                                                                                    \
+  [[nodiscard]] _CCCL_API friend iterator end(_ClassName& __sequence) noexcept(noexcept(__sequence.end()))             \
+  {                                                                                                                    \
+    return __sequence.end();                                                                                           \
+  }                                                                                                                    \
+  [[nodiscard]] _CCCL_API friend _ConstIter end(const _ClassName& __sequence) noexcept(noexcept(__sequence.end()))     \
+  {                                                                                                                    \
+    return __sequence.end();                                                                                           \
+  }                                                                                                                    \
+  [[nodiscard]] _CCCL_API friend _ConstIter cbegin(const _ClassName& __sequence) noexcept(                             \
+    noexcept(__sequence.begin()))                                                                                      \
+  {                                                                                                                    \
+    return __sequence.begin();                                                                                         \
+  }                                                                                                                    \
+  [[nodiscard]] _CCCL_API friend _ConstIter cend(const _ClassName& __sequence) noexcept(noexcept(__sequence.end()))    \
+  {                                                                                                                    \
+    return __sequence.end();                                                                                           \
+  }
+#define _CCCL_SYNTHESIZE_SEQUENCE_REVERSE_ACCESS(_ClassName, _ConstRevIter)                                          \
+  [[nodiscard]] _CCCL_API friend reverse_iterator rbegin(_ClassName& __sequence) noexcept(                           \
+    noexcept(__sequence.rbegin()))                                                                                   \
+  {                                                                                                                  \
+    return __sequence.rbegin();                                                                                      \
+  }                                                                                                                  \
+  [[nodiscard]] _CCCL_API friend _ConstRevIter rbegin(const _ClassName& __sequence) noexcept(                        \
+    noexcept(__sequence.rbegin()))                                                                                   \
+  {                                                                                                                  \
+    return __sequence.rbegin();                                                                                      \
+  }                                                                                                                  \
+  [[nodiscard]] _CCCL_API friend reverse_iterator rend(_ClassName& __sequence) noexcept(noexcept(__sequence.rend())) \
+  {                                                                                                                  \
+    return __sequence.rend();                                                                                        \
+  }                                                                                                                  \
+  [[nodiscard]] _CCCL_API friend _ConstRevIter rend(const _ClassName& __sequence) noexcept(                          \
+    noexcept(__sequence.rend()))                                                                                     \
+  {                                                                                                                  \
+    return __sequence.rend();                                                                                        \
+  }                                                                                                                  \
+  [[nodiscard]] _CCCL_API friend _ConstRevIter crbegin(const _ClassName& __sequence) noexcept(                       \
+    noexcept(__sequence.rbegin()))                                                                                   \
+  {                                                                                                                  \
+    return __sequence.rbegin();                                                                                      \
+  }                                                                                                                  \
+  [[nodiscard]] _CCCL_API friend _ConstRevIter crend(const _ClassName& __sequence) noexcept(                         \
+    noexcept(__sequence.rend()))                                                                                     \
+  {                                                                                                                  \
+    return __sequence.rend();                                                                                        \
+  }
+
+#endif // __CCCL_SEQUENCE_ACCESS_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/system_header.h

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_SYSTEM_HEADER_H
+#define __CCCL_SYSTEM_HEADER_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/is_non_narrowing_convertible.h> // IWYU pragma: export
+
+// Enforce that cccl headers are treated as system headers
+#if _CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC)
+#  define _CCCL_FORCE_SYSTEM_HEADER_GCC
+#elif _CCCL_COMPILER(CLANG)
+#  define _CCCL_FORCE_SYSTEM_HEADER_CLANG
+#elif _CCCL_COMPILER(MSVC)
+#  define _CCCL_FORCE_SYSTEM_HEADER_MSVC
+#endif // other compilers
+
+// Potentially enable that cccl headers are treated as system headers
+#if !defined(_CCCL_NO_SYSTEM_HEADER) && !(_CCCL_COMPILER(MSVC) && defined(_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING)) \
+  && !_CCCL_COMPILER(NVRTC) && !defined(_LIBCUDACXX_DISABLE_PRAGMA_GCC_SYSTEM_HEADER)
+#  if _CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC)
+#    define _CCCL_IMPLICIT_SYSTEM_HEADER_GCC
+#  elif _CCCL_COMPILER(CLANG)
+#    define _CCCL_IMPLICIT_SYSTEM_HEADER_CLANG
+#  elif _CCCL_COMPILER(MSVC)
+#    define _CCCL_IMPLICIT_SYSTEM_HEADER_MSVC
+#  endif // other compilers
+#endif // Use system header
+
+#endif // __CCCL_SYSTEM_HEADER_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/unreachable.h

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_UNREACHABLE_H
+#define __CCCL_UNREACHABLE_H
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_COMPILER(MSVC) && !_CCCL_DEVICE_COMPILATION()
+#  define _CCCL_UNREACHABLE() __assume(0)
+#else
+#  define _CCCL_UNREACHABLE() __builtin_unreachable()
+#endif
+
+#endif // __CCCL_UNREACHABLE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/version.h

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+// This file is somewhat automatically generated. Disable clang-format.
+// clang-format off
+
+
+#ifndef __CCCL_VERSION_H
+#define __CCCL_VERSION_H
+
+#define CCCL_VERSION 3005000
+#define CCCL_MAJOR_VERSION (CCCL_VERSION / 1000000)
+#define CCCL_MINOR_VERSION (((CCCL_VERSION / 1000) % 1000))
+#define CCCL_PATCH_VERSION (CCCL_VERSION % 1000)
+
+#if CCCL_PATCH_VERSION > 99
+#  error "CCCL patch version cannot be greater than 99 for compatibility with Thrust/CUB's MMMmmmpp format."
+#endif
+
+#endif // __CCCL_VERSION_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/visibility.h

@@ -0,0 +1,198 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CCCL_VISIBILITY_H
+#define __CCCL_VISIBILITY_H
+
+#ifndef _CUDA__CCCL_CONFIG
+#  error "<__cccl/visibility.h> should only be included in from <cuda/__cccl_config>"
+#endif // _CUDA__CCCL_CONFIG
+
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/system_header.h>
+
+// We want to ensure that all warning emitting from this header are suppressed
+#if defined(_CCCL_FORCE_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_FORCE_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_FORCE_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/attributes.h>
+#include <cuda/std/__cccl/cuda_capabilities.h>
+#include <cuda/std/__cccl/execution_space.h>
+#include <cuda/std/__cccl/os.h>
+
+// For unknown reasons, nvc++ need to selectively disable this warning
+// We do not want to use our usual macro because that would have push / pop semantics
+#if _CCCL_COMPILER(NVHPC)
+#  pragma nv_diag_suppress 1407
+#endif // _CCCL_COMPILER(NVHPC)
+
+// Enable us to hide kernels
+#if _CCCL_OS(WINDOWS) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_VISIBILITY_HIDDEN
+#else // ^^^ _CCCL_COMPILER(NVRTC) ^^^ / vvv _CCCL_COMPILER(NVRTC) vvv
+#  define _CCCL_VISIBILITY_HIDDEN __attribute__((__visibility__("hidden")))
+#endif // !_CCCL_COMPILER(NVRTC)
+
+#if _CCCL_COMPILER(NVRTC)
+#  define _CCCL_VISIBILITY_DEFAULT
+#elif _CCCL_OS(WINDOWS)
+#  define _CCCL_VISIBILITY_DEFAULT __declspec(dllimport)
+#else // ^^^ _CCCL_COMPILER(NVRTC) ^^^ / vvv !_CCCL_COMPILER(NVRTC) vvv
+#  define _CCCL_VISIBILITY_DEFAULT __attribute__((__visibility__("default")))
+#endif // !_CCCL_COMPILER(NVRTC)
+
+#if _CCCL_COMPILER(NVRTC)
+#  define _CCCL_VISIBILITY_EXPORT
+#elif _CCCL_OS(WINDOWS)
+#  define _CCCL_VISIBILITY_EXPORT __declspec(dllexport)
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_VISIBILITY_EXPORT _CCCL_VISIBILITY_DEFAULT
+#endif // !_CCCL_COMPILER(MSVC)
+
+#if _CCCL_OS(WINDOWS) || _CCCL_COMPILER(NVRTC)
+#  define _CCCL_TYPE_VISIBILITY_DEFAULT
+#  define _CCCL_TYPE_VISIBILITY_HIDDEN
+#elif _CCCL_HAS_ATTRIBUTE(__type_visibility__)
+#  define _CCCL_TYPE_VISIBILITY_DEFAULT __attribute__((__type_visibility__("default")))
+#  define _CCCL_TYPE_VISIBILITY_HIDDEN  __attribute__((__type_visibility__("hidden")))
+#else // ^^^ _CCCL_HAS_ATTRIBUTE(__type_visibility__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__type_visibility__) vvv
+#  define _CCCL_TYPE_VISIBILITY_DEFAULT _CCCL_VISIBILITY_DEFAULT
+#  define _CCCL_TYPE_VISIBILITY_HIDDEN  _CCCL_VISIBILITY_HIDDEN
+#endif // !_CCCL_COMPILER(NVRTC)
+
+#if _CCCL_COMPILER(MSVC)
+#  define _CCCL_FORCEINLINE __forceinline
+#  define _CCCL_FORCEINLINE_LAMBDA
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+#  define _CCCL_FORCEINLINE        __inline__ __attribute__((__always_inline__))
+#  define _CCCL_FORCEINLINE_LAMBDA __attribute__((__always_inline__))
+#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
+
+#if _CCCL_COMPILER(NVRTC)
+#  define _CCCL_NOINLINE __attribute__((noinline))
+#elif _CCCL_OS(WINDOWS)
+#  define _CCCL_NOINLINE __declspec(noinline)
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv _CCCL_COMPILER(MSVC) vvv
+// We can't use __noinline__ here because of CTK defining this macro.
+#  define _CCCL_NOINLINE __attribute__((noinline))
+#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
+
+#if _CCCL_DEVICE_COMPILATION()
+#  define _CCCL_NOINLINE_DEVICE _CCCL_NOINLINE
+#else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv
+#  define _CCCL_NOINLINE_DEVICE
+#endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^
+
+#if _CCCL_HAS_ATTRIBUTE(__exclude_from_explicit_instantiation__)
+#  define _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION __attribute__((__exclude_from_explicit_instantiation__))
+#else // ^^^ exclude_from_explicit_instantiation ^^^ / vvv !exclude_from_explicit_instantiation vvv
+// NVCC complains mightily about being unable to inline functions if we use _CCCL_FORCEINLINE here
+#  define _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#endif // !exclude_from_explicit_instantiation
+
+#if _CCCL_COMPILER(NVHPC) // NVHPC has issues with visibility attributes on symbols with internal linkage
+#  define _CCCL_HIDE_FROM_ABI inline
+#else // ^^^ _CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_COMPILER(NVHPC) vvv
+#  define _CCCL_HIDE_FROM_ABI _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION inline
+#endif // !_CCCL_COMPILER(NVHPC)
+
+// Note: we will allow the user to redefine _CCCL_KERNEL_ATTRIBUTES until CCCL 4.0, since they may have
+// redefined CUB_DETAIL_KERNEL_ATTRIBUTES or THRUST_DETAIL_KERNEL_ATTRIBUTES.
+#if !defined(_CCCL_KERNEL_ATTRIBUTES)
+#  define _CCCL_KERNEL_ATTRIBUTES __global__ _CCCL_VISIBILITY_HIDDEN
+#endif // !_CCCL_KERNEL_ATTRIBUTES
+
+#if defined(CUB_DETAIL_KERNEL_ATTRIBUTES) || defined(THRUST_DETAIL_KERNEL_ATTRIBUTES)
+#  error \
+    "Redefining CCCL's kernel attributes via CUB_DETAIL_KERNEL_ATTRIBUTES or THRUST_DETAIL_KERNEL_ATTRIBUTES is not allowed. If you absolutely rely on this, you can override them by defining _CCCL_KERNEL_ATTRIBUTES, but this will be disallowed in CCCL 4.0."
+#endif // !_CCCL_KERNEL_ATTRIBUTES
+
+//! @brief \c _CCCL_HIDE_FROM_ABI and \c _CCCL_FORCEINLINE cannot be used together because
+//! they both try to add `inline` to the function declaration. The following macros slice
+//! the function attributes differently to avoid this problem:
+//! - \c _CCCL_API declares the function host/device and hides the symbol from the ABI
+//! - \c _CCCL_NODEBUG_API does the same while also hiding the function from
+//!   debuggers and marking the function as \c inline.
+//! - \c _CCCL_TRIVIAL_API does the same as \c _CCCL_NODEBUG_API while also force-inlining
+//!   the function.
+#if _CCCL_COMPILER(NVHPC) // NVHPC has issues with visibility attributes on symbols with internal linkage
+#  define _CCCL_API             _CCCL_HOST_DEVICE
+#  define _CCCL_HOST_DEVICE_API _CCCL_HOST_DEVICE
+#  define _CCCL_HOST_API        _CCCL_HOST
+#  define _CCCL_DEVICE_API      _CCCL_DEVICE
+#  define _CCCL_TILE_API        _CCCL_TILE
+#else // ^^^ _CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_COMPILER(NVHPC) vvv
+#  define _CCCL_API             _CCCL_TILE _CCCL_HOST_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#  define _CCCL_HOST_DEVICE_API _CCCL_HOST_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#  define _CCCL_HOST_API        _CCCL_HOST _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#  define _CCCL_DEVICE_API      _CCCL_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#  define _CCCL_TILE_API        _CCCL_TILE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
+#endif // !_CCCL_COMPILER(NVHPC)
+
+//! @brief \c _CCCL_NODEBUG_API marks a function's visibility as hidden and causes
+//! debuggers to skip it. This is useful for functions like \c cuda::std::move that
+//! debuggers should not step into. If a \c _CCCL_NODEBUG_API function \c F calls a normal
+//! function \c G, stepping into \c F in a debugger will skip over \c F and step directly
+//! into \c G. In a stacktrace, \c F will still be shone, but you will not be able to
+//! set the debugger's active frame to \c F.
+#define _CCCL_NODEBUG_API        _CCCL_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline
+#define _CCCL_NODEBUG_HOST_API   _CCCL_HOST_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline
+#define _CCCL_NODEBUG_DEVICE_API _CCCL_DEVICE_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline
+
+//! @brief \c _CCCL_TRIVIAL_API force-inlines a function, marks its visibility as hidden,
+//! and causes debuggers to skip it. This is useful for trivial internal functions that do
+//! dispatching or other plumbing work. It is particularly useful in the definition of
+//! customization point objects.
+#define _CCCL_TRIVIAL_API        _CCCL_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE
+#define _CCCL_TRIVIAL_HOST_API   _CCCL_HOST_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE
+#define _CCCL_TRIVIAL_DEVICE_API _CCCL_DEVICE_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE
+
+// Some functions have their addresses appear in public types (e.g., in
+// `cuda::__overrides_for` specializations). If the function is declared
+// `__attribute__((visibility("hidden")))`, and if the address appears, say, in the type
+// of a member of a class that is declared `__attribute__((visibility("default")))`, GCC
+// complains bitterly. So we avoid declaring those functions `hidden`. Instead of the
+// typical `_CCCL_API` macro, we use `_CCCL_PUBLIC_API` for those functions.
+#if _CCCL_OS(WINDOWS)
+#  define _CCCL_PUBLIC_API        _CCCL_HOST_DEVICE
+#  define _CCCL_PUBLIC_HOST_API   _CCCL_HOST
+#  define _CCCL_PUBLIC_DEVICE_API _CCCL_DEVICE
+#else // ^^^ _CCCL_OS(WINDOWS) ^^^ / vvv !_CCCL_OS(WINDOWS) vvv
+#  define _CCCL_PUBLIC_API        _CCCL_HOST_DEVICE _CCCL_VISIBILITY_DEFAULT
+#  define _CCCL_PUBLIC_HOST_API   _CCCL_HOST _CCCL_VISIBILITY_DEFAULT
+#  define _CCCL_PUBLIC_DEVICE_API _CCCL_DEVICE _CCCL_VISIBILITY_DEFAULT
+#endif // !_CCCL_OS(WINDOWS)
+
+#ifdef _CCCL_DOXYGEN_INVOKED // Only for documentation
+//! If defined, usage of CUDA Dynamic Parallelism is disabled and APIs launching kernels can only be called from the
+//! host
+#  define CCCL_DISABLE_CDP
+#endif // _CCCL_DOXYGEN_INVOKED
+
+#if _CCCL_HAS_CDP()
+// We have CDP, so host and device APIs can call kernels
+#  define _CCCL_CDP_API _CCCL_API
+#else // ^^^ _CCCL_HAS_CDP() ^^^ / vvv !_CCCL_HAS_CDP() vvv
+// We don't have CDP, only host APIs can call kernels
+#  define _CCCL_CDP_API _CCCL_HOST_API
+#endif // ^^^ !_CCCL_HAS_CDP() ^^^
+
+//! _LIBCUDACXX_HIDE_FROM_ABI is for backwards compatibility for external projects.
+//! _CCCL_API and its variants are the preferred way to declare functions
+//! that should be hidden from the ABI.
+//! Defined here to suppress any warnings from the definition
+#define _LIBCUDACXX_HIDE_FROM_ABI _CCCL_API inline
+
+#endif // __CCCL_VISIBILITY_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/arithmetic.h

@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___CONCEPTS_ARITHMETIC_H
+#define _CUDA_STD___CONCEPTS_ARITHMETIC_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/is_arithmetic.h>
+#include <cuda/std/__type_traits/is_floating_point.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/is_signed_integer.h>
+#include <cuda/std/__type_traits/is_unsigned_integer.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+// [concepts.arithmetic], arithmetic concepts
+
+template <class _Tp>
+_CCCL_CONCEPT integral = is_integral_v<_Tp>;
+
+template <class _Tp>
+_CCCL_CONCEPT signed_integral = integral<_Tp> && is_signed_v<_Tp>;
+
+template <class _Tp>
+_CCCL_CONCEPT unsigned_integral = integral<_Tp> && !signed_integral<_Tp>;
+
+template <class _Tp>
+_CCCL_CONCEPT floating_point = is_floating_point_v<_Tp>;
+
+template <class _Tp>
+_CCCL_CONCEPT __cccl_signed_integer = __cccl_is_signed_integer_v<_Tp>;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___CONCEPTS_ARITHMETIC_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/assignable.h

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___CONCEPTS_ASSIGNABLE_H
+#define _CUDA_STD___CONCEPTS_ASSIGNABLE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/common_reference_with.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/same_as.h>
+#include <cuda/std/__type_traits/is_reference.h>
+#include <cuda/std/__type_traits/make_const_lvalue_ref.h>
+#include <cuda/std/__utility/forward.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if _CCCL_HAS_CONCEPTS()
+
+// [concept.assignable]
+
+template <class _Lhs, class _Rhs>
+concept assignable_from =
+  is_lvalue_reference_v<_Lhs> && common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>>
+  && requires(_Lhs __lhs, _Rhs&& __rhs) {
+       { __lhs = ::cuda::std::forward<_Rhs>(__rhs) } -> same_as<_Lhs>;
+     };
+
+#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv
+
+template <class _Lhs, class _Rhs>
+_CCCL_CONCEPT_FRAGMENT(
+  __assignable_from_,
+  requires(_Lhs __lhs,
+           _Rhs&& __rhs)(requires(is_lvalue_reference_v<_Lhs>),
+                         requires(common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>>),
+                         requires(same_as<_Lhs, decltype(__lhs = ::cuda::std::forward<_Rhs>(__rhs))>)));
+
+template <class _Lhs, class _Rhs>
+_CCCL_CONCEPT assignable_from = _CCCL_FRAGMENT(__assignable_from_, _Lhs, _Rhs);
+
+#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___CONCEPTS_ASSIGNABLE_H

qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/boolean_testable.h

@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___CONCEPTS_BOOLEAN_TESTABLE_H
+#define _CUDA_STD___CONCEPTS_BOOLEAN_TESTABLE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/convertible_to.h>
+#include <cuda/std/__utility/forward.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if _CCCL_HAS_CONCEPTS()
+
+// [concepts.booleantestable]
+
+template <class _Tp>
+concept __boolean_testable_impl = convertible_to<_Tp, bool>;
+
+template <class _Tp>
+concept __boolean_testable = __boolean_testable_impl<_Tp> && requires(_Tp&& __t) {
+  { !::cuda::std::forward<_Tp>(__t) } -> __boolean_testable_impl;
+};
+
+#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv
+
+template <class _Tp>
+_CCCL_CONCEPT __boolean_testable_impl = convertible_to<_Tp, bool>;
+
+template <class _Tp>
+_CCCL_CONCEPT_FRAGMENT(
+  __boolean_testable_,
+  requires(_Tp&& __t)(requires(__boolean_testable_impl<_Tp>),
+                      requires(__boolean_testable_impl<decltype(!::cuda::std::forward<_Tp>(__t))>)));
+
+template <class _Tp>
+_CCCL_CONCEPT __boolean_testable = _CCCL_FRAGMENT(__boolean_testable_, _Tp);
+
+#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___CONCEPTS_BOOLEAN_TESTABLE_H