From 8d6f9eaeb0ae277b30f976f61a33894b4c7cd8cb Mon Sep 17 00:00:00 2001 From: dylanyunlon Date: Thu, 13 Aug 2026 09:24:42 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20CCCL=20CachingDeviceAllocator=20preload?= =?UTF-8?q?=20=E2=80=94=20=E5=AE=8C=E6=95=B4=E4=BE=9D=E8=B5=96=E9=93=BE=20?= =?UTF-8?q?288=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 从 cccl_upstream 递归追踪 cub/util_allocator.cuh 的全部 include 依赖: cub/ 9 files (config, util_*, version, detect_cuda_runtime) cuda/ libcudacxx type_traits, concepts, algorithm, iterator... nv/ target macros, preprocessor 总计 288 个头文件 (1.4MB),打包到 include/ 目录,编译时 -I include 即可完全脱离 CCCL 原始目录结构。 .cu 文件直接 #include , 走原版 CUB CachingDeviceAllocator,零 mock。 BI-V100 参数: growth=2 bins=[8..32] max_cached=8GB/device --- .../cccl_preload/build_cccl_preload.sh | 116 +- .../cccl_preload/cccl_allocator_preload.cu | 465 +-- .../cccl_preload/include/cub/config.cuh | 29 + .../cub/detail/detect_cuda_runtime.cuh | 45 + .../include/cub/util_allocator.cuh | 901 +++++ .../cccl_preload/include/cub/util_arch.cuh | 219 ++ .../include/cub/util_cpp_dialect.cuh | 71 + .../cccl_preload/include/cub/util_debug.cuh | 187 ++ .../cccl_preload/include/cub/util_macro.cuh | 67 + .../include/cub/util_namespace.cuh | 172 + .../cccl_preload/include/cub/version.cuh | 65 + .../cccl_preload/include/cuda/__cccl_config | 38 + .../include/cuda/__cmath/ceil_div.h | 123 + .../include/cuda/__cmath/round_up.h | 104 + .../cuda/__device/compute_capability.h | 272 ++ .../cccl_preload/include/cuda/__fwd/complex.h | 48 + .../cccl_preload/include/cuda/__fwd/devices.h | 47 + .../include/cuda/__memory/address_space.h | 259 ++ .../include/cuda/__memory/check_address.h | 111 + .../cccl_preload/include/cuda/__nvtx/nvtx.h | 150 + .../cccl_preload/include/cuda/__nvtx/nvtx3.h | 2977 +++++++++++++++++ .../__ptx/instructions/generated/get_sreg.h | 949 ++++++ .../cuda/__ptx/instructions/get_sreg.h | 43 + .../include/cuda/__ptx/ptx_dot_variants.h | 230 ++ .../include/cuda/__ptx/ptx_helper_functions.h | 178 + .../__type_traits/is_trivially_copyable.h | 115 + .../include/cuda/std/__algorithm/clamp.h | 53 + .../include/cuda/std/__algorithm/comp.h | 58 + .../cuda/std/__algorithm/comp_ref_type.h | 85 + .../include/cuda/std/__algorithm/equal.h | 132 + .../include/cuda/std/__algorithm/fill_n.h | 51 + .../include/cuda/std/__algorithm/iter_swap.h | 85 + .../std/__algorithm/iterator_operations.h | 179 + .../std/__algorithm/lexicographical_compare.h | 70 + .../include/cuda/std/__algorithm/max.h | 63 + .../cuda/std/__algorithm/max_element.h | 67 + .../include/cuda/std/__algorithm/min.h | 63 + .../cuda/std/__algorithm/min_element.h | 87 + .../std/__algorithm/ranges_iterator_concept.h | 65 + .../cuda/std/__algorithm/swap_ranges.h | 78 + .../cuda/std/__algorithm/unwrap_iter.h | 95 + .../include/cuda/std/__bit/bit_cast.h | 86 + .../include/cuda/std/__cccl/architecture.h | 128 + .../include/cuda/std/__cccl/assert.h | 169 + .../include/cuda/std/__cccl/attributes.h | 221 ++ .../include/cuda/std/__cccl/builtin.h | 474 +++ .../include/cuda/std/__cccl/compiler.h | 238 ++ .../cuda/std/__cccl/cuda_capabilities.h | 118 + .../include/cuda/std/__cccl/cuda_toolkit.h | 56 + .../include/cuda/std/__cccl/deprecated.h | 87 + .../include/cuda/std/__cccl/diagnostic.h | 145 + .../include/cuda/std/__cccl/dialect.h | 230 ++ .../include/cuda/std/__cccl/epilogue.h | 390 +++ .../include/cuda/std/__cccl/exceptions.h | 42 + .../include/cuda/std/__cccl/execution_space.h | 87 + .../cuda/std/__cccl/extended_data_types.h | 148 + .../include/cuda/std/__cccl/host_std_lib.h | 61 + .../std/__cccl/is_non_narrowing_convertible.h | 71 + .../cccl_preload/include/cuda/std/__cccl/os.h | 120 + .../include/cuda/std/__cccl/preprocessor.h | 1366 ++++++++ .../include/cuda/std/__cccl/prologue.h | 348 ++ .../include/cuda/std/__cccl/ptx_isa.h | 369 ++ .../include/cuda/std/__cccl/rtti.h | 72 + .../include/cuda/std/__cccl/sequence_access.h | 83 + .../include/cuda/std/__cccl/system_header.h | 38 + .../include/cuda/std/__cccl/unreachable.h | 31 + .../include/cuda/std/__cccl/version.h | 26 + .../include/cuda/std/__cccl/visibility.h | 198 ++ .../include/cuda/std/__concepts/arithmetic.h | 56 + .../include/cuda/std/__concepts/assignable.h | 64 + .../cuda/std/__concepts/boolean_testable.h | 63 + .../cuda/std/__concepts/class_or_enum.h | 45 + .../std/__concepts/common_reference_with.h | 69 + .../cuda/std/__concepts/concept_macros.h | 389 +++ .../cuda/std/__concepts/constructible.h | 174 + .../cuda/std/__concepts/convertible_to.h | 70 + .../include/cuda/std/__concepts/copyable.h | 60 + .../cuda/std/__concepts/derived_from.h | 56 + .../cuda/std/__concepts/destructible.h | 76 + .../cuda/std/__concepts/equality_comparable.h | 98 + .../include/cuda/std/__concepts/invocable.h | 80 + .../include/cuda/std/__concepts/movable.h | 58 + .../include/cuda/std/__concepts/predicate.h | 54 + .../include/cuda/std/__concepts/regular.h | 54 + .../include/cuda/std/__concepts/relation.h | 77 + .../include/cuda/std/__concepts/same_as.h | 39 + .../include/cuda/std/__concepts/semiregular.h | 54 + .../include/cuda/std/__concepts/swappable.h | 209 ++ .../cuda/std/__concepts/totally_ordered.h | 101 + .../include/cuda/std/__cstddef/byte.h | 113 + .../include/cuda/std/__cstddef/types.h | 52 + .../include/cuda/std/__cstring/memcpy.h | 61 + .../cuda/std/__exception/exception_macros.h | 126 + .../include/cuda/std/__exception/terminate.h | 82 + .../cuda/std/__floating_point/format.h | 157 + .../cuda/std/__floating_point/properties.h | 229 ++ .../cuda/std/__floating_point/storage.h | 260 ++ .../cuda/std/__floating_point/traits.h | 171 + .../cuda/std/__functional/binary_function.h | 64 + .../include/cuda/std/__functional/identity.h | 57 + .../include/cuda/std/__functional/invoke.h | 298 ++ .../cuda/std/__functional/reference_wrapper.h | 116 + .../cuda/std/__functional/unary_function.h | 63 + .../cuda/std/__functional/unwrap_ref.h | 56 + .../cuda/std/__functional/weak_result_type.h | 262 ++ .../include/cuda/std/__fwd/array.h | 68 + .../include/cuda/std/__fwd/complex.h | 75 + .../include/cuda/std/__fwd/format.h | 120 + .../cccl_preload/include/cuda/std/__fwd/fp.h | 37 + .../cccl_preload/include/cuda/std/__fwd/get.h | 132 + .../include/cuda/std/__fwd/iterator.h | 45 + .../include/cuda/std/__fwd/pair.h | 53 + .../cuda/std/__fwd/reference_wrapper.h | 52 + .../include/cuda/std/__fwd/span.h | 45 + .../include/cuda/std/__fwd/subrange.h | 65 + .../include/cuda/std/__fwd/tuple.h | 52 + .../include/cuda/std/__host_stdlib/cstdio | 28 + .../include/cuda/std/__host_stdlib/math.h | 49 + .../include/cuda/std/__host_stdlib/memory | 36 + .../include/cuda/std/__host_stdlib/new | 29 + .../include/cuda/std/__host_stdlib/stdexcept | 28 + .../include/cuda/std/__internal/atomic.h | 55 + .../include/cuda/std/__internal/cpp_dialect.h | 44 + .../include/cuda/std/__internal/features.h | 127 + .../include/cuda/std/__internal/namespaces.h | 188 ++ .../include/cuda/std/__internal/pstl_config.h | 34 + .../include/cuda/std/__internal/thread_api.h | 58 + .../include/cuda/std/__internal/version.h | 52 + .../include/cuda/std/__iterator/access.h | 140 + .../include/cuda/std/__iterator/advance.h | 230 ++ .../include/cuda/std/__iterator/concepts.h | 718 ++++ .../include/cuda/std/__iterator/data.h | 61 + .../include/cuda/std/__iterator/distance.h | 130 + .../include/cuda/std/__iterator/empty.h | 53 + .../std/__iterator/incrementable_traits.h | 143 + .../include/cuda/std/__iterator/iter_move.h | 165 + .../include/cuda/std/__iterator/iter_swap.h | 185 + .../include/cuda/std/__iterator/iterator.h | 44 + .../cuda/std/__iterator/iterator_traits.h | 618 ++++ .../include/cuda/std/__iterator/next.h | 104 + .../include/cuda/std/__iterator/prev.h | 93 + .../cuda/std/__iterator/readable_traits.h | 156 + .../cuda/std/__iterator/reverse_access.h | 154 + .../cuda/std/__iterator/reverse_iterator.h | 375 +++ .../include/cuda/std/__iterator/size.h | 69 + .../cuda/std/__limits/numeric_limits.h | 612 ++++ .../cuda/std/__limits/numeric_limits_ext.h | 753 +++++ .../include/cuda/std/__memory/addressof.h | 96 + .../cuda/std/__memory/allocator_arg_t.h | 85 + .../cuda/std/__memory/pointer_traits.h | 247 ++ .../cuda/std/__memory/uses_allocator.h | 59 + .../include/cuda/std/__new/device_new.h | 30 + .../include/cuda/std/__ranges/access.h | 318 ++ .../include/cuda/std/__ranges/concepts.h | 303 ++ .../include/cuda/std/__ranges/data.h | 143 + .../cuda/std/__ranges/enable_borrowed_range.h | 41 + .../include/cuda/std/__ranges/enable_view.h | 78 + .../include/cuda/std/__ranges/size.h | 212 ++ .../cuda/std/__tuple_dir/make_tuple_types.h | 143 + .../cuda/std/__tuple_dir/sfinae_helpers.h | 94 + .../cuda/std/__tuple_dir/tuple_constraints.h | 643 ++++ .../cuda/std/__tuple_dir/tuple_element.h | 88 + .../cuda/std/__tuple_dir/tuple_indices.h | 44 + .../include/cuda/std/__tuple_dir/tuple_like.h | 103 + .../include/cuda/std/__tuple_dir/tuple_size.h | 101 + .../cuda/std/__tuple_dir/tuple_types.h | 53 + .../cuda/std/__type_traits/add_const.h | 40 + .../std/__type_traits/add_lvalue_reference.h | 62 + .../cuda/std/__type_traits/add_pointer.h | 65 + .../std/__type_traits/add_rvalue_reference.h | 62 + .../__type_traits/aggregate_members_all_of.h | 135 + .../cuda/std/__type_traits/always_false.h | 35 + .../cuda/std/__type_traits/common_reference.h | 262 ++ .../cuda/std/__type_traits/common_type.h | 173 + .../cuda/std/__type_traits/conditional.h | 65 + .../cuda/std/__type_traits/conjunction.h | 67 + .../include/cuda/std/__type_traits/copy_cv.h | 50 + .../cuda/std/__type_traits/copy_cvref.h | 148 + .../include/cuda/std/__type_traits/decay.h | 83 + .../cuda/std/__type_traits/disjunction.h | 77 + .../cuda/std/__type_traits/enable_if.h | 43 + .../include/cuda/std/__type_traits/extent.h | 67 + .../include/cuda/std/__type_traits/fold.h | 47 + .../std/__type_traits/integral_constant.h | 53 + .../cuda/std/__type_traits/is_aggregate.h | 42 + .../cuda/std/__type_traits/is_arithmetic.h | 42 + .../include/cuda/std/__type_traits/is_array.h | 62 + .../cuda/std/__type_traits/is_assignable.h | 78 + .../cuda/std/__type_traits/is_base_of.h | 42 + .../cuda/std/__type_traits/is_callable.h | 61 + .../include/cuda/std/__type_traits/is_class.h | 42 + .../cuda/std/__type_traits/is_comparable.h | 79 + .../cuda/std/__type_traits/is_complete.h | 53 + .../include/cuda/std/__type_traits/is_const.h | 60 + .../cuda/std/__type_traits/is_constructible.h | 174 + .../cuda/std/__type_traits/is_convertible.h | 211 ++ .../std/__type_traits/is_copy_assignable.h | 60 + .../std/__type_traits/is_copy_constructible.h | 58 + .../std/__type_traits/is_core_convertible.h | 47 + .../__type_traits/is_default_constructible.h | 40 + .../cuda/std/__type_traits/is_destructible.h | 115 + .../include/cuda/std/__type_traits/is_empty.h | 42 + .../include/cuda/std/__type_traits/is_enum.h | 42 + .../is_extended_floating_point.h | 84 + .../std/__type_traits/is_floating_point.h | 53 + .../cuda/std/__type_traits/is_function.h | 64 + .../is_implicitly_default_constructible.h | 61 + .../cuda/std/__type_traits/is_integral.h | 127 + .../is_member_function_pointer.h | 79 + .../__type_traits/is_member_object_pointer.h | 57 + .../std/__type_traits/is_member_pointer.h | 57 + .../std/__type_traits/is_move_assignable.h | 59 + .../std/__type_traits/is_move_constructible.h | 56 + .../std/__type_traits/is_nothrow_assignable.h | 70 + .../__type_traits/is_nothrow_constructible.h | 72 + .../__type_traits/is_nothrow_convertible.h | 59 + .../is_nothrow_copy_assignable.h | 60 + .../is_nothrow_copy_constructible.h | 58 + .../is_nothrow_default_constructible.h | 54 + .../__type_traits/is_nothrow_destructible.h | 82 + .../is_nothrow_move_assignable.h | 60 + .../is_nothrow_move_constructible.h | 57 + .../cuda/std/__type_traits/is_null_pointer.h | 43 + .../cuda/std/__type_traits/is_object.h | 57 + .../cuda/std/__type_traits/is_pointer.h | 60 + .../std/__type_traits/is_primary_template.h | 121 + .../cuda/std/__type_traits/is_reference.h | 89 + .../cuda/std/__type_traits/is_referenceable.h | 55 + .../include/cuda/std/__type_traits/is_same.h | 88 + .../cuda/std/__type_traits/is_scalar.h | 60 + .../cuda/std/__type_traits/is_signed.h | 76 + .../std/__type_traits/is_signed_integer.h | 59 + .../cuda/std/__type_traits/is_swappable.h | 211 ++ .../std/__type_traits/is_trivially_copyable.h | 42 + .../std/__type_traits/is_unbounded_array.h | 60 + .../include/cuda/std/__type_traits/is_union.h | 42 + .../cuda/std/__type_traits/is_unsigned.h | 76 + .../std/__type_traits/is_unsigned_integer.h | 59 + .../std/__type_traits/is_valid_expansion.h | 41 + .../include/cuda/std/__type_traits/is_void.h | 70 + .../cuda/std/__type_traits/is_volatile.h | 60 + .../include/cuda/std/__type_traits/lazy.h | 35 + .../std/__type_traits/make_const_lvalue_ref.h | 36 + .../cuda/std/__type_traits/make_signed.h | 155 + .../cuda/std/__type_traits/make_unsigned.h | 166 + .../include/cuda/std/__type_traits/nat.h | 39 + .../include/cuda/std/__type_traits/negation.h | 43 + .../reference_constructs_from_temporary.h | 57 + .../std/__type_traits/remove_all_extents.h | 71 + .../cuda/std/__type_traits/remove_const.h | 59 + .../cuda/std/__type_traits/remove_cv.h | 57 + .../cuda/std/__type_traits/remove_cvref.h | 57 + .../cuda/std/__type_traits/remove_extent.h | 65 + .../cuda/std/__type_traits/remove_pointer.h | 73 + .../cuda/std/__type_traits/remove_reference.h | 72 + .../cuda/std/__type_traits/remove_volatile.h | 58 + .../cuda/std/__type_traits/sfinae_traits.h | 63 + .../cuda/std/__type_traits/type_identity.h | 40 + .../cuda/std/__type_traits/type_list.h | 1098 ++++++ .../include/cuda/std/__type_traits/type_set.h | 131 + .../cuda/std/__type_traits/underlying_type.h | 52 + .../include/cuda/std/__type_traits/void_t.h | 34 + .../include/cuda/std/__utility/auto_cast.h | 34 + .../cuda/std/__utility/convert_to_integral.h | 101 + .../include/cuda/std/__utility/declval.h | 77 + .../include/cuda/std/__utility/exchange.h | 46 + .../include/cuda/std/__utility/forward.h | 90 + .../cuda/std/__utility/integer_sequence.h | 308 ++ .../include/cuda/std/__utility/move.h | 140 + .../include/cuda/std/__utility/pair.h | 943 ++++++ .../cuda/std/__utility/piecewise_construct.h | 37 + .../include/cuda/std/__utility/priority_tag.h | 40 + .../include/cuda/std/__utility/swap.h | 64 + .../cuda/std/__utility/to_underlying.h | 40 + .../include/cuda/std/__utility/undefined.h | 34 + .../include/cuda/std/__utility/unreachable.h | 37 + .../cccl_preload/include/cuda/std/array | 553 +++ .../cccl_preload/include/cuda/std/cassert | 27 + .../cccl_preload/include/cuda/std/cfloat | 59 + .../cccl_preload/include/cuda/std/climits | 61 + .../cccl_preload/include/cuda/std/cstddef | 28 + .../cccl_preload/include/cuda/std/cstdint | 204 ++ .../include/cuda/std/detail/__config | 22 + .../include/cuda/std/initializer_list | 48 + .../cccl_preload/include/cuda/std/limits | 28 + .../cccl_preload/include/cuda/std/version | 243 ++ .../cccl_preload/include/cuda/version | 16 + .../include/nv/detail/__preprocessor | 169 + .../include/nv/detail/__target_macros | 794 +++++ .../cccl_preload/include/nv/target | 250 ++ 290 files changed, 39904 insertions(+), 455 deletions(-) create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/config.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/detail/detect_cuda_runtime.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/util_allocator.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/util_arch.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/util_cpp_dialect.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/util_debug.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/util_macro.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/util_namespace.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cub/version.cuh create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__cccl_config create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__cmath/ceil_div.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__cmath/round_up.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__device/compute_capability.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__fwd/complex.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__fwd/devices.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__memory/address_space.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__memory/check_address.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx3.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/get_sreg.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/get_sreg.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_dot_variants.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_helper_functions.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/__type_traits/is_trivially_copyable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/clamp.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp_ref_type.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/equal.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/fill_n.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iter_swap.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iterator_operations.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/lexicographical_compare.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max_element.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min_element.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/ranges_iterator_concept.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/swap_ranges.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/unwrap_iter.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__bit/bit_cast.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/architecture.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/assert.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/attributes.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/builtin.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/compiler.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_capabilities.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_toolkit.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/deprecated.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/diagnostic.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/dialect.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/epilogue.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/exceptions.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/execution_space.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/extended_data_types.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/host_std_lib.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/is_non_narrowing_convertible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/os.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/preprocessor.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/prologue.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/ptx_isa.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/rtti.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/sequence_access.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/system_header.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/unreachable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/version.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/visibility.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/arithmetic.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/boolean_testable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/class_or_enum.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/common_reference_with.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/concept_macros.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/convertible_to.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/copyable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/derived_from.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/destructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/equality_comparable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/invocable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/movable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/predicate.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/regular.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/relation.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/same_as.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/semiregular.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/swappable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/totally_ordered.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/byte.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/types.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__cstring/memcpy.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/exception_macros.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/terminate.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/format.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/properties.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/storage.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/traits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/binary_function.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/identity.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/invoke.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/reference_wrapper.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unary_function.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unwrap_ref.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/weak_result_type.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/array.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/complex.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/format.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/fp.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/get.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/iterator.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/pair.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/reference_wrapper.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/span.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/subrange.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/tuple.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/cstdio create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/math.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/memory create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/new create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/stdexcept create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/atomic.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/cpp_dialect.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/features.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/namespaces.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/pstl_config.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/thread_api.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/version.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/access.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/advance.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/concepts.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/data.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/distance.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/empty.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/incrementable_traits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_move.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_swap.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator_traits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/next.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/prev.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/readable_traits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_access.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_iterator.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/size.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__limits/numeric_limits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__limits/numeric_limits_ext.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__memory/addressof.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__memory/allocator_arg_t.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__memory/pointer_traits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__memory/uses_allocator.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__new/device_new.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__ranges/access.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__ranges/concepts.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__ranges/data.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__ranges/enable_borrowed_range.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__ranges/enable_view.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__ranges/size.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/make_tuple_types.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/sfinae_helpers.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/tuple_constraints.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/tuple_element.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/tuple_indices.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/tuple_like.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/tuple_size.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__tuple_dir/tuple_types.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/add_const.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/add_lvalue_reference.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/add_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/add_rvalue_reference.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/aggregate_members_all_of.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/always_false.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/common_reference.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/common_type.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/conditional.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/conjunction.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/copy_cv.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/copy_cvref.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/decay.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/disjunction.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/enable_if.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/extent.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/fold.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/integral_constant.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_aggregate.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_arithmetic.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_array.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_base_of.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_callable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_class.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_comparable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_complete.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_const.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_convertible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_copy_assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_copy_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_core_convertible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_default_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_destructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_empty.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_enum.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_extended_floating_point.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_floating_point.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_function.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_implicitly_default_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_integral.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_member_function_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_member_object_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_member_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_move_assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_move_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_convertible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_copy_assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_copy_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_default_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_destructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_move_assignable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_nothrow_move_constructible.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_null_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_object.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_primary_template.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_reference.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_referenceable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_same.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_scalar.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_signed.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_signed_integer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_swappable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_trivially_copyable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_unbounded_array.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_union.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_unsigned.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_unsigned_integer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_valid_expansion.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_void.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/is_volatile.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/lazy.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/make_const_lvalue_ref.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/make_signed.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/make_unsigned.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/nat.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/negation.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/reference_constructs_from_temporary.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_all_extents.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_const.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_cv.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_cvref.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_extent.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_pointer.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_reference.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/remove_volatile.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/sfinae_traits.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/type_identity.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/type_list.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/type_set.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/underlying_type.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__type_traits/void_t.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/auto_cast.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/convert_to_integral.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/declval.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/exchange.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/forward.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/integer_sequence.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/move.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/pair.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/piecewise_construct.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/priority_tag.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/swap.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/to_underlying.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/undefined.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/__utility/unreachable.h create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/array create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/cassert create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/cfloat create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/climits create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/cstddef create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/cstdint create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/detail/__config create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/initializer_list create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/limits create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/std/version create mode 100644 qwen3_6_scripts/cccl_preload/include/cuda/version create mode 100644 qwen3_6_scripts/cccl_preload/include/nv/detail/__preprocessor create mode 100644 qwen3_6_scripts/cccl_preload/include/nv/detail/__target_macros create mode 100644 qwen3_6_scripts/cccl_preload/include/nv/target diff --git a/qwen3_6_scripts/cccl_preload/build_cccl_preload.sh b/qwen3_6_scripts/cccl_preload/build_cccl_preload.sh index 0d7c53c7..b97fde7c 100755 --- a/qwen3_6_scripts/cccl_preload/build_cccl_preload.sh +++ b/qwen3_6_scripts/cccl_preload/build_cccl_preload.sh @@ -1,41 +1,43 @@ #!/usr/bin/env bash -# Build libcccl_allocator.so — LD_PRELOAD .so for CUB CachingDeviceAllocator +# Build libcccl_allocator.so +# +# Full CCCL dependency chain (288 headers) in ./include/ +# Source: cccl_upstream/cub/cub/util_allocator.cuh + transitive deps # # Usage: # bash build_cccl_preload.sh [output_dir] -# -# On BI-V100 with CoreX SDK: -# bash build_cccl_preload.sh /workspace/qwen3_6_scripts/cccl_preload -# -# The .so intercepts cudaMalloc/cudaFree and routes through CUB's -# caching allocator, bypassing CoreX's "expandable segment not supported" -# ASSERT in CUDACachingAllocator.cpp:545. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" OUTPUT_DIR="${1:-${SCRIPT_DIR}}" SRC="${SCRIPT_DIR}/cccl_allocator_preload.cu" +INC="${SCRIPT_DIR}/include" OUT="${OUTPUT_DIR}/libcccl_allocator.so" -# Find CoreX clang++ (preferred) or system g++ -if [[ -x /usr/local/corex-3.2.3/bin/clang++ ]]; then - CXX=/usr/local/corex-3.2.3/bin/clang++ - echo "[build] Using CoreX clang++: ${CXX}" -elif [[ -x /usr/local/corex/bin/clang++ ]]; then - CXX=/usr/local/corex/bin/clang++ - echo "[build] Using CoreX clang++ (alt): ${CXX}" -else - CXX=g++ - echo "[build] CoreX clang++ not found, falling back to g++" -fi +[[ -d "${INC}/cub" ]] || { echo "CCCL include tree missing: ${INC}/cub"; exit 2; } +[[ -d "${INC}/cuda" ]] || { echo "CCCL include tree missing: ${INC}/cuda"; exit 2; } -# Find CUDA include path +# Find compiler +CXX="" +for candidate in \ + /usr/local/corex-3.2.3/bin/clang++ \ + /usr/local/corex/bin/clang++ \ + /usr/local/corex/lib64/clang/16/bin/clang++ \ + ; do + if [[ -x "${candidate}" ]]; then + CXX="${candidate}" + break + fi +done +[[ -n "${CXX}" ]] || { CXX=g++; echo "[build] no CoreX clang++, falling back to g++"; } +echo "[build] CXX=${CXX}" + +# Find CUDA headers (for cuda_runtime_api.h) CUDA_INC="" for candidate in \ /usr/local/corex/include \ /usr/local/cuda/include \ - /usr/local/corex/lib64/clang/16/include \ ; do if [[ -f "${candidate}/cuda_runtime_api.h" ]]; then CUDA_INC="${candidate}" @@ -43,7 +45,7 @@ for candidate in \ fi done -# Find CUDA lib path for linking +# Find CUDA libs CUDA_LIB="" for candidate in \ /usr/local/corex/lib64 \ @@ -55,57 +57,43 @@ for candidate in \ fi done -if [[ -z "${CUDA_INC}" ]]; then - echo "[WARN] cuda_runtime_api.h not found — trying compile anyway" -fi - echo "[build] CUDA include: ${CUDA_INC:-system}" -echo "[build] CUDA lib: ${CUDA_LIB:-system}" -echo "[build] Source: ${SRC}" -echo "[build] Output: ${OUT}" +echo "[build] CUDA lib: ${CUDA_LIB:-system}" +echo "[build] CCCL include: ${INC} ($(find "${INC}" -type f | wc -l) files)" +echo "[build] Source: ${SRC}" +echo "[build] Output: ${OUT}" + +COMMON_FLAGS=( + -shared -fPIC -O2 -std=c++17 + -I"${INC}" + ${CUDA_INC:+-I"${CUDA_INC}"} + ${CUDA_LIB:+-L"${CUDA_LIB}"} + -lcudart -ldl + # Suppress CCCL warnings that don't affect correctness + -Wno-unused-function + -Wno-unknown-pragmas + # CUB needs these for non-NVCC compilers + -D_CCCL_COMPILER_GCC=1 + -D__CUDA_ARCH_LIST__=700 + -DCUB_DISABLE_NAMESPACE_MAGIC + -DCUB_WRAPPED_NAMESPACE=cccl_preload +) -# Build as shared library -# -x cuda or -x c++ depending on compiler if [[ "${CXX}" == *clang++* ]]; then - # CoreX clang++ can compile .cu natively - ${CXX} \ - -shared -fPIC \ - -O2 \ - ${CUDA_INC:+-I"${CUDA_INC}"} \ - ${CUDA_LIB:+-L"${CUDA_LIB}"} \ - -lcudart \ - -ldl \ - -std=c++17 \ - -o "${OUT}" \ - "${SRC}" + "${CXX}" "${COMMON_FLAGS[@]}" -x c++ -o "${OUT}" "${SRC}" 2>&1 else - # g++ needs .cu renamed or treated as C++ - # cuda_runtime_api.h should still work with host compiler - ${CXX} \ - -shared -fPIC \ - -O2 \ - ${CUDA_INC:+-I"${CUDA_INC}"} \ - ${CUDA_LIB:+-L"${CUDA_LIB}"} \ - -lcudart \ - -ldl \ - -std=c++17 \ - -x c++ \ - -o "${OUT}" \ - "${SRC}" + "${CXX}" "${COMMON_FLAGS[@]}" -x c++ -o "${OUT}" "${SRC}" 2>&1 fi if [[ -f "${OUT}" ]]; then - SIZE=$(stat -c%s "${OUT}" 2>/dev/null || stat -f%z "${OUT}" 2>/dev/null || echo "?") + SIZE=$(stat -c%s "${OUT}" 2>/dev/null || echo "?") + echo "" echo "[build] SUCCESS: ${OUT} (${SIZE} bytes)" echo "" - echo "Usage:" - echo " LD_PRELOAD=${OUT} CCCL_ALLOC_DEBUG=1 python3 -c 'import torch; t=torch.zeros(1024, device=\"cuda\")'" - echo "" - echo "In computility-run.yaml, add to env:" - echo " - name: LD_PRELOAD" - echo " value: /workspace/qwen3_6_scripts/cccl_preload/libcccl_allocator.so" - echo " - name: PYTORCH_CUDA_ALLOC_CONF" - echo " value: expandable_segments:True" + echo "Test:" + echo " LD_PRELOAD=${OUT} CCCL_ALLOC_DEBUG=1 \\" + echo " PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \\" + echo " python3 verify_preload.py" else echo "[build] FAILED" exit 1 diff --git a/qwen3_6_scripts/cccl_preload/cccl_allocator_preload.cu b/qwen3_6_scripts/cccl_preload/cccl_allocator_preload.cu index 2812d2ae..50061632 100644 --- a/qwen3_6_scripts/cccl_preload/cccl_allocator_preload.cu +++ b/qwen3_6_scripts/cccl_preload/cccl_allocator_preload.cu @@ -1,409 +1,90 @@ /* * cccl_allocator_preload.cu * - * LD_PRELOAD .so that replaces PyTorch's CUDA memory allocator with - * CUB's CachingDeviceAllocator (extracted from CCCL upstream). + * LD_PRELOAD .so — CUB CachingDeviceAllocator from CCCL upstream. + * Full dependency chain (288 files) extracted into include/. * - * Purpose: CoreX's CUDACachingAllocator.cpp:545 asserts - * "expandable segment not supported". Instead of patching libtorch, - * we intercept cudaMalloc/cudaFree at the dynamic linker level and - * route them through CUB's battle-tested caching allocator. + * Intercepts cudaMalloc/cudaFree, routes through CUB's geometric-bin + * caching allocator. Strips expandable_segments from + * PYTORCH_CUDA_ALLOC_CONF before libtorch reads it. * - * Source: cccl_upstream/cub/cub/util_allocator.cuh - * License: BSD-3 (NVIDIA/CUB) - * - * Build (on BI-V100 with CoreX clang++): - * bash build_cccl_preload.sh - * - * Usage: - * LD_PRELOAD=/workspace/qwen3_6_scripts/cccl_preload/libcccl_allocator.so \ - * CCCL_ALLOC_DEBUG=0 \ - * PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512 \ - * python3 -m vllm.entrypoints.openai.api_server ... + * Source: CCCL cub/cub/util_allocator.cuh (BSD-3, NVIDIA) + * Build: bash build_cccl_preload.sh */ -#include +/* ---- CCCL include chain (288 files from cccl_upstream) ---- */ +#include + +/* ---- System ---- */ #include #include #include #include -#include -#include -#include +#include /* ======================================================================== - * CUB CachingDeviceAllocator — extracted from CCCL - * cccl_upstream/cub/cub/util_allocator.cuh + * Configuration for BI-V100 (32GB × 4 cards) * - * All CUB/CCCL macro dependencies replaced with plain C++. + * CUB CachingDeviceAllocator parameters: + * bin_growth = 2 (power-of-2 bins: 256B, 512B, 1KB, ... 4GB) + * min_bin = 8 (2^8 = 256B minimum allocation) + * max_bin = 32 (2^32 = 4GB maximum cached bin) + * max_cached = 8GB per device + * + * More granular bins (growth=2) than CUB default (growth=8) because + * PyTorch tensor sizes vary widely in inference. * ======================================================================== */ -static bool g_cccl_debug = false; +static constexpr unsigned int ALLOC_BIN_GROWTH = 2; +static constexpr unsigned int ALLOC_MIN_BIN = 8; /* 256 bytes */ +static constexpr unsigned int ALLOC_MAX_BIN = 32; /* 4 GB */ +static constexpr size_t ALLOC_MAX_CACHED = (size_t)8 * 1024 * 1024 * 1024; /* 8GB */ -#define CcclDebug(e) (e) -#define CcclLog(...) \ - do { \ - if (g_cccl_debug) { \ - fprintf(stderr, "[cccl_alloc] "); \ - fprintf(stderr, __VA_ARGS__); \ - } \ - } while (0) - -struct CachingDeviceAllocator -{ - static constexpr unsigned int INVALID_BIN = (unsigned int) -1; - static constexpr size_t INVALID_SIZE = (size_t) -1; - static constexpr int INVALID_DEVICE_ORDINAL = -1; - - struct BlockDescriptor - { - void* d_ptr; - size_t bytes; - unsigned int bin; - int device; - cudaStream_t associated_stream; - cudaEvent_t ready_event; - - BlockDescriptor(void* d_ptr_, int device_) - : d_ptr(d_ptr_), bytes(0), bin(INVALID_BIN), device(device_), - associated_stream(nullptr), ready_event(nullptr) {} - - BlockDescriptor(int device_) - : d_ptr(nullptr), bytes(0), bin(INVALID_BIN), device(device_), - associated_stream(nullptr), ready_event(nullptr) {} - - static bool PtrCompare(const BlockDescriptor& a, const BlockDescriptor& b) { - return (a.device == b.device) ? (a.d_ptr < b.d_ptr) : (a.device < b.device); - } - static bool SizeCompare(const BlockDescriptor& a, const BlockDescriptor& b) { - return (a.device == b.device) ? (a.bytes < b.bytes) : (a.device < b.device); - } - }; - - using Compare = bool (*)(const BlockDescriptor&, const BlockDescriptor&); - - struct TotalBytes { size_t free; size_t live; TotalBytes() : free(0), live(0) {} }; - - using CachedBlocks = std::multiset; - using BusyBlocks = std::multiset; - using GpuCachedBytes = std::map; - - static unsigned int IntPow(unsigned int base, unsigned int exp) { - unsigned int retval = 1; - while (exp > 0) { - if (exp & 1) retval *= base; - base *= base; - exp >>= 1; - } - return retval; - } - - void NearestPowerOf(unsigned int& power, size_t& rounded_bytes, - unsigned int base, size_t value) { - power = 0; - rounded_bytes = 1; - if (value * base < value) { - power = sizeof(size_t) * 8; - rounded_bytes = size_t(0) - 1; - return; - } - while (rounded_bytes < value) { - rounded_bytes *= base; - power++; - } - } - - std::mutex mutex; - unsigned int bin_growth; - unsigned int min_bin; - unsigned int max_bin; - size_t min_bin_bytes; - size_t max_bin_bytes; - size_t max_cached_bytes; - bool skip_cleanup; - GpuCachedBytes cached_bytes; - CachedBlocks cached_blocks; - BusyBlocks live_blocks; - - /* - * Constructor tuned for BI-V100 (32GB per card, 4 cards): - * bin_growth=8, min_bin=3 (512B), max_bin=13 (~550MB) - * max_cached_bytes = 4GB per device (reasonable for 32GB card) - * - * This replaces PyTorch's expandable_segments with a proven - * geometric-bin caching strategy from CUB/CCCL. - */ - CachingDeviceAllocator() - : bin_growth(8) - , min_bin(3) /* 8^3 = 512B minimum allocation */ - , max_bin(13) /* 8^13 = ~550MB maximum cached bin */ - , min_bin_bytes(IntPow(8, 3)) - , max_bin_bytes(IntPow(8, 13)) - , max_cached_bytes((size_t)4 * 1024 * 1024 * 1024) /* 4GB per device */ - , skip_cleanup(true) /* CoreX may tear down CUDA before our dtor */ - , cached_blocks(BlockDescriptor::SizeCompare) - , live_blocks(BlockDescriptor::PtrCompare) - { - CcclLog("CachingDeviceAllocator init: bin_growth=%u min_bin=%u " - "max_bin=%u max_cached=%.1fGB\n", - bin_growth, min_bin, max_bin, - (double)max_cached_bytes / (1024.0*1024.0*1024.0)); - } - - /* ---- Real cudaMalloc/cudaFree via dlsym(RTLD_NEXT) ---- */ - using RealMalloc_t = cudaError_t (*)(void**, size_t); - using RealFree_t = cudaError_t (*)(void*); - - static RealMalloc_t get_real_malloc() { - static RealMalloc_t fn = (RealMalloc_t)dlsym(RTLD_NEXT, "cudaMalloc"); - return fn; - } - static RealFree_t get_real_free() { - static RealFree_t fn = (RealFree_t)dlsym(RTLD_NEXT, "cudaFree"); - return fn; - } - - cudaError_t DeviceAllocate(int device, void** d_ptr, size_t bytes, - cudaStream_t active_stream = nullptr) - { - *d_ptr = nullptr; - int entrypoint_device = INVALID_DEVICE_ORDINAL; - cudaError_t error = cudaSuccess; - - if (device == INVALID_DEVICE_ORDINAL) { - error = cudaGetDevice(&entrypoint_device); - if (error != cudaSuccess) return error; - device = entrypoint_device; - } - - bool found = false; - BlockDescriptor search_key(device); - search_key.associated_stream = active_stream; - NearestPowerOf(search_key.bin, search_key.bytes, bin_growth, bytes); - - if (search_key.bin > max_bin) { - search_key.bin = INVALID_BIN; - search_key.bytes = bytes; - } else { - mutex.lock(); - if (search_key.bin < min_bin) { - search_key.bin = min_bin; - search_key.bytes = min_bin_bytes; - } - - CachedBlocks::iterator block_itr = cached_blocks.lower_bound(search_key); - while ((block_itr != cached_blocks.end()) && - (block_itr->device == device) && - (block_itr->bin == search_key.bin)) - { - bool is_reusable = false; - if (active_stream == block_itr->associated_stream) { - is_reusable = true; - } else { - cudaError_t event_status = cudaEventQuery(block_itr->ready_event); - if (event_status != cudaErrorNotReady) { - is_reusable = true; - } - } - - if (is_reusable) { - found = true; - search_key = *block_itr; - search_key.associated_stream = active_stream; - live_blocks.insert(search_key); - cached_bytes[device].free -= search_key.bytes; - cached_bytes[device].live += search_key.bytes; - - CcclLog("reuse %p (%zu bytes) dev=%d\n", - search_key.d_ptr, search_key.bytes, device); - cached_blocks.erase(block_itr); - break; - } - block_itr++; - } - mutex.unlock(); - } - - if (!found) { - if (device != entrypoint_device) { - if (entrypoint_device == INVALID_DEVICE_ORDINAL) - cudaGetDevice(&entrypoint_device); - cudaSetDevice(device); - } - - /* Use real cudaMalloc, not ourselves */ - error = get_real_malloc()(&search_key.d_ptr, search_key.bytes); - - if (error == cudaErrorMemoryAllocation) { - CcclLog("OOM for %zu bytes on dev=%d, freeing cache...\n", - search_key.bytes, device); - cudaGetLastError(); /* reset */ - - mutex.lock(); - BlockDescriptor free_key(device); - CachedBlocks::iterator block_itr = cached_blocks.lower_bound(free_key); - while ((block_itr != cached_blocks.end()) && - (block_itr->device == device)) - { - error = get_real_free()(block_itr->d_ptr); - if (error != cudaSuccess) break; - cudaEventDestroy(block_itr->ready_event); - cached_bytes[device].free -= block_itr->bytes; - block_itr = cached_blocks.erase(block_itr); - } - mutex.unlock(); - - if (error != cudaSuccess) return error; - error = get_real_malloc()(&search_key.d_ptr, search_key.bytes); - if (error != cudaSuccess) return error; - } else if (error != cudaSuccess) { - return error; - } - - cudaEventCreateWithFlags(&search_key.ready_event, cudaEventDisableTiming); - - mutex.lock(); - live_blocks.insert(search_key); - cached_bytes[device].live += search_key.bytes; - mutex.unlock(); - - CcclLog("alloc %p (%zu bytes, bin=%u) dev=%d\n", - search_key.d_ptr, search_key.bytes, search_key.bin, device); - - if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && - (entrypoint_device != device)) - cudaSetDevice(entrypoint_device); - } - - *d_ptr = search_key.d_ptr; - return cudaSuccess; - } - - cudaError_t DeviceAllocate(void** d_ptr, size_t bytes, - cudaStream_t active_stream = nullptr) { - return DeviceAllocate(INVALID_DEVICE_ORDINAL, d_ptr, bytes, active_stream); - } - - cudaError_t DeviceFree(int device, void* d_ptr) - { - int entrypoint_device = INVALID_DEVICE_ORDINAL; - cudaError_t error = cudaSuccess; - - if (d_ptr == nullptr) return cudaSuccess; - - if (device == INVALID_DEVICE_ORDINAL) { - error = cudaGetDevice(&entrypoint_device); - if (error != cudaSuccess) return error; - device = entrypoint_device; - } - - mutex.lock(); - bool recached = false; - BlockDescriptor search_key(d_ptr, device); - BusyBlocks::iterator block_itr = live_blocks.find(search_key); - - if (block_itr != live_blocks.end()) { - search_key = *block_itr; - live_blocks.erase(block_itr); - cached_bytes[device].live -= search_key.bytes; - - if ((search_key.bin != INVALID_BIN) && - (cached_bytes[device].free + search_key.bytes <= max_cached_bytes)) - { - recached = true; - cached_blocks.insert(search_key); - cached_bytes[device].free += search_key.bytes; - CcclLog("cache %p (%zu bytes) dev=%d\n", - d_ptr, search_key.bytes, device); - } - } - mutex.unlock(); - - if (device != entrypoint_device) { - if (entrypoint_device == INVALID_DEVICE_ORDINAL) - cudaGetDevice(&entrypoint_device); - cudaSetDevice(device); - } - - if (recached) { - cudaEventRecord(search_key.ready_event, search_key.associated_stream); - } else { - /* Not tracked or cache full — real free */ - CcclLog("free %p dev=%d (not cached)\n", d_ptr, device); - error = get_real_free()(d_ptr); - if (block_itr != live_blocks.end()) - cudaEventDestroy(search_key.ready_event); - } - - if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && - (entrypoint_device != device)) - cudaSetDevice(entrypoint_device); - - return error; - } - - cudaError_t DeviceFree(void* d_ptr) { - return DeviceFree(INVALID_DEVICE_ORDINAL, d_ptr); - } - - cudaError_t FreeAllCached() - { - cudaError_t error = cudaSuccess; - int entrypoint_device = INVALID_DEVICE_ORDINAL; - int current_device = INVALID_DEVICE_ORDINAL; - - mutex.lock(); - while (!cached_blocks.empty()) { - CachedBlocks::iterator begin = cached_blocks.begin(); - if (entrypoint_device == INVALID_DEVICE_ORDINAL) - cudaGetDevice(&entrypoint_device); - if (begin->device != current_device) { - cudaSetDevice(begin->device); - current_device = begin->device; - } - get_real_free()(begin->d_ptr); - cudaEventDestroy(begin->ready_event); - cached_bytes[current_device].free -= begin->bytes; - cached_blocks.erase(begin); - } - mutex.unlock(); - - if (entrypoint_device != INVALID_DEVICE_ORDINAL) - cudaSetDevice(entrypoint_device); - return error; - } -}; - -/* ======================================================================== - * Global singleton + LD_PRELOAD intercepts - * ======================================================================== */ - -static CachingDeviceAllocator& get_allocator() { - static CachingDeviceAllocator instance; +/* ---- Global allocator singleton ---- */ +static cub::CachingDeviceAllocator& get_allocator() { + static cub::CachingDeviceAllocator instance( + ALLOC_BIN_GROWTH, + ALLOC_MIN_BIN, + ALLOC_MAX_BIN, + ALLOC_MAX_CACHED, + true /* skip_cleanup: CoreX may tear down CUDA before our dtor */ + ); return instance; } static bool g_preload_active = false; +static bool g_debug = false; -/* Called once at .so load time */ +/* ---- Real cudaMalloc/cudaFree via dlsym(RTLD_NEXT) ---- */ +using RealMalloc_t = cudaError_t (*)(void**, size_t); +using RealFree_t = cudaError_t (*)(void*); + +static RealMalloc_t get_real_malloc() { + static RealMalloc_t fn = (RealMalloc_t)dlsym(RTLD_NEXT, "cudaMalloc"); + return fn; +} +static RealFree_t get_real_free() { + static RealFree_t fn = (RealFree_t)dlsym(RTLD_NEXT, "cudaFree"); + return fn; +} + +/* ======================================================================== + * Constructor: runs at LD_PRELOAD load time + * ======================================================================== */ __attribute__((constructor)) static void cccl_preload_init() { const char* debug_env = getenv("CCCL_ALLOC_DEBUG"); - g_cccl_debug = (debug_env && atoi(debug_env) > 0); + g_debug = (debug_env && atoi(debug_env) > 0); const char* disable_env = getenv("CCCL_ALLOC_DISABLE"); if (disable_env && atoi(disable_env) > 0) { fprintf(stderr, "[cccl_alloc] DISABLED by CCCL_ALLOC_DISABLE=1\n"); - g_preload_active = false; return; } - /* Strip expandable_segments from PYTORCH_CUDA_ALLOC_CONF - * so CoreX's allocator doesn't hit the assert. - * We handle the caching ourselves. */ + /* Strip expandable_segments from PYTORCH_CUDA_ALLOC_CONF */ const char* alloc_conf = getenv("PYTORCH_CUDA_ALLOC_CONF"); if (alloc_conf) { - /* Build a new conf string without expandable_segments */ std::string conf(alloc_conf); std::string clean; size_t pos = 0; @@ -411,49 +92,51 @@ static void cccl_preload_init() { size_t comma = conf.find(',', pos); if (comma == std::string::npos) comma = conf.size(); std::string token = conf.substr(pos, comma - pos); - /* Skip expandable_segments:* */ if (token.find("expandable_segments") == std::string::npos) { if (!clean.empty()) clean += ","; clean += token; } pos = comma + 1; } - if (clean.empty()) { + if (clean.empty()) unsetenv("PYTORCH_CUDA_ALLOC_CONF"); - } else { + else setenv("PYTORCH_CUDA_ALLOC_CONF", clean.c_str(), 1); - } - fprintf(stderr, "[cccl_alloc] stripped expandable_segments from " - "PYTORCH_CUDA_ALLOC_CONF: \"%s\" -> \"%s\"\n", + + fprintf(stderr, "[cccl_alloc] PYTORCH_CUDA_ALLOC_CONF: \"%s\" -> \"%s\"\n", alloc_conf, clean.empty() ? "(unset)" : clean.c_str()); } - /* Force-initialize the allocator singleton */ - (void)get_allocator(); + /* Initialize allocator */ + auto& alloc = get_allocator(); + if (g_debug) { + alloc.debug = true; + } + g_preload_active = true; - fprintf(stderr, "[cccl_alloc] LD_PRELOAD active — CUB CachingDeviceAllocator " - "replacing cudaMalloc/cudaFree\n"); + fprintf(stderr, + "[cccl_alloc] LD_PRELOAD active — CUB CachingDeviceAllocator " + "(growth=%u, bins=[%u..%u], max_cached=%.1fGB)\n", + ALLOC_BIN_GROWTH, ALLOC_MIN_BIN, ALLOC_MAX_BIN, + (double)ALLOC_MAX_CACHED / (1024.0*1024.0*1024.0)); } -/* ---- cudaMalloc intercept ---- */ +/* ======================================================================== + * cudaMalloc / cudaFree intercepts + * ======================================================================== */ + extern "C" cudaError_t cudaMalloc(void** devPtr, size_t size) { if (!g_preload_active) { - /* Fallback to real cudaMalloc during init or if disabled */ - static auto real_fn = (CachingDeviceAllocator::RealMalloc_t) - dlsym(RTLD_NEXT, "cudaMalloc"); - return real_fn(devPtr, size); + return get_real_malloc()(devPtr, size); } return get_allocator().DeviceAllocate(devPtr, size); } -/* ---- cudaFree intercept ---- */ extern "C" cudaError_t cudaFree(void* devPtr) { if (!g_preload_active || devPtr == nullptr) { - static auto real_fn = (CachingDeviceAllocator::RealFree_t) - dlsym(RTLD_NEXT, "cudaFree"); - return real_fn(devPtr); + return get_real_free()(devPtr); } return get_allocator().DeviceFree(devPtr); } diff --git a/qwen3_6_scripts/cccl_preload/include/cub/config.cuh b/qwen3_6_scripts/cccl_preload/include/cub/config.cuh new file mode 100644 index 00000000..a3bc8443 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/config.cuh @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/** + * \file + * Static configuration header for the CUB project. + */ + +#pragma once + +// For _CCCL_IMPLICIT_SYSTEM_HEADER +#include // IWYU pragma: export + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export + +#if !_CCCL_COMPILER(NVRTC) +# include +#endif // !_CCCL_COMPILER(NVRTC) diff --git a/qwen3_6_scripts/cccl_preload/include/cub/detail/detect_cuda_runtime.cuh b/qwen3_6_scripts/cccl_preload/include/cub/detail/detect_cuda_runtime.cuh new file mode 100644 index 00000000..ef53fe85 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/detail/detect_cuda_runtime.cuh @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/** + * @file + * Utilities for CUDA dynamic parallelism. + */ + +#pragma once + +// We cannot use `cub/config.cuh` here due to circular dependencies +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes: +//! Defined if RDC is enabled and CUB_DISABLE_CDP is not defined. +//! Deprecated [Since 3.2] +# define CUB_RDC_ENABLED + +//! If defined, support for device-side usage of CUB is disabled. +//! Deprecated [Since 3.2]. Use CCCL_DISABLE_CDP instead. +# define CUB_DISABLE_CDP + +//! Execution space for functions that use the CUDA runtime API, e.g. to launch kernels. Such functions are `__host__ +//! __device__` when compiling with RDC, otherwise only `__host__`. +//! Deprecated [Since 3.2] +# define CUB_RUNTIME_FUNCTION +#else // Non-doxygen pass: + +# if _CCCL_HAS_CDP() +# define CUB_RDC_ENABLED +# endif // _CCCL_HAS_CDP() + +# ifndef CUB_RUNTIME_FUNCTION +# define CUB_RUNTIME_FUNCTION _CCCL_CDP_API +# endif // CUB_RUNTIME_FUNCTION predefined +#endif // Do not document diff --git a/qwen3_6_scripts/cccl_preload/include/cub/util_allocator.cuh b/qwen3_6_scripts/cccl_preload/include/cub/util_allocator.cuh new file mode 100644 index 00000000..6b2d26d4 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/util_allocator.cuh @@ -0,0 +1,901 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/****************************************************************************** + * Simple caching allocator for device memory allocations. The allocator is + * thread-safe and capable of managing device allocations on multiple devices. + ******************************************************************************/ + +#pragma once + +#include + +#ifndef CCCL_DISABLE_NVRTC_COMPATIBILITY_CHECK +# if _CCCL_COMPILER(NVRTC) +# error \ + "Including is not supported when compiling with NVRTC, which supports device code only. You can define CCCL_DISABLE_NVRTC_COMPATIBILITY_CHECK to disable this warning." +# endif // _CCCL_COMPILER(NVRTC) +#endif // CCCL_DISABLE_NVRTC_COMPATIBILITY_CHECK + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +#include +#include +#include + +CUB_NAMESPACE_BEGIN + +/****************************************************************************** + * CachingDeviceAllocator (host use) + ******************************************************************************/ + +/** + * @brief A simple caching allocator for device memory allocations. + * + * @par Overview + * The allocator is thread-safe and stream-safe and is capable of managing cached + * device allocations on multiple devices. It behaves as follows: + * + * @par + * - Allocations from the allocator are associated with an @p active_stream. Once freed, + * the allocation becomes available immediately for reuse within the @p active_stream + * with which it was associated with during allocation, and it becomes available for + * reuse within other streams when all prior work submitted to @p active_stream has completed. + * - Allocations are categorized and cached by bin size. A new allocation request of + * a given size will only consider cached allocations within the corresponding bin. + * - Bin limits progress geometrically in accordance with the growth factor + * @p bin_growth provided during construction. Unused device allocations within + * a larger bin cache are not reused for allocation requests that categorize to + * smaller bin sizes. + * - Allocation requests below ( @p bin_growth ^ @p min_bin ) are rounded up to + * ( @p bin_growth ^ @p min_bin ). + * - Allocations above ( @p bin_growth ^ @p max_bin ) are not rounded up to the nearest + * bin and are simply freed when they are deallocated instead of being returned + * to a bin-cache. + * - If the total storage of cached allocations on a given device will exceed + * @p max_cached_bytes, allocations for that device are simply freed when they are + * deallocated instead of being returned to their bin-cache. + * + * @par + * For example, the default-constructed CachingDeviceAllocator is configured with: + * - @p bin_growth = 8 + * - @p min_bin = 3 + * - @p max_bin = 7 + * - @p max_cached_bytes = 6MB - 1B + * + * @par + * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB + * and sets a maximum of 6,291,455 cached bytes per device + * + */ +struct CachingDeviceAllocator +{ + //--------------------------------------------------------------------- + // Constants + //--------------------------------------------------------------------- + + /// Out-of-bounds bin + static constexpr unsigned int INVALID_BIN = (unsigned int) -1; + + /// Invalid size + static constexpr size_t INVALID_SIZE = (size_t) -1; + +#ifndef _CCCL_DOXYGEN_INVOKED // Do not document + + /// Invalid device ordinal + static constexpr int INVALID_DEVICE_ORDINAL = -1; + + //--------------------------------------------------------------------- + // Type definitions and helper types + //--------------------------------------------------------------------- + + /** + * Descriptor for device memory allocations + */ + struct BlockDescriptor + { + // Device pointer + void* d_ptr; + + // Size of allocation in bytes + size_t bytes; + + // Bin enumeration + unsigned int bin; + + // device ordinal + int device; + + // Associated associated_stream + cudaStream_t associated_stream; + + // Signal when associated stream has run to the point at which this block was freed + cudaEvent_t ready_event; + + // Constructor (suitable for searching maps for a specific block, given its pointer and + // device) + BlockDescriptor(void* d_ptr, int device) + : d_ptr(d_ptr) + , bytes(0) + , bin(INVALID_BIN) + , device(device) + , associated_stream(nullptr) + , ready_event(nullptr) + {} + + // Constructor (suitable for searching maps for a range of suitable blocks, given a device) + BlockDescriptor(int device) + : d_ptr(nullptr) + , bytes(0) + , bin(INVALID_BIN) + , device(device) + , associated_stream(nullptr) + , ready_event(nullptr) + {} + + // Comparison functor for comparing device pointers + static bool PtrCompare(const BlockDescriptor& a, const BlockDescriptor& b) + { + if (a.device == b.device) + { + return (a.d_ptr < b.d_ptr); + } + else + { + return (a.device < b.device); + } + } + + // Comparison functor for comparing allocation sizes + static bool SizeCompare(const BlockDescriptor& a, const BlockDescriptor& b) + { + if (a.device == b.device) + { + return (a.bytes < b.bytes); + } + else + { + return (a.device < b.device); + } + } + }; + + /// BlockDescriptor comparator function interface + using Compare = bool (*)(const BlockDescriptor&, const BlockDescriptor&); + + class TotalBytes + { + public: + size_t free; + size_t live; + TotalBytes() + { + free = live = 0; + } + }; + + /// Set type for cached blocks (ordered by size) + using CachedBlocks = std::multiset; + + /// Set type for live blocks (ordered by ptr) + using BusyBlocks = std::multiset; + + /// Map type of device ordinals to the number of cached bytes cached by each device + using GpuCachedBytes = std::map; + + //--------------------------------------------------------------------- + // Utility functions + //--------------------------------------------------------------------- + + /** + * Integer pow function for unsigned base and exponent + */ + static unsigned int IntPow(unsigned int base, unsigned int exp) + { + unsigned int retval = 1; + while (exp > 0) + { + if (exp & 1) + { + retval = retval * base; // multiply the result by the current base + } + base = base * base; // square the base + exp = exp >> 1; // divide the exponent in half + } + return retval; + } + + /** + * Round up to the nearest power-of + */ + void NearestPowerOf(unsigned int& power, size_t& rounded_bytes, unsigned int base, size_t value) + { + power = 0; + rounded_bytes = 1; + + if (value * base < value) + { + // Overflow + power = sizeof(size_t) * 8; + rounded_bytes = size_t(0) - 1; + return; + } + + while (rounded_bytes < value) + { + rounded_bytes *= base; + power++; + } + } + + //--------------------------------------------------------------------- + // Fields + //--------------------------------------------------------------------- + + /// Mutex for thread-safety + std::mutex mutex; + + /// Geometric growth factor for bin-sizes + unsigned int bin_growth; + + /// Minimum bin enumeration + unsigned int min_bin; + + /// Maximum bin enumeration + unsigned int max_bin; + + /// Minimum bin size + size_t min_bin_bytes; + + /// Maximum bin size + size_t max_bin_bytes; + + /// Maximum aggregate cached bytes per device + size_t max_cached_bytes; + + /// Whether or not to skip a call to FreeAllCached() when destructor is called. + /// (The CUDA runtime may have already shut down for statically declared allocators) + const bool skip_cleanup; + + /// Whether or not to print (de)allocation events to stdout + bool debug; + + /// Map of device ordinal to aggregate cached bytes on that device + GpuCachedBytes cached_bytes; + + /// Set of cached device allocations available for reuse + CachedBlocks cached_blocks; + + /// Set of live device allocations currently in use + BusyBlocks live_blocks; + +#endif // _CCCL_DOXYGEN_INVOKED + + //--------------------------------------------------------------------- + // Methods + //--------------------------------------------------------------------- + + /** + * @brief Constructor. + * + * @param bin_growth + * Geometric growth factor for bin-sizes + * + * @param min_bin + * Minimum bin (default is bin_growth ^ 1) + * + * @param max_bin + * Maximum bin (default is no max bin) + * + * @param max_cached_bytes + * Maximum aggregate cached bytes per device (default is no limit) + * + * @param skip_cleanup + * Whether or not to skip a call to @p FreeAllCached() when the destructor is called (default + * is to deallocate) + */ + CachingDeviceAllocator( + unsigned int bin_growth, + unsigned int min_bin = 1, + unsigned int max_bin = INVALID_BIN, + size_t max_cached_bytes = INVALID_SIZE, + bool skip_cleanup = false) + : bin_growth(bin_growth) + , min_bin(min_bin) + , max_bin(max_bin) + , min_bin_bytes(IntPow(bin_growth, min_bin)) + , max_bin_bytes(IntPow(bin_growth, max_bin)) + , max_cached_bytes(max_cached_bytes) + , skip_cleanup(skip_cleanup) + , debug(false) + , cached_blocks(BlockDescriptor::SizeCompare) + , live_blocks(BlockDescriptor::PtrCompare) + {} + + /** + * @brief Default constructor. + * + * Configured with: + * @par + * - @p bin_growth = 8 + * - @p min_bin = 3 + * - @p max_bin = 7 + * - @p max_cached_bytes = ( @p bin_growth ^ @p max_bin) * 3 ) - 1 = 6,291,455 bytes + * + * which delineates five bin-sizes: 512B, 4KB, 32KB, 256KB, and 2MB and + * sets a maximum of 6,291,455 cached bytes per device + */ + CachingDeviceAllocator(bool skip_cleanup = false, bool debug = false) + : bin_growth(8) + , min_bin(3) + , max_bin(7) + , min_bin_bytes(IntPow(bin_growth, min_bin)) + , max_bin_bytes(IntPow(bin_growth, max_bin)) + , max_cached_bytes((max_bin_bytes * 3) - 1) + , skip_cleanup(skip_cleanup) + , debug(debug) + , cached_blocks(BlockDescriptor::SizeCompare) + , live_blocks(BlockDescriptor::PtrCompare) + {} + + /** + * @brief Sets the limit on the number bytes this allocator is allowed to cache per device. + * + * Changing the ceiling of cached bytes does not cause any allocations (in-use or + * cached-in-reserve) to be freed. See \p FreeAllCached(). + */ + cudaError_t SetMaxCachedBytes(size_t max_cached_bytes_) + { + // Lock + mutex.lock(); + +#ifdef CUB_DEBUG_LOG + _CubLog( + "Changing max_cached_bytes (%lld -> %lld)\n", (long long) this->max_cached_bytes, (long long) max_cached_bytes_); +#endif + + this->max_cached_bytes = max_cached_bytes_; + + // Unlock + mutex.unlock(); + + return cudaSuccess; + } + + /** + * @brief Provides a suitable allocation of device memory for the given size on the specified + * device. + * + * Once freed, the allocation becomes available immediately for reuse within the @p + * active_stream with which it was associated with during allocation, and it becomes available + * for reuse within other streams when all prior work submitted to @p active_stream has + * completed. + * + * @param[in] device + * Device on which to place the allocation + * + * @param[out] d_ptr + * Reference to pointer to the allocation + * + * @param[in] bytes + * Minimum number of bytes for the allocation + * + * @param[in] active_stream + * The stream to be associated with this allocation + */ + cudaError_t DeviceAllocate(int device, void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr) + { + *d_ptr = nullptr; + int entrypoint_device = INVALID_DEVICE_ORDINAL; + cudaError_t error = cudaSuccess; + + if (device == INVALID_DEVICE_ORDINAL) + { + error = CubDebug(cudaGetDevice(&entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + + device = entrypoint_device; + } + + // Create a block descriptor for the requested allocation + bool found = false; + BlockDescriptor search_key(device); + search_key.associated_stream = active_stream; + NearestPowerOf(search_key.bin, search_key.bytes, bin_growth, bytes); + + if (search_key.bin > max_bin) + { + // Bin is greater than our maximum bin: allocate the request + // exactly and give out-of-bounds bin. It will not be cached + // for reuse when returned. + search_key.bin = INVALID_BIN; + search_key.bytes = bytes; + } + else + { + // Search for a suitable cached allocation: lock + mutex.lock(); + + if (search_key.bin < min_bin) + { + // Bin is less than minimum bin: round up + search_key.bin = min_bin; + search_key.bytes = min_bin_bytes; + } + + // Iterate through the range of cached blocks on the same device in the same bin + CachedBlocks::iterator block_itr = cached_blocks.lower_bound(search_key); + while ((block_itr != cached_blocks.end()) && (block_itr->device == device) && (block_itr->bin == search_key.bin)) + { + // To prevent races with reusing blocks returned by the host but still + // in use by the device, only consider cached blocks that are + // either (from the active stream) or (from an idle stream) + bool is_reusable = false; + if (active_stream == block_itr->associated_stream) + { + is_reusable = true; + } + else + { + const cudaError_t event_status = cudaEventQuery(block_itr->ready_event); + if (event_status != cudaErrorNotReady) + { + CubDebug(event_status); + is_reusable = true; + } + } + + if (is_reusable) + { + // Reuse existing cache block. Insert into live blocks. + found = true; + search_key = *block_itr; + search_key.associated_stream = active_stream; + live_blocks.insert(search_key); + + // Remove from free blocks + cached_bytes[device].free -= search_key.bytes; + cached_bytes[device].live += search_key.bytes; + +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d reused cached block at %p (%lld bytes) for stream %lld (previously associated with " + "stream %lld).\n", + device, + search_key.d_ptr, + (long long) search_key.bytes, + (long long) search_key.associated_stream, + (long long) block_itr->associated_stream); +#endif + + cached_blocks.erase(block_itr); + + break; + } + block_itr++; + } + + // Done searching: unlock + mutex.unlock(); + } + + // Allocate the block if necessary + if (!found) + { + // Set runtime's current device to specified device (entrypoint may not be set) + if (device != entrypoint_device) + { + error = CubDebug(cudaGetDevice(&entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + + error = CubDebug(cudaSetDevice(device)); + if (cudaSuccess != error) + { + return error; + } + } + + // Attempt to allocate + error = CubDebug(cudaMalloc(&search_key.d_ptr, search_key.bytes)); + if (error == cudaErrorMemoryAllocation) + { + // The allocation attempt failed: free all cached blocks on device and retry +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d failed to allocate %lld bytes for stream %lld, retrying after freeing cached allocations", + device, + (long long) search_key.bytes, + (long long) search_key.associated_stream); +#endif + + error = cudaSuccess; // Reset the error we will return + cudaGetLastError(); // Reset CUDART's error + + // Lock + mutex.lock(); + + // Iterate the range of free blocks on the same device + BlockDescriptor free_key(device); + CachedBlocks::iterator block_itr = cached_blocks.lower_bound(free_key); + + while ((block_itr != cached_blocks.end()) && (block_itr->device == device)) + { + // No need to worry about synchronization with the device: cudaFree is + // blocking and will synchronize across all kernels executing + // on the current device + + // Free device memory and destroy stream event. + error = CubDebug(cudaFree(block_itr->d_ptr)); + if (cudaSuccess != error) + { + break; + } + + error = CubDebug(cudaEventDestroy(block_itr->ready_event)); + if (cudaSuccess != error) + { + break; + } + + // Reduce balance and erase entry + cached_bytes[device].free -= block_itr->bytes; + +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d freed %lld bytes.\n\t\t %lld available blocks cached (%lld bytes), %lld live blocks " + "(%lld bytes) outstanding.\n", + device, + (long long) block_itr->bytes, + (long long) cached_blocks.size(), + (long long) cached_bytes[device].free, + (long long) live_blocks.size(), + (long long) cached_bytes[device].live); +#endif + + block_itr = cached_blocks.erase(block_itr); + } + + // Unlock + mutex.unlock(); + + // Return under error + if (error) + { + return error; + } + + // Try to allocate again + error = CubDebug(cudaMalloc(&search_key.d_ptr, search_key.bytes)); + if (cudaSuccess != error) + { + return error; + } + } + + // Create ready event + error = CubDebug(cudaEventCreateWithFlags(&search_key.ready_event, cudaEventDisableTiming)); + + if (cudaSuccess != error) + { + return error; + } + + // Insert into live blocks + mutex.lock(); + live_blocks.insert(search_key); + cached_bytes[device].live += search_key.bytes; + mutex.unlock(); + +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d allocated new device block at %p (%lld bytes associated with stream %lld).\n", + device, + search_key.d_ptr, + (long long) search_key.bytes, + (long long) search_key.associated_stream); +#endif + + // Attempt to revert back to previous device if necessary + if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device)) + { + error = CubDebug(cudaSetDevice(entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + } + } + + // Copy device pointer to output parameter + *d_ptr = search_key.d_ptr; + +#ifdef CUB_DEBUG_LOG + if (debug) + { + _CubLog("\t\t%lld available blocks cached (%lld bytes), %lld live blocks outstanding(%lld bytes).\n", + (long long) cached_blocks.size(), + (long long) cached_bytes[device].free, + (long long) live_blocks.size(), + (long long) cached_bytes[device].live); + } +#endif + + return error; + } + + /** + * @brief Provides a suitable allocation of device memory for the given size on the current + * device. + * + * Once freed, the allocation becomes available immediately for reuse within the @p + * active_stream with which it was associated with during allocation, and it becomes available + * for reuse within other streams when all prior work submitted to @p active_stream has + * completed. + * + * @param[out] d_ptr + * Reference to pointer to the allocation + * + * @param[in] bytes + * Minimum number of bytes for the allocation + * + * @param[in] active_stream + * The stream to be associated with this allocation + */ + cudaError_t DeviceAllocate(void** d_ptr, size_t bytes, cudaStream_t active_stream = nullptr) + { + return DeviceAllocate(INVALID_DEVICE_ORDINAL, d_ptr, bytes, active_stream); + } + + /** + * @brief Frees a live allocation of device memory on the specified device, returning it to the + * allocator. + * + * Once freed, the allocation becomes available immediately for reuse within the + * @p active_stream with which it was associated with during allocation, and it becomes + * available for reuse within other streams when all prior work submitted to @p active_stream + * has completed. + */ + cudaError_t DeviceFree(int device, void* d_ptr) + { + int entrypoint_device = INVALID_DEVICE_ORDINAL; + cudaError_t error = cudaSuccess; + + if (device == INVALID_DEVICE_ORDINAL) + { + error = CubDebug(cudaGetDevice(&entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + device = entrypoint_device; + } + + // Lock + mutex.lock(); + + // Find corresponding block descriptor + bool recached = false; + BlockDescriptor search_key(d_ptr, device); + BusyBlocks::iterator block_itr = live_blocks.find(search_key); + if (block_itr != live_blocks.end()) + { + // Remove from live blocks + search_key = *block_itr; + live_blocks.erase(block_itr); + cached_bytes[device].live -= search_key.bytes; + + // Keep the returned allocation if bin is valid and we won't exceed the max cached threshold + if ((search_key.bin != INVALID_BIN) && (cached_bytes[device].free + search_key.bytes <= max_cached_bytes)) + { + // Insert returned allocation into free blocks + recached = true; + cached_blocks.insert(search_key); + cached_bytes[device].free += search_key.bytes; + +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d returned %lld bytes from associated stream %lld.\n\t\t %lld available blocks cached (%lld " + "bytes), %lld live blocks outstanding. (%lld bytes)\n", + device, + (long long) search_key.bytes, + (long long) search_key.associated_stream, + (long long) cached_blocks.size(), + (long long) cached_bytes[device].free, + (long long) live_blocks.size(), + (long long) cached_bytes[device].live); +#endif + } + } + + // Unlock + mutex.unlock(); + + // First set to specified device (entrypoint may not be set) + if (device != entrypoint_device) + { + error = CubDebug(cudaGetDevice(&entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + + error = CubDebug(cudaSetDevice(device)); + if (cudaSuccess != error) + { + return error; + } + } + + if (recached) + { + // Insert the ready event in the associated stream (must have current device set properly) + error = CubDebug(cudaEventRecord(search_key.ready_event, search_key.associated_stream)); + if (cudaSuccess != error) + { + return error; + } + } + + if (!recached) + { + // Free the allocation from the runtime and cleanup the event. + error = CubDebug(cudaFree(d_ptr)); + if (cudaSuccess != error) + { + return error; + } + + error = CubDebug(cudaEventDestroy(search_key.ready_event)); + if (cudaSuccess != error) + { + return error; + } + +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d freed %lld bytes from associated stream %lld.\n\t\t %lld available blocks cached (%lld " + "bytes), %lld live blocks (%lld bytes) outstanding.\n", + device, + (long long) search_key.bytes, + (long long) search_key.associated_stream, + (long long) cached_blocks.size(), + (long long) cached_bytes[device].free, + (long long) live_blocks.size(), + (long long) cached_bytes[device].live); +#endif + } + + // Reset device + if ((entrypoint_device != INVALID_DEVICE_ORDINAL) && (entrypoint_device != device)) + { + error = CubDebug(cudaSetDevice(entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + } + + return error; + } + + /** + * @brief Frees a live allocation of device memory on the current device, returning it to the + * allocator. + * + * Once freed, the allocation becomes available immediately for reuse within the @p + * active_stream with which it was associated with during allocation, and it becomes available + * for reuse within other streams when all prior work submitted to @p active_stream has + * completed. + */ + cudaError_t DeviceFree(void* d_ptr) + { + return DeviceFree(INVALID_DEVICE_ORDINAL, d_ptr); + } + + /** + * @brief Frees all cached device allocations on all devices + */ + cudaError_t FreeAllCached() + { + cudaError_t error = cudaSuccess; + int entrypoint_device = INVALID_DEVICE_ORDINAL; + int current_device = INVALID_DEVICE_ORDINAL; + + mutex.lock(); + + while (!cached_blocks.empty()) + { + // Get first block + CachedBlocks::iterator begin = cached_blocks.begin(); + + // Get entry-point device ordinal if necessary + if (entrypoint_device == INVALID_DEVICE_ORDINAL) + { + error = CubDebug(cudaGetDevice(&entrypoint_device)); + if (cudaSuccess != error) + { + break; + } + } + + // Set current device ordinal if necessary + if (begin->device != current_device) + { + error = CubDebug(cudaSetDevice(begin->device)); + if (cudaSuccess != error) + { + break; + } + current_device = begin->device; + } + + // Free device memory + error = CubDebug(cudaFree(begin->d_ptr)); + if (cudaSuccess != error) + { + break; + } + + error = CubDebug(cudaEventDestroy(begin->ready_event)); + if (cudaSuccess != error) + { + break; + } + + // Reduce balance and erase entry + const size_t block_bytes = begin->bytes; + cached_bytes[current_device].free -= block_bytes; + cached_blocks.erase(begin); + +#ifdef CUB_DEBUG_LOG + _CubLog("\tDevice %d freed %lld bytes.\n\t\t %lld available blocks cached (%lld bytes), %lld live blocks (%lld " + "bytes) outstanding.\n", + current_device, + (long long) block_bytes, + (long long) cached_blocks.size(), + (long long) cached_bytes[current_device].free, + (long long) live_blocks.size(), + (long long) cached_bytes[current_device].live); +#endif + } + + mutex.unlock(); + + // Attempt to revert back to entry-point device if necessary + if (entrypoint_device != INVALID_DEVICE_ORDINAL) + { + error = CubDebug(cudaSetDevice(entrypoint_device)); + if (cudaSuccess != error) + { + return error; + } + } + + return error; + } + + /** + * @brief Destructor + */ + virtual ~CachingDeviceAllocator() + { + if (!skip_cleanup) + { + FreeAllCached(); + } + } +}; + +CUB_NAMESPACE_END diff --git a/qwen3_6_scripts/cccl_preload/include/cub/util_arch.cuh b/qwen3_6_scripts/cccl_preload/include/cub/util_arch.cuh new file mode 100644 index 00000000..c5b38bb3 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/util_arch.cuh @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2025, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/** + * \file + * Static architectural properties by SM version. + */ + +#pragma once + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include // IWYU pragma: export +#include +#include + +#include +#include +#include +#include +#include +#include + +// Legacy include; this functionality used to be defined in here. +#include + +CUB_NAMESPACE_BEGIN + +#ifndef _CCCL_DOXYGEN_INVOKED // Do not document + +/// In device code, CUB_PTX_ARCH expands to the PTX version for which we are +/// compiling. In host code, CUB_PTX_ARCH's value is implementation defined. +# ifndef CUB_PTX_ARCH +// deprecated in 3.1 +# if _CCCL_CUDA_COMPILER(NVHPC) +// NV_TARGET_MINIMUM_SM_INTEGER is the oldest target PTX version, and is defined when compiling both host code and +// device code. +# define CUB_PTX_ARCH (NV_TARGET_MINIMUM_SM_INTEGER * 10) +# else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv +# define CUB_PTX_ARCH _CCCL_PTX_ARCH() +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^ +# endif + +/// Maximum number of devices supported. +# ifndef CUB_MAX_DEVICES +//! Deprecated [Since 3.0] +# define CUB_MAX_DEVICES (128) +# endif +static_assert(CUB_MAX_DEVICES > 0, "CUB_MAX_DEVICES must be greater than 0."); + +/// Number of threads per warp +# ifndef CUB_LOG_WARP_THREADS +//! Deprecated [Since 3.0] +# define CUB_LOG_WARP_THREADS(unused) (5) +//! Deprecated [Since 3.0] +# define CUB_WARP_THREADS(unused) (1 << CUB_LOG_WARP_THREADS(0)) + +//! Deprecated [Since 3.0] +# define CUB_PTX_WARP_THREADS CUB_WARP_THREADS(0) +//! Deprecated [Since 3.0] +# define CUB_PTX_LOG_WARP_THREADS CUB_LOG_WARP_THREADS(0) +# endif + +/// Number of smem banks +# ifndef CUB_LOG_SMEM_BANKS +//! Deprecated [Since 3.0] +# define CUB_LOG_SMEM_BANKS(unused) (5) +//! Deprecated [Since 3.0] +# define CUB_SMEM_BANKS(unused) (1 << CUB_LOG_SMEM_BANKS(0)) + +//! Deprecated [Since 3.0] +# define CUB_PTX_LOG_SMEM_BANKS CUB_LOG_SMEM_BANKS(0) +//! Deprecated [Since 3.0] +# define CUB_PTX_SMEM_BANKS CUB_SMEM_BANKS +# endif + +/// Oversubscription factor +# ifndef CUB_SUBSCRIPTION_FACTOR +//! Deprecated [Since 3.0] +# define CUB_SUBSCRIPTION_FACTOR(unused) (5) +//! Deprecated [Since 3.0] +# define CUB_PTX_SUBSCRIPTION_FACTOR CUB_SUBSCRIPTION_FACTOR(0) +# endif + +/// Prefer padding overhead vs X-way conflicts greater than this threshold +# ifndef CUB_PREFER_CONFLICT_OVER_PADDING +//! Deprecated [Since 3.0] +# define CUB_PREFER_CONFLICT_OVER_PADDING(unused) (1) +//! Deprecated [Since 3.0] +# define CUB_PTX_PREFER_CONFLICT_OVER_PADDING CUB_PREFER_CONFLICT_OVER_PADDING(0) +# endif + +namespace detail +{ +inline constexpr int max_devices = CUB_MAX_DEVICES; +inline constexpr int warp_threads = CUB_PTX_WARP_THREADS; +inline constexpr int log2_warp_threads = CUB_PTX_LOG_WARP_THREADS; +inline constexpr int smem_banks = CUB_SMEM_BANKS(0); +inline constexpr int log2_smem_banks = CUB_PTX_LOG_SMEM_BANKS; + +inline constexpr int subscription_factor = CUB_PTX_SUBSCRIPTION_FACTOR; +inline constexpr bool prefer_conflict_over_padding = CUB_PTX_PREFER_CONFLICT_OVER_PADDING; + +// The maximum amount of shared memory available per thread block for eternity. Every current and future CUDA +// architecture has and will have at least this amount of shared memory. This is also the maximum size of total static +// shared memory in a kernel. Note that dynamic shared memory may be larger than this amount. +static constexpr ::cuda::std::size_t max_smem_per_block = 48 * 1024; + +// The size in bytes of the largest machine word that can be atomically read/written in a single instruction, so we can +// use it to pass messages from one thread to another using strong loads (acquire) and stores (release). +inline constexpr int largest_atomic_message_size = 16; + +struct scaling_result +{ + int items_per_thread; + int threads_per_block; +}; + +[[nodiscard]] _CCCL_HOST_DEVICE_API inline constexpr auto +scale_reg_bound(int nominal_4B_threads_per_block, int nominal_4B_items_per_thread, int target_type_size) + -> scaling_result +{ + const int items_per_thread = + (::cuda::std::max) (1, nominal_4B_items_per_thread * 4 / (::cuda::std::max) (4, target_type_size)); + const int threads_per_block = + (::cuda::std::min) (nominal_4B_threads_per_block, + ::cuda::ceil_div(int{max_smem_per_block} / (target_type_size * items_per_thread), 32) * 32); + return {items_per_thread, threads_per_block}; +} + +template +struct RegBoundScaling +{ +private: + static constexpr auto result = + scale_reg_bound(Nominal4ByteThreadsPerBlock, Nominal4ByteItemsPerThread, int{sizeof(T)}); + +public: + static constexpr int ITEMS_PER_THREAD = result.items_per_thread; + static constexpr int BLOCK_THREADS = result.threads_per_block; +}; + +[[nodiscard]] _CCCL_HOST_DEVICE_API inline constexpr auto +scale_mem_bound(int nominal_4B_threads_per_block, int nominal_4B_items_per_thread, int target_type_size) + -> scaling_result +{ + const int items_per_thread = + ::cuda::std::clamp(nominal_4B_items_per_thread * 4 / target_type_size, 1, nominal_4B_items_per_thread * 2); + const int threads_per_block = + (::cuda::std::min) (nominal_4B_threads_per_block, + ::cuda::round_up(int{max_smem_per_block} / (target_type_size * items_per_thread), 32)); + return {items_per_thread, threads_per_block}; +} + +template +struct MemBoundScaling +{ +private: + static constexpr auto result = + scale_mem_bound(Nominal4ByteThreadsPerBlock, Nominal4ByteItemsPerThread, int{sizeof(T)}); + +public: + static constexpr int ITEMS_PER_THREAD = result.items_per_thread; + static constexpr int BLOCK_THREADS = result.threads_per_block; +}; + +template +struct NoScaling +{ + static constexpr int ITEMS_PER_THREAD = Nominal4ByteItemsPerThread; + static constexpr int BLOCK_THREADS = Nominal4ByteThreadsPerBlock; +}; + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr ::cuda::compute_capability current_tuning_cc() noexcept +{ +# if _CCCL_CUDA_COMPILER(NVHPC) + return ::cuda::compute_capability(NV_TARGET_MINIMUM_SM_INTEGER); +# elif _CCCL_DEVICE_COMPILATION() + return ::cuda::device::current_compute_capability(); +# else + // clang 22+ supports __CUDA_ARCH_LIST__ and also instantiates tuning policies inside kernels during the **host** + // pass (e.g. to compute the value for __launch_bounds__), where we rely on current_tuning_cc(), which is then passed + // to the policy selector. In the rare case that the policy selector is an adapter over a policy hub and invokes + // ChainedPolicy (e.g. test cub.test.device.histogram_custom_policy_hub.lid_0), it will fail to compile during + // constant evaluation, since it cannot find a policy for a PTX version of zero. As a workaround, we return the oldest + // CC we are compiling for during the host pass. And for consistency, we do the same for all compilers. +# if _CCCL_CUDA_COMPILER(CLANG) + return ::cuda::__target_compute_capabilities().front(); +# else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv + return {}; +# endif // ^^^ !_CCCL_CUDA_COMPILER(CLANG) ^^^ +# endif +} + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto select_policy(::cuda::compute_capability cc) +{ + return PolicySelector{}(cc); +} + +template +[[nodiscard]] _CCCL_DEVICE_API constexpr auto current_policy() +{ + return select_policy(current_tuning_cc()); +} +} // namespace detail +#endif // Do not document + +CUB_NAMESPACE_END diff --git a/qwen3_6_scripts/cccl_preload/include/cub/util_cpp_dialect.cuh b/qwen3_6_scripts/cccl_preload/include/cub/util_cpp_dialect.cuh new file mode 100644 index 00000000..34fbcc46 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/util_cpp_dialect.cuh @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +//! @file +//! Detect the version of the C++ standard used by the compiler. + +#pragma once + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#ifndef _CCCL_DOXYGEN_INVOKED // Do not document + +// Deprecation warnings may be silenced by defining the following macros. These +// may be combined. +// - CCCL_IGNORE_DEPRECATED_COMPILER +// Ignore deprecation warnings when using deprecated compilers. Compiling +// with deprecated C++ dialects will still issue warnings. + +//! Deprecated [Since 3.0] +# define CUB_CPP_DIALECT _CCCL_STD_VER + +// Define CUB_COMPILER_DEPRECATION macro: +# if _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define CUB_COMP_DEPR_IMPL(msg) _CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " #msg)) +# else // clang / gcc: +# define CUB_COMP_DEPR_IMPL(msg) _CCCL_PRAGMA(GCC warning #msg) +# endif + +// Compiler checks: +// clang-format off +# define CUB_COMPILER_DEPRECATION(REQ) \ + CUB_COMP_DEPR_IMPL(CUB requires at least REQ. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message.) + +# define CUB_COMPILER_DEPRECATION_SOFT(REQ, CUR) \ + CUB_COMP_DEPR_IMPL( \ + CUB requires at least REQ. CUR is deprecated but still supported. CUR support will be removed in a \ + future release. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message.) +// clang-format on + +# ifndef CCCL_IGNORE_DEPRECATED_COMPILER +# if _CCCL_COMPILER(GCC, <, 7) +CUB_COMPILER_DEPRECATION(GCC 7.0); +# elif _CCCL_COMPILER(CLANG, <, 7) +CUB_COMPILER_DEPRECATION(Clang 7.0); +# elif _CCCL_COMPILER(MSVC, <, 19, 10) +// <2017. Hard upgrade message: +CUB_COMPILER_DEPRECATION(MSVC 2019(19.20 / 16.0 / 14.20)); +# endif +# endif // CCCL_IGNORE_DEPRECATED_COMPILER + +# undef CUB_COMPILER_DEPRECATION_SOFT +# undef CUB_COMPILER_DEPRECATION + +// C++17 dialect check: +# ifndef CCCL_IGNORE_DEPRECATED_CPP_DIALECT +# if _CCCL_STD_VER < 2017 +# error CUB requires at least C++17. Define CCCL_IGNORE_DEPRECATED_CPP_DIALECT to suppress this message. +# endif // _CCCL_STD_VER < 2017 +# endif + +# undef CUB_COMP_DEPR_IMPL + +#endif // !_CCCL_DOXYGEN_INVOKED diff --git a/qwen3_6_scripts/cccl_preload/include/cub/util_debug.cuh b/qwen3_6_scripts/cccl_preload/include/cub/util_debug.cuh new file mode 100644 index 00000000..bb58c46a --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/util_debug.cuh @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/** + * \file + * Error and event logging routines. + * + * The following macros definitions are supported: + * - \p CUB_LOG. Simple event messages are printed to \p stdout. + */ + +#pragma once + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes: + +/** + * @def CUB_DEBUG_LOG + * + * Causes kernel launch configurations to be printed to the console + */ +# define CUB_DEBUG_LOG + +/** + * @def CUB_DEBUG_SYNC + * + * Causes synchronization of the stream after every kernel launch to check + * for errors. Also causes kernel launch configurations to be printed to the + * console. + */ +# define CUB_DEBUG_SYNC + +/** + * @def CUB_DEBUG_ALL + * + * Causes host and device-side precondition assertions to be checked. Apart + * from that, causes synchronization of the stream after every kernel launch to + * check for errors. Also causes kernel launch configurations to be printed to + * the console. + */ +# define CUB_DEBUG_ALL + +#endif // _CCCL_DOXYGEN_INVOKED + +// CUB_DEBUG_SYNC also enables CUB_DEBUG_LOG +#ifdef CUB_DEBUG_SYNC +# ifndef CUB_DEBUG_LOG +# define CUB_DEBUG_LOG +# endif +#endif + +// CUB_DEBUG_ALL = CUB_DEBUG_LOG + CUB_DEBUG_SYNC +#ifdef CUB_DEBUG_ALL +# ifndef CUB_DEBUG_LOG +# define CUB_DEBUG_LOG +# endif // CUB_DEBUG_LOG +# ifndef CUB_DEBUG_SYNC +# define CUB_DEBUG_SYNC +# endif // CUB_DEBUG_SYNC +#endif // CUB_DEBUG_ALL + +/// CUB error reporting macro (prints error messages to stderr) +#if (defined(DEBUG) || defined(_DEBUG)) && !defined(CUB_STDERR) +# define CUB_STDERR +#endif + +#if defined(CUB_STDERR) || defined(CUB_DEBUG_LOG) +# include +#endif + +CUB_NAMESPACE_BEGIN + +/** + * \brief %If \p CUB_STDERR is defined and \p error is not \p cudaSuccess, the + * corresponding error message is printed to \p stderr (or \p stdout in device + * code) along with the supplied source context. + * + * \return The CUDA error. + */ +_CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t +Debug(cudaError_t error, [[maybe_unused]] const char* filename, [[maybe_unused]] int line) +{ + // Clear the global CUDA error state which may have been set by the last + // call. Otherwise, errors may "leak" to unrelated kernel launches. + + // clang-format off + #ifndef CUB_RDC_ENABLED + #define CUB_TEMP_DEVICE_CODE + #else + #define CUB_TEMP_DEVICE_CODE last_error = cudaGetLastError() + #endif + + cudaError_t last_error = cudaSuccess; + + NV_IF_ELSE_TARGET( + NV_IS_HOST, + (last_error = cudaGetLastError();), + (CUB_TEMP_DEVICE_CODE;) + ); + + #undef CUB_TEMP_DEVICE_CODE + // clang-format on + + if (error == cudaSuccess && last_error != cudaSuccess) + { + error = last_error; + } + +#ifdef CUB_STDERR + if (error) + { + NV_IF_ELSE_TARGET( + NV_IS_HOST, + (fprintf(stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error)); + fflush(stderr);), + (printf("CUDA error %d [block (%d,%d,%d) thread (%d,%d,%d), %s, %d]\n", + error, + blockIdx.z, + blockIdx.y, + blockIdx.x, + threadIdx.z, + threadIdx.y, + threadIdx.x, + filename, + line);)); + } +#endif + + return error; +} + +/** + * \brief Debug macro + */ +#ifndef CubDebug +# define CubDebug(e) CUB_NS_QUALIFIER::Debug((cudaError_t) (e), __FILE__, __LINE__) +#endif + +/** + * \brief Debug macro with exit + */ +#ifndef CubDebugExit +# define CubDebugExit(e) \ + if (CUB_NS_QUALIFIER::Debug((cudaError_t) (e), __FILE__, __LINE__)) \ + { \ + exit(1); \ + } +#endif + +/** + * \brief Log macro for printf statements. + */ +#if !defined(_CubLog) +# if _CCCL_HOSTJIT() +# define _CubLog(format, ...) (void(0)) +# else // ^^^ _CCCL_HOSTJIT() ^^^ / vvv !_CCCL_HOSTJIT() vvv +# define _CubLog(format, ...) \ + do \ + { \ + NV_IF_ELSE_TARGET( \ + NV_IS_HOST, \ + (printf(format, __VA_ARGS__);), \ + (printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, \ + blockIdx.z, \ + blockIdx.y, \ + blockIdx.x, \ + threadIdx.z, \ + threadIdx.y, \ + threadIdx.x, \ + __VA_ARGS__);)); \ + } while (false) +# endif // !_CCCL_HOSTJIT() +#endif // !defined(_CubLog) + +CUB_NAMESPACE_END diff --git a/qwen3_6_scripts/cccl_preload/include/cub/util_macro.cuh b/qwen3_6_scripts/cccl_preload/include/cub/util_macro.cuh new file mode 100644 index 00000000..8b9c43bd --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/util_macro.cuh @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2024, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/****************************************************************************** + * Common C/C++ macro utilities + ******************************************************************************/ + +#pragma once + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include // IWYU pragma: export +#include // IWYU pragma: export + +CUB_NAMESPACE_BEGIN + +#ifdef _CCCL_DOXYGEN_INVOKED +# define CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION +#endif + +/** + * @def CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION + * If defined, the default suppression of kernel visibility attribute warning is disabled. + */ +#if !defined(CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION) +_CCCL_DIAG_SUPPRESS_GCC("-Wattributes") +_CCCL_DIAG_SUPPRESS_CLANG("-Wattributes") +# if !_CCCL_CUDA_COMPILER(NVHPC) +_CCCL_DIAG_SUPPRESS_NVHPC(attribute_requires_external_linkage) +# endif // !_CCCL_CUDA_COMPILER(NVHPC) +#endif // !CUB_DISABLE_KERNEL_VISIBILITY_WARNING_SUPPRESSION + +#ifndef CUB_DEFINE_KERNEL_GETTER +# define CUB_DEFINE_KERNEL_GETTER(name, ...) \ + _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr decltype(&__VA_ARGS__) name() \ + { \ + return &__VA_ARGS__; \ + } +#endif + +// TODO(bgruber): drop in CCCL 4.0 when we drop the public dispatchers +#ifndef CUB_DEFINE_SUB_POLICY_GETTER +# define CUB_DEFINE_SUB_POLICY_GETTER(name) \ + _CCCL_HOST_DEVICE static constexpr auto name() \ + { \ + return MakePolicyWrapper(typename StaticPolicyT::name##Policy()); \ + } +#endif + +#if defined(CUB_DEFINE_RUNTIME_POLICIES) +# define CUB_DETAIL_STATIC_ISH_ASSERT(expr, msg) _CCCL_ASSERT(expr, msg) +# define CUB_DETAIL_CONSTEXPR_ISH +#else // ^^^ CUB_DEFINE_RUNTIME_POLICIES ^^^ / vvv !CUB_DEFINE_RUNTIME_POLICIES vvv +# define CUB_DETAIL_STATIC_ISH_ASSERT(expr, msg) static_assert(expr, msg); +# define CUB_DETAIL_CONSTEXPR_ISH constexpr +#endif // !(CUB_DEFINE_RUNTIME_POLICIES) + +CUB_NAMESPACE_END diff --git a/qwen3_6_scripts/cccl_preload/include/cub/util_namespace.cuh b/qwen3_6_scripts/cccl_preload/include/cub/util_namespace.cuh new file mode 100644 index 00000000..60ddf454 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/util_namespace.cuh @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/** + * \file util_namespace.cuh + * \brief Utilities that allow `cub::` to be placed inside an + * application-specific namespace. + */ + +#pragma once + +// This is not used by this file; this is a hack so that we can detect the +// CUB version from Thrust on older versions of CUB that did not have +// version.cuh. +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +// Prior to 1.13.1, only the PREFIX/POSTFIX macros were used. Notify users +// that they must now define the qualifier macro, too. +#if (defined(CUB_NS_PREFIX) || defined(CUB_NS_POSTFIX)) && !defined(CUB_NS_QUALIFIER) +# error CUB requires a definition of CUB_NS_QUALIFIER when CUB_NS_PREFIX/POSTFIX are defined. +#endif + +#ifdef _CCCL_DOXYGEN_INVOKED +# define THRUST_CUB_WRAPPED_NAMESPACE +#endif + +/** + * \def THRUST_CUB_WRAPPED_NAMESPACE + * If defined, this value will be used as the name of a namespace that wraps the + * `thrust::` and `cub::` namespaces. + * This macro should not be used with any other CUB namespace macros. + */ +#ifdef THRUST_CUB_WRAPPED_NAMESPACE +# define CUB_WRAPPED_NAMESPACE THRUST_CUB_WRAPPED_NAMESPACE +#endif + +#ifdef _CCCL_DOXYGEN_INVOKED +# define CUB_WRAPPED_NAMESPACE +#endif + +/** + * \def CUB_WRAPPED_NAMESPACE + * If defined, this value will be used as the name of a namespace that wraps the + * `cub::` namespace. + * If THRUST_CUB_WRAPPED_NAMESPACE is set, this will inherit that macro's value. + * This macro should not be used with any other CUB namespace macros. + */ +#ifdef CUB_WRAPPED_NAMESPACE +# define CUB_NS_PREFIX \ + namespace CUB_WRAPPED_NAMESPACE \ + { +# define CUB_NS_POSTFIX } + +# define CUB_NS_QUALIFIER ::CUB_WRAPPED_NAMESPACE::cub +#endif + +#ifdef _CCCL_DOXYGEN_INVOKED +# define CUB_NS_PREFIX +#endif + +/** + * \def CUB_NS_PREFIX + * This macro is inserted prior to all `namespace cub { ... }` blocks. It is + * derived from CUB_WRAPPED_NAMESPACE, if set, and will be empty otherwise. + * It may be defined by users, in which case CUB_NS_PREFIX, + * CUB_NS_POSTFIX, and CUB_NS_QUALIFIER must all be set consistently. + */ +#ifndef CUB_NS_PREFIX +# define CUB_NS_PREFIX +#endif + +#ifdef _CCCL_DOXYGEN_INVOKED +# define CUB_NS_POSTFIX +#endif + +/** + * \def CUB_NS_POSTFIX + * This macro is inserted following the closing braces of all + * `namespace cub { ... }` block. It is defined appropriately when + * CUB_WRAPPED_NAMESPACE is set, and will be empty otherwise. It may be + * defined by users, in which case CUB_NS_PREFIX, CUB_NS_POSTFIX, and + * CUB_NS_QUALIFIER must all be set consistently. + */ +#ifndef CUB_NS_POSTFIX +# define CUB_NS_POSTFIX +#endif + +#ifdef _CCCL_DOXYGEN_INVOKED +# define CUB_NS_QUALIFIER +#endif + +/** + * \def CUB_NS_QUALIFIER + * This macro is used to qualify members of cub:: when accessing them from + * outside of their namespace. By default, this is just `::cub`, and will be + * set appropriately when CUB_WRAPPED_NAMESPACE is defined. This macro may be + * defined by users, in which case CUB_NS_PREFIX, CUB_NS_POSTFIX, and + * CUB_NS_QUALIFIER must all be set consistently. + */ +#ifndef CUB_NS_QUALIFIER +# define CUB_NS_QUALIFIER ::cub +#endif + +#if defined(CUB_DISABLE_NAMESPACE_MAGIC) || defined(CUB_WRAPPED_NAMESPACE) +# if !defined(CUB_WRAPPED_NAMESPACE) +# if !defined(CUB_IGNORE_NAMESPACE_MAGIC_ERROR) +# error "Disabling namespace magic is unsafe without wrapping namespace" +# endif // !defined(CUB_IGNORE_NAMESPACE_MAGIC_ERROR) +# endif // !defined(CUB_WRAPPED_NAMESPACE) +# define CUB_DETAIL_MAGIC_NS_BEGIN +# define CUB_DETAIL_MAGIC_NS_END +#else // not defined(CUB_DISABLE_NAMESPACE_MAGIC) +# if defined(_NVHPC_CUDA) +# define CUB_DETAIL_MAGIC_NS_BEGIN \ + inline namespace _CCCL_PP_CAT( \ + _CCCL_PP_CAT(_CCCL_PP_CAT(_V_, CUB_VERSION), _CCCL_PP_SPLICE_WITH(_, _SM, NV_TARGET_SM_INTEGER_LIST)), _NVHPC) \ + { +# define CUB_DETAIL_MAGIC_NS_END } +# else // not defined(_NVHPC_CUDA) +# define CUB_DETAIL_MAGIC_NS_BEGIN \ + inline namespace _CCCL_PP_CAT(_CCCL_PP_CAT(_V_, CUB_VERSION), _CCCL_PP_SPLICE_WITH(_, _SM, __CUDA_ARCH_LIST__)) \ + { +# define CUB_DETAIL_MAGIC_NS_END } +# endif // not defined(_NVHPC_CUDA) +#endif // not defined(CUB_DISABLE_NAMESPACE_MAGIC) + +/** + * \def CUB_NAMESPACE_BEGIN + * This macro is used to open a `cub::` namespace block, along with any + * enclosing namespaces requested by CUB_WRAPPED_NAMESPACE, etc. + * This macro is defined by CUB and may not be overridden. + */ +#define CUB_NAMESPACE_BEGIN \ + CUB_NS_PREFIX \ + namespace cub \ + { \ + CUB_DETAIL_MAGIC_NS_BEGIN + +/** + * \def CUB_NAMESPACE_END + * This macro is used to close a `cub::` namespace block, along with any + * enclosing namespaces requested by CUB_WRAPPED_NAMESPACE, etc. + * This macro is defined by CUB and may not be overridden. + */ +#define CUB_NAMESPACE_END \ + CUB_DETAIL_MAGIC_NS_END \ + } /* end namespace cub */ \ + CUB_NS_POSTFIX + +// Declare these namespaces here for the purpose of Doxygenating them +CUB_NS_PREFIX + +/*! \namespace cub + * \brief \p cub is the top-level namespace which contains all CUB + * functions and types. + */ +namespace cub +{ +} + +CUB_NS_POSTFIX diff --git a/qwen3_6_scripts/cccl_preload/include/cub/version.cuh b/qwen3_6_scripts/cccl_preload/include/cub/version.cuh new file mode 100644 index 00000000..986c2575 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cub/version.cuh @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. +// SPDX-License-Identifier: BSD-3 + +/*! \file version.cuh + * \brief Compile-time macros encoding CUB release version + * + * is the only CUB header that is guaranteed to + * change with every CUB release. + * + */ + +#pragma once + +// For _CCCL_IMPLICIT_SYSTEM_HEADER +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +/*! \def CUB_VERSION + * \brief The preprocessor macro \p CUB_VERSION encodes the version + * number of the CUB library as MMMmmmpp. + * + * \note CUB_VERSION is formatted as `MMMmmmpp`, which differs from `CCCL_VERSION` that uses `MMMmmmppp`. + * + * CUB_VERSION % 100 is the sub-minor version. + * CUB_VERSION / 100 % 1000 is the minor version. + * CUB_VERSION / 100000 is the major version. + */ +#define CUB_VERSION 300500 // macro expansion with ## requires this to be a single value + +/*! \def CUB_MAJOR_VERSION + * \brief The preprocessor macro \p CUB_MAJOR_VERSION encodes the + * major version number of the CUB library. + */ +#define CUB_MAJOR_VERSION (CUB_VERSION / 100000) + +/*! \def CUB_MINOR_VERSION + * \brief The preprocessor macro \p CUB_MINOR_VERSION encodes the + * minor version number of the CUB library. + */ +#define CUB_MINOR_VERSION (CUB_VERSION / 100 % 1000) + +/*! \def CUB_SUBMINOR_VERSION + * \brief The preprocessor macro \p CUB_SUBMINOR_VERSION encodes the + * sub-minor version number of the CUB library. + */ +#define CUB_SUBMINOR_VERSION (CUB_VERSION % 100) + +/*! \def CUB_PATCH_NUMBER + * \brief The preprocessor macro \p CUB_PATCH_NUMBER encodes the + * patch number of the CUB library. + */ +#define CUB_PATCH_NUMBER 0 + +static_assert(CUB_MAJOR_VERSION == CCCL_MAJOR_VERSION); +static_assert(CUB_MINOR_VERSION == CCCL_MINOR_VERSION); +static_assert(CUB_SUBMINOR_VERSION == CCCL_PATCH_VERSION); diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__cccl_config b/qwen3_6_scripts/cccl_preload/include/cuda/__cccl_config new file mode 100644 index 00000000..4ff35e24 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__cccl_config @@ -0,0 +1,38 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA__CCCL_CONFIG +#define _CUDA__CCCL_CONFIG + +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export + +#endif // _CUDA__CCCL_CONFIG diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__cmath/ceil_div.h b/qwen3_6_scripts/cccl_preload/include/cuda/__cmath/ceil_div.h new file mode 100644 index 00000000..139ebf34 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__cmath/ceil_div.h @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___CMATH_CEIL_DIV_H +#define _CUDA___CMATH_CEIL_DIV_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA + +//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder +//! @param __a The dividend +//! @param __b The divisor +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>) +[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, _Up> ceil_div(const _Tp __a, const _Up __b) noexcept +{ + _CCCL_ASSERT(__b > _Up{0}, "cuda::ceil_div: 'b' must be positive"); + if constexpr (::cuda::std::is_signed_v<_Tp>) + { + _CCCL_ASSERT(__a >= _Tp{0}, "cuda::ceil_div: 'a' must be non negative"); + } + using _Common = ::cuda::std::common_type_t<_Tp, _Up>; + using _Prom = decltype(_Tp{} / _Up{}); + using _UProm = ::cuda::std::make_unsigned_t<_Prom>; + auto __a1 = static_cast<_UProm>(__a); + auto __b1 = static_cast<_UProm>(__b); + if constexpr (::cuda::std::is_signed_v<_Prom>) + { + return static_cast<_Common>((__a1 + __b1 - 1) / __b1); + } + else + { + _CCCL_IF_CONSTEVAL_DEFAULT + { + const auto __res = __a1 / __b1; + return static_cast<_Common>(__res + (__res * __b1 != __a1)); + } + else + { + // the ::min method is faster even if __b is a compile-time constant + NV_IF_ELSE_TARGET(NV_IS_DEVICE, + (return static_cast<_Common>(::cuda::std::min(__a1, 1 + ((__a1 - 1) / __b1)));), + (const auto __res = __a1 / __b1; // + return static_cast<_Common>(__res + (__res * __b1 != __a1));)) + } + } +} + +//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder, \p __b is an enum +//! @param __a The dividend +//! @param __b The divisor +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>) +[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, ::cuda::std::underlying_type_t<_Up>> +ceil_div(const _Tp __a, const _Up __b) noexcept +{ + return ::cuda::ceil_div(__a, ::cuda::std::to_underlying(__b)); +} + +//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder, \p __b is an enum +//! @param __a The dividend +//! @param __b The divisor +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>) +[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, _Up> +ceil_div(const _Tp __a, const _Up __b) noexcept +{ + return ::cuda::ceil_div(::cuda::std::to_underlying(__a), __b); +} + +//! @brief Divides two numbers \p __a and \p __b, rounding up if there is a remainder, \p __b is an enum +//! @param __a The dividend +//! @param __b The divisor +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>) +[[nodiscard]] +_CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, ::cuda::std::underlying_type_t<_Up>> +ceil_div(const _Tp __a, const _Up __b) noexcept +{ + return ::cuda::ceil_div(::cuda::std::to_underlying(__a), ::cuda::std::to_underlying(__b)); +} + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // _CUDA___CMATH_CEIL_DIV_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__cmath/round_up.h b/qwen3_6_scripts/cccl_preload/include/cuda/__cmath/round_up.h new file mode 100644 index 00000000..91e8f3f3 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__cmath/round_up.h @@ -0,0 +1,104 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___CMATH_ROUND_UP_H +#define _CUDA___CMATH_ROUND_UP_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA + +//! @brief Round the number \p __a to the next multiple of \p __b +//! @param __a The input number +//! @param __b The multiplicand +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>) +[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, _Up> round_up(const _Tp __a, const _Up __b) noexcept +{ + _CCCL_ASSERT(__b > _Up{0}, "cuda::round_up: 'b' must be positive"); + if constexpr (::cuda::std::is_signed_v<_Tp>) + { + _CCCL_ASSERT(__a >= _Tp{0}, "cuda::round_up: 'a' must be non negative"); + } + using _Common = ::cuda::std::common_type_t<_Tp, _Up>; + using _Prom = decltype(_Tp{} / _Up{}); + auto __c = ::cuda::ceil_div(static_cast<_Prom>(__a), static_cast<_Prom>(__b)); + _CCCL_ASSERT(static_cast<_Common>(__c) <= ::cuda::std::numeric_limits<_Common>::max() / static_cast<_Common>(__b), + "cuda::round_up: result overflow"); + return static_cast<_Common>(static_cast<_Prom>(__c) * static_cast<_Prom>(__b)); +} + +//! @brief Round the number \p __a to the next multiple of \p __b +//! @param __a The input number +//! @param __b The multiplicand +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>) +[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<_Tp, ::cuda::std::underlying_type_t<_Up>> +round_up(const _Tp __a, const _Up __b) noexcept +{ + return ::cuda::round_up(__a, ::cuda::std::to_underlying(__b)); +} + +//! @brief Round the number \p __a to the next multiple of \p __b +//! @param __a The input number +//! @param __b The multiplicand +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_integral_v<_Up>) +[[nodiscard]] _CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, _Up> +round_up(const _Tp __a, const _Up __b) noexcept +{ + return ::cuda::round_up(::cuda::std::to_underlying(__a), __b); +} + +//! @brief Round the number \p __a to the next multiple of \p __b +//! @param __a The input number +//! @param __b The multiplicand +//! @pre \p __a must be non-negative +//! @pre \p __b must be positive +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES(::cuda::std::is_enum_v<_Tp> _CCCL_AND ::cuda::std::is_enum_v<_Up>) +[[nodiscard]] +_CCCL_API constexpr ::cuda::std::common_type_t<::cuda::std::underlying_type_t<_Tp>, ::cuda::std::underlying_type_t<_Up>> +round_up(const _Tp __a, const _Up __b) noexcept +{ + return ::cuda::round_up(::cuda::std::to_underlying(__a), ::cuda::std::to_underlying(__b)); +} + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // _CUDA___CMATH_ROUND_UP_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__device/compute_capability.h b/qwen3_6_scripts/cccl_preload/include/cuda/__device/compute_capability.h new file mode 100644 index 00000000..fcf7407d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__device/compute_capability.h @@ -0,0 +1,272 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___DEVICE_COMPUTE_CAPABILITY_H +#define _CUDA___DEVICE_COMPUTE_CAPABILITY_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA + +//! @brief Type representing the CUDA compute capability. +class compute_capability +{ +public: + int __cc_{}; //!< The stored compute capability in format 10 * major + minor. + + _CCCL_HIDE_FROM_ABI constexpr compute_capability() noexcept = default; + + //! @brief Constructs the object from compute capability \c __cc. The expected format is 10 * major + minor. + //! + //! @param __cc Compute capability. + _CCCL_HOST_DEVICE_API explicit constexpr compute_capability(int __cc) noexcept + : __cc_{__cc} + {} + + //! @brief Constructs the object by combining the \c __major and \c __minor compute capability. + //! + //! @param __major The major compute capability. + //! @param __minor The minor compute capability. Must be less than 10. + _CCCL_HOST_DEVICE_API constexpr compute_capability(int __major, int __minor) noexcept + : __cc_{10 * __major + __minor} + { + _CCCL_ASSERT(__minor < 10, "invalid minor compute capability"); + } + + //! @brief Constructs the object from the architecture id. + //! + //! @param __arch_id The architecture id. + _CCCL_HOST_DEVICE_API explicit constexpr compute_capability(arch_id __arch_id) noexcept + { + const auto __val = ::cuda::std::to_underlying(__arch_id); + if (__val > __arch_specific_id_multiplier) + { + __cc_ = __val / __arch_specific_id_multiplier; + } + else + { + __cc_ = __val; + } + } + + _CCCL_HIDE_FROM_ABI constexpr compute_capability(const compute_capability&) noexcept = default; + + _CCCL_HIDE_FROM_ABI constexpr compute_capability& operator=(const compute_capability& __other) noexcept = default; + + //! @brief Gets the stored compute capability. + //! + //! @return The stored compute capability in format 10 * major + minor. + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int get() const noexcept + { + return __cc_; + } + + //! @brief Gets the major compute capability. + //! + //! @return Major compute capability. + //! + //! @deprecated This symbol is deprecated because it collides with major(...) macro defined in and + //! will be removed in next major release. Use cc.major_cap() instead. + [[nodiscard]] + CCCL_DEPRECATED_BECAUSE("This symbol is deprecated because it collides with major(...) macro defined in " + " and will be removed in next major release. Use cc.major_cap() instead.") + _CCCL_HOST_DEVICE_API constexpr int major() const noexcept + { + return major_cap(); + } + + //! @brief Gets the major compute capability. + //! + //! @return Major compute capability. + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int major_cap() const noexcept + { + return __cc_ / 10; + } + + //! @brief Gets the minor compute capability. + //! + //! @return Minor compute capability. The value is always less than 10. + //! + //! @deprecated This symbol is deprecated because it collides with minor(...) macro defined in and + //! will be removed in next major release. Use cc.minor_cap() instead. + [[nodiscard]] + CCCL_DEPRECATED_BECAUSE("This symbol is deprecated because it collides with minor(...) macro defined in " + " and will be removed in next major release. Use cc.minor_cap() instead.") + _CCCL_HOST_DEVICE_API constexpr int minor() const noexcept + { + return minor_cap(); + } + + //! @brief Gets the minor compute capability. + //! + //! @return Minor compute capability. The value is always less than 10. + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int minor_cap() const noexcept + { + return __cc_ % 10; + } + + //! @brief Conversion operator to \c int. + //! + //! @return The stored compute capability in format 10 * major + minor. + _CCCL_HOST_DEVICE_API explicit constexpr operator int() const noexcept + { + return __cc_; + } + + //! @brief Equality operator. + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator==(compute_capability __lhs, compute_capability __rhs) noexcept + { + return __lhs.__cc_ == __rhs.__cc_; + } + + //! @brief Inequality operator. + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator!=(compute_capability __lhs, compute_capability __rhs) noexcept + { + return __lhs.__cc_ != __rhs.__cc_; + } + + //! @brief Less than operator. + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator<(compute_capability __lhs, compute_capability __rhs) noexcept + { + return __lhs.__cc_ < __rhs.__cc_; + } + + //! @brief Less than or equal to operator. + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator<=(compute_capability __lhs, compute_capability __rhs) noexcept + { + return __lhs.__cc_ <= __rhs.__cc_; + } + + //! @brief Greater than operator. + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator>(compute_capability __lhs, compute_capability __rhs) noexcept + { + return __lhs.__cc_ > __rhs.__cc_; + } + + //! @brief Greater than or equal to operator. + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator>=(compute_capability __lhs, compute_capability __rhs) noexcept + { + return __lhs.__cc_ >= __rhs.__cc_; + } +}; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __make_all_compute_capabilities() noexcept +{ + return ::cuda::std::array{compute_capability{_Vs}...}; +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __all_compute_capabilities() noexcept +{ + return ::cuda::__make_all_compute_capabilities<_CCCL_KNOWN_CUDA_ARCH_LIST>(); +} + +#if _CCCL_CUDA_COMPILATION() +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __make_cc_list() noexcept +{ +# if defined(__CUDA_ARCH_LIST__) + return ::cuda::std::array{compute_capability{_Vs / 10}...}; +# elif defined(NV_TARGET_SM_INTEGER_LIST) + return ::cuda::std::array{compute_capability{_Vs}...}; +# else // ^^^ has arch list ^^^ / vvv no arch list vvv + static_assert(::cuda::std::__always_false_v, + "This function can be instantiated only when __CUDA_ARCH_LIST__ or NV_TARGET_SM_INTEGER_LIST are " + "defined"); +# endif // ^^^ no arch list ^^^ +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __target_compute_capabilities() noexcept +{ +# if defined(__CUDA_ARCH_LIST__) + return ::cuda::__make_cc_list<__CUDA_ARCH_LIST__>(); +# elif defined(NV_TARGET_SM_INTEGER_LIST) + return ::cuda::__make_cc_list(); +# else // ^^^ has arch list ^^^ / vvv no arch list vvv + // Fallback to a list of all compute capabilities. + return ::cuda::__all_compute_capabilities(); +# endif // ^^^ no arch list ^^^ +} +#endif // _CCCL_CUDA_COMPILATION() + +_CCCL_END_NAMESPACE_CUDA + +#if __cpp_lib_format >= 201907L +_CCCL_BEGIN_NAMESPACE_STD + +template +struct formatter<::cuda::compute_capability, _CharT> : private formatter +{ + template + _CCCL_HOST_API constexpr auto parse(_ParseCtx& __ctx) + { + return __ctx.begin(); + } + + template + _CCCL_HOST_API auto format(const ::cuda::compute_capability& __cc, _FmtCtx& __ctx) const + { + return formatter::format(__cc.get(), __ctx); + } +}; + +_CCCL_END_NAMESPACE_STD +#endif // __cpp_lib_format >= 201907L + +// todo: specialize cuda::std::formatter for cuda::compute_capability + +#if _CCCL_CUDA_COMPILATION() + +_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE + +//! @brief Returns the \c cuda::compute_capability that is currently being compiled. +//! +//! @note This API cannot be used in constexpr context when compiling with nvc++ in CUDA mode. +[[nodiscard]] _CCCL_DEVICE_API inline _CCCL_TARGET_CONSTEXPR ::cuda::compute_capability +current_compute_capability() noexcept +{ +# if _CCCL_CUDA_COMPILER(NVHPC) + return ::cuda::compute_capability{__builtin_current_device_sm()}; +# elif _CCCL_DEVICE_COMPILATION() + return ::cuda::compute_capability{__CUDA_ARCH__ / 10}; +# else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv + return {}; +# endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^ +} + +_CCCL_END_NAMESPACE_CUDA_DEVICE + +#endif // _CCCL_CUDA_COMPILATION() + +#include + +#endif // _CUDA___DEVICE_COMPUTE_CAPABILITY_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__fwd/complex.h b/qwen3_6_scripts/cccl_preload/include/cuda/__fwd/complex.h new file mode 100644 index 00000000..bc299c1c --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__fwd/complex.h @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___FWD_COMPLEX_H +#define _CUDA___FWD_COMPLEX_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT complex; + +// __is_cuda_complex_v + +template +inline constexpr bool __is_cuda_complex_v = false; +template +inline constexpr bool __is_cuda_complex_v = __is_cuda_complex_v<_Tp>; +template +inline constexpr bool __is_cuda_complex_v = __is_cuda_complex_v<_Tp>; +template +inline constexpr bool __is_cuda_complex_v = __is_cuda_complex_v<_Tp>; +template +inline constexpr bool __is_cuda_complex_v> = true; + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // _CUDA___FWD_COMPLEX_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__fwd/devices.h b/qwen3_6_scripts/cccl_preload/include/cuda/__fwd/devices.h new file mode 100644 index 00000000..3158b66e --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__fwd/devices.h @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___FWD_DEVICES_H +#define _CUDA___FWD_DEVICES_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA + +#if _CCCL_HAS_CTK() +class __physical_device; +class device_ref; +template <::cudaDeviceAttr _Attr> +struct __dev_attr; +#endif // _CCCL_HAS_CTK() + +struct arch_traits_t; +class compute_capability; +enum class arch_id : int; + +inline constexpr int __arch_specific_id_multiplier = 100000; + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // _CUDA___FWD_DEVICES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__memory/address_space.h b/qwen3_6_scripts/cccl_preload/include/cuda/__memory/address_space.h new file mode 100644 index 00000000..383b71e3 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__memory/address_space.h @@ -0,0 +1,259 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___MEMORY_ADDRESS_SPACE_H +#define _CUDA___MEMORY_ADDRESS_SPACE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_CUDA_COMPILATION() + +# include +# include + +# include + +# include + +_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE + +//! @brief Address space enumeration for CUDA device code. +//! +//! See https://docs.nvidia.com/cuda/parallel-thread-execution/#state-spaces for more details. +enum class address_space +{ + global, //!< Global state space + shared, //!< Shared state space + constant, //!< Constant state space + local, //!< Local state space + grid_constant, //!< Kernel function parameter in the parameter state space + cluster_shared, //!< Cluster shared window within the shared state space + __max, +}; + +[[nodiscard]] _CCCL_DEVICE_API constexpr bool __cccl_is_valid_address_space(address_space __space) noexcept +{ + const auto __v = ::cuda::std::to_underlying(__space); + return __v >= 0 && __v < ::cuda::std::to_underlying(address_space::__max); +} + +[[nodiscard]] _CCCL_DEVICE_API inline bool __is_smem_valid_ptr(const void* __ptr) noexcept +{ + NV_IF_TARGET(NV_PROVIDES_SM_90, (return __ptr != nullptr;), (return true;)); +} + +//! @brief Checks if the given pointer is from the specified address state space. +//! @param __ptr The address to check. +//! @param __space The address state space to check against. +//! @return `true` if the pointer is from the specified address space, `false` otherwise. +[[nodiscard]] _CCCL_DEVICE_API inline bool __internal_is_address_from(const void* __ptr, address_space __space) noexcept +{ + _CCCL_ASSERT(::cuda::device::__cccl_is_valid_address_space(__space), "invalid address space"); + // NVCC and NVRTC < 12.3 have problems tracking the address space of pointers, fallback to inline PTX for them + switch (__space) + { + case address_space::global: { +# if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) + unsigned __ret; + asm volatile( + "{\n\t" + " .reg .pred p;\n\t" + " isspacep.global p, %1;\n\t" + " selp.u32 %0, 1, 0, p;\n\t" + "}\n\t" + : "=r"(__ret) + : "l"(__ptr)); + return static_cast(__ret); +# else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ / + // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv + bool __p = static_cast(::__isGlobal(__ptr)); + if (__p) + { + _CCCL_ASSUME(__p); + } + return __p; +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ + } + case address_space::constant: { +# if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) + unsigned __ret; + asm volatile( + "{\n\t" + " .reg .pred p;\n\t" + " isspacep.const p, %1;\n\t" + " selp.u32 %0, 1, 0, p;\n\t" + "}\n\t" + : "=r"(__ret) + : "l"(__ptr)); + return static_cast(__ret); +# else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ / + // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv + bool __p = static_cast(::__isConstant(__ptr)); + if (__p) + { + _CCCL_ASSUME(__p); + } + return __p; +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ + } + case address_space::local: { + // __isLocal is buggy until CUDA 13.1, see nvbug 5254298 +# if _CCCL_CUDA_COMPILER(NVCC, <, 13, 1) || _CCCL_CUDA_COMPILER(NVRTC, <, 13, 1) + unsigned __ret; + asm volatile( + "{\n\t" + " .reg .pred p;\n\t" + " isspacep.local p, %1;\n\t" + " selp.u32 %0, 1, 0, p;\n\t" + "}\n\t" + : "=r"(__ret) + : "l"(__ptr)); + return static_cast(__ret); +# else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 13, 1) || _CCCL_CUDA_COMPILER(NVRTC, <, 13, 1) ^^^ / + // vvv !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) vvv + bool __p = static_cast(::__isLocal(__ptr)); + if (__p) + { + _CCCL_ASSUME(__p); + } + return __p; +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) ^^^ + } + case address_space::grid_constant: { +# if _CCCL_CUDA_COMPILER(NVCC, >=, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 3) + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_70, + (bool __p = static_cast(::__isGridConstant(__ptr)); // + if (__p) // + { // + _CCCL_ASSUME(__p); // + } // + return __p;), + (return false;)) +# else // ^^^ has functional __isGridConstant() ^^^ / vvv no functional __isGridConstant() vvv + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_70, + (unsigned __ret; // + asm volatile("{\n\t" + " .reg .pred p;\n\t" + " isspacep.param p, %1;\n\t" + " selp.u32 %0, 1, 0, p;\n\t" + "}\n\t" : "=r"(__ret) : "l"(__ptr)); + return static_cast(__ret);), + (return false;)) +# endif // ^^^ no functional __isGridConstant() ^^^ + } + case address_space::cluster_shared: { +# if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_90, + (unsigned __ret; // + asm volatile("{\n\t" + " .reg .pred p;\n\t" + " isspacep.shared::cluster p, %1;\n\t" + " selp.u32 %0, 1, 0, p;\n\t" + "}\n\t" : "=r"(__ret) : "l"(__ptr)); + return static_cast(__ret);), + ([[fallthrough]]; /* to `case shared:` */)) +# else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ / + // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_90, + (bool __p = static_cast(::__isClusterShared(__ptr)); // + if (__p) // + { // + _CCCL_ASSUME(__p); // + } // + return __p;), + ([[fallthrough]]; /* to `case shared:` */)) +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ + } + case address_space::shared: { + // smem can start at address 0x0 before sm_90 +# if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) + unsigned __ret; + asm volatile( + "{\n\t" + " .reg .pred p;\n\t" + " isspacep.shared p, %1;\n\t" + " selp.u32 %0, 1, 0, p;\n\t" + "}\n\t" + : "=r"(__ret) + : "l"(__ptr)); + return static_cast(__ret); +# else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ / + // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv + bool __p = static_cast(::__isShared(__ptr)); + if (__p) + { + _CCCL_ASSUME(__p); + } + return __p; +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ + } + default: + return false; + } +} + +//! @brief Checks if the given pointer is from the specified address state space. +//! @param __ptr The address to check. +//! @param __space The address state space to check against. +//! @return `true` if the pointer is from the specified address space, `false` otherwise. +[[nodiscard]] _CCCL_DEVICE_API inline bool is_address_from(const void* __ptr, address_space __space) noexcept +{ + // The debug assertions intentionally differ but compile out in release builds. + // NOLINTBEGIN(bugprone-branch-clone) + if (__space == address_space::shared) + { + _CCCL_ASSERT(::cuda::device::__is_smem_valid_ptr(__ptr), "invalid pointer"); + } + else + { + _CCCL_ASSERT(__ptr != nullptr, "invalid pointer"); + } + // NOLINTEND(bugprone-branch-clone) + return ::cuda::device::__internal_is_address_from(__ptr, __space); +} + +//! @brief Checks if the given pointer is from the specified address state space. +//! @param __ptr The address to check. +//! @param __space The address state space to check against. +//! @return `true` if the pointer is from the specified address space, `false` otherwise. +[[nodiscard]] _CCCL_DEVICE_API inline bool is_address_from(const volatile void* __ptr, address_space __space) noexcept +{ + return ::cuda::device::is_address_from(const_cast(__ptr), __space); +} + +//! @brief Checks if the given object is from the specified address state space. +//! @param __obj The object to check. +//! @param __space The address state space to check against. +//! @return `true` if the object is from the specified address space, `false` otherwise. +template +[[nodiscard]] _CCCL_DEVICE_API inline bool is_object_from(_Tp& __obj, address_space __space) noexcept +{ + return ::cuda::device::is_address_from(::cuda::std::addressof(__obj), __space); +} + +_CCCL_END_NAMESPACE_CUDA_DEVICE + +# include + +#endif // _CCCL_CUDA_COMPILATION() + +#endif // _CUDA___MEMORY_ADDRESS_SPACE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__memory/check_address.h b/qwen3_6_scripts/cccl_preload/include/cuda/__memory/check_address.h new file mode 100644 index 00000000..cd223c26 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__memory/check_address.h @@ -0,0 +1,111 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___MEMORY_IS_VALID_ADDRESS +#define _CUDA___MEMORY_IS_VALID_ADDRESS + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#if _CCCL_CUDA_COMPILATION() +# include +# include +#endif // _CCCL_CUDA_COMPILATION() + +#include + +#include + +#if _CCCL_CUDA_COMPILATION() + +_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE + +[[nodiscard]] _CCCL_DEVICE_API inline bool +__is_smem_valid_address_range(const void* __ptr, ::cuda::std::size_t __n) noexcept +{ + if (!::cuda::device::__is_smem_valid_ptr(__ptr)) + { + return false; + } + if (!::cuda::device::__internal_is_address_from(__ptr, ::cuda::device::address_space::shared)) + { + return false; + } + // if __ptr is a shared memory pointer, __ptr + __n must also be a valid shared memory pointer + if (!::cuda::device::__internal_is_address_from( + reinterpret_cast(__ptr) + __n, ::cuda::device::address_space::shared)) + { + return false; + } + return (__n <= ::cuda::ptx::get_sreg_total_smem_size()); +} + +_CCCL_END_NAMESPACE_CUDA_DEVICE + +#endif // _CCCL_CUDA_COMPILATION() + +_CCCL_BEGIN_NAMESPACE_CUDA + +[[nodiscard]] _CCCL_API inline bool __is_valid_address_range(const void* __ptr, ::cuda::std::size_t __n) noexcept +{ + if (__n == 0) + { + return false; + } + + // use (~::cuda::std::uintptr_t{0}) instead of cuda::std::numeric_limits::max() to avoid + // circular dependency because: + // numeric_limits -> bit_cast -> cstring -> check_address + // also includes cuda/std/limits + const auto __limit = (~::cuda::std::uintptr_t{0}) - static_cast<::cuda::std::uintptr_t>(__n); + + if (reinterpret_cast<::cuda::std::uintptr_t>(__ptr) > __limit) + { + return false; + } + NV_IF_TARGET(NV_IS_DEVICE, ({ + if (::cuda::device::__internal_is_address_from(__ptr, ::cuda::device::address_space::shared) + && !::cuda::device::__is_smem_valid_address_range(__ptr, __n)) + { + return false; + } + })); + return (__ptr != nullptr); +} + +[[nodiscard]] _CCCL_API inline bool __is_valid_address(const void* __ptr) noexcept +{ + return ::cuda::__is_valid_address_range(__ptr, 0); +} + +[[nodiscard]] _CCCL_API inline bool +__are_ptrs_overlapping(const void* __ptr_lhs, const void* __ptr_rhs, ::cuda::std::size_t __n) noexcept +{ + const auto __ptr1_start = static_cast(__ptr_lhs); + const auto __ptr2_start = static_cast(__ptr_rhs); + const auto __ptr1_end = __ptr1_start + __n; + const auto __ptr2_end = __ptr2_start + __n; + return __ptr1_start < __ptr2_end && __ptr2_start < __ptr1_end; +} + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // _CUDA___MEMORY_IS_VALID_ADDRESS diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx.h b/qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx.h new file mode 100644 index 00000000..5e7780c9 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx.h @@ -0,0 +1,150 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___NVTX_NVTX_H +#define _CUDA___NVTX_NVTX_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes: +//! When this macro is defined, no NVTX ranges are emitted by CCCL +# define CCCL_DISABLE_NVTX +#endif // _CCCL_DOXYGEN_INVOKED + +#define _CCCL_HAS_NVTX3() 0 + +// Enable the functionality of this header if: +// * The NVTX3 C API is available in CTK +// * NVTX is not explicitly disabled (via CCCL_DISABLE_NVTX or NVTX_DISABLE) +// * the compiler is not nvc++ (NVTX3 uses module as an identifier, which trips up NVHPC, fixed in CTK >= 13.0) +// * the compiler is not NVRTC +#if __has_include() && !defined(CCCL_DISABLE_NVTX) && !defined(NVTX_DISABLE) \ + && (!_CCCL_COMPILER(NVHPC) || _CCCL_CTK_AT_LEAST(13, 0)) \ + && !_CCCL_COMPILER(NVRTC) + +// Since NVTX 3.2, the NVTX headers can declare themselves as system headers by declaring the following macro: +# ifdef NVTX_AS_SYSTEM_HEADER +# define NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER +# else // NVTX_AS_SYSTEM_HEADER +# define NVTX_AS_SYSTEM_HEADER +# endif // NVTX_AS_SYSTEM_HEADER + +// Include our NVTX3 C++ wrapper if not available from the CTK or not provided by the user +// Note: NVTX3 is available in the CTK since 12.9, so we can drop our copy once this is the minimum supported version +# if __has_include() +# include +# else // __has_include() +# include +# endif // __has_include() + +# ifndef NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER +# undef NVTX_AS_SYSTEM_HEADER +# endif // NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER +# undef NVTX_AS_SYSTEM_HEADER_DEFINED_BY_USER + +// We expect the NVTX3 V1 C++ API to be available when nvtx3.hpp is available. This should work, because newer versions +// of NVTX3 will continue to declare previous API versions. See also: +// https://github.com/NVIDIA/NVTX/blob/release-v3/c/include/nvtx3/nvtx3.hpp#L2835-L2841. +# ifdef NVTX3_CPP_DEFINITIONS_V1_0 +# undef _CCCL_HAS_NVTX3 +# define _CCCL_HAS_NVTX3() 1 +# else // NVTX3_CPP_DEFINITIONS_V1_0 +// If this happens NVTX3 changed in a way we did not anticipate, and we need to get in touch with them +# if _CCCL_COMPILER(MSVC) +# pragma message( \ + "warning: nvtx3.h is available but does not define the V1 API. This is odd. Please open a GitHub issue at: https://github.com/NVIDIA/cccl/issues.") +# else +# warning nvtx3.h is available but does not define the V1 API. This is odd. Please open a GitHub issue at: https://github.com/NVIDIA/cccl/issues. +# endif +# endif // NVTX3_CPP_DEFINITIONS_V1_0 +#endif // __has_include() && !defined(CCCL_DISABLE_NVTX) && !defined(NVTX_DISABLE) && + // (!_CCCL_COMPILER(NVHPC)) && !_CCCL_COMPILER(NVRTC) + +#if _CCCL_HAS_NVTX3() +# include + +_CCCL_BEGIN_NAMESPACE_CUDA +struct __nvtx_cccl_domain +{ + static constexpr const char* name{"CCCL"}; +}; + +using __nvtx_cccl_range = ::nvtx3::v1::scoped_range_in<__nvtx_cccl_domain>; + +// this type ensures that no NVTX range code is emitted in device code +struct __nvtx_cccl_optional_range_host_only +{ + bool __engaged = false; + alignas(__nvtx_cccl_range) unsigned char __storage[sizeof(__nvtx_cccl_range)]; + + __nvtx_cccl_optional_range_host_only() = default; + + _CCCL_HOST_API void __start(const ::nvtx3::v1::event_attributes& __attributes) + { + ::new (__storage) __nvtx_cccl_range(__attributes); + __engaged = true; + } + + _CCCL_API ~__nvtx_cccl_optional_range_host_only() + { + NV_IF_TARGET(NV_IS_HOST, ({ + if (__engaged) + { + reinterpret_cast<__nvtx_cccl_range*>(__storage)->~__nvtx_cccl_range(); + } + })); + } +}; +_CCCL_END_NAMESPACE_CUDA + +// Hook for the NestedNVTXRangeGuard from the unit tests +# ifndef _CCCL_BEFORE_NVTX_RANGE_SCOPE +# define _CCCL_BEFORE_NVTX_RANGE_SCOPE(name) +# endif // !CCCL_DETAIL_BEFORE_NVTX_RANGE_SCOPE + +# if _CCCL_HOST_COMPILATION() +// Conditionally inserts a NVTX range starting here until the end of the current function scope in host code. Does +// nothing in device code. +// The __nvtx_cccl_optional_range_host_only type (a simplified optional) is needed to defer the construction of the +// NVTX range and message string registration (static variables) into a region running only on the host, while +// preserving the semantic scope where the range is declared. +# define _CCCL_NVTX_RANGE_SCOPE_IF(condition, name) \ + _CCCL_BEFORE_NVTX_RANGE_SCOPE(name) \ + ::cuda::__nvtx_cccl_optional_range_host_only __cuda_nvtx3_range; \ + NV_IF_TARGET( \ + NV_IS_HOST, ({ \ + static const ::nvtx3::v1::registered_string_in<::cuda::__nvtx_cccl_domain> __cuda_nvtx3_func_name{name}; \ + static const ::nvtx3::v1::event_attributes __cuda_nvtx3_func_attr{__cuda_nvtx3_func_name}; \ + if (condition) \ + { \ + __cuda_nvtx3_range.__start(__cuda_nvtx3_func_attr); \ + } \ + })) +# else // ^^^ _CCCL_HOST_COMPILATION() ^^^ / vvv !_CCCL_HOST_COMPILATION() vvv +# define _CCCL_NVTX_RANGE_SCOPE_IF(condition, name) +# endif // ^^^ !_CCCL_HOST_COMPILATION() ^^^ + +# define _CCCL_NVTX_RANGE_SCOPE(name) _CCCL_NVTX_RANGE_SCOPE_IF(true, name) + +# include + +#else // _CCCL_HAS_NVTX3() +# define _CCCL_NVTX_RANGE_SCOPE_IF(condition, name) +# define _CCCL_NVTX_RANGE_SCOPE(name) +#endif // _CCCL_HAS_NVTX3() + +#endif // _CUDA___NVTX_NVTX_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx3.h b/qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx3.h new file mode 100644 index 00000000..6e49c0f4 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__nvtx/nvtx3.h @@ -0,0 +1,2977 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2020-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Licensed under the Apache License v2.0 with LLVM Exceptions. + * See https://nvidia.github.io/NVTX/LICENSE.txt for license information. + */ + +/* Temporary helper #defines, #undef'ed at end of header */ +#define NVTX3_CPP_VERSION_MAJOR 1 +#define NVTX3_CPP_VERSION_MINOR 0 + +/* This section handles the decision of whether to provide unversioned symbols. + * If NVTX3_CPP_REQUIRE_EXPLICIT_VERSION is #defined, unversioned symbols are + * not provided, and explicit-version symbols such as nvtx3::v1::scoped_range + * and NVTX3_V1_FUNC_RANGE must be used. By default, the first #include of this + * header will define the unversioned symbols such as nvtx3::scoped_range and + * NVTX3_FUNC_RANGE. Subsequently including a different major version of this + * header without #defining NVTX3_CPP_REQUIRE_EXPLICIT_VERSION triggers an error + * since the symbols would conflict. Subsequently including of a different + * minor version within the same major version is allowed. Functionality of + * minor versions is cumulative, regardless of include order. + * + * Since NVTX3_CPP_REQUIRE_EXPLICIT_VERSION allows all combinations of versions + * to coexist without problems within a translation unit, the recommended best + * practice for instrumenting header-based libraries with NVTX C++ Wrappers is + * is to #define NVTX3_CPP_REQUIRE_EXPLICIT_VERSION before including nvtx3.hpp, + * #undef it afterward, and only use explicit-version symbols. This is not + * necessary in common cases, such as instrumenting a standalone application, or + * static/shared libraries in .cpp files or headers private to those projects. + */ +/* clang-format off */ +#if !defined(NVTX3_CPP_REQUIRE_EXPLICIT_VERSION) + /* Define macro used by all definitions in this header to indicate the + * unversioned symbols should be defined in addition to the versioned ones. + */ + #define NVTX3_INLINE_THIS_VERSION + + #if !defined(NVTX3_CPP_INLINED_VERSION_MAJOR) + /* First occurrence of this header in the translation unit. Define macros + * indicating which version shall be used for unversioned symbols. + */ + + /** + * @brief Semantic major version number for NVTX C++ wrappers of unversioned symbols + * + * Breaking changes may occur between major versions, and different major versions + * cannot provide unversioned symbols in the same translation unit (.cpp file). + * + * Note: If NVTX3_CPP_REQUIRE_EXPLICIT_VERSION is defined, this macro is not defined. + * + * Not to be confused with the version number of the NVTX core library. + */ + #define NVTX3_CPP_INLINED_VERSION_MAJOR 1 // NVTX3_CPP_VERSION_MAJOR + + /** + * @brief Semantic minor version number for NVTX C++ wrappers of unversioned symbols + * + * No breaking changes occur between minor versions -- minor version changes within + * a major version are purely additive. + * + * Note: If NVTX3_CPP_REQUIRE_EXPLICIT_VERSION is defined, this macro is not defined. + * + * Not to be confused with the version number of the NVTX core library. + */ + #define NVTX3_CPP_INLINED_VERSION_MINOR 0 // NVTX3_CPP_VERSION_MINOR + #elif NVTX3_CPP_INLINED_VERSION_MAJOR != NVTX3_CPP_VERSION_MAJOR + /* Unsupported case -- cannot define unversioned symbols for different major versions + * in the same translation unit. + */ + #error \ + "Two different major versions of the NVTX C++ Wrappers are being included in a single .cpp file, with unversioned symbols enabled in both. Only one major version can enable unversioned symbols in a .cpp file. To disable unversioned symbols, #define NVTX3_CPP_REQUIRE_EXPLICIT_VERSION before #including nvtx3.hpp, and use the explicit-version symbols instead -- this is the preferred way to use nvtx3.hpp from a header file." + #elif (NVTX3_CPP_INLINED_VERSION_MAJOR == NVTX3_CPP_VERSION_MAJOR) && \ + (NVTX3_CPP_INLINED_VERSION_MINOR < NVTX3_CPP_VERSION_MINOR) + /* An older minor version of the same major version already defined unversioned + * symbols. The new features provided in this header will be inlined + * redefine the minor version macro to this header's version. + */ + #undef NVTX3_CPP_INLINED_VERSION_MINOR + #define NVTX3_CPP_INLINED_VERSION_MINOR 0 // NVTX3_CPP_VERSION_MINOR + // else, already have this version or newer, nothing to do + #endif +#endif +/* clang-format on */ + +/** + * @file nvtx3.hpp + * + * @brief Provides C++ constructs making the NVTX library safer and easier to + * use with zero overhead. + */ + +/** + * \mainpage + * \tableofcontents + * + * \section QUICK_START Quick Start + * + * To add NVTX ranges to your code, use the `nvtx3::scoped_range` RAII object. A + * range begins when the object is created, and ends when the object is + * destroyed. + * + * \code{.cpp} + * #include "nvtx3.hpp" + * void some_function() { + * // Begins a NVTX range with the message "some_function" + * // The range ends when some_function() returns and `r` is destroyed + * nvtx3::scoped_range r{"some_function"}; + * + * for(int i = 0; i < 6; ++i) { + * nvtx3::scoped_range loop{"loop range"}; + * std::this_thread::sleep_for(std::chrono::seconds{1}); + * } + * } // Range ends when `r` is destroyed + * \endcode + * + * The example code above generates the following timeline view in Nsight + * Systems: + * + * \image html + * https://raw.githubusercontent.com/NVIDIA/NVTX/release-v3/docs/images/example_range.png + * + * Alternatively, use the \ref MACROS like `NVTX3_FUNC_RANGE()` to add + * ranges to your code that automatically use the name of the enclosing function + * as the range's message. + * + * \code{.cpp} + * #include "nvtx3.hpp" + * void some_function() { + * // Creates a range with a message "some_function" that ends when the + * // enclosing function returns + * NVTX3_FUNC_RANGE(); + * ... + * } + * \endcode + * + * + * \section Overview + * + * The NVTX library provides a set of functions for users to annotate their code + * to aid in performance profiling and optimization. These annotations provide + * information to tools like Nsight Systems to improve visualization of + * application timelines. + * + * \ref RANGES are one of the most commonly used NVTX constructs for annotating + * a span of time. For example, imagine a user wanted to see every time a + * function, `my_function`, is called and how long it takes to execute. This can + * be accomplished with an NVTX range created on the entry to the function and + * terminated on return from `my_function` using the push/pop C APIs: + * + * \code{.cpp} + * void my_function(...) { + * nvtxRangePushA("my_function"); // Begins NVTX range + * // do work + * nvtxRangePop(); // Ends NVTX range + * } + * \endcode + * + * One of the challenges with using the NVTX C API is that it requires manually + * terminating the end of the range with `nvtxRangePop`. This can be challenging + * if `my_function()` has multiple returns or can throw exceptions as it + * requires calling `nvtxRangePop()` before all possible return points. + * + * NVTX C++ solves this inconvenience through the "RAII" technique by providing + * a `nvtx3::scoped_range` class that begins a range at construction and ends + * the range on destruction. The above example then becomes: + * + * \code{.cpp} + * void my_function(...) { + * nvtx3::scoped_range r{"my_function"}; // Begins NVTX range + * // do work + * } // Range ends on exit from `my_function` when `r` is destroyed + * \endcode + * + * The range object `r` is deterministically destroyed whenever `my_function` + * returns---ending the NVTX range without manual intervention. For more + * information, see \ref RANGES and `nvtx3::scoped_range_in`. + * + * Another inconvenience of the NVTX C APIs are the several constructs where the + * user is expected to initialize an object at the beginning of an application + * and reuse that object throughout the lifetime of the application. For example + * see domains, categories, and registered messages. + * + * Example: + * \code{.cpp} + * nvtxDomainHandle_t D = nvtxDomainCreateA("my domain"); + * // Reuse `D` throughout the rest of the application + * \endcode + * + * This can be problematic if the user application or library does not have an + * explicit initialization function called before all other functions to + * ensure that these long-lived objects are initialized before being used. + * + * NVTX C++ makes use of the "construct on first use" technique to alleviate + * this inconvenience. In short, a function local static object is constructed + * upon the first invocation of a function and returns a reference to that + * object on all future invocations. See the documentation for `nvtx3::domain`, + * `nvtx3::named_category`, `nvtx3::registered_string`, and + * https://isocpp.org/wiki/faq/ctors#static-init-order-on-first-use for more + * information. + * + * Using construct on first use, the above example becomes: + * \code{.cpp} + * struct my_domain{ static constexpr char const* name{"my domain"}; }; + * + * // The first invocation of `domain::get` for the type `my_domain` will + * // construct a `nvtx3::domain` object and return a reference to it. Future + * // invocations simply return a reference. + * nvtx3::domain const& D = nvtx3::domain::get(); + * \endcode + * For more information about NVTX and how it can be used, see + * https://docs.nvidia.com/cuda/profiler-users-guide/index.html#nvtx and + * https://devblogs.nvidia.com/cuda-pro-tip-generate-custom-application-profile-timelines-nvtx/ + * for more information. + * + * \section RANGES Ranges + * + * Ranges are used to describe a span of time during the execution of an + * application. Common examples are using ranges to annotate the time it takes + * to execute a function or an iteration of a loop. + * + * NVTX C++ uses RAII to automate the generation of ranges that are tied to the + * lifetime of objects. Similar to `std::lock_guard` in the C++ Standard + * Template Library. + * + * \subsection scoped_range Scoped Range + * + * `nvtx3::scoped_range_in` is a class that begins a range upon construction + * and ends the range at destruction. This is one of the most commonly used + * constructs in NVTX C++ and is useful for annotating spans of time on a + * particular thread. These ranges can be nested to arbitrary depths. + * + * `nvtx3::scoped_range` is an alias for a `nvtx3::scoped_range_in` in the + * global NVTX domain. For more information about Domains, see \ref DOMAINS. + * + * Various attributes of a range can be configured constructing a + * `nvtx3::scoped_range_in` with a `nvtx3::event_attributes` object. For + * more information, see \ref ATTRIBUTES. + * + * Example: + * + * \code{.cpp} + * void some_function() { + * // Creates a range for the duration of `some_function` + * nvtx3::scoped_range r{}; + * + * while(true) { + * // Creates a range for every loop iteration + * // `loop_range` is nested inside `r` + * nvtx3::scoped_range loop_range{}; + * } + * } + * \endcode + * + * \subsection unique_range Unique Range + * + * `nvtx3::unique_range` is similar to `nvtx3::scoped_range`, with a few key differences: + * - `unique_range` objects can be destroyed in any order whereas `scoped_range` objects must be + * destroyed in exact reverse creation order + * - `unique_range` can start and end on different threads + * - `unique_range` is movable + * - `unique_range` objects can be constructed as heap objects + * + * There is extra overhead associated with `unique_range` constructs and therefore use of + * `nvtx3::scoped_range_in` should be preferred. + * + * \section MARKS Marks + * + * `nvtx3::mark` annotates an instantaneous point in time with a "marker". + * + * Unlike a "range" which has a beginning and an end, a marker is a single event + * in an application, such as detecting a problem: + * + * \code{.cpp} + * bool success = do_operation(...); + * if (!success) { + * nvtx3::mark("operation failed!"); + * } + * \endcode + * + * \section DOMAINS Domains + * + * Similar to C++ namespaces, domains allow for scoping NVTX events. By default, + * all NVTX events belong to the "global" domain. Libraries and applications + * should scope their events to use a custom domain to differentiate where the + * events originate from. + * + * It is common for a library or application to have only a single domain and + * for the name of that domain to be known at compile time. Therefore, Domains + * in NVTX C++ are represented by _tag types_. + * + * For example, to define a custom domain, simply define a new concrete type + * (a `class` or `struct`) with a `static` member called `name` that contains + * the desired name of the domain. + * + * \code{.cpp} + * struct my_domain{ static constexpr char const* name{"my domain"}; }; + * \endcode + * + * For any NVTX C++ construct that can be scoped to a domain, the type + * `my_domain` can be passed as an explicit template argument to scope it to + * the custom domain. + * + * The tag type `nvtx3::domain::global` represents the global NVTX domain. + * + * \code{.cpp} + * // By default, `scoped_range_in` belongs to the global domain + * nvtx3::scoped_range_in<> r0{}; + * + * // Alias for a `scoped_range_in` in the global domain + * nvtx3::scoped_range r1{}; + * + * // `r` belongs to the custom domain + * nvtx3::scoped_range_in r{}; + * \endcode + * + * When using a custom domain, it is recommended to define type aliases for NVTX + * constructs in the custom domain. + * \code{.cpp} + * using my_scoped_range = nvtx3::scoped_range_in; + * using my_registered_string = nvtx3::registered_string_in; + * using my_named_category = nvtx3::named_category_in; + * \endcode + * + * See `nvtx3::domain` for more information. + * + * \section ATTRIBUTES Event Attributes + * + * NVTX events can be customized with various attributes to provide additional + * information (such as a custom message) or to control visualization of the + * event (such as the color used). These attributes can be specified per-event + * via arguments to a `nvtx3::event_attributes` object. + * + * NVTX events can be customized via four "attributes": + * - \ref COLOR : color used to visualize the event in tools. + * - \ref MESSAGES : Custom message string. + * - \ref PAYLOAD : User-defined numerical value. + * - \ref CATEGORY : Intra-domain grouping. + * + * It is possible to construct a `nvtx3::event_attributes` from any number of + * attribute objects (nvtx3::color, nvtx3::message, nvtx3::payload, + * nvtx3::category) in any order. If an attribute is not specified, a tool + * specific default value is used. See `nvtx3::event_attributes` for more + * information. + * + * \code{.cpp} + * // Set message, same as passing nvtx3::message{"message"} + * nvtx3::event_attributes attr{"message"}; + * + * // Set message and color + * nvtx3::event_attributes attr{"message", nvtx3::rgb{127, 255, 0}}; + * + * // Set message, color, payload, category + * nvtx3::event_attributes attr{"message", + * nvtx3::rgb{127, 255, 0}, + * nvtx3::payload{42}, + * nvtx3::category{1}}; + * + * // Same as above -- can use any order of arguments + * nvtx3::event_attributes attr{nvtx3::payload{42}, + * nvtx3::category{1}, + * "message", + * nvtx3::rgb{127, 255, 0}}; + * + * // Multiple arguments of the same type are allowed, but only the first is + * // used -- in this example, payload is set to 42: + * nvtx3::event_attributes attr{ nvtx3::payload{42}, nvtx3::payload{7} }; + * + * // Using the nvtx3 namespace in a local scope makes the syntax more succinct: + * using namespace nvtx3; + * event_attributes attr{"message", rgb{127, 255, 0}, payload{42}, category{1}}; + * \endcode + * + * \subsection MESSAGES message + * + * `nvtx3::message` sets the message string for an NVTX event. + * + * Example: + * \code{.cpp} + * // Create an `event_attributes` with the message "my message" + * nvtx3::event_attributes attr{nvtx3::message{"my message"}}; + * + * // strings and string literals implicitly assumed to be a `nvtx3::message` + * nvtx3::event_attributes attr{"my message"}; + * \endcode + * + * \subsubsection REGISTERED_MESSAGE Registered Messages + * + * Associating a `nvtx3::message` with an event requires copying the contents of + * the message every time the message is used, i.e., copying the entire message + * string. This may cause non-trivial overhead in performance sensitive code. + * + * To eliminate this overhead, NVTX allows registering a message string, + * yielding a "handle" that is inexpensive to copy that may be used in place of + * a message string. When visualizing the events, tools such as Nsight Systems + * will take care of mapping the message handle to its string. + * + * A message should be registered once and the handle reused throughout the rest + * of the application. This can be done by either explicitly creating static + * `nvtx3::registered_string` objects, or using the + * `nvtx3::registered_string::get` construct on first use helper (recommended). + * + * Similar to \ref DOMAINS, `nvtx3::registered_string::get` requires defining a + * custom tag type with a static `message` member whose value will be the + * contents of the registered string. + * + * Example: + * \code{.cpp} + * // Explicitly constructed, static `registered_string` in my_domain: + * static registered_string_in static_message{"my message"}; + * + * // Or use construct on first use: + * // Define a tag type with a `message` member string to register + * struct my_message{ static constexpr char const* message{ "my message" }; }; + * + * // Uses construct on first use to register the contents of + * // `my_message::message` + * auto& msg = nvtx3::registered_string_in::get(); + * \endcode + * + * \subsection COLOR color + * + * Associating a `nvtx3::color` with an event allows controlling how the event + * is visualized in a tool such as Nsight Systems. This is a convenient way to + * visually differentiate among different events. + * + * \code{.cpp} + * // Define a color via rgb color values + * nvtx3::color c{nvtx3::rgb{127, 255, 0}}; + * nvtx3::event_attributes attr{c}; + * + * // rgb color values can be passed directly to an `event_attributes` + * nvtx3::event_attributes attr1{nvtx3::rgb{127,255,0}}; + * \endcode + * + * \subsection CATEGORY category + * + * A `nvtx3::category` is simply an integer id that allows for fine-grain + * grouping of NVTX events. For example, one might use separate categories for + * IO, memory allocation, compute, etc. + * + * \code{.cpp} + * nvtx3::event_attributes{nvtx3::category{1}}; + * \endcode + * + * \subsubsection NAMED_CATEGORIES Named Categories + * + * Associates a `name` string with a category `id` to help differentiate among + * categories. + * + * For any given category id `Id`, a `named_category{Id, "name"}` should only + * be constructed once and reused throughout an application. This can be done by + * either explicitly creating static `nvtx3::named_category` objects, or using + * the `nvtx3::named_category::get` construct on first use helper (recommended). + * + * Similar to \ref DOMAINS, `nvtx3::named_category::get` requires defining a + * custom tag type with static `name` and `id` members. + * + * \code{.cpp} + * // Explicitly constructed, static `named_category` in my_domain: + * static nvtx3::named_category_in static_category{42, "my category"}; + * + * // Or use construct on first use: + * // Define a tag type with `name` and `id` members + * struct my_category { + * static constexpr char const* name{"my category"}; // category name + * static constexpr uint32_t id{42}; // category id + * }; + * + * // Use construct on first use to name the category id `42` + * // with name "my category": + * auto& cat = named_category_in::get(); + * + * // Range `r` associated with category id `42` + * nvtx3::event_attributes attr{cat}; + * \endcode + * + * \subsection PAYLOAD payload + * + * Allows associating a user-defined numerical value with an event. + * + * \code{.cpp} + * // Constructs a payload from the `int32_t` value 42 + * nvtx3:: event_attributes attr{nvtx3::payload{42}}; + * \endcode + * + * + * \section EXAMPLE Example + * + * Putting it all together: + * \code{.cpp} + * // Define a custom domain tag type + * struct my_domain{ static constexpr char const* name{"my domain"}; }; + * + * // Define a named category tag type + * struct my_category{ + * static constexpr char const* name{"my category"}; + * static constexpr uint32_t id{42}; + * }; + * + * // Define a registered string tag type + * struct my_message{ static constexpr char const* message{"my message"}; }; + * + * // For convenience, use aliases for domain scoped objects + * using my_scoped_range = nvtx3::scoped_range_in; + * using my_registered_string = nvtx3::registered_string_in; + * using my_named_category = nvtx3::named_category_in; + * + * // Default values for all attributes + * nvtx3::event_attributes attr{}; + * my_scoped_range r0{attr}; + * + * // Custom (unregistered) message, and unnamed category + * nvtx3::event_attributes attr1{"message", nvtx3::category{2}}; + * my_scoped_range r1{attr1}; + * + * // Alternatively, pass arguments of `event_attributes` constructor directly + * // to `my_scoped_range` + * my_scoped_range r2{"message", nvtx3::category{2}}; + * + * // construct on first use a registered string + * auto& msg = my_registered_string::get(); + * + * // construct on first use a named category + * auto& cat = my_named_category::get(); + * + * // Use registered string and named category with a custom payload + * my_scoped_range r3{msg, cat, nvtx3::payload{42}}; + * + * // Any number of arguments in any order + * my_scoped_range r{nvtx3::rgb{127, 255,0}, msg}; + * + * \endcode + * \section MACROS Convenience Macros + * + * Oftentimes users want to quickly and easily add NVTX ranges to their library + * or application to aid in profiling and optimization. + * + * A convenient way to do this is to use the \ref NVTX3_FUNC_RANGE and + * \ref NVTX3_FUNC_RANGE_IN macros. These macros take care of constructing an + * `nvtx3::scoped_range_in` with the name of the enclosing function as the + * range's message. + * + * \code{.cpp} + * void some_function() { + * // Automatically generates an NVTX range for the duration of the function + * // using "some_function" as the event's message. + * NVTX3_FUNC_RANGE(); + * } + * \endcode + * + */ + +/* Temporary helper #defines, removed with #undef at end of header */ + +/* Some compilers do not correctly support SFINAE, which is used in this API + * to detect common usage errors and provide clearer error messages (by using + * static_assert) than the compiler would produce otherwise. These compilers + * will generate errors while compiling this file such as: + * + * error: 'name' is not a member of 'nvtx3::v1::domain::global' + * + * The following compiler versions are known to have this problem, and so are + * set by default to disable the SFINAE-based checks: + * + * - All MSVC versions prior to VS2017 Update 7 (15.7) + * - GCC 8.1-8.3 (the problem was fixed in GCC 8.4) + * + * If you find your compiler hits this problem, you can work around it by + * defining NVTX3_USE_CHECKED_OVERLOADS_FOR_GET to 0 before including this + * header, or you can add a check for your compiler version to this #if. + * Also, please report the issue on the NVTX GitHub page. + */ +#if !defined(NVTX3_USE_CHECKED_OVERLOADS_FOR_GET) +# if defined(_MSC_VER) && _MSC_VER < 1914 || defined(__GNUC__) && __GNUC__ == 8 && __GNUC_MINOR__ < 4 +# define NVTX3_USE_CHECKED_OVERLOADS_FOR_GET 0 +# else +# define NVTX3_USE_CHECKED_OVERLOADS_FOR_GET 1 +# endif +# define NVTX3_USE_CHECKED_OVERLOADS_FOR_GET_DEFINED_HERE +#endif + +/* Within this header, nvtx3::NVTX3_VERSION_NAMESPACE resolves to nvtx3::vX, + * where "X" is the major version number. */ +#define NVTX3_CONCAT(A, B) A##B +#define NVTX3_NAMESPACE_FOR(VERSION) NVTX3_CONCAT(v, VERSION) +#define NVTX3_VERSION_NAMESPACE NVTX3_NAMESPACE_FOR(NVTX3_CPP_VERSION_MAJOR) + +/* Avoid duplicating #if defined(NVTX3_INLINE_THIS_VERSION) for namespaces + * in each minor version by making a macro to use unconditionally, which + * resolves to "inline" or nothing as appropriate. */ +#if defined(NVTX3_INLINE_THIS_VERSION) +# define NVTX3_INLINE_IF_REQUESTED inline +#else +# define NVTX3_INLINE_IF_REQUESTED +#endif + +/* Enables the use of constexpr when support for C++14 constexpr is present. + * + * Initialization of a class member that is a union to a specific union member + * can only be done in the body of a constructor, not in a member initializer + * list. A constexpr constructor must have an empty body until C++14, so there + * is no way to make an initializer of a member union constexpr in C++11. This + * macro allows making functions constexpr in C++14 or newer, but non-constexpr + * in C++11 compilation. It is used here on constructors that initialize their + * member unions. + */ +#if __cpp_constexpr >= 201304L +# define NVTX3_CONSTEXPR_IF_CPP14 constexpr +#else +# define NVTX3_CONSTEXPR_IF_CPP14 +#endif + +// Macro wrappers for C++ attributes +#if !defined(__has_cpp_attribute) +# define __has_cpp_attribute(x) 0 +#endif +#if __has_cpp_attribute(maybe_unused) +# define NVTX3_MAYBE_UNUSED [[maybe_unused]] +#else +# define NVTX3_MAYBE_UNUSED +#endif +#if __has_cpp_attribute(nodiscard) +# define NVTX3_NO_DISCARD [[nodiscard]] +#else +# define NVTX3_NO_DISCARD +#endif + +/* Use a macro for static asserts, which defaults to static_assert, but that + * testing tools can replace with a logging function. For example: + * #define NVTX3_STATIC_ASSERT(c, m) \ + * do { if (!(c)) printf("static_assert would fail: %s\n", m); } while (0) + */ +#if !defined(NVTX3_STATIC_ASSERT) +# define NVTX3_STATIC_ASSERT(condition, message) static_assert(condition, message) +# define NVTX3_STATIC_ASSERT_DEFINED_HERE +#endif + +/* Implementation sections, enclosed in guard macros for each minor version */ + +#ifndef NVTX3_CPP_DEFINITIONS_V1_0 +# define NVTX3_CPP_DEFINITIONS_V1_0 + +# include + +# include +# include +# include +# include + +# include + +namespace nvtx3 +{ +NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE +{ + namespace detail + { + template + struct always_false : std::false_type + {}; + + template + struct has_name : std::false_type + {}; + template + struct has_name : std::true_type + {}; + + template + struct has_id : std::false_type + {}; + template + struct has_id : std::true_type + {}; + + template + struct has_message : std::false_type + {}; + template + struct has_message : std::true_type + {}; + + template + struct is_c_string : std::false_type + {}; + template + struct is_c_string::value + || std::is_convertible::value>::type> : std::true_type + {}; + + template + using is_uint32 = std::is_same::type, uint32_t>; + } // namespace detail + + /** + * @brief `domain`s allow for grouping NVTX events into a single scope to + * differentiate them from events in other `domain`s. + * + * By default, all NVTX constructs are placed in the "global" NVTX domain. + * + * A custom `domain` may be used in order to differentiate a library's or + * application's NVTX events from other events. + * + * `domain`s are expected to be long-lived and unique to a library or + * application. As such, it is assumed a domain's name is known at compile + * time. Therefore, all NVTX constructs that can be associated with a domain + * require the domain to be specified via a *type* `D` passed as an + * explicit template parameter. + * + * The type `domain::global` may be used to indicate that the global NVTX + * domain should be used. + * + * None of the C++ NVTX constructs require the user to manually construct a + * `domain` object. Instead, if a custom domain is desired, the user is + * expected to define a type `D` that contains a member + * `D::name` which resolves to either a `char const*` or `wchar_t + * const*`. The value of `D::name` is used to name and uniquely + * identify the custom domain. + * + * Upon the first use of an NVTX construct associated with the type + * `D`, the "construct on first use" pattern is used to construct a + * function local static `domain` object. All future NVTX constructs + * associated with `D` will use a reference to the previously + * constructed `domain` object. See `domain::get`. + * + * Example: + * \code{.cpp} + * // The type `my_domain` defines a `name` member used to name and identify + * // the `domain` object identified by `my_domain`. + * struct my_domain{ static constexpr char const* name{"my_domain"}; }; + * + * // The NVTX range `r` will be grouped with all other NVTX constructs + * // associated with `my_domain`. + * nvtx3::scoped_range_in r{}; + * + * // An alias can be created for a `scoped_range_in` in the custom domain + * using my_scoped_range = nvtx3::scoped_range_in; + * my_scoped_range my_range{}; + * + * // `domain::global` indicates that the global NVTX domain is used + * nvtx3::scoped_range_in r2{}; + * + * // For convenience, `nvtx3::scoped_range` is an alias for a range in the + * // global domain + * nvtx3::scoped_range r3{}; + * \endcode + */ + class domain + { + public: + domain(domain const&) = delete; + domain& operator=(domain const&) = delete; + domain(domain&&) = delete; + domain& operator=(domain&&) = delete; + + /** + * @brief Tag type for the "global" NVTX domain. + * + * This type may be passed as a template argument to any function/class + * expecting a type to identify a domain to indicate that the global domain + * should be used. + * + * All NVTX events in the global domain across all libraries and + * applications will be grouped together. + * + */ + struct global + {}; + +# if NVTX3_USE_CHECKED_OVERLOADS_FOR_GET + /** + * @brief Returns reference to an instance of a function local static + * `domain` object. + * + * Uses the "construct on first use" idiom to safely ensure the `domain` + * object is initialized exactly once upon first invocation of + * `domain::get()`. All following invocations will return a + * reference to the previously constructed `domain` object. See + * https://isocpp.org/wiki/faq/ctors#static-init-order-on-first-use + * + * None of the constructs in this header require the user to directly invoke + * `domain::get`. It is automatically invoked when constructing objects like + * a `scoped_range_in` or `category`. Advanced users may wish to use + * `domain::get` for the convenience of the "construct on first use" idiom + * when using domains with their own use of the NVTX C API. + * + * This function is thread-safe as of C++11. If two or more threads call + * `domain::get` concurrently, exactly one of them is guaranteed + * to construct the `domain` object and the other(s) will receive a + * reference to the object after it is fully constructed. + * + * The domain's name is specified via the type `D` pass as an + * explicit template parameter. `D` is required to contain a + * member `D::name` that resolves to either a `char const*` or + * `wchar_t const*`. The value of `D::name` is used to name and + * uniquely identify the `domain`. + * + * Example: + * \code{.cpp} + * // The type `my_domain` defines a `name` member used to name and identify + * // the `domain` object identified by `my_domain`. + * struct my_domain{ static constexpr char const* name{"my domain"}; }; + * + * auto& D1 = domain::get(); // First invocation constructs a + * // `domain` with the name "my domain" + * + * auto& D2 = domain::get(); // Quickly returns reference to + * // previously constructed `domain`. + * \endcode + * + * @tparam D Type that contains a `D::name` member used to + * name the `domain` object. + * @return Reference to the `domain` corresponding to the type `D`. + */ + template ::value, int>::type = 0> + NVTX3_NO_DISCARD static domain const& get() noexcept + { + static domain const d(D::name); + return d; + } + + /** + * @brief Overload of `domain::get` to provide a clear compile error when + * `D` has a `name` member that is not directly convertible to either + * `char const*` or `wchar_t const*`. + */ + template ::value, int>::type = 0> + NVTX3_NO_DISCARD static domain const& get() noexcept + { + NVTX3_STATIC_ASSERT(detail::always_false::value, + "Type used to identify an NVTX domain must contain a static constexpr member " + "called 'name' of type const char* or const wchar_t* -- 'name' member is not " + "convertible to either of those types"); + static domain const unused; + return unused; // Function must compile for static_assert to be triggered + } + + /** + * @brief Overload of `domain::get` to provide a clear compile error when + * `D` does not have a `name` member. + */ + template ::value, int>::type = 0> + NVTX3_NO_DISCARD static domain const& get() noexcept + { + NVTX3_STATIC_ASSERT(detail::always_false::value, + "Type used to identify an NVTX domain must contain a static constexpr member " + "called 'name' of type const char* or const wchar_t* -- 'name' member is missing"); + static domain const unused; + return unused; // Function must compile for static_assert to be triggered + } +# else + template + NVTX3_NO_DISCARD static domain const& get() noexcept + { + static domain const d(D::name); + return d; + } +# endif + + /** + * @brief Conversion operator to `nvtxDomainHandle_t`. + * + * Allows transparently passing a domain object into an API expecting a + * native `nvtxDomainHandle_t` object. + */ + operator nvtxDomainHandle_t() const noexcept + { + return _domain; + } + + private: + /** + * @brief Construct a new domain with the specified `name`. + * + * This constructor is private as it is intended that `domain` objects only + * be created through the `domain::get` function. + * + * @param name A unique name identifying the domain + */ + explicit domain(char const* name) noexcept + : _domain{nvtxDomainCreateA(name)} + {} + + /** + * @brief Construct a new domain with the specified `name`. + * + * This constructor is private as it is intended that `domain` objects only + * be created through the `domain::get` function. + * + * @param name A unique name identifying the domain + */ + explicit domain(wchar_t const* name) noexcept + : _domain{nvtxDomainCreateW(name)} + {} + + /** + * @brief Construct a new domain with the specified `name`. + * + * This constructor is private as it is intended that `domain` objects only + * be created through the `domain::get` function. + * + * @param name A unique name identifying the domain + */ + explicit domain(std::string const& name) noexcept + : domain{name.c_str()} + {} + + /** + * @brief Construct a new domain with the specified `name`. + * + * This constructor is private as it is intended that `domain` objects only + * be created through the `domain::get` function. + * + * @param name A unique name identifying the domain + */ + explicit domain(std::wstring const& name) noexcept + : domain{name.c_str()} + {} + + /** + * @brief Default constructor creates a `domain` representing the + * "global" NVTX domain. + * + * All events not associated with a custom `domain` are grouped in the + * "global" NVTX domain. + * + */ + constexpr domain() noexcept {} + + /** + * @brief Intentionally avoid calling nvtxDomainDestroy on the `domain` object. + * + * No currently-available tools attempt to free domain resources when the + * nvtxDomainDestroy function is called, due to the thread-safety and + * efficiency challenges of freeing thread-local storage for other threads. + * Since libraries may be disallowed from introducing static destructors, + * and destroying the domain is likely to have no effect, the destructor + * for `domain` intentionally chooses to not destroy the domain. + * + * In a situation where domain destruction is necessary, either manually + * call nvtxDomainDestroy on the domain's handle, or make a class that + * derives from `domain` and calls nvtxDomainDestroy in its destructor. + */ + ~domain() = default; + + private: + nvtxDomainHandle_t const _domain{}; ///< The `domain`s NVTX handle + }; + + /** + * @brief Returns reference to the `domain` object that represents the global + * NVTX domain. + * + * This specialization for `domain::global` returns a default constructed, + * `domain` object for use when the "global" domain is desired. + * + * All NVTX events in the global domain across all libraries and applications + * will be grouped together. + * + * @return Reference to the `domain` corresponding to the global NVTX domain. + * + */ + template <> + NVTX3_NO_DISCARD inline domain const& domain::get() noexcept + { + static domain const d{}; + return d; + } + + /** + * @brief Indicates the values of the red, green, and blue color channels for + * an RGB color to use as an event attribute (assumes no transparency). + * + */ + struct rgb + { + /// Type used for component values + using component_type = uint8_t; + + /** + * @brief Construct a rgb with red, green, and blue channels + * specified by `red_`, `green_`, and `blue_`, respectively. + * + * Valid values are in the range `[0,255]`. + * + * @param red_ Value of the red channel + * @param green_ Value of the green channel + * @param blue_ Value of the blue channel + */ + constexpr rgb(component_type red_, component_type green_, component_type blue_) noexcept + : red{red_} + , green{green_} + , blue{blue_} + {} + + component_type red{}; ///< Red channel value + component_type green{}; ///< Green channel value + component_type blue{}; ///< Blue channel value + }; + + /** + * @brief Indicates the value of the alpha, red, green, and blue color + * channels for an ARGB color to use as an event attribute. + * + */ + struct argb final : rgb + { + /** + * @brief Construct an argb with alpha, red, green, and blue channels + * specified by `alpha_`, `red_`, `green_`, and `blue_`, respectively. + * + * Valid values are in the range `[0,255]`. + * + * @param alpha_ Value of the alpha channel (opacity) + * @param red_ Value of the red channel + * @param green_ Value of the green channel + * @param blue_ Value of the blue channel + * + */ + constexpr argb(component_type alpha_, component_type red_, component_type green_, component_type blue_) noexcept + : rgb{red_, green_, blue_} + , alpha{alpha_} + {} + + component_type alpha{}; ///< Alpha channel value + }; + + /** + * @brief Represents a custom color that can be associated with an NVTX event + * via its `event_attributes`. + * + * Specifying colors for NVTX events is a convenient way to visually + * differentiate among different events in a visualization tool such as Nsight + * Systems. + * + */ + class color + { + public: + /// Type used for the color's value + using value_type = uint32_t; + + /** + * @brief Constructs a `color` using the value provided by `hex_code`. + * + * `hex_code` is expected to be a 4 byte argb hex code. + * + * The most significant byte indicates the value of the alpha channel + * (opacity) (0-255) + * + * The next byte indicates the value of the red channel (0-255) + * + * The next byte indicates the value of the green channel (0-255) + * + * The least significant byte indicates the value of the blue channel + * (0-255) + * + * @param hex_code The hex code used to construct the `color` + */ + constexpr explicit color(value_type hex_code) noexcept + : _value{hex_code} + {} + + /** + * @brief Construct a `color` using the alpha, red, green, blue components + * in `argb`. + * + * @param argb_ The alpha, red, green, blue components of the desired `color` + */ + constexpr color(argb argb_) noexcept + : color{from_bytes_msb_to_lsb(argb_.alpha, argb_.red, argb_.green, argb_.blue)} + {} + + /** + * @brief Construct a `color` using the red, green, blue components in + * `rgb`. + * + * Uses maximum value for the alpha channel (opacity) of the `color`. + * + * @param rgb_ The red, green, blue components of the desired `color` + */ + constexpr color(rgb rgb_) noexcept + : color{from_bytes_msb_to_lsb(0xFF, rgb_.red, rgb_.green, rgb_.blue)} + {} + + /** + * @brief Returns the `color`s argb hex code + * + */ + constexpr value_type get_value() const noexcept + { + return _value; + } + + /** + * @brief Return the NVTX color type of the color. + * + */ + constexpr nvtxColorType_t get_type() const noexcept + { + return _type; + } + + color() = delete; + ~color() = default; + color(color const&) = default; + color& operator=(color const&) = default; + color(color&&) = default; + color& operator=(color&&) = default; + + private: + /** + * @brief Constructs an unsigned, 4B integer from the component bytes in + * most to least significant byte order. + * + */ + constexpr static value_type + from_bytes_msb_to_lsb(uint8_t byte3, uint8_t byte2, uint8_t byte1, uint8_t byte0) noexcept + { + return uint32_t{byte3} << 24 | uint32_t{byte2} << 16 | uint32_t{byte1} << 8 | uint32_t{byte0}; + } + + value_type _value{}; ///< color's argb color code + nvtxColorType_t _type{NVTX_COLOR_ARGB}; ///< NVTX color type code + }; + + /** + * @brief Object for intra-domain grouping of NVTX events. + * + * A `category` is simply an integer id that allows for fine-grain grouping of + * NVTX events. For example, one might use separate categories for IO, memory + * allocation, compute, etc. + * + * Example: + * \code{.cpp} + * nvtx3::category cat1{1}; + * + * // Range `r1` belongs to the category identified by the value `1`. + * nvtx3::scoped_range r1{cat1}; + * + * // Range `r2` belongs to the same category as `r1` + * nvtx3::scoped_range r2{nvtx3::category{1}}; + * \endcode + * + * To associate a name string with a category id, see `named_category`. + * + */ + class category + { + public: + /// Type used for `category`s integer id. + using id_type = uint32_t; + + /** + * @brief Construct a `category` with the specified `id`. + * + * The `category` will be unnamed and identified only by its `id` value. + * + * All `category`s in a domain sharing the same `id` are equivalent. + * + * @param[in] id The `category`'s identifying value + */ + constexpr explicit category(id_type id) noexcept + : id_{id} + {} + + /** + * @brief Returns the id of the category. + * + */ + constexpr id_type get_id() const noexcept + { + return id_; + } + + category() = delete; + ~category() = default; + category(category const&) = default; + category& operator=(category const&) = default; + category(category&&) = default; + category& operator=(category&&) = default; + + private: + id_type id_{}; ///< category's unique identifier + }; + + /** + * @brief A `category` with an associated name string. + * + * Associates a `name` string with a category `id` to help differentiate among + * categories. + * + * For any given category id `Id`, a `named_category(Id, "name")` should only + * be constructed once and reused throughout an application. This can be done + * by either explicitly creating static `named_category` objects, or using the + * `named_category::get` construct on first use helper (recommended). + * + * Creating two or more `named_category` objects with the same value for `id` + * in the same domain results in undefined behavior. + * + * Similarly, behavior is undefined when a `named_category` and `category` + * share the same value of `id`. + * + * Example: + * \code{.cpp} + * // Explicitly constructed, static `named_category` in global domain: + * static nvtx3::named_category static_category{42, "my category"}; + * + * // Range `r` associated with category id `42` + * nvtx3::scoped_range r{static_category}; + * + * // OR use construct on first use: + * + * // Define a type with `name` and `id` members + * struct my_category { + * static constexpr char const* name{"my category"}; // category name + * static constexpr uint32_t id{42}; // category id + * }; + * + * // Use construct on first use to name the category id `42` + * // with name "my category" + * auto& cat = named_category_in::get(); + * + * // Range `r` associated with category id `42` + * nvtx3::scoped_range r{cat}; + * \endcode + * + * `named_category_in`'s association of a name to a category id is local to + * the domain specified by the type `D`. An id may have a different name in + * another domain. + * + * @tparam D Type containing `name` member used to identify the `domain` to + * which the `named_category_in` belongs. Else, `domain::global` to indicate + * that the global NVTX domain should be used. + */ + template + class named_category_in final : public category + { + public: +# if NVTX3_USE_CHECKED_OVERLOADS_FOR_GET + /** + * @brief Returns a global instance of a `named_category_in` as a + * function-local static. + * + * Creates a `named_category_in` with name and id specified by the contents + * of a type `C`. `C::name` determines the name and `C::id` determines the + * category id. + * + * This function is useful for constructing a named `category` exactly once + * and reusing the same instance throughout an application. + * + * Example: + * \code{.cpp} + * // Define a type with `name` and `id` members + * struct my_category { + * static constexpr char const* name{"my category"}; // category name + * static constexpr uint32_t id{42}; // category id + * }; + * + * // Use construct on first use to name the category id `42` + * // with name "my category" + * auto& cat = named_category_in::get(); + * + * // Range `r` associated with category id `42` + * nvtx3::scoped_range r{cat}; + * \endcode + * + * Uses the "construct on first use" idiom to safely ensure the `category` + * object is initialized exactly once. See + * https://isocpp.org/wiki/faq/ctors#static-init-order-on-first-use + * + * @tparam C Type containing a member `C::name` that resolves to either a + * `char const*` or `wchar_t const*` and `C::id`. + */ + template < + typename C, + typename std::enable_if::value && detail::is_uint32::value, + int>::type = 0> + static named_category_in const& get() noexcept + { + static named_category_in const cat(C::id, C::name); + return cat; + } + + /** + * @brief Overload of `named_category_in::get` to provide a clear compile error + * when `C` has the required `name` and `id` members, but they are not the + * required types. `name` must be directly convertible to `char const*` or + * `wchar_t const*`, and `id` must be `uint32_t`. + */ + template ::value + || !detail::is_uint32::value, + int>::type = 0> + NVTX3_NO_DISCARD static named_category_in const& get() noexcept + { + NVTX3_STATIC_ASSERT(detail::is_c_string::value, + "Type used to name an NVTX category must contain a static constexpr member " + "called 'name' of type const char* or const wchar_t* -- 'name' member is not " + "convertible to either of those types"); + NVTX3_STATIC_ASSERT(detail::is_uint32::value, + "Type used to name an NVTX category must contain a static constexpr member " + "called 'id' of type uint32_t -- 'id' member is the wrong type"); + static named_category_in const unused; + return unused; // Function must compile for static_assert to be triggered + } + + /** + * @brief Overload of `named_category_in::get` to provide a clear compile error + * when `C` does not have the required `name` and `id` members. + */ + template ::value || !detail::has_id::value, int>::type = 0> + NVTX3_NO_DISCARD static named_category_in const& get() noexcept + { + NVTX3_STATIC_ASSERT(detail::has_name::value, + "Type used to name an NVTX category must contain a static constexpr member " + "called 'name' of type const char* or const wchar_t* -- 'name' member is missing"); + NVTX3_STATIC_ASSERT(detail::has_id::value, + "Type used to name an NVTX category must contain a static constexpr member " + "called 'id' of type uint32_t -- 'id' member is missing"); + static named_category_in const unused; + return unused; // Function must compile for static_assert to be triggered + } +# else + template + NVTX3_NO_DISCARD static named_category_in const& get() noexcept + { + static named_category_in const cat(C::id, C::name); + return cat; + } +# endif + + private: + // Default constructor is only used internally for static_assert(false) cases. + named_category_in() noexcept + : category{0} + {} + + public: + /** + * @brief Construct a `named_category_in` with the specified `id` and `name`. + * + * The name `name` will be registered with `id`. + * + * Every unique value of `id` should only be named once. + * + * @param[in] id The category id to name + * @param[in] name The name to associated with `id` + */ + named_category_in(id_type id, char const* name) noexcept + : category{id} + { +# ifndef NVTX_DISABLE + nvtxDomainNameCategoryA(domain::get(), get_id(), name); +# else + (void) id; + (void) name; +# endif + } + + /** + * @brief Construct a `named_category_in` with the specified `id` and `name`. + * + * The name `name` will be registered with `id`. + * + * Every unique value of `id` should only be named once. + * + * @param[in] id The category id to name + * @param[in] name The name to associated with `id` + */ + named_category_in(id_type id, wchar_t const* name) noexcept + : category{id} + { +# ifndef NVTX_DISABLE + nvtxDomainNameCategoryW(domain::get(), get_id(), name); +# else + (void) id; + (void) name; +# endif + } + }; + + /** + * @brief Alias for a `named_category_in` in the global NVTX domain. + * + */ + using named_category = named_category_in; + + /** + * @brief A message registered with NVTX. + * + * Normally, associating a `message` with an NVTX event requires copying the + * contents of the message string. This may cause non-trivial overhead in + * highly performance sensitive regions of code. + * + * message registration is an optimization to lower the overhead of + * associating a message with an NVTX event. Registering a message yields a + * handle that is inexpensive to copy that may be used in place of a message + * string. + * + * A particular message should only be registered once and the handle + * reused throughout the rest of the application. This can be done by either + * explicitly creating static `registered_string_in` objects, or using the + * `registered_string_in::get` construct on first use helper (recommended). + * + * Example: + * \code{.cpp} + * // Explicitly constructed, static `registered_string` in my_domain: + * static registered_string_in static_message{"message"}; + * + * // "message" is associated with the range `r` + * nvtx3::scoped_range r{static_message}; + * + * // Or use construct on first use: + * + * // Define a type with a `message` member that defines the contents of the + * // registered string + * struct my_message{ static constexpr char const* message{ "my message" }; }; + * + * // Uses construct on first use to register the contents of + * // `my_message::message` + * auto& msg = registered_string_in::get(); + * + * // "my message" is associated with the range `r` + * nvtx3::scoped_range r{msg}; + * \endcode + * + * `registered_string_in`s are local to a particular domain specified via + * the type `D`. + * + * @tparam D Type containing `name` member used to identify the `domain` to + * which the `registered_string_in` belongs. Else, `domain::global` to indicate + * that the global NVTX domain should be used. + */ + template + class registered_string_in + { + public: +# if NVTX3_USE_CHECKED_OVERLOADS_FOR_GET + /** + * @brief Returns a global instance of a `registered_string_in` as a function + * local static. + * + * Provides a convenient way to register a message with NVTX without having + * to explicitly register the message. + * + * Upon first invocation, constructs a `registered_string_in` whose contents + * are specified by `message::message`. + * + * All future invocations will return a reference to the object constructed + * in the first invocation. + * + * Example: + * \code{.cpp} + * // Define a type with a `message` member that defines the contents of the + * // registered string + * struct my_message{ static constexpr char const* message{ "my message" }; + * }; + * + * // Uses construct on first use to register the contents of + * // `my_message::message` + * auto& msg = registered_string_in::get(); + * + * // "my message" is associated with the range `r` + * nvtx3::scoped_range r{msg}; + * \endcode + * + * @tparam M Type required to contain a member `M::message` that + * resolves to either a `char const*` or `wchar_t const*` used as the + * registered string's contents. + * @return Reference to a `registered_string_in` associated with the type `M`. + */ + template ::value, int>::type = 0> + NVTX3_NO_DISCARD static registered_string_in const& get() noexcept + { + static registered_string_in const regstr(M::message); + return regstr; + } + + /** + * @brief Overload of `registered_string_in::get` to provide a clear compile error + * when `M` has a `message` member that is not directly convertible to either + * `char const*` or `wchar_t const*`. + */ + template ::value, int>::type = 0> + NVTX3_NO_DISCARD static registered_string_in const& get() noexcept + { + NVTX3_STATIC_ASSERT(detail::always_false::value, + "Type used to register an NVTX string must contain a static constexpr member " + "called 'message' of type const char* or const wchar_t* -- 'message' member is " + "not convertible to either of those types"); + static registered_string_in const unused; + return unused; // Function must compile for static_assert to be triggered + } + + /** + * @brief Overload of `registered_string_in::get` to provide a clear compile error when + * `M` does not have a `message` member. + */ + template ::value, int>::type = 0> + NVTX3_NO_DISCARD static registered_string_in const& get() noexcept + { + NVTX3_STATIC_ASSERT(detail::always_false::value, + "Type used to register an NVTX string must contain a static constexpr member " + "called 'message' of type const char* or const wchar_t* -- 'message' member " + "is missing"); + static registered_string_in const unused; + return unused; // Function must compile for static_assert to be triggered + } +# else + template + NVTX3_NO_DISCARD static registered_string_in const& get() noexcept + { + static registered_string_in const regstr(M::message); + return regstr; + } +# endif + + /** + * @brief Constructs a `registered_string_in` from the specified `msg` string. + * + * Registers `msg` with NVTX and associates a handle with the registered + * message. + * + * A particular message should should only be registered once and the handle + * reused throughout the rest of the application. + * + * @param msg The contents of the message + */ + explicit registered_string_in(char const* msg) noexcept + : handle_{nvtxDomainRegisterStringA(domain::get(), msg)} + {} + + /** + * @brief Constructs a `registered_string_in` from the specified `msg` string. + * + * Registers `msg` with NVTX and associates a handle with the registered + * message. + * + * A particular message should should only be registered once and the handle + * reused throughout the rest of the application. + * + * @param msg The contents of the message + */ + explicit registered_string_in(std::string const& msg) noexcept + : registered_string_in{msg.c_str()} + {} + + /** + * @brief Constructs a `registered_string_in` from the specified `msg` string. + * + * Registers `msg` with NVTX and associates a handle with the registered + * message. + * + * A particular message should should only be registered once and the handle + * reused throughout the rest of the application. + * + * @param msg The contents of the message + */ + explicit registered_string_in(wchar_t const* msg) noexcept + : handle_{nvtxDomainRegisterStringW(domain::get(), msg)} + {} + + /** + * @brief Constructs a `registered_string_in` from the specified `msg` string. + * + * Registers `msg` with NVTX and associates a handle with the registered + * message. + * + * A particular message should only be registered once and the handle + * reused throughout the rest of the application. + * + * @param msg The contents of the message + */ + explicit registered_string_in(std::wstring const& msg) noexcept + : registered_string_in{msg.c_str()} + {} + + /** + * @brief Returns the registered string's handle + * + */ + nvtxStringHandle_t get_handle() const noexcept + { + return handle_; + } + + private: + // Default constructor is only used internally for static_assert(false) cases. + registered_string_in() noexcept {} + + public: + ~registered_string_in() = default; + registered_string_in(registered_string_in const&) = default; + registered_string_in& operator=(registered_string_in const&) = default; + registered_string_in(registered_string_in&&) = default; + registered_string_in& operator=(registered_string_in&&) = default; + + private: + nvtxStringHandle_t handle_{}; ///< The handle returned from + ///< registering the message with NVTX + }; + + /** + * @brief Alias for a `registered_string_in` in the global NVTX domain. + * + */ + using registered_string = registered_string_in; + + /** + * @brief Allows associating a message string with an NVTX event via + * its `EventAttribute`s. + * + * Associating a `message` with an NVTX event through its `event_attributes` + * allows for naming events to easily differentiate them from other events. + * + * Every time an NVTX event is created with an associated `message`, the + * contents of the message string must be copied. This may cause non-trivial + * overhead in highly performance sensitive sections of code. Use of a + * `nvtx3::registered_string` is recommended in these situations. + * + * Example: + * \code{.cpp} + * // Creates an `event_attributes` with message "message 0" + * nvtx3::event_attributes attr0{nvtx3::message{"message 0"}}; + * + * // `range0` contains message "message 0" + * nvtx3::scoped_range range0{attr0}; + * + * // `std::string` and string literals are implicitly assumed to be + * // the contents of an `nvtx3::message` + * // Creates an `event_attributes` with message "message 1" + * nvtx3::event_attributes attr1{"message 1"}; + * + * // `range1` contains message "message 1" + * nvtx3::scoped_range range1{attr1}; + * + * // `range2` contains message "message 2" + * nvtx3::scoped_range range2{nvtx3::message{"message 2"}}; + * + * // `std::string` and string literals are implicitly assumed to be + * // the contents of an `nvtx3::message` + * // `range3` contains message "message 3" + * nvtx3::scoped_range range3{"message 3"}; + * \endcode + */ + class message + { + public: + using value_type = nvtxMessageValue_t; + + /** + * @brief Construct a `message` whose contents are specified by `msg`. + * + * @param msg The contents of the message + */ + NVTX3_CONSTEXPR_IF_CPP14 message(char const* msg) noexcept + : type_{NVTX_MESSAGE_TYPE_ASCII} + { + value_.ascii = msg; + } + + /** + * @brief Construct a `message` whose contents are specified by `msg`. + * + * @param msg The contents of the message + */ + message(std::string const& msg) noexcept + : message{msg.c_str()} + {} + + /** + * @brief Disallow construction for `std::string` r-value + * + * `message` is a non-owning type and therefore cannot take ownership of an + * r-value. Therefore, constructing from an r-value is disallowed to prevent + * a dangling pointer. + * + */ + message(std::string&&) = delete; + + /** + * @brief Construct a `message` whose contents are specified by `msg`. + * + * @param msg The contents of the message + */ + NVTX3_CONSTEXPR_IF_CPP14 message(wchar_t const* msg) noexcept + : type_{NVTX_MESSAGE_TYPE_UNICODE} + { + value_.unicode = msg; + } + + /** + * @brief Construct a `message` whose contents are specified by `msg`. + * + * @param msg The contents of the message + */ + message(std::wstring const& msg) noexcept + : message{msg.c_str()} + {} + + /** + * @brief Disallow construction for `std::wstring` r-value + * + * `message` is a non-owning type and therefore cannot take ownership of an + * r-value. Therefore, constructing from an r-value is disallowed to prevent + * a dangling pointer. + * + */ + message(std::wstring&&) = delete; + + /** + * @brief Construct a `message` from a `registered_string_in`. + * + * @tparam D Type containing `name` member used to identify the `domain` + * to which the `registered_string_in` belongs. Else, `domain::global` to + * indicate that the global NVTX domain should be used. + * @param msg The message that has already been registered with NVTX. + */ + template + NVTX3_CONSTEXPR_IF_CPP14 message(registered_string_in const& msg) noexcept + : type_{NVTX_MESSAGE_TYPE_REGISTERED} + { + value_.registered = msg.get_handle(); + } + + /** + * @brief Construct a `message` from NVTX C API type and value. + * + * @param type nvtxMessageType_t enum value indicating type of the payload + * @param value nvtxMessageValue_t union containing message + */ + constexpr message(nvtxMessageType_t const& type, nvtxMessageValue_t const& value) noexcept + : type_{type} + , value_(value) + {} + + /** + * @brief Construct a `message` from NVTX C API registered string handle. + * + * @param handle nvtxStringHandle_t value of registered string handle + */ + NVTX3_CONSTEXPR_IF_CPP14 message(nvtxStringHandle_t handle) noexcept + : type_{NVTX_MESSAGE_TYPE_REGISTERED} + { + value_.registered = handle; + } + + /** + * @brief Return the union holding the value of the message. + * + */ + constexpr value_type get_value() const noexcept + { + return value_; + } + + /** + * @brief Return the type information about the value the union holds. + * + */ + constexpr nvtxMessageType_t get_type() const noexcept + { + return type_; + } + + private: + nvtxMessageType_t type_{}; ///< message type + nvtxMessageValue_t value_{}; ///< message contents + }; + + /** + * @brief A numerical value that can be associated with an NVTX event via + * its `event_attributes`. + * + * Example: + * \code{.cpp} + * // Constructs a payload from the int32_t value 42 + * nvtx3:: event_attributes attr{nvtx3::payload{42}}; + * + * // `range0` will have an int32_t payload of 42 + * nvtx3::scoped_range range0{attr}; + * + * // range1 has double payload of 3.14 + * nvtx3::scoped_range range1{nvtx3::payload{3.14}}; + * \endcode + */ + class payload + { + public: + using value_type = typename nvtxEventAttributes_v2::payload_t; + + /** + * @brief Construct a `payload` from a signed, 8 byte integer. + * + * @param value Value to use as contents of the payload + */ + NVTX3_CONSTEXPR_IF_CPP14 explicit payload(int64_t value) noexcept + : type_{NVTX_PAYLOAD_TYPE_INT64} + , value_{} + { + value_.llValue = value; + } + + /** + * @brief Construct a `payload` from a signed, 4 byte integer. + * + * @param value Value to use as contents of the payload + */ + NVTX3_CONSTEXPR_IF_CPP14 explicit payload(int32_t value) noexcept + : type_{NVTX_PAYLOAD_TYPE_INT32} + , value_{} + { + value_.iValue = value; + } + + /** + * @brief Construct a `payload` from an unsigned, 8 byte integer. + * + * @param value Value to use as contents of the payload + */ + NVTX3_CONSTEXPR_IF_CPP14 explicit payload(uint64_t value) noexcept + : type_{NVTX_PAYLOAD_TYPE_UNSIGNED_INT64} + , value_{} + { + value_.ullValue = value; + } + + /** + * @brief Construct a `payload` from an unsigned, 4 byte integer. + * + * @param value Value to use as contents of the payload + */ + NVTX3_CONSTEXPR_IF_CPP14 explicit payload(uint32_t value) noexcept + : type_{NVTX_PAYLOAD_TYPE_UNSIGNED_INT32} + , value_{} + { + value_.uiValue = value; + } + + /** + * @brief Construct a `payload` from a single-precision floating point + * value. + * + * @param value Value to use as contents of the payload + */ + NVTX3_CONSTEXPR_IF_CPP14 explicit payload(float value) noexcept + : type_{NVTX_PAYLOAD_TYPE_FLOAT} + , value_{} + { + value_.fValue = value; + } + + /** + * @brief Construct a `payload` from a double-precision floating point + * value. + * + * @param value Value to use as contents of the payload + */ + NVTX3_CONSTEXPR_IF_CPP14 explicit payload(double value) noexcept + : type_{NVTX_PAYLOAD_TYPE_DOUBLE} + , value_{} + { + value_.dValue = value; + } + + /** + * @brief Construct a `payload` from NVTX C API type and value. + * + * @param type nvtxPayloadType_t enum value indicating type of the payload + * @param value nvtxEventAttributes_t::payload_t union containing payload + */ + constexpr payload(nvtxPayloadType_t const& type, value_type const& value) noexcept + : type_{type} + , value_(value) + {} + + /** + * @brief Return the union holding the value of the payload + * + */ + constexpr value_type get_value() const noexcept + { + return value_; + } + + /** + * @brief Return the information about the type the union holds. + * + */ + constexpr nvtxPayloadType_t get_type() const noexcept + { + return type_; + } + + private: + nvtxPayloadType_t type_; ///< Type of the payload value + value_type value_; ///< Union holding the payload value + }; + + /** + * @brief Describes the attributes of a NVTX event. + * + * NVTX events can be customized via four "attributes": + * + * - color: color used to visualize the event in tools such as Nsight + * Systems. See `color`. + * - message: Custom message string. See `message`. + * - payload: User-defined numerical value. See `payload`. + * - category: Intra-domain grouping. See `category`. + * + * These component attributes are specified via an `event_attributes` object. + * See `nvtx3::color`, `nvtx3::message`, `nvtx3::payload`, and + * `nvtx3::category` for how these individual attributes are constructed. + * + * While it is possible to specify all four attributes, it is common to want + * to only specify a subset of attributes and use default values for the + * others. For convenience, `event_attributes` can be constructed from any + * number of attribute components in any order. + * + * Example: + * \code{.cpp} + * // Set message, same as using nvtx3::message{"message"} + * event_attributes attr{"message"}; + * + * // Set message and color + * event_attributes attr{"message", nvtx3::rgb{127, 255, 0}}; + * + * // Set message, color, payload, category + * event_attributes attr{"message", + * nvtx3::rgb{127, 255, 0}, + * nvtx3::payload{42}, + * nvtx3::category{1}}; + * + * // Same as above -- can use any order of arguments + * event_attributes attr{nvtx3::payload{42}, + * nvtx3::category{1}, + * "message", + * nvtx3::rgb{127, 255, 0}}; + * + * // Multiple arguments of the same type are allowed, but only the first is + * // used -- in this example, payload is set to 42: + * event_attributes attr{ nvtx3::payload{42}, nvtx3::payload{7} }; + * + * // Range `r` will be customized according the attributes in `attr` + * nvtx3::scoped_range r{attr}; + * + * // For convenience, `event_attributes` constructor arguments may be passed + * // to the `scoped_range_in` constructor -- they are forwarded to the + * // `event_attributes` constructor + * nvtx3::scoped_range r{nvtx3::payload{42}, nvtx3::category{1}, "message"}; + * + * // Using the nvtx3 namespace in a local scope makes the syntax more succinct: + * using namespace nvtx3; + * scoped_range r{payload{42}, category{1}, "message"}; + * \endcode + * + */ + class event_attributes + { + public: + using value_type = nvtxEventAttributes_t; + + /** + * @brief Default constructor creates an `event_attributes` with no + * category, color, payload, nor message. + */ + constexpr event_attributes() noexcept + : attributes_{ + NVTX_VERSION, // version + sizeof(nvtxEventAttributes_t), // size + 0, // category + NVTX_COLOR_UNKNOWN, // color type + 0, // color value + NVTX_PAYLOAD_UNKNOWN, // payload type + 0, // reserved 4B + {0}, // payload value (union) + NVTX_MESSAGE_UNKNOWN, // message type + {0} // message value (union) + } + {} + + /** + * @brief Variadic constructor where the first argument is a `category`. + * + * Sets the value of the `EventAttribute`s category based on `c` and + * forwards the remaining variadic parameter pack to the next constructor. + * + */ + template + NVTX3_CONSTEXPR_IF_CPP14 explicit event_attributes(category const& c, Args const&... args) noexcept + : event_attributes(args...) + { + attributes_.category = c.get_id(); + } + + /** + * @brief Variadic constructor where the first argument is a `color`. + * + * Sets the value of the `EventAttribute`s color based on `c` and forwards + * the remaining variadic parameter pack to the next constructor. + * + */ + template + NVTX3_CONSTEXPR_IF_CPP14 explicit event_attributes(color const& c, Args const&... args) noexcept + : event_attributes(args...) + { + attributes_.color = c.get_value(); + attributes_.colorType = c.get_type(); + } + + /** + * @brief Variadic constructor where the first argument is a `payload`. + * + * Sets the value of the `EventAttribute`s payload based on `p` and forwards + * the remaining variadic parameter pack to the next constructor. + * + */ + template + NVTX3_CONSTEXPR_IF_CPP14 explicit event_attributes(payload const& p, Args const&... args) noexcept + : event_attributes(args...) + { + attributes_.payload = p.get_value(); + attributes_.payloadType = p.get_type(); + } + + /** + * @brief Variadic constructor where the first argument is a `message`. + * + * Sets the value of the `EventAttribute`s message based on `m` and forwards + * the remaining variadic parameter pack to the next constructor. + * + */ + template + NVTX3_CONSTEXPR_IF_CPP14 explicit event_attributes(message const& m, Args const&... args) noexcept + : event_attributes(args...) + { + attributes_.message = m.get_value(); + attributes_.messageType = m.get_type(); + } + + ~event_attributes() = default; + event_attributes(event_attributes const&) = default; + event_attributes& operator=(event_attributes const&) = default; + event_attributes(event_attributes&&) = default; + event_attributes& operator=(event_attributes&&) = default; + + /** + * @brief Get raw pointer to underlying NVTX attributes object. + * + */ + constexpr value_type const* get() const noexcept + { + return &attributes_; + } + + private: + value_type attributes_{}; ///< The NVTX attributes structure + }; + + /** + * @brief A RAII object for creating a NVTX range local to a thread within a + * domain. + * + * When constructed, begins a nested NVTX range on the calling thread in the + * specified domain. Upon destruction, ends the NVTX range. + * + * Behavior is undefined if a `scoped_range_in` object is + * created/destroyed on different threads. + * + * `scoped_range_in` is neither movable nor copyable. + * + * `scoped_range_in`s may be nested within other ranges. + * + * The domain of the range is specified by the template type parameter `D`. + * By default, the `domain::global` is used, which scopes the range to the + * global NVTX domain. The convenience alias `scoped_range` is provided for + * ranges scoped to the global domain. + * + * A custom domain can be defined by creating a type, `D`, with a static + * member `D::name` whose value is used to name the domain associated with + * `D`. `D::name` must resolve to either `char const*` or `wchar_t const*` + * + * Example: + * \code{.cpp} + * // Define a type `my_domain` with a member `name` used to name the domain + * // associated with the type `my_domain`. + * struct my_domain{ + * static constexpr char const* name{"my domain"}; + * }; + * \endcode + * + * Usage: + * \code{.cpp} + * nvtx3::scoped_range_in r1{"range 1"}; // Range in my domain + * + * // Three equivalent ways to make a range in the global domain: + * nvtx3::scoped_range_in r2{"range 2"}; + * nvtx3::scoped_range_in<> r3{"range 3"}; + * nvtx3::scoped_range r4{"range 4"}; + * + * // Create an alias to succinctly make ranges in my domain: + * using my_scoped_range = nvtx3::scoped_range_in; + * + * my_scoped_range r3{"range 3"}; + * \endcode + */ + template + class NVTX3_MAYBE_UNUSED scoped_range_in + { + public: + /** + * @brief Construct a `scoped_range_in` with the specified + * `event_attributes` + * + * Example: + * \code{cpp} + * nvtx3::event_attributes attr{"msg", nvtx3::rgb{127,255,0}}; + * nvtx3::scoped_range range{attr}; // Creates a range with message contents + * // "msg" and green color + * \endcode + * + * @param[in] attr `event_attributes` that describes the desired attributes + * of the range. + */ + explicit scoped_range_in(event_attributes const& attr) noexcept + { +# ifndef NVTX_DISABLE + nvtxDomainRangePushEx(domain::get(), attr.get()); +# else + (void) attr; +# endif + } + + /** + * @brief Constructs a `scoped_range_in` from the constructor arguments + * of an `event_attributes`. + * + * Forwards the arguments `args...` to construct an + * `event_attributes` object. The `event_attributes` object is then + * associated with the `scoped_range_in`. + * + * For more detail, see `event_attributes` documentation. + * + * Example: + * \code{cpp} + * // Creates a range with message "message" and green color + * nvtx3::scoped_range r{"message", nvtx3::rgb{127,255,0}}; + * \endcode + * + * @param[in] args Arguments to used to construct an `event_attributes` associated with this + * range. + * + */ + template + explicit scoped_range_in(Args const&... args) noexcept + : scoped_range_in{event_attributes{args...}} + {} + + /** + * @brief Default constructor creates a `scoped_range_in` with no + * message, color, payload, nor category. + * + */ + scoped_range_in() noexcept + : scoped_range_in{event_attributes{}} + {} + + /** + * @brief Delete `operator new` to disallow heap allocated objects. + * + * `scoped_range_in` must follow RAII semantics to guarantee proper push/pop semantics. + * + */ + void* operator new(std::size_t) = delete; + + scoped_range_in(scoped_range_in const&) = delete; + scoped_range_in& operator=(scoped_range_in const&) = delete; + scoped_range_in(scoped_range_in&&) = delete; + scoped_range_in& operator=(scoped_range_in&&) = delete; + + /** + * @brief Destroy the scoped_range_in, ending the NVTX range event. + */ + ~scoped_range_in() noexcept + { +# ifndef NVTX_DISABLE + nvtxDomainRangePop(domain::get()); +# endif + } + }; + + /** + * @brief Alias for a `scoped_range_in` in the global NVTX domain. + * + */ + using scoped_range = scoped_range_in; + + namespace detail + { + /// @cond internal + template + class NVTX3_MAYBE_UNUSED optional_scoped_range_in + { + public: + optional_scoped_range_in() = default; + + void begin(event_attributes const& attr) noexcept + { +# ifndef NVTX_DISABLE + // This class is not meant to be part of the public NVTX C++ API and should + // only be used in the `NVTX3_FUNC_RANGE_IF` and `NVTX3_FUNC_RANGE_IF_IN` + // macros. However, to prevent developers from misusing this class, make + // sure to not start multiple ranges. + if (initialized) + { + return; + } + + nvtxDomainRangePushEx(domain::get(), attr.get()); + initialized = true; +# endif + } + + ~optional_scoped_range_in() noexcept + { +# ifndef NVTX_DISABLE + if (initialized) + { + nvtxDomainRangePop(domain::get()); + } +# endif + } + + void* operator new(std::size_t) = delete; + optional_scoped_range_in(optional_scoped_range_in const&) = delete; + optional_scoped_range_in& operator=(optional_scoped_range_in const&) = delete; + optional_scoped_range_in(optional_scoped_range_in&&) = delete; + optional_scoped_range_in& operator=(optional_scoped_range_in&&) = delete; + + private: +# ifndef NVTX_DISABLE + bool initialized = false; +# endif + }; + /// @endcond + } // namespace detail + + /** + * @brief Handle used for correlating explicit range start and end events. + * + * A handle is "null" if it does not correspond to any range. + * + */ + struct range_handle + { + /// Type used for the handle's value + using value_type = nvtxRangeId_t; + + /** + * @brief Construct a `range_handle` from the given id. + * + */ + constexpr explicit range_handle(value_type id) noexcept + : _range_id{id} + {} + + /** + * @brief Constructs a null range handle. + * + * A null range_handle corresponds to no range. Calling `end_range` on a + * null handle is undefined behavior when a tool is active. + * + */ + constexpr range_handle() noexcept = default; + + /** + * @brief Checks whether this handle is null + * + * Provides contextual conversion to `bool`. + * + * \code{cpp} + * range_handle handle{}; + * if (handle) {...} + * \endcode + * + */ + constexpr explicit operator bool() const noexcept + { + return get_value() != null_range_id; + } + + /** + * @brief Implicit conversion from `nullptr` constructs a null handle. + * + * Satisfies the "NullablePointer" requirement to make `range_handle` comparable with `nullptr`. + * + */ + constexpr range_handle(std::nullptr_t) noexcept {} + + /** + * @brief Returns the `range_handle`'s value + * + * @return value_type The handle's value + */ + constexpr value_type get_value() const noexcept + { + return _range_id; + } + + private: + /// Sentinel value for a null handle that corresponds to no range + static constexpr value_type null_range_id = nvtxRangeId_t{0}; + + value_type _range_id{null_range_id}; ///< The underlying NVTX range id + }; + + /** + * @brief Compares two range_handles for equality + * + * @param lhs The first range_handle to compare + * @param rhs The second range_handle to compare + */ + inline constexpr bool operator==(range_handle lhs, range_handle rhs) noexcept + { + return lhs.get_value() == rhs.get_value(); + } + + /** + * @brief Compares two range_handles for inequality + * + * @param lhs The first range_handle to compare + * @param rhs The second range_handle to compare + */ + inline constexpr bool operator!=(range_handle lhs, range_handle rhs) noexcept + { + return !(lhs == rhs); + } + + /** + * @brief Manually begin an NVTX range. + * + * Explicitly begins an NVTX range and returns a unique handle. To end the + * range, pass the handle to `end_range_in()`. + * + * `nvtx3::start_range(...)` is equivalent to `nvtx3::start_range_in<>(...)` and + * `nvtx3::start_range_in(...)`. + * + * `start_range_in/end_range_in` are the most explicit and lowest level APIs + * provided for creating ranges. Use of `nvtx3::unique_range_in` should be + * preferred unless one is unable to tie the range to the lifetime of an object. + * + * Example: + * \code{.cpp} + * nvtx3::event_attributes attr{"msg", nvtx3::rgb{127,255,0}}; + * // Manually begin a range + * nvtx3::range_handle h = nvtx3::start_range_in(attr); + * ... + * nvtx3::end_range_in(h); // End the range + * \endcode + * + * @tparam D Type containing `name` member used to identify the `domain` + * to which the range belongs. Else, `domain::global` to indicate that the + * global NVTX domain should be used. + * @param[in] attr `event_attributes` that describes the desired attributes + * of the range. + * @return Unique handle to be passed to `end_range_in` to end the range. + */ + template + NVTX3_NO_DISCARD inline range_handle start_range_in(event_attributes const& attr) noexcept + { +# ifndef NVTX_DISABLE + return range_handle{nvtxDomainRangeStartEx(domain::get(), attr.get())}; +# else + (void) attr; + return {}; +# endif + } + + /** + * @brief Manually begin an NVTX range. + * + * Explicitly begins an NVTX range and returns a unique handle. To end the + * range, pass the handle to `end_range_in()`. + * + * `nvtx3::start_range(...)` is equivalent to `nvtx3::start_range_in<>(...)` and + * `nvtx3::start_range_in(...)`. + * + * `start_range_in/end_range_in` are the most explicit and lowest level APIs + * provided for creating ranges. Use of `nvtx3::unique_range_in` should be + * preferred unless one is unable to tie the range to the lifetime of an object. + * + * This overload uses `args...` to construct an `event_attributes` to + * associate with the range. For more detail, see `event_attributes`. + * + * Example: + * \code{cpp} + * // Manually begin a range + * nvtx3::range_handle h = nvtx3::start_range_in("msg", nvtx3::rgb{127,255,0}); + * ... + * nvtx3::end_range_in(h); // Ends the range + * \endcode + * + * @tparam D Type containing `name` member used to identify the `domain` + * to which the range belongs. Else, `domain::global` to indicate that the + * global NVTX domain should be used. + * @param[in] args Variadic parameter pack of the arguments for an `event_attributes`. + * @return Unique handle to be passed to `end_range` to end the range. + */ + template + NVTX3_NO_DISCARD inline range_handle start_range_in(Args const&... args) noexcept + { +# ifndef NVTX_DISABLE + return start_range_in(event_attributes{args...}); +# else + return {}; +# endif + } + + /** + * @brief Manually begin an NVTX range in the global domain. + * + * Explicitly begins an NVTX range and returns a unique handle. To end the + * range, pass the handle to `end_range()`. + * + * `nvtx3::start_range(...)` is equivalent to `nvtx3::start_range_in<>(...)` and + * `nvtx3::start_range_in(...)`. + * + * `start_range/end_range` are the most explicit and lowest level APIs + * provided for creating ranges. Use of `nvtx3::unique_range` should be + * preferred unless one is unable to tie the range to the lifetime of an object. + * + * Example: + * \code{.cpp} + * nvtx3::event_attributes attr{"msg", nvtx3::rgb{127,255,0}}; + * // Manually begin a range + * nvtx3::range_handle h = nvtx3::start_range(attr); + * ... + * nvtx3::end_range(h); // End the range + * \endcode + * + * @param[in] attr `event_attributes` that describes the desired attributes + * of the range. + * @return Unique handle to be passed to `end_range_in` to end the range. + */ + NVTX3_NO_DISCARD inline range_handle start_range(event_attributes const& attr) noexcept + { +# ifndef NVTX_DISABLE + return start_range_in(attr); +# else + (void) attr; + return {}; +# endif + } + + /** + * @brief Manually begin an NVTX range in the global domain. + * + * Explicitly begins an NVTX range and returns a unique handle. To end the + * range, pass the handle to `end_range_in()`. + * + * `nvtx3::start_range(...)` is equivalent to `nvtx3::start_range_in<>(...)` and + * `nvtx3::start_range_in(...)`. + * + * `start_range_in/end_range_in` are the most explicit and lowest level APIs + * provided for creating ranges. Use of `nvtx3::unique_range_in` should be + * preferred unless one is unable to tie the range to the lifetime of an object. + * + * This overload uses `args...` to construct an `event_attributes` to + * associate with the range. For more detail, see `event_attributes`. + * + * Example: + * \code{cpp} + * // Manually begin a range + * nvtx3::range_handle h = nvtx3::start_range("msg", nvtx3::rgb{127,255,0}); + * ... + * nvtx3::end_range(h); // Ends the range + * \endcode + * + * @param[in] args Variadic parameter pack of the arguments for an `event_attributes`. + * @return Unique handle to be passed to `end_range` to end the range. + */ + template + NVTX3_NO_DISCARD inline range_handle start_range(Args const&... args) noexcept + { +# ifndef NVTX_DISABLE + return start_range_in(args...); +# else + return {}; +# endif + } + + /** + * @brief Manually end the range associated with the handle `r` in domain `D`. + * + * Explicitly ends the NVTX range indicated by the handle `r` returned from a + * prior call to `start_range_in`. The range may end on a different thread + * from where it began. + * + * @tparam D Type containing `name` member used to identify the `domain` to + * which the range belongs. Else, `domain::global` to indicate that the global + * NVTX domain should be used. + * @param r Handle to a range started by a prior call to `start_range_in`. + * + * @warning The domain type specified as template parameter to this function + * must be the same that was specified on the associated `start_range_in` call. + */ + template + inline void end_range_in(range_handle r) noexcept + { +# ifndef NVTX_DISABLE + nvtxDomainRangeEnd(domain::get(), r.get_value()); +# else + (void) r; +# endif + } + + /** + * @brief Manually end the range associated with the handle `r` in the global + * domain. + * + * Explicitly ends the NVTX range indicated by the handle `r` returned from a + * prior call to `start_range`. The range may end on a different thread from + * where it began. + * + * @param r Handle to a range started by a prior call to `start_range`. + * + * @warning The domain type specified as template parameter to this function + * must be the same that was specified on the associated `start_range` call. + */ + inline void end_range(range_handle r) noexcept + { +# ifndef NVTX_DISABLE + end_range_in(r); +# else + (void) r; +# endif + } + + /** + * @brief A RAII object for creating a NVTX range within a domain that can + * be created and destroyed on different threads. + * + * When constructed, begins a NVTX range in the specified domain. Upon + * destruction, ends the NVTX range. + * + * Similar to `nvtx3::scoped_range_in`, with a few key differences: + * - `unique_range` objects can be destroyed in an order whereas `scoped_range` objects must be + * destroyed in exact reverse creation order + * - `unique_range` can start and end on different threads + * - `unique_range` is movable + * - `unique_range` objects can be constructed as heap objects + * + * There is extra overhead associated with `unique_range` constructs and therefore use of + * `nvtx3::scoped_range_in` should be preferred. + * + * @tparam D Type containing `name` member used to identify the `domain` + * to which the `unique_range_in` belongs. Else, `domain::global` to + * indicate that the global NVTX domain should be used. + */ + template + class NVTX3_MAYBE_UNUSED unique_range_in + { + public: + /** + * @brief Construct a new unique_range_in object with the specified event attributes + * + * Example: + * \code{cpp} + * nvtx3::event_attributes attr{"msg", nvtx3::rgb{127,255,0}}; + * nvtx3::unique_range_in range{attr}; // Creates a range with message contents + * // "msg" and green color + * \endcode + * + * @param[in] attr `event_attributes` that describes the desired attributes + * of the range. + */ + explicit unique_range_in(event_attributes const& attr) noexcept + : handle_{start_range_in(attr)} + {} + + /** + * @brief Constructs a `unique_range_in` from the constructor arguments + * of an `event_attributes`. + * + * Forwards the arguments `args...` to construct an + * `event_attributes` object. The `event_attributes` object is then + * associated with the `unique_range_in`. + * + * For more detail, see `event_attributes` documentation. + * + * Example: + * \code{.cpp} + * // Creates a range with message "message" and green color + * nvtx3::unique_range_in<> r{"message", nvtx3::rgb{127,255,0}}; + * \endcode + * + * @param[in] args Variadic parameter pack of arguments to construct an `event_attributes` + * associated with this range. + */ + template + explicit unique_range_in(Args const&... args) noexcept + : unique_range_in{event_attributes{args...}} + {} + + /** + * @brief Default constructor creates a `unique_range_in` with no + * message, color, payload, nor category. + * + */ + constexpr unique_range_in() noexcept + : unique_range_in{event_attributes{}} + {} + + /** + * @brief Destroy the `unique_range_in` ending the range. + * + */ + ~unique_range_in() noexcept = default; + + /** + * @brief Move constructor allows taking ownership of the NVTX range from + * another `unique_range_in`. + * + * @param other The range to take ownership of + */ + unique_range_in(unique_range_in&& other) noexcept = default; + + /** + * @brief Move assignment operator allows taking ownership of an NVTX range + * from another `unique_range_in`. + * + * @param other The range to take ownership of + */ + unique_range_in& operator=(unique_range_in&& other) noexcept = default; + + /// Copy construction is not allowed to prevent multiple objects from owning + /// the same range handle + unique_range_in(unique_range_in const&) = delete; + + /// Copy assignment is not allowed to prevent multiple objects from owning the + /// same range handle + unique_range_in& operator=(unique_range_in const&) = delete; + + private: + struct end_range_handle + { + using pointer = range_handle; /// Override the pointer type of the unique_ptr + void operator()(range_handle h) const noexcept + { + end_range_in(h); + } + }; + + /// Range handle used to correlate the start/end of the range + std::unique_ptr handle_; + }; + + /** + * @brief Alias for a `unique_range_in` in the global NVTX domain. + * + */ + using unique_range = unique_range_in; + + /** + * @brief Annotates an instantaneous point in time with a "marker", using the + * attributes specified by `attr`. + * + * Unlike a "range" which has a beginning and an end, a marker is a single event + * in an application, such as detecting a problem: + * + * \code{.cpp} + * bool success = do_operation(...); + * if (!success) { + * nvtx3::event_attributes attr{"operation failed!", nvtx3::rgb{255,0,0}}; + * nvtx3::mark_in(attr); + * } + * \endcode + * + * Note that nvtx3::mark_in is a function, not a class like scoped_range_in. + * + * @tparam D Type containing `name` member used to identify the `domain` + * to which the `unique_range_in` belongs. Else, `domain::global` to + * indicate that the global NVTX domain should be used. + * @param[in] attr `event_attributes` that describes the desired attributes + * of the mark. + */ + template + inline void mark_in(event_attributes const& attr) noexcept + { +# ifndef NVTX_DISABLE + nvtxDomainMarkEx(domain::get(), attr.get()); +# else + (void) (attr); +# endif + } + + /** + * @brief Annotates an instantaneous point in time with a "marker", using the + * arguments to construct an `event_attributes`. + * + * Unlike a "range" which has a beginning and an end, a marker is a single event + * in an application, such as detecting a problem: + * + * \code{.cpp} + * bool success = do_operation(...); + * if (!success) { + * nvtx3::mark_in("operation failed!", nvtx3::rgb{255,0,0}); + * } + * \endcode + * + * Note that nvtx3::mark_in is a function, not a class like scoped_range_in. + * + * Forwards the arguments `args...` to construct an `event_attributes` object. + * The attributes are then associated with the marker. For more detail, see + * the `event_attributes` documentation. + * + * @tparam D Type containing `name` member used to identify the `domain` + * to which the `unique_range_in` belongs. Else `domain::global` to + * indicate that the global NVTX domain should be used. + * @param[in] args Variadic parameter pack of arguments to construct an `event_attributes` + * associated with this range. + * + */ + template + inline void mark_in(Args const&... args) noexcept + { +# ifndef NVTX_DISABLE + mark_in(event_attributes{args...}); +# endif + } + + /** + * @brief Annotates an instantaneous point in time with a "marker", using the + * attributes specified by `attr`, in the global domain. + * + * Unlike a "range" which has a beginning and an end, a marker is a single event + * in an application, such as detecting a problem: + * + * \code{.cpp} + * bool success = do_operation(...); + * if (!success) { + * nvtx3::event_attributes attr{"operation failed!", nvtx3::rgb{255,0,0}}; + * nvtx3::mark(attr); + * } + * \endcode + * + * Note that nvtx3::mark is a function, not a class like scoped_range. + * + * @param[in] attr `event_attributes` that describes the desired attributes + * of the mark. + */ + inline void mark(event_attributes const& attr) noexcept + { +# ifndef NVTX_DISABLE + mark_in(attr); +# endif + } + + /** + * @brief Annotates an instantaneous point in time with a "marker", using the + * arguments to construct an `event_attributes`, in the global domain. + * + * Unlike a "range" which has a beginning and an end, a marker is a single event + * in an application, such as detecting a problem: + * + * \code{.cpp} + * bool success = do_operation(...); + * if (!success) { + * nvtx3::mark("operation failed!", nvtx3::rgb{255,0,0}); + * } + * \endcode + * + * Note that nvtx3::mark is a function, not a class like scoped_range. + * + * Forwards the arguments `args...` to construct an `event_attributes` object. + * The attributes are then associated with the marker. For more detail, see + * the `event_attributes` documentation. + * + * @param[in] args Variadic parameter pack of arguments to construct an + * `event_attributes` associated with this range. + * + */ + template + inline void mark(Args const&... args) noexcept + { +# ifndef NVTX_DISABLE + mark_in(args...); +# endif + } + +} // namespace NVTX3_VERSION_NAMESPACE +} // namespace nvtx3 + +# ifndef NVTX_DISABLE +/** + * @brief Convenience macro for generating a range in the specified `domain` + * from the lifetime of a function + * + * This macro is useful for generating an NVTX range in `domain` from + * the entry point of a function to its exit. It is intended to be the first + * line of the function. + * + * Constructs a static `registered_string_in` using the name of the immediately + * enclosing function returned by `__func__` and constructs a + * `nvtx3::scoped_range` using the registered function name as the range's + * message. + * + * Example: + * \code{.cpp} + * struct my_domain{static constexpr char const* name{"my_domain"};}; + * + * void foo(...) { + * NVTX3_FUNC_RANGE_IN(my_domain); // Range begins on entry to foo() + * // do stuff + * ... + * } // Range ends on return from foo() + * \endcode + * + * @param[in] D Type containing `name` member used to identify the + * `domain` to which the `registered_string_in` belongs. Else, + * `domain::global` to indicate that the global NVTX domain should be used. + */ +# define NVTX3_V1_FUNC_RANGE_IN(D) \ + static ::nvtx3::v1::registered_string_in const nvtx3_func_name__{__func__}; \ + static ::nvtx3::v1::event_attributes const nvtx3_func_attr__{nvtx3_func_name__}; \ + ::nvtx3::v1::scoped_range_in const nvtx3_range__{nvtx3_func_attr__}; + +/** + * @brief Convenience macro for generating a range in the specified `domain` + * from the lifetime of a function if the given boolean expression evaluates + * to true. + * + * Similar to `NVTX3_V1_FUNC_RANGE_IN(D)`, the only difference being that + * `NVTX3_V1_FUNC_RANGE_IF_IN(D, C)` only generates a range if the given boolean + * expression evaluates to true. + * + * @param[in] D Type containing `name` member used to identify the + * `domain` to which the `registered_string_in` belongs. Else, + * `domain::global` to indicate that the global NVTX domain should be used. + * + * @param[in] C Boolean expression used to determine if a range should be + * generated. + */ +# define NVTX3_V1_FUNC_RANGE_IF_IN(D, C) \ + ::nvtx3::v1::detail::optional_scoped_range_in optional_nvtx3_range__; \ + if (C) \ + { \ + static ::nvtx3::v1::registered_string_in const nvtx3_func_name__{__func__}; \ + static ::nvtx3::v1::event_attributes const nvtx3_func_attr__{nvtx3_func_name__}; \ + optional_nvtx3_range__.begin(nvtx3_func_attr__); \ + } +# else +# define NVTX3_V1_FUNC_RANGE_IN(D) +# define NVTX3_V1_FUNC_RANGE_IF_IN(D, C) +# endif // NVTX_DISABLE + +/** + * @brief Convenience macro for generating a range in the global domain from the + * lifetime of a function. + * + * This macro is useful for generating an NVTX range in the global domain from + * the entry point of a function to its exit. It is intended to be the first + * line of the function. + * + * Constructs a static `registered_string_in` using the name of the immediately + * enclosing function returned by `__func__` and constructs a + * `nvtx3::scoped_range` using the registered function name as the range's + * message. + * + * Example: + * \code{.cpp} + * void foo(...) { + * NVTX3_FUNC_RANGE(); // Range begins on entry to foo() + * // do stuff + * ... + * } // Range ends on return from foo() + * \endcode + */ +# define NVTX3_V1_FUNC_RANGE() NVTX3_V1_FUNC_RANGE_IN(::nvtx3::v1::domain::global) + +/** + * @brief Convenience macro for generating a range in the global domain from the + * lifetime of a function if the given boolean expression evaluates to true. + * + * Similar to `NVTX3_V1_FUNC_RANGE()`, the only difference being that + * `NVTX3_V1_FUNC_RANGE_IF(C)` only generates a range if the given boolean + * expression evaluates to true. + * + * @param[in] C Boolean expression used to determine if a range should be + * generated. + */ +# define NVTX3_V1_FUNC_RANGE_IF(C) NVTX3_V1_FUNC_RANGE_IF_IN(::nvtx3::v1::domain::global, C) + +/* When inlining this version, versioned macros must have unversioned aliases. + * For each NVTX3_Vx_ #define, make an NVTX3_ alias of it here.*/ +# if defined(NVTX3_INLINE_THIS_VERSION) +/* clang format off */ +# define NVTX3_FUNC_RANGE NVTX3_V1_FUNC_RANGE +# define NVTX3_FUNC_RANGE_IF NVTX3_V1_FUNC_RANGE_IF +# define NVTX3_FUNC_RANGE_IN NVTX3_V1_FUNC_RANGE_IN +# define NVTX3_FUNC_RANGE_IF_IN NVTX3_V1_FUNC_RANGE_IF_IN +/* clang format on */ +# endif + +#endif // NVTX3_CPP_DEFINITIONS_V1_0 + +/* Add functionality for new minor versions here, by copying the above section enclosed + * in #ifndef NVTX3_CPP_DEFINITIONS_Vx_y, and incrementing the minor version. This code + * is an example of how additions for version 1.2 would look, indented for clarity. Note + * that the versioned symbols and macros are always provided, and the unversioned symbols + * are only provided if NVTX3_INLINE_THIS_VERSION was defined at the top of this header. + * + * \code{.cpp} + * #ifndef NVTX3_CPP_DEFINITIONS_V1_2 + * #define NVTX3_CPP_DEFINITIONS_V1_2 + * namespace nvtx3 { + * NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE { + * class new_class {}; + * inline void new_function() {} + * } + * } + * + * // Macros must have the major version in their names: + * #define NVTX3_V1_NEW_MACRO_A() ... + * #define NVTX3_V1_NEW_MACRO_B() ... + * + * // If inlining, make aliases for the macros with the version number omitted + * #if defined(NVTX3_INLINE_THIS_VERSION) + * #define NVTX3_NEW_MACRO_A NVTX3_V1_NEW_MACRO_A + * #define NVTX3_NEW_MACRO_B NVTX3_V1_NEW_MACRO_B + * #endif + * #endif // NVTX3_CPP_DEFINITIONS_V1_2 + * \endcode + */ + +/* Undefine all temporarily-defined unversioned macros, which would conflict with + * subsequent includes of different versions of this header. */ +#undef NVTX3_CPP_VERSION_MAJOR +#undef NVTX3_CPP_VERSION_MINOR +#undef NVTX3_CONCAT +#undef NVTX3_NAMESPACE_FOR +#undef NVTX3_VERSION_NAMESPACE +#undef NVTX3_INLINE_IF_REQUESTED +#undef NVTX3_CONSTEXPR_IF_CPP14 +#undef NVTX3_MAYBE_UNUSED +#undef NVTX3_NO_DISCARD + +#if defined(NVTX3_INLINE_THIS_VERSION) +# undef NVTX3_INLINE_THIS_VERSION +#endif + +#if defined(NVTX3_USE_CHECKED_OVERLOADS_FOR_GET_DEFINED_HERE) +# undef NVTX3_USE_CHECKED_OVERLOADS_FOR_GET_DEFINED_HERE +# undef NVTX3_USE_CHECKED_OVERLOADS_FOR_GET +#endif + +#if defined(NVTX3_STATIC_ASSERT_DEFINED_HERE) +# undef NVTX3_STATIC_ASSERT_DEFINED_HERE +# undef NVTX3_STATIC_ASSERT +#endif diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/get_sreg.h b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/get_sreg.h new file mode 100644 index 00000000..d0441d2f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/generated/get_sreg.h @@ -0,0 +1,949 @@ +// This file was automatically generated. Do not edit. + +#ifndef _CUDA_PTX_GENERATED_GET_SREG_H_ +#define _CUDA_PTX_GENERATED_GET_SREG_H_ + +/* +// mov.u32 sreg_value, %%tid.x; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_tid_x(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_x() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%tid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%tid.y; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_tid_y(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_y() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%tid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%tid.z; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_tid_z(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_tid_z() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%tid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%ntid.x; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_ntid_x(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_x() +{ + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%ntid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%ntid.y; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_ntid_y(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_y() +{ + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%ntid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%ntid.z; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_ntid_z(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ntid_z() +{ + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%ntid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%laneid; // PTX ISA 13 +template +__device__ static inline uint32_t get_sreg_laneid(); +*/ +#if __cccl_ptx_isa >= 130 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_laneid() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%laneid;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 130 + +/* +// mov.u32 sreg_value, %%warpid; // PTX ISA 13 +template +__device__ static inline uint32_t get_sreg_warpid(); +*/ +#if __cccl_ptx_isa >= 130 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_warpid() +{ + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%warpid;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 130 + +/* +// mov.u32 sreg_value, %%nwarpid; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_nwarpid(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nwarpid() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%nwarpid;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%ctaid.x; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_ctaid_x(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_x() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%ctaid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%ctaid.y; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_ctaid_y(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_y() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%ctaid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%ctaid.z; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_ctaid_z(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_ctaid_z() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%ctaid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%nctaid.x; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_nctaid_x(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_x() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%nctaid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%nctaid.y; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_nctaid_y(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_y() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%nctaid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%nctaid.z; // PTX ISA 20 +template +__device__ static inline uint32_t get_sreg_nctaid_z(); +*/ +#if __cccl_ptx_isa >= 200 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nctaid_z() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%nctaid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%smid; // PTX ISA 13 +template +__device__ static inline uint32_t get_sreg_smid(); +*/ +#if __cccl_ptx_isa >= 130 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_smid() +{ + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%smid;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 130 + +/* +// mov.u32 sreg_value, %%nsmid; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_nsmid(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nsmid() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%nsmid;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u64 sreg_value, %%gridid; // PTX ISA 30 +template +__device__ static inline uint64_t get_sreg_gridid(); +*/ +#if __cccl_ptx_isa >= 300 +template +_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_gridid() +{ + ::cuda::std::uint64_t __sreg_value; + asm("mov.u64 %0, %%gridid;" : "=l"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 300 + +/* +// mov.pred sreg_value, %%is_explicit_cluster; // PTX ISA 78, SM_90 +template +__device__ static inline bool get_sreg_is_explicit_cluster(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline bool get_sreg_is_explicit_cluster() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("{\n\t .reg .pred P_OUT; \n\t" + "mov.pred P_OUT, %%is_explicit_cluster;\n\t" + "selp.b32 %0, 1, 0, P_OUT; \n" + "}" + : "=r"(__sreg_value) + : + :); + return static_cast(__sreg_value); +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__(); + return false; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%clusterid.x; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_clusterid_x(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_x() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%clusterid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%clusterid.y; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_clusterid_y(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_y() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%clusterid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%clusterid.z; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_clusterid_z(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clusterid_z() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%clusterid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%nclusterid.x; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_nclusterid_x(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_x() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%nclusterid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%nclusterid.y; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_nclusterid_y(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_y() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%nclusterid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%nclusterid.z; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_nclusterid_z(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_nclusterid_z() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%nclusterid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_ctaid.x; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_ctaid_x(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_x() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_ctaid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_ctaid.y; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_ctaid_y(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_y() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_ctaid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_ctaid.z; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_ctaid_z(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctaid_z() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_ctaid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_nctaid.x; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_nctaid_x(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_x() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_nctaid.x;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_nctaid.y; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_nctaid_y(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_y() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_nctaid.y;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_nctaid.z; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_nctaid_z(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctaid_z() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_nctaid.z;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_ctarank; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_ctarank(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_ctarank() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_ctarank;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%cluster_nctarank; // PTX ISA 78, SM_90 +template +__device__ static inline uint32_t get_sreg_cluster_nctarank(); +*/ +#if __cccl_ptx_isa >= 780 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_cluster_nctarank() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%cluster_nctarank;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 780 + +/* +// mov.u32 sreg_value, %%lanemask_eq; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_lanemask_eq(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_eq() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%lanemask_eq;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%lanemask_le; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_lanemask_le(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_le() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%lanemask_le;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%lanemask_lt; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_lanemask_lt(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_lt() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%lanemask_lt;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%lanemask_ge; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_lanemask_ge(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_ge() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%lanemask_ge;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%lanemask_gt; // PTX ISA 20, SM_35 +template +__device__ static inline uint32_t get_sreg_lanemask_gt(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_lanemask_gt() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%lanemask_gt;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u32 sreg_value, %%clock; // PTX ISA 10 +template +__device__ static inline uint32_t get_sreg_clock(); +*/ +#if __cccl_ptx_isa >= 100 +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clock() +{ + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%clock;" : "=r"(__sreg_value) : :); + return __sreg_value; +} +#endif // __cccl_ptx_isa >= 100 + +/* +// mov.u32 sreg_value, %%clock_hi; // PTX ISA 50, SM_35 +template +__device__ static inline uint32_t get_sreg_clock_hi(); +*/ +#if __cccl_ptx_isa >= 500 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_clock_hi() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%clock_hi;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 500 + +/* +// mov.u64 sreg_value, %%clock64; // PTX ISA 20, SM_35 +template +__device__ static inline uint64_t get_sreg_clock64(); +*/ +#if __cccl_ptx_isa >= 200 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_clock64() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint64_t __sreg_value; + asm volatile("mov.u64 %0, %%clock64;" : "=l"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 200 + +/* +// mov.u64 sreg_value, %%globaltimer; // PTX ISA 31, SM_35 +template +__device__ static inline uint64_t get_sreg_globaltimer(); +*/ +#if __cccl_ptx_isa >= 310 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_globaltimer() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint64_t __sreg_value; + asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 310 + +/* +// mov.u32 sreg_value, %%globaltimer_lo; // PTX ISA 31, SM_35 +template +__device__ static inline uint32_t get_sreg_globaltimer_lo(); +*/ +#if __cccl_ptx_isa >= 310 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_globaltimer_lo() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%globaltimer_lo;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 310 + +/* +// mov.u32 sreg_value, %%globaltimer_hi; // PTX ISA 31, SM_35 +template +__device__ static inline uint32_t get_sreg_globaltimer_hi(); +*/ +#if __cccl_ptx_isa >= 310 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_globaltimer_hi() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm volatile("mov.u32 %0, %%globaltimer_hi;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 310 + +/* +// mov.u32 sreg_value, %%total_smem_size; // PTX ISA 41, SM_35 +template +__device__ static inline uint32_t get_sreg_total_smem_size(); +*/ +#if __cccl_ptx_isa >= 410 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_total_smem_size() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%total_smem_size;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 410 + +/* +// mov.u32 sreg_value, %%aggr_smem_size; // PTX ISA 81, SM_90 +template +__device__ static inline uint32_t get_sreg_aggr_smem_size(); +*/ +#if __cccl_ptx_isa >= 810 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_aggr_smem_size() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 900 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%aggr_smem_size;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 810 + +/* +// mov.u32 sreg_value, %%dynamic_smem_size; // PTX ISA 41, SM_35 +template +__device__ static inline uint32_t get_sreg_dynamic_smem_size(); +*/ +#if __cccl_ptx_isa >= 410 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint32_t get_sreg_dynamic_smem_size() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 350 + ::cuda::std::uint32_t __sreg_value; + asm("mov.u32 %0, %%dynamic_smem_size;" : "=r"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 410 + +/* +// mov.u64 sreg_value, %%current_graph_exec; // PTX ISA 80, SM_50 +template +__device__ static inline uint64_t get_sreg_current_graph_exec(); +*/ +#if __cccl_ptx_isa >= 800 +extern "C" _CCCL_DEVICE void __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__(); +template +_CCCL_DEVICE static inline ::cuda::std::uint64_t get_sreg_current_graph_exec() +{ +# if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 500 + ::cuda::std::uint64_t __sreg_value; + asm("mov.u64 %0, %%current_graph_exec;" : "=l"(__sreg_value) : :); + return __sreg_value; +# else + // Unsupported architectures will have a linker error with a semi-decent error message + __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__(); + return 0; +# endif +} +#endif // __cccl_ptx_isa >= 800 + +#endif // _CUDA_PTX_GENERATED_GET_SREG_H_ diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/get_sreg.h b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/get_sreg.h new file mode 100644 index 00000000..1ac117c2 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/instructions/get_sreg.h @@ -0,0 +1,43 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_PTX_GET_SREG_H_ +#define _CUDA_PTX_GET_SREG_H_ + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include // __CUDA_MINIMUM_ARCH__ and friends + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_PTX + +// 10. Special Registers +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#special-registers +#include + +_CCCL_END_NAMESPACE_CUDA_PTX + +#include + +#endif // _CUDA_PTX_GET_SREG_H_ diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_dot_variants.h b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_dot_variants.h new file mode 100644 index 00000000..d923a548 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_dot_variants.h @@ -0,0 +1,230 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// WARNING: The source of truth for this file is libcuda-ptx. Do not modify without syncing with libcuda-ptx. + +#ifndef _CUDA_PTX_DOT_VARIANTS_H_ +#define _CUDA_PTX_DOT_VARIANTS_H_ + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +/* + * Public integral constant types and values for ".variant"s: + * + * - .sem: acquire, release, .. + * - .space: global, shared, constant, .. + * - .scope: cta, cluster, gpu, .. + * - .op: add, min, cas, .. + * + * For each .variant, the code below defines: + * - An enum `dot_variant` with each possible value + * - A type template `variant_t` + * - Types `variant_A_t`, ..., `variant_Z_t` + * - Constexpr values `variant_A` of type `variant_A_t` + * + * These types enable specifying fine-grained overloads of a PTX binding. If a + * binding can handle multiple variants, then it is defined as: + * + * template + * [...] void ptx_binding(variant_t __v) { ... } + * + * If it only handles a single variant, then it is defined as: + * + * [...] void ptx_binding(variant_A __v) { ... } + * + * If two variants have different behaviors or return types (see .space + * overloads of mbarrier.arrive.expect_tx for an example), then these can be + * provided as separate overloads of the same function: + * + * [...] void ptx_binding(variant_A __v) { ... } + * [...] int ptx_binding(variant_B __v) { ... } + * + */ + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_PTX + +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#operation-types +enum class dot_sem +{ + acq_rel, + acquire, + relaxed, + release, + sc, + weak +}; + +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#state-spaces +enum class dot_space +{ + global, + cluster, // The PTX spelling is shared::cluster + shared, // The PTX spelling is shared::cta + + // The following state spaces are unlikely to be used in cuda::ptx in the near + // future, so they are not exposed: + + // reg, + // sreg, + // const_mem, // Using const_mem as `const` is reserved in C++. + // local, + // param, + // tex // deprecated +}; + +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#scope +enum class dot_scope +{ + cta, + cluster, + gpu, + sys +}; + +enum class dot_op +{ + add, + dec, + inc, + max, + min, + and_op, // Using and_op, as `and, or, xor` are reserved in C++. + or_op, + xor_op, + cas, + exch +}; + +enum class dot_cta_group +{ + cta_group_1, + cta_group_2 +}; + +enum class dot_kind +{ + f16, + f8f6f4, + i8, + mxf4, + mxf4nvf4, + mxf8f6f4, + tf32 +}; + +template +using sem_t = ::cuda::std::integral_constant; +using sem_acq_rel_t = sem_t; +using sem_acquire_t = sem_t; +using sem_relaxed_t = sem_t; +using sem_release_t = sem_t; +using sem_sc_t = sem_t; +using sem_weak_t = sem_t; + +[[maybe_unused]] static constexpr sem_acq_rel_t sem_acq_rel{}; +[[maybe_unused]] static constexpr sem_acquire_t sem_acquire{}; +[[maybe_unused]] static constexpr sem_relaxed_t sem_relaxed{}; +[[maybe_unused]] static constexpr sem_release_t sem_release{}; +[[maybe_unused]] static constexpr sem_sc_t sem_sc{}; +[[maybe_unused]] static constexpr sem_weak_t sem_weak{}; + +template +using space_t = ::cuda::std::integral_constant; +using space_global_t = space_t; +using space_shared_t = space_t; +using space_cluster_t = space_t; + +[[maybe_unused]] static constexpr space_global_t space_global{}; +[[maybe_unused]] static constexpr space_shared_t space_shared{}; +[[maybe_unused]] static constexpr space_cluster_t space_cluster{}; + +template +using scope_t = ::cuda::std::integral_constant; +using scope_cluster_t = scope_t; +using scope_cta_t = scope_t; +using scope_gpu_t = scope_t; +using scope_sys_t = scope_t; + +[[maybe_unused]] static constexpr scope_cluster_t scope_cluster{}; +[[maybe_unused]] static constexpr scope_cta_t scope_cta{}; +[[maybe_unused]] static constexpr scope_gpu_t scope_gpu{}; +[[maybe_unused]] static constexpr scope_sys_t scope_sys{}; + +template +using op_t = ::cuda::std::integral_constant; +using op_add_t = op_t; +using op_dec_t = op_t; +using op_inc_t = op_t; +using op_max_t = op_t; +using op_min_t = op_t; +using op_and_op_t = op_t; +using op_or_op_t = op_t; +using op_xor_op_t = op_t; +using op_cas_t = op_t; +using op_exch_t = op_t; + +[[maybe_unused]] static constexpr op_add_t op_add{}; +[[maybe_unused]] static constexpr op_dec_t op_dec{}; +[[maybe_unused]] static constexpr op_inc_t op_inc{}; +[[maybe_unused]] static constexpr op_max_t op_max{}; +[[maybe_unused]] static constexpr op_min_t op_min{}; +[[maybe_unused]] static constexpr op_and_op_t op_and_op{}; +[[maybe_unused]] static constexpr op_or_op_t op_or_op{}; +[[maybe_unused]] static constexpr op_xor_op_t op_xor_op{}; +[[maybe_unused]] static constexpr op_cas_t op_cas{}; +[[maybe_unused]] static constexpr op_exch_t op_exch{}; + +template +using cta_group_t = ::cuda::std::integral_constant; +using cta_group_1_t = cta_group_t; +using cta_group_2_t = cta_group_t; + +[[maybe_unused]] static constexpr cta_group_1_t cta_group_1{}; +[[maybe_unused]] static constexpr cta_group_2_t cta_group_2{}; + +template +using kind_t = ::cuda::std::integral_constant; +using kind_f16_t = kind_t; +using kind_f8f6f4_t = kind_t; +using kind_i8_t = kind_t; +using kind_mxf4_t = kind_t; +using kind_mxf4nvf4_t = kind_t; +using kind_mxf8f6f4_t = kind_t; +using kind_tf32_t = kind_t; + +[[maybe_unused]] static constexpr kind_f16_t kind_f16{}; +[[maybe_unused]] static constexpr kind_f8f6f4_t kind_f8f6f4{}; +[[maybe_unused]] static constexpr kind_i8_t kind_i8{}; +[[maybe_unused]] static constexpr kind_mxf4_t kind_mxf4{}; +[[maybe_unused]] static constexpr kind_mxf4nvf4_t kind_mxf4nvf4{}; +[[maybe_unused]] static constexpr kind_mxf8f6f4_t kind_mxf8f6f4{}; +[[maybe_unused]] static constexpr kind_tf32_t kind_tf32{}; + +template +using n32_t = ::cuda::std::integral_constant; + +_CCCL_END_NAMESPACE_CUDA_PTX + +#include + +#endif // _CUDA_PTX_DOT_VARIANTS_H_ diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_helper_functions.h b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_helper_functions.h new file mode 100644 index 00000000..08f8a7f9 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__ptx/ptx_helper_functions.h @@ -0,0 +1,178 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_PTX_HELPER_FUNCTIONS_H_ +#define _CUDA_PTX_HELPER_FUNCTIONS_H_ + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#if _CCCL_CUDA_COMPILATION() + +# include + +# if defined(__CUDACC__) || defined(_NVHPC_CUDA) || defined(__CUDACC_RTC__) +# define _CUDA_PTX_CUDACC_MAJOR() __CUDACC_VER_MAJOR__ +# elif defined(__CUDA__) && defined(__clang__) +# define _CUDA_PTX_CUDACC_MAJOR() (CUDA_VERSION / 1000) +# endif // ^^^ has cuda compiler ^^^ + +# if !defined(_LIBCUDA_PTX_ARCH_SPECIFIC) +# if defined(__CUDA_ARCH_SPECIFIC__) +# define _LIBCUDA_PTX_ARCH_SPECIFIC() __CUDA_ARCH_SPECIFIC__ +# else +# if defined(__CUDA_ARCH_FEAT_SM90_ALL) +# define _LIBCUDA_PTX_ARCH_SPECIFIC() 900 +# elif defined(__CUDA_ARCH_FEAT_SM100_ALL) +# define _LIBCUDA_PTX_ARCH_SPECIFIC() 1000 +# elif defined(__CUDA_ARCH_FEAT_SM103_ALL) +# define _LIBCUDA_PTX_ARCH_SPECIFIC() 1030 +# elif defined(__CUDA_ARCH_FEAT_SM120_ALL) +# define _LIBCUDA_PTX_ARCH_SPECIFIC() 1200 +# else +# define _LIBCUDA_PTX_ARCH_SPECIFIC() 0 +# endif +# endif // ^^^ !defined(__CUDA_ARCH_SPECIFIC__) +# endif // ^^^ !defined(_LIBCUDA_PTX_ARCH_SPECIFIC) + +# if !defined(__CUDA_HAS_ARCH_FAMILY_SPECIFIC) + +# define __CUDA_HAS_ARCH_FAMILY_SPECIFIC(N) false + +# endif // !defined(__CUDA_HAS_ARCH_FAMILY_SPECIFIC) + +_CCCL_BEGIN_NAMESPACE_CUDA_PTX + +# if _CUDA_PTX_CUDACC_MAJOR() < 13 +struct alignas(32) longlong4_32a +{ + long long x, y, z, w; +}; +struct alignas(32) ulonglong4_32a +{ + unsigned long long x, y, z, w; +}; +struct alignas(32) double4_32a +{ + double x, y, z, w; +}; +# else +using ::double4_32a; +using ::longlong4_32a; +using ::ulonglong4_32a; +# endif // _CUDA_PTX_CUDACC_MAJOR() < 13 + +/************************************************************* + * + * Conversion from generic pointer -> state space "pointer" + * + **************************************************************/ +_CCCL_DEVICE_API inline ::cuda::std::uint32_t __as_ptr_smem(const void* __ptr) +{ + // Consider adding debug asserts here. + return static_cast<::cuda::std::uint32_t>(::__cvta_generic_to_shared(__ptr)); +} + +_CCCL_DEVICE_API inline ::cuda::std::uint32_t __as_ptr_dsmem(const void* __ptr) +{ + // No difference in implementation to __as_ptr_smem. + return __as_ptr_smem(__ptr); +} + +_CCCL_DEVICE_API inline ::cuda::std::uint32_t __as_ptr_remote_dsmem(const void* __ptr) +{ + // No difference in implementation to __as_ptr_smem. + // Consider adding debug asserts here. + return __as_ptr_smem(__ptr); +} + +_CCCL_DEVICE_API inline ::cuda::std::uint64_t __as_ptr_gmem(const void* __ptr) +{ + // Consider adding debug asserts here. + return static_cast<::cuda::std::uint64_t>(::__cvta_generic_to_global(__ptr)); +} + +/************************************************************* + * + * Conversion from state space "pointer" -> generic pointer + * + **************************************************************/ +template +_CCCL_DEVICE_API _Tp* __from_ptr_smem(::cuda::std::size_t __ptr) +{ + // Consider adding debug asserts here. + return reinterpret_cast<_Tp*>(::__cvta_shared_to_generic(__ptr)); +} + +template +_CCCL_DEVICE_API _Tp* __from_ptr_dsmem(::cuda::std::size_t __ptr) +{ + // Consider adding debug asserts here. + return __from_ptr_smem<_Tp>(__ptr); +} + +template +_CCCL_DEVICE_API _Tp* __from_ptr_remote_dsmem(::cuda::std::size_t __ptr) +{ + // Consider adding debug asserts here. + return __from_ptr_smem<_Tp>(__ptr); +} + +template +_CCCL_DEVICE_API _Tp* __from_ptr_gmem(::cuda::std::size_t __ptr) +{ + // Consider adding debug asserts here. + return reinterpret_cast<_Tp*>(::__cvta_global_to_generic(__ptr)); +} + +/************************************************************* + * + * Conversion to and from b8 type + * + **************************************************************/ + +template +_CCCL_DEVICE_API uint32_t __b8_as_u32(_B8 __val) +{ + static_assert(sizeof(_B8) == 1); + ::cuda::std::uint32_t __u32 = 0; + ::memcpy(&__u32, &__val, 1); + return __u32; +} + +template +_CCCL_DEVICE_API _B8 __u32_as_b8(uint32_t __u32) +{ + static_assert(sizeof(_B8) == 1); + _B8 b8; + ::memcpy(&b8, &__u32, 1); + return b8; +} + +_CCCL_END_NAMESPACE_CUDA_PTX + +# include + +#endif // _CCCL_CUDA_COMPILATION() + +#endif // _CUDA_PTX_HELPER_FUNCTIONS_H_ diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/__type_traits/is_trivially_copyable.h b/qwen3_6_scripts/cccl_preload/include/cuda/__type_traits/is_trivially_copyable.h new file mode 100644 index 00000000..5b1ab962 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/__type_traits/is_trivially_copyable.h @@ -0,0 +1,115 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CUDA__TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H +#define __CUDA__TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA + +template +inline constexpr bool __is_aggregate_trivially_copyable_v = false; + +template +inline constexpr bool __is_trivially_copyable_v = + ::cuda::std::is_trivially_copyable_v<_Tp> || __is_aggregate_trivially_copyable_v<_Tp>; + +#if _CCCL_HAS_NVFP16() + +template <> +inline constexpr bool __is_trivially_copyable_v<::__half> = true; + +template <> +inline constexpr bool __is_trivially_copyable_v<::__half2> = true; + +#endif // _CCCL_HAS_NVFP16() + +#if _CCCL_HAS_NVBF16() + +template <> +inline constexpr bool __is_trivially_copyable_v<::__nv_bfloat16> = true; +template <> +inline constexpr bool __is_trivially_copyable_v<::__nv_bfloat162> = true; + +#endif // _CCCL_HAS_NVBF16() + +template +inline constexpr bool __is_trivially_copyable_v<_Tp[]> = __is_trivially_copyable_v<_Tp>; + +template +inline constexpr bool __is_trivially_copyable_v<_Tp[_Size]> = __is_trivially_copyable_v<_Tp>; + +template +inline constexpr bool __is_trivially_copyable_v<::cuda::std::array<_Tp, _Size>> = __is_trivially_copyable_v<_Tp>; + +template +inline constexpr bool __is_trivially_copyable_v<::cuda::std::pair<_T1, _T2>> = + __is_trivially_copyable_v<_T1> && __is_trivially_copyable_v<_T2>; + +template +inline constexpr bool __is_trivially_copyable_v<::cuda::std::tuple<_Ts...>> = (__is_trivially_copyable_v<_Ts> && ...); + +template +inline constexpr bool __is_trivially_copyable_v> = true; + +template +inline constexpr bool __is_trivially_copyable_v<::cuda::std::complex<_Tp>> = true; + +// if all the previous conditions fail, check if the type is an aggregate and all its members are trivially copyable +template +using __is_trivially_copyable_callable = ::cuda::std::bool_constant<__is_trivially_copyable_v<_Tp>>; + +template +inline constexpr bool __is_aggregate_trivially_copyable_v< + _Tp, + ::cuda::std::enable_if_t<::cuda::std::is_aggregate_v<_Tp> && !::cuda::std::is_trivially_copyable_v<_Tp>>> = + ::cuda::std::__aggregate_all_of_v<__is_trivially_copyable_callable, _Tp>; + +//---------------------------------------------------------------------------------------------------------------------- +// public traits + +template +inline constexpr bool is_trivially_copyable_v = __is_trivially_copyable_v<_Tp>; + +template +inline constexpr bool is_trivially_copyable_v = is_trivially_copyable_v<_Tp>; + +// defined as alias so users cannot specialize it (they should specialize the variable template instead) +template +using is_trivially_copyable = ::cuda::std::bool_constant>; + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // __CUDA__TYPE_TRAITS_IS_TRIVIALLY_COPYABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/clamp.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/clamp.h new file mode 100644 index 00000000..260b9223 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/clamp.h @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_CLAMP_H +#define _CUDA_STD___ALGORITHM_CLAMP_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr const _Tp& +clamp(const _Tp& __v _CCCL_LIFETIMEBOUND, + const _Tp& __lo _CCCL_LIFETIMEBOUND, + const _Tp& __hi _CCCL_LIFETIMEBOUND, + _Compare __comp) +{ + _CCCL_ASSERT(!__comp(__hi, __lo), "Bad bounds passed to cuda::std::clamp"); + return __comp(__v, __lo) ? __lo : __comp(__hi, __v) ? __hi : __v; +} + +template +[[nodiscard]] _CCCL_API constexpr const _Tp& +clamp(const _Tp& __v _CCCL_LIFETIMEBOUND, const _Tp& __lo _CCCL_LIFETIMEBOUND, const _Tp& __hi _CCCL_LIFETIMEBOUND) +{ + _CCCL_ASSERT(!(__hi < __lo), "Bad bounds passed to cuda::std::clamp"); + return __v < __lo ? __lo : __hi < __v ? __hi : __v; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_CLAMP_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp.h new file mode 100644 index 00000000..140b1633 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp.h @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_COMP_H +#define _CUDA_STD___ALGORITHM_COMP_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#if defined(_LIBCUDACXX_HAS_STRING) +# include +#endif // _LIBCUDACXX_HAS_STRING + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __equal_to +{ + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API constexpr bool operator()(const _T1& __lhs, const _T2& __rhs) const + noexcept(noexcept(__lhs == __rhs)) + { + return __lhs == __rhs; + } +}; + +struct __less +{ + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API constexpr bool operator()(const _Tp& __lhs, const _Up& __rhs) const + noexcept(noexcept(__lhs < __rhs)) + { + return __lhs < __rhs; + } +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_COMP_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp_ref_type.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp_ref_type.h new file mode 100644 index 00000000..b892fb6b --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/comp_ref_type.h @@ -0,0 +1,85 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_COMP_REF_TYPE_H +#define _CUDA_STD___ALGORITHM_COMP_REF_TYPE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct __debug_less +{ + _Compare& __comp_; + _CCCL_API constexpr __debug_less(_Compare& __c) + : __comp_(__c) + {} + + template + [[nodiscard]] _CCCL_API constexpr bool operator()(const _Tp& __x, const _Up& __y) + { + bool __r = __comp_(__x, __y); + if (__r) + { + __do_compare_assert(0, __y, __x); + } + return __r; + } + + template + [[nodiscard]] _CCCL_API constexpr bool operator()(_Tp& __x, _Up& __y) + { + bool __r = __comp_(__x, __y); + if (__r) + { + __do_compare_assert(0, __y, __x); + } + return __r; + } + + template + _CCCL_API constexpr decltype((void) declval<_Compare&>()(declval<_LHS&>(), declval<_RHS&>())) + __do_compare_assert(int, [[maybe_unused]] _LHS& __l, [[maybe_unused]] _RHS& __r) + { + _CCCL_ASSERT(!__comp_(__l, __r), "Comparator does not induce a strict weak ordering"); + } + + template + _CCCL_API constexpr void __do_compare_assert(long, _LHS&, _RHS&) + {} +}; + +// Pass the comparator by lvalue reference. Or in debug mode, using a +// debugging wrapper that stores a reference. +#ifdef _CCCL_ENABLE_DEBUG_MODE +template +using __comp_ref_type = __debug_less<_Comp>; +#else // ^^^ _LIBCUDACXX_ENABLE_DEBUG_MODE ^^^ / vvv !_LIBCUDACXX_ENABLE_DEBUG_MODE vvv +template +using __comp_ref_type = _Comp&; +#endif // !_LIBCUDACXX_ENABLE_DEBUG_MODE + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_COMP_REF_TYPE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/equal.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/equal.h new file mode 100644 index 00000000..b708bcab --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/equal.h @@ -0,0 +1,132 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_EQUAL_H +#define _CUDA_STD___ALGORITHM_EQUAL_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr bool +equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) +{ + bool __result = true; + for (; __first1 != __last1; ++__first1, (void) ++__first2) + { + if (!__pred(*__first1, *__first2)) + { + __result = false; + break; + } + } + return __result; +} + +template +[[nodiscard]] _CCCL_API constexpr bool equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2) +{ + return ::cuda::std::equal(__first1, __last1, __first2, __equal_to{}); +} + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr bool __equal( + _InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _BinaryPredicate __pred, + input_iterator_tag, + input_iterator_tag) +{ + bool __result = true; + for (; __first1 != __last1 && __first2 != __last2; ++__first1, (void) ++__first2) + { + if (!__pred(*__first1, *__first2)) + { + __result = false; + break; + } + } + return __result && __first1 == __last1 && __first2 == __last2; +} + +template +[[nodiscard]] _CCCL_API constexpr bool __equal( + _RandomAccessIterator1 __first1, + _RandomAccessIterator1 __last1, + _RandomAccessIterator2 __first2, + _RandomAccessIterator2 __last2, + _BinaryPredicate __pred, + random_access_iterator_tag, + random_access_iterator_tag) +{ + if (__last1 - __first1 != __last2 - __first2) + { + return false; + } + return ::cuda::std::equal<_RandomAccessIterator1, _RandomAccessIterator2, add_lvalue_reference_t<_BinaryPredicate>>( + __first1, __last1, __first2, __pred); +} + +template +[[nodiscard]] _CCCL_API constexpr bool +equal(_InputIterator1 __first1, + _InputIterator1 __last1, + _InputIterator2 __first2, + _InputIterator2 __last2, + _BinaryPredicate __pred) +{ + return ::cuda::std::__equal>( + __first1, + __last1, + __first2, + __last2, + __pred, + __iterator_traits_category_or_concept_t<_InputIterator1>(), + __iterator_traits_category_or_concept_t<_InputIterator2>()); +} + +template +[[nodiscard]] _CCCL_API constexpr bool +equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) +{ + return ::cuda::std::__equal( + __first1, + __last1, + __first2, + __last2, + __equal_to{}, + __iterator_traits_category_or_concept_t<_InputIterator1>(), + __iterator_traits_category_or_concept_t<_InputIterator2>()); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_EQUAL_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/fill_n.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/fill_n.h new file mode 100644 index 00000000..f10148bc --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/fill_n.h @@ -0,0 +1,51 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_FILL_N_H +#define _CUDA_STD___ALGORITHM_FILL_N_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr _OutputIterator __fill_n(_OutputIterator __first, _Size __n, const _Tp& __value_) +{ + for (; __n > 0; ++__first, (void) --__n) + { + *__first = __value_; + } + return __first; +} + +template +_CCCL_API constexpr _OutputIterator fill_n(_OutputIterator __first, _Size __n, const _Tp& __value_) +{ + return ::cuda::std::__fill_n(__first, __convert_to_integral(__n), __value_); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_FILL_N_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iter_swap.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iter_swap.h new file mode 100644 index 00000000..52b473dd --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iter_swap.h @@ -0,0 +1,85 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_ITER_SWAP_H +#define _CUDA_STD___ALGORITHM_ITER_SWAP_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +//! Intentionally not an algorithm to avoid breaking types that pull in `::std::iter_swap` via ADL +_CCCL_BEGIN_NAMESPACE_CPO(__iter_swap) +// "Poison pill" overload to intentionally create ambiguity with the unconstrained +// `std::iter_swap` function. +template +void iter_swap(_ForwardIterator1, _ForwardIterator2) = delete; + +template +_CCCL_CONCEPT __unqualified_iter_swap = + _CCCL_REQUIRES_EXPR((_ForwardIterator1, _ForwardIterator2), _ForwardIterator1&& __a, _ForwardIterator2&& __b)( + iter_swap(::cuda::std::forward<_ForwardIterator1>(__a), ::cuda::std::forward<_ForwardIterator2>(__b))); + +template +_CCCL_CONCEPT __readable_swappable = + _CCCL_REQUIRES_EXPR((_ForwardIterator1, _ForwardIterator2), _ForwardIterator1 __a, _ForwardIterator2 __b)( + requires(!__unqualified_iter_swap<_ForwardIterator1, _ForwardIterator2>), swap(*__a, *__b)); + +struct __fn +{ + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _ForwardIterator1, class _ForwardIterator2) + _CCCL_REQUIRES(__unqualified_iter_swap<_ForwardIterator1, _ForwardIterator2>) + _CCCL_API constexpr void operator()(_ForwardIterator1&& __a, _ForwardIterator2&& __b) const + noexcept(noexcept(iter_swap(::cuda::std::declval<_ForwardIterator1>(), ::cuda::std::declval<_ForwardIterator2>()))) + { + (void) iter_swap(::cuda::std::forward<_ForwardIterator1>(__a), ::cuda::std::forward<_ForwardIterator2>(__b)); + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _ForwardIterator1, class _ForwardIterator2) + _CCCL_REQUIRES(__readable_swappable<_ForwardIterator1, _ForwardIterator2>) + _CCCL_API constexpr void operator()(_ForwardIterator1&& __a, _ForwardIterator2&& __b) const + noexcept(noexcept(swap(*::cuda::std::declval<_ForwardIterator1>(), *::cuda::std::declval<_ForwardIterator2>()))) + { + swap(*__a, *__b); + } +}; + +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +// This is a global constant to avoid breaking types that pull in `::std::iter_swap` via ADL +_CCCL_GLOBAL_CONSTANT auto iter_swap = __iter_swap::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __iter_swap_cpo = __iter_swap::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_ITER_SWAP_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iterator_operations.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iterator_operations.h new file mode 100644 index 00000000..829a95b6 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/iterator_operations.h @@ -0,0 +1,179 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_ITERATOR_OPERATIONS_H +#define _CUDA_STD___ALGORITHM_ITERATOR_OPERATIONS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct _IterOps; + +struct _RangeAlgPolicy +{}; + +template <> +struct _IterOps<_RangeAlgPolicy> +{ + template + using __value_type = iter_value_t<_Iter>; + + template + using __difference_type = iter_difference_t<_Iter>; + + static constexpr auto advance = ::cuda::std::ranges::__advance_cpo{}; + static constexpr auto distance = ::cuda::std::ranges::__distance_cpo{}; + static constexpr auto __iter_move = ::cuda::std::ranges::__iter_move_cpo{}; + static constexpr auto iter_swap = ::cuda::std::ranges::__iter_swap_cpo{}; + static constexpr auto next = ::cuda::std::ranges::__next_cpo{}; + static constexpr auto prev = ::cuda::std::ranges::__prev_cpo{}; + static constexpr auto __advance_to = ::cuda::std::ranges::__advance_cpo{}; +}; + +struct _ClassicAlgPolicy +{}; + +template <> +struct _IterOps<_ClassicAlgPolicy> +{ + template + using __value_type = typename iterator_traits<_Iter>::value_type; + + template + using __difference_type = typename iterator_traits<_Iter>::difference_type; + + // advance + template + _CCCL_API constexpr static void advance(_Iter& __iter, _Distance __count) + { + ::cuda::std::advance(__iter, __count); + } + + // distance + template + _CCCL_API constexpr static typename iterator_traits<_Iter>::difference_type distance(_Iter __first, _Iter __last) + { + return ::cuda::std::distance(__first, __last); + } + + template + using __deref_t = decltype(*::cuda::std::declval<_Iter&>()); + + template + using __move_t = decltype(::cuda::std::move(*::cuda::std::declval<_Iter&>())); + + template + _CCCL_API constexpr static void __validate_iter_reference() + { + static_assert( + is_same_v<__deref_t<_Iter>, typename iterator_traits>::reference>, + "It looks like your iterator's `iterator_traits::reference` does not match the return type of " + "dereferencing the iterator, i.e., calling `*it`. This is undefined behavior according to [input.iterators] " + "and can lead to dangling reference issues at runtime, so we are flagging this."); + } + + // iter_move + _CCCL_EXEC_CHECK_DISABLE + template >, int> = 0> + _CCCL_API constexpr static + // If the result of dereferencing `_Iter` is a reference type, deduce the result of calling `::cuda::std::move` on + // it. Note that the C++03 mode doesn't support `decltype(auto)` as the return type. + __move_t<_Iter> + __iter_move(_Iter&& __i) + { + __validate_iter_reference<_Iter>(); + + return ::cuda::std::move(*::cuda::std::forward<_Iter>(__i)); + } + + _CCCL_EXEC_CHECK_DISABLE + template >, int> = 0> + _CCCL_API constexpr static + // If the result of dereferencing `_Iter` is a value type, deduce the return value of this function to also be a + // value -- otherwise, after `operator*` returns a temporary, this function would return a dangling reference to + // that temporary. Note that the C++03 mode doesn't support `auto` as the return type. + __deref_t<_Iter> + __iter_move(_Iter&& __i) + { + __validate_iter_reference<_Iter>(); + + return *::cuda::std::forward<_Iter>(__i); + } + + // iter_swap + template + _CCCL_API constexpr static void iter_swap(_Iter1&& __a, _Iter2&& __b) + { + ::cuda::std::__iter_swap_cpo{}(::cuda::std::forward<_Iter1>(__a), ::cuda::std::forward<_Iter2>(__b)); + } + + // next + template + _CCCL_API static constexpr _Iterator next(_Iterator, _Iterator __last) + { + return __last; + } + + template + _CCCL_API static constexpr remove_cvref_t<_Iter> next(_Iter&& __it, __difference_type> __n = 1) + { + return ::cuda::std::next(::cuda::std::forward<_Iter>(__it), __n); + } + + // prev + template + _CCCL_API static constexpr remove_cvref_t<_Iter> prev(_Iter&& __iter, __difference_type> __n = 1) + { + return ::cuda::std::prev(::cuda::std::forward<_Iter>(__iter), __n); + } + + _CCCL_EXEC_CHECK_DISABLE + template + _CCCL_API static constexpr void __advance_to(_Iter& __first, _Iter __last) + { + __first = __last; + } +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_ITERATOR_OPERATIONS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/lexicographical_compare.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/lexicographical_compare.h new file mode 100644 index 00000000..39b146d0 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/lexicographical_compare.h @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H +#define _CUDA_STD___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr bool __lexicographical_compare( + _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp) +{ + bool __result = false; + for (; __first2 != __last2; ++__first1, (void) ++__first2) + { + if (__first1 == __last1 || __comp(*__first1, *__first2)) + { + __result = true; + break; + } + if (__comp(*__first2, *__first1)) + { + break; + } + } + return __result; +} + +template +[[nodiscard]] _CCCL_API constexpr bool lexicographical_compare( + _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp) +{ + return __lexicographical_compare<__comp_ref_type<_Compare>>(__first1, __last1, __first2, __last2, __comp); +} + +template +[[nodiscard]] _CCCL_API constexpr bool lexicographical_compare( + _InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2) +{ + return ::cuda::std::lexicographical_compare(__first1, __last1, __first2, __last2, __less{}); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_LEXICOGRAPHICAL_COMPARE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max.h new file mode 100644 index 00000000..bf1008da --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max.h @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_MAX_H +#define _CUDA_STD___ALGORITHM_MAX_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr const _Tp& +max(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND, _Compare __comp) +{ + return __comp(__a, __b) ? __b : __a; +} + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr const _Tp& max(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND) +{ + return __a < __b ? __b : __a; +} + +template +[[nodiscard]] _CCCL_API constexpr _Tp max(initializer_list<_Tp> __t, _Compare __comp) +{ + return *::cuda::std::__max_element<__comp_ref_type<_Compare>>(__t.begin(), __t.end(), __comp); +} + +template +[[nodiscard]] _CCCL_API constexpr _Tp max(initializer_list<_Tp> __t) +{ + return *::cuda::std::max_element(__t.begin(), __t.end(), __less{}); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_MAX_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max_element.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max_element.h new file mode 100644 index 00000000..bba33f1b --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/max_element.h @@ -0,0 +1,67 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_MAX_ELEMENT_H +#define _CUDA_STD___ALGORITHM_MAX_ELEMENT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr _ForwardIterator __max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) +{ + static_assert(__has_forward_traversal<_ForwardIterator>, "::cuda::std::max_element requires a ForwardIterator"); + if (__first != __last) + { + _ForwardIterator __i = __first; + while (++__i != __last) + { + if (__comp(*__first, *__i)) + { + __first = __i; + } + } + } + return __first; +} + +template +[[nodiscard]] _CCCL_API constexpr _ForwardIterator +max_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) +{ + return ::cuda::std::__max_element<__comp_ref_type<_Compare>>(__first, __last, __comp); +} + +template +[[nodiscard]] _CCCL_API constexpr _ForwardIterator max_element(_ForwardIterator __first, _ForwardIterator __last) +{ + return ::cuda::std::max_element(__first, __last, __less{}); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_MAX_ELEMENT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min.h new file mode 100644 index 00000000..5335d7b8 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min.h @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_MIN_H +#define _CUDA_STD___ALGORITHM_MIN_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr const _Tp& +min(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND, _Compare __comp) +{ + return __comp(__b, __a) ? __b : __a; +} + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr const _Tp& min(const _Tp& __a _CCCL_LIFETIMEBOUND, const _Tp& __b _CCCL_LIFETIMEBOUND) +{ + return __b < __a ? __b : __a; +} + +template +[[nodiscard]] _CCCL_API constexpr _Tp min(initializer_list<_Tp> __t, _Compare __comp) +{ + return *::cuda::std::__min_element<__comp_ref_type<_Compare>>(__t.begin(), __t.end(), __comp); +} + +template +[[nodiscard]] _CCCL_API constexpr _Tp min(initializer_list<_Tp> __t) +{ + return *::cuda::std::min_element(__t.begin(), __t.end(), __less{}); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_MIN_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min_element.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min_element.h new file mode 100644 index 00000000..99185fed --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/min_element.h @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_MIN_ELEMENT_H +#define _CUDA_STD___ALGORITHM_MIN_ELEMENT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp, _Proj& __proj) +{ + if (__first == __last) + { + return __first; + } + + _Iter __i = __first; + while (++__i != __last) + { + if (::cuda::std::invoke(__comp, ::cuda::std::invoke(__proj, *__i), ::cuda::std::invoke(__proj, *__first))) + { + __first = __i; + } + } + + return __first; +} + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr _Iter __min_element(_Iter __first, _Sent __last, _Comp __comp) +{ + auto __proj = identity(); + return ::cuda::std::__min_element<_Comp>(::cuda::std::move(__first), ::cuda::std::move(__last), __comp, __proj); +} + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr _ForwardIterator +min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp) +{ + static_assert(__has_forward_traversal<_ForwardIterator>, "std::min_element requires a ForwardIterator"); + static_assert(__is_callable<_Compare, decltype(*__first), decltype(*__first)>::value, + "The comparator has to be callable"); + + return ::cuda::std::__min_element<__comp_ref_type<_Compare>>( + ::cuda::std::move(__first), ::cuda::std::move(__last), __comp); +} + +template +[[nodiscard]] _CCCL_API constexpr _ForwardIterator min_element(_ForwardIterator __first, _ForwardIterator __last) +{ + return ::cuda::std::min_element(__first, __last, __less{}); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_MIN_ELEMENT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/ranges_iterator_concept.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/ranges_iterator_concept.h new file mode 100644 index 00000000..07d7172d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/ranges_iterator_concept.h @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_RANGES_ITERATOR_CONCEPT_H +#define _CUDA_STD___ALGORITHM_RANGES_ITERATOR_CONCEPT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES + +template +_CCCL_API constexpr auto __get_iterator_concept() +{ + using _Iter = remove_cvref_t<_IterMaybeQualified>; + + if constexpr (contiguous_iterator<_Iter>) + { + return contiguous_iterator_tag(); + } + else if constexpr (random_access_iterator<_Iter>) + { + return random_access_iterator_tag(); + } + else if constexpr (bidirectional_iterator<_Iter>) + { + return bidirectional_iterator_tag(); + } + else if constexpr (forward_iterator<_Iter>) + { + return forward_iterator_tag(); + } + else if constexpr (input_iterator<_Iter>) + { + return input_iterator_tag(); + } +} + +template +using __iterator_concept = decltype(__get_iterator_concept<_Iter>()); + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +#include + +#endif // _CUDA_STD___ALGORITHM_RANGES_ITERATOR_CONCEPT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/swap_ranges.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/swap_ranges.h new file mode 100644 index 00000000..ae6e6611 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/swap_ranges.h @@ -0,0 +1,78 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_SWAP_RANGES_H +#define _CUDA_STD___ALGORITHM_SWAP_RANGES_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// 2+2 iterators: the shorter size will be used. +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr pair<_ForwardIterator1, _ForwardIterator2> +__swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2, _Sentinel2 __last2) +{ + while (__first1 != __last1 && __first2 != __last2) + { + _IterOps<_AlgPolicy>::iter_swap(__first1, __first2); + ++__first1; + ++__first2; + } + + return pair<_ForwardIterator1, _ForwardIterator2>(::cuda::std::move(__first1), ::cuda::std::move(__first2)); +} + +// 2+1 iterators: size2 >= size1. +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr pair<_ForwardIterator1, _ForwardIterator2> +__swap_ranges(_ForwardIterator1 __first1, _Sentinel1 __last1, _ForwardIterator2 __first2) +{ + while (__first1 != __last1) + { + _IterOps<_AlgPolicy>::iter_swap(__first1, __first2); + ++__first1; + ++__first2; + } + + return pair<_ForwardIterator1, _ForwardIterator2>(::cuda::std::move(__first1), ::cuda::std::move(__first2)); +} + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr _ForwardIterator2 +swap_ranges(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) +{ + return ::cuda::std::__swap_ranges<_ClassicAlgPolicy>( + ::cuda::std::move(__first1), ::cuda::std::move(__last1), ::cuda::std::move(__first2)) + .second; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_SWAP_RANGES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/unwrap_iter.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/unwrap_iter.h new file mode 100644 index 00000000..b0bc5313 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__algorithm/unwrap_iter.h @@ -0,0 +1,95 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ALGORITHM_UNWRAP_ITER_H +#define _CUDA_STD___ALGORITHM_UNWRAP_ITER_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// TODO: Change the name of __unwrap_iter_impl to something more appropriate +// The job of __unwrap_iter is to remove iterator wrappers (like reverse_iterator or __wrap_iter), +// to reduce the number of template instantiations and to enable pointer-based optimizations e.g. in ::cuda::std::copy. +// In debug mode, we don't do this. +// +// Some algorithms (e.g. ::cuda::std::copy, but not ::cuda::std::sort) need to convert an +// "unwrapped" result back into the original iterator type. Doing that is the job of __rewrap_iter. + +// Default case - we can't unwrap anything +template > +struct __unwrap_iter_impl +{ + _CCCL_EXEC_CHECK_DISABLE + static _CCCL_API constexpr _Iter __rewrap(_Iter, _Iter __iter) + { + return __iter; + } + _CCCL_EXEC_CHECK_DISABLE + static _CCCL_API constexpr _Iter __unwrap(_Iter __i) noexcept + { + return __i; + } +}; + +// It's a contiguous iterator, so we can use a raw pointer instead +template +struct __unwrap_iter_impl<_Iter, true> +{ + using _ToAddressT = decltype(::cuda::std::__to_address(::cuda::std::declval<_Iter>())); + + _CCCL_EXEC_CHECK_DISABLE + static _CCCL_API constexpr _Iter __rewrap(_Iter __orig_iter, _ToAddressT __unwrapped_iter) + { + return __orig_iter + (__unwrapped_iter - ::cuda::std::__to_address(__orig_iter)); + } + + _CCCL_EXEC_CHECK_DISABLE + static _CCCL_API constexpr _ToAddressT __unwrap(_Iter __i) noexcept + { + return ::cuda::std::__to_address(__i); + } +}; + +template , enable_if_t, int> = 0> +_CCCL_API constexpr decltype(_Impl::__unwrap(::cuda::std::declval<_Iter>())) __unwrap_iter(_Iter __i) noexcept +{ + return _Impl::__unwrap(__i); +} + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr _OrigIter __rewrap_iter(_OrigIter __orig_iter, _Iter __iter) noexcept +{ + return _Impl::__rewrap(::cuda::std::move(__orig_iter), ::cuda::std::move(__iter)); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ALGORITHM_UNWRAP_ITER_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__bit/bit_cast.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__bit/bit_cast.h new file mode 100644 index 00000000..7cd58887 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__bit/bit_cast.h @@ -0,0 +1,86 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024-26 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___BIT_BIT_CAST_H +#define _CUDA_STD___BIT_BIT_CAST_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +// MSVC supports __builtin_bit_cast from 19.25 on +#if _CCCL_CHECK_BUILTIN(builtin_bit_cast) || _CCCL_COMPILER(MSVC, >, 19, 25) +# define _CCCL_BUILTIN_BIT_CAST(...) __builtin_bit_cast(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_bit_cast) + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if defined(_CCCL_BUILTIN_BIT_CAST) +# define _CCCL_CONSTEXPR_BIT_CAST constexpr +# define _CCCL_HAS_CONSTEXPR_BIT_CAST() 1 +#else // ^^^ _CCCL_BUILTIN_BIT_CAST ^^^ / vvv !_CCCL_BUILTIN_BIT_CAST vvv +# define _CCCL_CONSTEXPR_BIT_CAST +# define _CCCL_HAS_CONSTEXPR_BIT_CAST() 0 +#endif // !_CCCL_BUILTIN_BIT_CAST + +#if _CCCL_COMPILER(GCC, >=, 8) +_CCCL_DIAG_PUSH +_CCCL_DIAG_SUPPRESS_GCC("-Wclass-memaccess") +#endif // _CCCL_COMPILER(GCC, >=, 8) + +template +[[nodiscard]] _CCCL_API inline _To __bit_cast_memcpy(const _From& __from) noexcept +{ + static_assert(::cuda::std::is_default_constructible_v<_To>, + "bit_cast memcpy fallback requires the destination type to be default constructible"); + _To __temp; + ::cuda::std::memcpy(&__temp, &__from, sizeof(_To)); + return __temp; +} + +#if _CCCL_COMPILER(GCC, >=, 8) +_CCCL_DIAG_POP +#endif // _CCCL_COMPILER(GCC, >=, 8) + +_CCCL_TEMPLATE(class _To, class _From) +_CCCL_REQUIRES((sizeof(_To) == sizeof(_From)) _CCCL_AND(::cuda::is_trivially_copyable_v<_To>) + _CCCL_AND(::cuda::is_trivially_copyable_v<_From>)) +[[nodiscard]] _CCCL_API inline _CCCL_CONSTEXPR_BIT_CAST _To bit_cast(const _From& __from) noexcept +{ +#if defined(_CCCL_BUILTIN_BIT_CAST) + if constexpr (::cuda::std::is_trivially_copyable_v<_To> && ::cuda::std::is_trivially_copyable_v<_From>) + { + return _CCCL_BUILTIN_BIT_CAST(_To, __from); + } + else +#endif // _CCCL_BUILTIN_BIT_CAST + { + return ::cuda::std::__bit_cast_memcpy<_To>(__from); + } +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___BIT_BIT_CAST_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/architecture.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/architecture.h new file mode 100644 index 00000000..807b6b63 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/architecture.h @@ -0,0 +1,128 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_ARCH_H +#define __CCCL_ARCH_H + +#include +#include + +// The header provides the following macros to determine the host architecture: +// +// _CCCL_HOST_ARCH(ARM64) ARM64 +// _CCCL_HOST_ARCH(X86_64) X86 64 bit +// CCCL_HOST_ARCH(ARM64) ARM64 +// CCCL_HOST_ARCH(X86_64) X86 64 bit + +// Determine the host architecture + +// Arm 64-bit +#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) /*emulation*/) +# define _CCCL_HOST_ARCH_ARM64_() 1 +#else +# define _CCCL_HOST_ARCH_ARM64_() 0 +#endif + +// X86 64-bit + +// _M_X64 is defined even if we are compiling in Arm64 emulation mode +#if (defined(_M_X64) && !defined(_M_ARM64EC)) || defined(__amd64__) || defined(__x86_64__) +# define _CCCL_HOST_ARCH_X86_64_() 1 +#else +# define _CCCL_HOST_ARCH_X86_64_() 0 +#endif + +#define _CCCL_HOST_ARCH(...) _CCCL_HOST_ARCH_##__VA_ARGS__##_() + +//! @def CCCL_HOST_ARCH(ARCH) /* implementation defined */ +//! +//! @brief Detect the current host architecture. +//! +//! @param ARCH The name of the host architecture to test. +//! +//! @note This macro is made available when including any libcu++ header. Users that wish to +//! include the smallest possible header for this macro should include ``. +//! +//! For supported host architectures, the macro expands to an implementation-defined true value +//! if the current host architecture matches, or false otherwise. These values may be used in +//! boolean expressions (preprocessor or otherwise), but no other guarantees are made. +//! +//! Available values for `ARCH` include: +//! +//! - ``ARM64``: ARM 64-bit, including MSVC ARM64EC emulation. +//! - ``X86_64``: X86 64-bit. This is false when compiling in MSVC ARM64EC emulation mode. +//! +//! Passing any other value will result in an undefined expansion, which may or may not be +//! diagnosed by the compiler. +//! +//! @par Example +//! @code +//! #define MY_OTHER_MACRO 1 +//! +//! // Expansion value can be used in ordinary macro conditionals +//! #if CCCL_HOST_ARCH(X86_64) && MY_OTHER_MACRO +//! // ... +//! #endif +//! +//! // Can be negated as usual +//! #if !CCCL_HOST_ARCH(ARM64) +//! // ... +//! #endif +//! @endcode +//! +//! @return true if the specified host architecture is being compiled for, false otherwise. +#ifdef _CCCL_DOXYGEN_INVOKED +# define CCCL_HOST_ARCH(ARCH) /* implementation defined */ +#else +# define CCCL_HOST_ARCH(__arch__) _CCCL_HOST_ARCH_##__arch__##_() +#endif + +// Note: the public API is single-arg to constrain the API and allow for future expansion. The +// implementation is duplicated to guard against the architecture targets being accidentally +// defined by the user. + +// Determine the endianness + +#define _CCCL_ENDIAN_LITTLE() 0xDEAD +#define _CCCL_ENDIAN_BIG() 0xFACE +#define _CCCL_ENDIAN_PDP() 0xBEEF + +#if _CCCL_COMPILER(NVRTC) || (_CCCL_COMPILER(MSVC) && (_CCCL_HOST_ARCH(X86_64) || _CCCL_HOST_ARCH(ARM64))) \ + || __LITTLE_ENDIAN__ +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE() +#elif __BIG_ENDIAN__ +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG() +#elif defined(__BYTE_ORDER__) +# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE() +# elif __BYTE_ORDER__ == __ORDER_PDP_ENDIAN__ +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_PDP() +# elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG() +# endif // __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#elif __has_include() +# include +# if __BYTE_ORDER == __LITTLE_ENDIAN +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE() +# elif __BYTE_ORDER == __PDP_ENDIAN +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_PDP() +# elif __BYTE_ORDER == __BIG_ENDIAN +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_BIG() +# endif // __BYTE_ORDER == __BIG_ENDIAN +#endif // ^^^ has endian.h ^^^ + +#if !defined(_CCCL_ENDIAN_NATIVE) +_CCCL_WARNING("failed to determine the endianness of the host architecture, defaulting to little-endian") +# define _CCCL_ENDIAN_NATIVE() _CCCL_ENDIAN_LITTLE() +#endif // !_CCCL_ENDIAN_NATIVE + +#define _CCCL_ENDIAN(_NAME) (_CCCL_ENDIAN_NATIVE() == _CCCL_ENDIAN_##_NAME()) + +#endif // __CCCL_ARCH_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/assert.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/assert.h new file mode 100644 index 00000000..8fd3dc9b --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/assert.h @@ -0,0 +1,169 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_ASSERT_H +#define __CCCL_ASSERT_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#if _CCCL_HOSTED() +# include +#endif // _CCCL_HOSTED() + +#include + +#if defined(_DEBUG) || defined(DEBUG) +# ifndef _CCCL_ENABLE_DEBUG_MODE +# define _CCCL_ENABLE_DEBUG_MODE +# endif // !_CCCL_ENABLE_DEBUG_MODE +#endif // _DEBUG || DEBUG + +// Automatically enable assertions when debug mode is enabled +#ifdef _CCCL_ENABLE_DEBUG_MODE +# ifndef CCCL_ENABLE_ASSERTIONS +# define CCCL_ENABLE_ASSERTIONS +# endif // !CCCL_ENABLE_ASSERTIONS +#endif // _CCCL_ENABLE_DEBUG_MODE + +//! Ensure that we switch on host assertions when all assertions are enabled +#ifndef CCCL_ENABLE_HOST_ASSERTIONS +# ifdef CCCL_ENABLE_ASSERTIONS +# define CCCL_ENABLE_HOST_ASSERTIONS +# endif // CCCL_ENABLE_ASSERTIONS +#endif // !CCCL_ENABLE_HOST_ASSERTIONS + +//! Ensure that we switch on device assertions when all assertions are enabled +#ifndef CCCL_ENABLE_DEVICE_ASSERTIONS +# if defined(CCCL_ENABLE_ASSERTIONS) || defined(__CUDACC_DEBUG__) +# define CCCL_ENABLE_DEVICE_ASSERTIONS +# endif // CCCL_ENABLE_ASSERTIONS +#endif // !CCCL_ENABLE_DEVICE_ASSERTIONS + +//! Use the different standard library implementations to implement host side asserts +//! _CCCL_ASSERT_IMPL_HOST should never be used directly +#if _CCCL_OS(QNX) +# define _CCCL_ASSERT_IMPL_HOST(expression, message) ((void) 0) +#elif _CCCL_COMPILER(NVRTC) // There is no host standard library in nvrtc +# define _CCCL_ASSERT_IMPL_HOST(expression, message) ((void) 0) +#elif __has_include() && _CCCL_OS(WINDOWS) // Windows uses _STL_VERIFY from +# include +# define _CCCL_ASSERT_IMPL_HOST(expression, message) _STL_VERIFY(expression, message) +#else // ^^^ MSVC STL ^^^ / vvv !MSVC STL vvv +# ifdef NDEBUG +// Reintroduce the __assert_fail / __assert_rtn declaration +extern "C" { +# if !_CCCL_CUDA_COMPILER(CLANG) +_CCCL_HOST_DEVICE +# endif // !_CCCL_CUDA_COMPILER(CLANG) +# if _CCCL_OS(APPLE) +void __assert_rtn(const char* __function, const char* __assertion, const char* __file, unsigned int __line) noexcept + __attribute__((__noreturn__)); +# else // ^^^ _CCCL_OS(APPLE) ^^^ / vvv !_CCCL_OS(APPLE) ^^^ +void __assert_fail(const char* __assertion, const char* __file, unsigned int __line, const char* __function) noexcept + __attribute__((__noreturn__)); +# endif // !_CCCL_OS(APPLE) +} +# endif // NDEBUG + +# if _CCCL_OS(APPLE) +# define _CCCL_ASSERT_IMPL_HOST(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : __assert_rtn(__func__, __FILE__, __LINE__, message) +# elif _CCCL_OS(ANDROID) +# define _CCCL_ASSERT_IMPL_HOST(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : __assert2(__FILE__, __LINE__, __func__, message) +# else // ^^^ _CCCL_OS(APPLE) ^^^ / vvv !_CCCL_OS(APPLE) ^^^ +# define _CCCL_ASSERT_IMPL_HOST(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : __assert_fail(message, __FILE__, __LINE__, __func__) +# endif // !_CCCL_OS(APPLE) +#endif // !MSVC STL + +//! Use custom implementations with nvcc on device and the host ones with clang-cuda and nvhpc +//! _CCCL_ASSERT_IMPL_DEVICE should never be used directly +#if _CCCL_OS(QNX) || _CCCL_OS(APPLE) +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) ((void) 0) +#elif _CCCL_COMPILER(NVRTC) +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : __assertfail(message, __FILE__, __LINE__, __func__, sizeof(char)) +#elif _CCCL_CUDA_COMPILER(NVCC) //! Use __assert_fail to implement device side asserts +# if _CCCL_COMPILER(MSVC) +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : _wassert(_CRT_WIDE(#message), __FILEW__, __LINE__) +# elif _CCCL_OS(ANDROID) +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : __assert2(__FILE__, __LINE__, __func__, message) +# else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) \ + _CCCL_BUILTIN_EXPECT(static_cast(expression), 1) \ + ? (void) 0 : __assert_fail(message, __FILE__, __LINE__, __func__) +# endif // !_CCCL_COMPILER(MSVC) +#elif _CCCL_CUDA_COMPILATION() +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message) +#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv +# define _CCCL_ASSERT_IMPL_DEVICE(expression, message) ((void) 0) +#endif // !_CCCL_CUDA_COMPILATION() + +//! _CCCL_ASSERT_HOST is enabled conditionally depending on CCCL_ENABLE_HOST_ASSERTIONS +#ifdef CCCL_ENABLE_HOST_ASSERTIONS +# define _CCCL_ASSERT_HOST(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message) +#else // ^^^ CCCL_ENABLE_HOST_ASSERTIONS ^^^ / vvv !CCCL_ENABLE_HOST_ASSERTIONS vvv +# define _CCCL_ASSERT_HOST(expression, message) ((void) 0) +#endif // !CCCL_ENABLE_HOST_ASSERTIONS + +//! _CCCL_ASSERT_DEVICE is enabled conditionally depending on CCCL_ENABLE_DEVICE_ASSERTIONS +#ifdef CCCL_ENABLE_DEVICE_ASSERTIONS +# define _CCCL_ASSERT_DEVICE(expression, message) _CCCL_ASSERT_IMPL_DEVICE(expression, message) +#else // ^^^ CCCL_ENABLE_DEVICE_ASSERTIONS ^^^ / vvv !CCCL_ENABLE_DEVICE_ASSERTIONS vvv +# define _CCCL_ASSERT_DEVICE(expression, message) ((void) 0) +#endif // !CCCL_ENABLE_DEVICE_ASSERTIONS + +//! _CCCL_VERIFY is enabled unconditionally and reserved for critical checks that are required to always be on +//! _CCCL_ASSERT is enabled conditionally depending on CCCL_ENABLE_HOST_ASSERTIONS and CCCL_ENABLE_DEVICE_ASSERTIONS +#if _CCCL_CUDA_COMPILER(NVHPC) // NVHPC can't have different behavior for host and device. + // The host version of the assert will also work in device code. +# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message) +# if defined(CCCL_ENABLE_HOST_ASSERTIONS) || defined(CCCL_ENABLE_DEVICE_ASSERTIONS) +# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message) +# else +# define _CCCL_ASSERT(expression, message) ((void) 0) +# endif +#elif _CCCL_CUDA_COMPILATION() +# if _CCCL_DEVICE_COMPILATION() +# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_DEVICE(expression, message) +# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_DEVICE(expression, message) +# else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv +# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message) +# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message) +# endif // !_CCCL_DEVICE_COMPILATION() +#else // ^^^ _CCCL_CUDA_COMPILATION() ^^^ / vvv !_CCCL_CUDA_COMPILATION() vvv +# define _CCCL_VERIFY(expression, message) _CCCL_ASSERT_IMPL_HOST(expression, message) +# define _CCCL_ASSERT(expression, message) _CCCL_ASSERT_HOST(expression, message) +#endif // !_CCCL_CUDA_COMPILATION() + +#endif // __CCCL_ASSERT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/attributes.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/attributes.h new file mode 100644 index 00000000..bc8bb34d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/attributes.h @@ -0,0 +1,221 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_ATTRIBUTES_H +#define __CCCL_ATTRIBUTES_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +#ifdef __has_attribute +# define _CCCL_HAS_ATTRIBUTE(__x) __has_attribute(__x) +#else // ^^^ __has_attribute ^^^ / vvv !__has_attribute vvv +# define _CCCL_HAS_ATTRIBUTE(__x) 0 +#endif // !__has_attribute + +#ifdef __has_cpp_attribute +# define _CCCL_HAS_CPP_ATTRIBUTE(__x) __has_cpp_attribute(__x) +#else // ^^^ __has_cpp_attribute ^^^ / vvv !__has_cpp_attribute vvv +# define _CCCL_HAS_CPP_ATTRIBUTE(__x) 0 +#endif // !__has_cpp_attribute + +#ifdef __has_declspec_attribute +# define _CCCL_HAS_DECLSPEC_ATTRIBUTE(__x) __has_declspec_attribute(__x) +#else // ^^^ __has_declspec_attribute ^^^ / vvv !__has_declspec_attribute vvv +# define _CCCL_HAS_DECLSPEC_ATTRIBUTE(__x) 0 +#endif // !__has_declspec_attribute + +// MSVC needs extra help with empty base classes +#if _CCCL_COMPILER(MSVC) || _CCCL_HAS_DECLSPEC_ATTRIBUTE(empty_bases) +# define _CCCL_DECLSPEC_EMPTY_BASES __declspec(empty_bases) +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_DECLSPEC_EMPTY_BASES +#endif // !_CCCL_COMPILER(MSVC) + +#if _CCCL_HAS_ATTRIBUTE(__nodebug__) +# define _CCCL_NODEBUG __attribute__((__nodebug__)) +#else // ^^^ _CCCL_HAS_ATTRIBUTE(__nodebug__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__nodebug__) vvv +# define _CCCL_NODEBUG +#endif // !_CCCL_HAS_ATTRIBUTE(__nodebug__) + +// Debuggers do not step into functions marked with __attribute__((__artificial__)). This +// is useful for small wrapper functions that just dispatch to other functions and that +// are inlined into the caller. +#if _CCCL_HAS_ATTRIBUTE(__artificial__) && !_CCCL_CUDA_COMPILER(NVCC) +# define _CCCL_ARTIFICIAL __attribute__((__artificial__)) +#else // ^^^ _CCCL_HAS_ATTRIBUTE(__artificial__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__artificial__) vvv +# define _CCCL_ARTIFICIAL +#endif // !_CCCL_HAS_ATTRIBUTE(__artificial__) + +// The nodebug attribute flattens aliases down to the actual type rather typename meow::type +#if _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_NODEBUG_ALIAS _CCCL_NODEBUG +#else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv +# define _CCCL_NODEBUG_ALIAS +#endif // !_CCCL_CUDA_COMPILER(CLANG) + +// _CCCL_ASSUME +// NVCC does not properly respect [[assume()]], so use __builtin_assume, see nvbug5458663 +#if _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION() +# define _CCCL_ASSUME(...) __builtin_assume(__VA_ARGS__) +#elif _CCCL_HAS_CPP_ATTRIBUTE(assume) +# define _CCCL_ASSUME(...) [[assume(__VA_ARGS__)]] +#else +# define _CCCL_ASSUME(...) _CCCL_BUILTIN_ASSUME(__VA_ARGS__) +#endif + +#if _CCCL_TILE_COMPILATION() // nvbug6100910: __builtin_assume is not supported in tile mode +# undef _CCCL_ASSUME +# define _CCCL_ASSUME(...) +#endif // _CCCL_TILE_COMPILATION() + +// _CCCL_CONST + +#if _CCCL_HAS_CPP_ATTRIBUTE(__gnu__::__const__) +# define _CCCL_CONST [[__gnu__::__const__]] +#else // ^^^ has gnu::const ^^^ / vvv no gnu::const vvv +# define _CCCL_CONST _CCCL_PURE +#endif // ^^^ no gnu::const ^^^ + +// _CCCL_DIAGNOSE_IF + +#if _CCCL_HAS_ATTRIBUTE(__diagnose_if__) +# define _CCCL_DIAGNOSE_IF(_COND, _MSG, _TYPE) __attribute__((__diagnose_if__(_COND, _MSG, _TYPE))) +#else // ^^^ _CCCL_HAS_ATTRIBUTE(diagnose_if) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(diagnose_if) vvv +# define _CCCL_DIAGNOSE_IF(_COND, _MSG, _TYPE) +#endif // !_CCCL_HAS_ATTRIBUTE(diagnose_if) + +// _CCCL_INTRINSIC + +// MSVC provides a way to mark functions as intrinsic provided the function's body consists of a single +// return statement of a cast expression (e.g., move(x) or forward(u)). +#if _CCCL_COMPILER(MSVC) && _CCCL_HAS_CPP_ATTRIBUTE(msvc::intrinsic) +# define _CCCL_INTRINSIC [[msvc::intrinsic]] +#else +# define _CCCL_INTRINSIC +#endif + +// _CCCL_PURE + +#if _CCCL_CUDA_COMPILER(NVCC, >=, 12, 5) +# define _CCCL_PURE __nv_pure__ +#elif _CCCL_HAS_CPP_ATTRIBUTE(__gnu__::__pure__) +# define _CCCL_PURE [[__gnu__::__pure__]] +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_PURE __declspec(noalias) +#else +# define _CCCL_PURE +#endif + +// _CCCL_NO_CFI + +#if !_CCCL_COMPILER(GCC) +# define _CCCL_NO_CFI _CCCL_NO_SANITIZE("cfi") +#else +# define _CCCL_NO_CFI +#endif + +// _CCCL_NO_SANITIZE + +#if _CCCL_HAS_ATTRIBUTE(__no_sanitize__) +# define _CCCL_NO_SANITIZE(_STR) __attribute__((__no_sanitize__(_STR))) +#else // ^^^ _CCCL_HAS_ATTRIBUTE(no_sanitize) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(no_sanitize) vvv +# define _CCCL_NO_SANITIZE(_STR) +#endif // !_CCCL_HAS_ATTRIBUTE(no_sanitize) + +// _CCCL_NO_SPECIALIZATIONS + +#if _CCCL_HAS_CPP_ATTRIBUTE(clang::__no_specializations__) +# define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG) [[clang::__no_specializations__(_MSG)]] +# define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 1 +#elif _CCCL_HAS_CPP_ATTRIBUTE(msvc::no_specializations) +# define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG) [[msvc::no_specializations(_MSG)]] +# define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 1 +#else // ^^^ has attribute no_specializations ^^^ / vvv hasn't attribute no_specializations vvv +# define _CCCL_NO_SPECIALIZATIONS_BECAUSE(_MSG) +# define _CCCL_HAS_ATTRIBUTE_NO_SPECIALIZATIONS() 0 +#endif // ^^^ hasn't attribute no_specializations ^^^ + +#define _CCCL_NO_SPECIALIZATIONS \ + _CCCL_NO_SPECIALIZATIONS_BECAUSE("Users are not allowed to specialize this cccl entity") + +// _CCCL_LIFETIMEBOUND + +#if _CCCL_HAS_CPP_ATTRIBUTE(clang::lifetimebound) || _CCCL_COMPILER(CLANG) +# define _CCCL_LIFETIMEBOUND [[clang::lifetimebound]] +#elif _CCCL_HAS_CPP_ATTRIBUTE(msvc::lifetimebound) || _CCCL_COMPILER(MSVC, >=, 19, 37) +# define _CCCL_LIFETIMEBOUND [[msvc::lifetimebound]] +#else +# define _CCCL_LIFETIMEBOUND +#endif + +// _CCCL_NO_UNIQUE_ADDRESS + +#if _CCCL_COMPILER(MSVC) || _CCCL_HAS_CPP_ATTRIBUTE(no_unique_address) < 201803L +// MSVC implementation has lead to multiple issues with silent runtime corruption when passing data into kernels +# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0 +# define _CCCL_NO_UNIQUE_ADDRESS +#elif _CCCL_HAS_CPP_ATTRIBUTE(no_unique_address) +# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 1 +# define _CCCL_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else +# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0 +# define _CCCL_NO_UNIQUE_ADDRESS +#endif + +// Passing objects with nested [[no_unique_address]] to kernels leads to data corruption. +// This is caused by cudafe++ not honoring [[no_unique_address]] when compiling for C++17 +// with clang as the host compiler. See nvbug 5265027 for more details. +#if _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() && _CCCL_COMPILER(CLANG) && _CCCL_STD_VER < 2020 \ + && _CCCL_CUDA_COMPILER(NVCC) +# undef _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS +# undef _CCCL_NO_UNIQUE_ADDRESS +# define _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() 0 +# define _CCCL_NO_UNIQUE_ADDRESS +#endif // _CCCL_HAS_ATTRIBUTE_NO_UNIQUE_ADDRESS() && _CCCL_COMPILER(CLANG) + +// _CCCL_PREFERRED_NAME + +#if _CCCL_HAS_ATTRIBUTE(__preferred_name__) +# define _CCCL_PREFERRED_NAME(x) __attribute__((__preferred_name__(x))) +#else +# define _CCCL_PREFERRED_NAME(x) +#endif + +#if _CCCL_HAS_ATTRIBUTE(__require_constant_initialization__) +# define _CCCL_REQUIRE_CONSTANT_INITIALIZATION __attribute__((__require_constant_initialization__)) +#else +# define _CCCL_REQUIRE_CONSTANT_INITIALIZATION +#endif + +// _CCCL_RESTRICT + +#if _CCCL_COMPILER(MSVC) // vvv _CCCL_COMPILER(MSVC) vvv +# define _CCCL_RESTRICT __restrict +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_RESTRICT __restrict__ +#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^ + +#include + +#endif // __CCCL_ATTRIBUTES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/builtin.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/builtin.h new file mode 100644 index 00000000..5580b83f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/builtin.h @@ -0,0 +1,474 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_BUILTIN_H +#define __CCCL_BUILTIN_H + +#include +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +//! This file consolidates all compiler builtin detection for CCCL. +//! +//! To work around older compilers not supporting `__has_builtin` we use `_CCCL_CHECK_BUILTIN` that detects more +//! cases +//! +//! * We work around old clang versions (before clang-10) not supporting __has_builtin via _CCCL_CHECK_BUILTIN +//! * We work around old intel versions (before 2021.3) not supporting __has_builtin via _CCCL_CHECK_BUILTIN +//! * We work around old nvhpc versions (before 2022.11) not supporting __has_builtin via _CCCL_CHECK_BUILTIN +//! * MSVC needs manual handling, has no real way of checking builtins so all is manual +//! * GCC needs manual handling, before gcc-10 as that finally supports __has_builtin +//! +//! In case compiler support for a builtin is advertised but leads to regressions we explicitly undef the macro +//! +//! Finally, because `_CCCL_CHECK_BUILTIN` may lead to false positives, we move detection of new builtins over towards +//! just using _CCCL_HAS_BUILTIN + +#ifdef __has_builtin +# define _CCCL_HAS_BUILTIN(__x) __has_builtin(__x) +#else // ^^^ __has_builtin ^^^ / vvv !__has_builtin vvv +# define _CCCL_HAS_BUILTIN(__x) 0 +#endif // !__has_builtin + +#ifdef __has_feature +# define _CCCL_HAS_FEATURE(__x) __has_feature(__x) +#else // ^^^ __has_feature ^^^ / vvv !__has_feature vvv +# define _CCCL_HAS_FEATURE(__x) 0 +#endif // !__has_feature + +// '__is_identifier' returns '0' if '__x' is a reserved identifier provided by the compiler and '1' otherwise. +#ifdef __is_identifier +# define _CCCL_IS_IDENTIFIER(__x) __is_identifier(__x) +#else // ^^^ __is_identifier ^^^ / vvv !__is_identifier vvv +# define _CCCL_IS_IDENTIFIER(__x) 1 +#endif // !__is_identifier + +#define _CCCL_HAS_KEYWORD(__x) !(_CCCL_IS_IDENTIFIER(__x)) + +// https://bugs.llvm.org/show_bug.cgi?id=44517 +#define _CCCL_CHECK_BUILTIN(__x) (_CCCL_HAS_BUILTIN(__##__x) || _CCCL_HAS_KEYWORD(__##__x) || _CCCL_HAS_FEATURE(__x)) + +// NVCC has issues with function pointers +#if _CCCL_HAS_BUILTIN(__add_lvalue_reference) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_ADD_LVALUE_REFERENCE(...) __add_lvalue_reference(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__add_lvalue_reference) + +// NVCC has issues with function pointers +#if _CCCL_HAS_BUILTIN(__add_pointer) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_ADD_POINTER(...) __add_pointer(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__add_pointer) + +// NVCC has issues with function pointers +#if _CCCL_HAS_BUILTIN(__add_rvalue_reference) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_ADD_RVALUE_REFERENCE(...) __add_rvalue_reference(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__add_rvalue_reference) + +// TODO: Enable using the builtin __array_rank when https://llvm.org/PR57133 is resolved +#if 0 // _CCCL_CHECK_BUILTIN(array_rank) +# define _CCCL_BUILTIN_ARRAY_RANK(...) __array_rank(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(array_rank) + +// nvhpc has a bug where it supports __builtin_addressof but does not mark it via _CCCL_CHECK_BUILTIN +#if _CCCL_CHECK_BUILTIN(builtin_addressof) || _CCCL_COMPILER(GCC, >=, 7) || _CCCL_COMPILER(MSVC) \ + || _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC, >=, 12, 3) +# define _CCCL_BUILTIN_ADDRESSOF(...) __builtin_addressof(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_addressof) + +#if _CCCL_CHECK_BUILTIN(builtin_assume) || _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(NVHPC) +# define _CCCL_BUILTIN_ASSUME(...) __builtin_assume(__VA_ARGS__) +#elif _CCCL_COMPILER(GCC, >=, 13) +# define _CCCL_BUILTIN_ASSUME(...) __attribute__((__assume__(__VA_ARGS__))) +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_BUILTIN_ASSUME(...) __assume(__VA_ARGS__) +#else +# define _CCCL_BUILTIN_ASSUME(...) +#endif // _CCCL_CHECK_BUILTIN(builtin_assume) + +#if _CCCL_TILE_COMPILATION() // nvbug6100910: __builtin_assume is not supported in tile mode +# undef _CCCL_BUILTIN_ASSUME +# define _CCCL_BUILTIN_ASSUME(...) +#endif // _CCCL_TILE_COMPILATION() + +#if _CCCL_HAS_BUILTIN(__builtin_assume_aligned) || _CCCL_COMPILER(MSVC, >=, 19, 23) || _CCCL_COMPILER(GCC) +# define _CCCL_BUILTIN_ASSUME_ALIGNED(...) __builtin_assume_aligned(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__builtin_assume_aligned) + +#if _CCCL_CHECK_BUILTIN(builtin_constant_p) || _CCCL_COMPILER(GCC) +# define _CCCL_BUILTIN_CONSTANT_P(...) __builtin_constant_p(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_constant_p) + +#if _CCCL_CHECK_BUILTIN(builtin_expect) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC) +# define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) __builtin_expect(_EXPR, _VAL) +#else // ^^^ has __builtin_expect ^^^ / vvv no __builtin_expect vvv +# define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) (_EXPR) +#endif // ^^^ no __builtin_expect ^^^ + +#if _CCCL_TILE_COMPILATION() // nvbug6100927: __builtin_expect is unsupported in tile mode +# undef _CCCL_BUILTIN_EXPECT +# define _CCCL_BUILTIN_EXPECT(_EXPR, _VAL) (_EXPR) +#endif // _CCCL_TILE_COMPILATION() + +#if _CCCL_CHECK_BUILTIN(builtin_huge_valf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_HUGE_VALF() __builtin_huge_valf() +#endif // _CCCL_CHECK_BUILTIN(builtin_huge_valf) + +#if _CCCL_CHECK_BUILTIN(builtin_huge_val) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_HUGE_VAL() __builtin_huge_val() +#endif // _CCCL_CHECK_BUILTIN(builtin_huge_val) + +#if _CCCL_CHECK_BUILTIN(builtin_huge_vall) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_HUGE_VALL() __builtin_huge_vall() +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_BUILTIN_HUGE_VALL() static_cast(__builtin_huge_val()) +#endif // _CCCL_CHECK_BUILTIN(builtin_huge_vall) + +#if _CCCL_HAS_FLOAT128() +# if _CCCL_CHECK_BUILTIN(builtin_huge_valf128) || _CCCL_COMPILER(GCC, >=, 7) +# define _CCCL_BUILTIN_HUGE_VALF128() __builtin_huge_valf128() +# endif // _CCCL_CHECK_BUILTIN(builtin_huge_valf128) || _CCCL_COMPILER(GCC, >=, 7) + +// nvcc does not implement __builtin_huge_valf128 +# if _CCCL_CUDA_COMPILER(NVCC) +# undef _CCCL_BUILTIN_HUGE_VALF128 +# endif // _CCCL_CUDA_COMPILER(NVCC) +#endif // _CCCL_HAS_FLOAT128() + +#if _CCCL_CHECK_BUILTIN(builtin_is_constant_evaluated) || _CCCL_COMPILER(GCC, >=, 9) || _CCCL_COMPILER(MSVC, >, 19, 24) +# define _CCCL_BUILTIN_IS_CONSTANT_EVALUATED(...) __builtin_is_constant_evaluated(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_is_constant_evaluated) + +#if _CCCL_TILE_COMPILATION() // nvbug6067464: __builtin_is_constant_evaluated is unsupported in tile mode +# undef _CCCL_BUILTIN_IS_CONSTANT_EVALUATED +#endif // _CCCL_TILE_COMPILATION() + +#if _CCCL_CHECK_BUILTIN(builtin_is_corresponding_member) +# define _CCCL_BUILTIN_IS_CORRESPONDING_MEMBER(_C1, _C2, _MPtr1, _MPtr2) \ + __builtin_is_corresponding_member(_MPtr1, _MPtr2) +#elif _CCCL_COMPILER(MSVC, >=, 19, 29) +// using __is_corresponding_member with msvc outside of constexpr context causes linker errors, see +// https://developercommunity.visualstudio.com/t/Using-compiler-builtins-causes-linking-n/10888080 +// # define _CCCL_BUILTIN_IS_CORRESPONDING_MEMBER(_C1, _C2, _MPtr1, _MPtr2) __is_corresponding_member(_C1, _C2, _MPtr1, +// _MPtr2) +#endif // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 29) ^^^ + +#if _CCCL_CHECK_BUILTIN(builtin_is_pointer_interconvertible_with_class) +# define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS(_S, _MPtr) \ + __builtin_is_pointer_interconvertible_with_class(_MPtr) +#elif _CCCL_COMPILER(MSVC, >=, 19, 29) +// using __is_pointer_interconvertible_with_class with msvc outside of constexpr context causes linker errors, see +// https://developercommunity.visualstudio.com/t/Using-compiler-builtins-causes-linking-n/10888080 +// # define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS(_S, _MPtr) +// __is_pointer_interconvertible_with_class(_S, _MPtr) +#endif // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 29) ^^^ + +#if _CCCL_CHECK_BUILTIN(builtin_is_virtual_base_of) +# define _CCCL_BUILTIN_IS_VIRTUAL_BASE_OF(...) __builtin_is_virtual_base_of(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_is_virtual_base_of) + +// nvcc < 13.3 doesn't implement __builtin_is_virtual_base_of +#if _CCCL_CUDA_COMPILER(NVCC, <, 13, 3) +# undef _CCCL_BUILTIN_IS_VIRTUAL_BASE_OF +#endif // _CCCL_CUDA_COMPILER(NVCC, <, 13, 3) + +#if _CCCL_CHECK_BUILTIN(builtin_nanf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_NANF(...) __builtin_nanf(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_nanf) + +#if _CCCL_CHECK_BUILTIN(builtin_nan) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_NAN(...) __builtin_nan(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_nan) + +#if _CCCL_CHECK_BUILTIN(builtin_nanl) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_NANL(...) __builtin_nanl(__VA_ARGS__) +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_BUILTIN_NANL(...) static_cast(__builtin_nan(__VA_ARGS__)) +#endif // _CCCL_CHECK_BUILTIN(builtin_nanl) + +#if _CCCL_HAS_FLOAT128() +# if _CCCL_CHECK_BUILTIN(builtin_nanf128) || _CCCL_COMPILER(GCC, >=, 7) +# define _CCCL_BUILTIN_NANF128(...) __builtin_nanf128(__VA_ARGS__) +# endif // _CCCL_CHECK_BUILTIN(builtin_nanf128) || _CCCL_COMPILER(GCC, >=, 7) + +// nvcc does not implement __builtin_nanf128 +# if _CCCL_CUDA_COMPILER(NVCC) +# undef _CCCL_BUILTIN_NANF128 +# endif // _CCCL_CUDA_COMPILER(NVCC) +#endif // _CCCL_HAS_FLOAT128() + +#if _CCCL_CHECK_BUILTIN(builtin_nansf) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_NANSF(...) __builtin_nansf(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_nansf) + +#if _CCCL_CHECK_BUILTIN(builtin_nans) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_NANS(...) __builtin_nans(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_nans) + +#if _CCCL_CHECK_BUILTIN(builtin_nansl) || _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_BUILTIN_NANSL(...) __builtin_nansl(__VA_ARGS__) +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_BUILTIN_NANSL(...) static_cast(__builtin_nans(__VA_ARGS__)) +#endif // _CCCL_CHECK_BUILTIN(builtin_nansl) + +#if _CCCL_HAS_FLOAT128() +# if _CCCL_CHECK_BUILTIN(builtin_nansf128) || _CCCL_COMPILER(GCC, >=, 7) +# define _CCCL_BUILTIN_NANSF128(...) __builtin_nansf128(__VA_ARGS__) +# endif // _CCCL_CHECK_BUILTIN(builtin_nansf128) || _CCCL_COMPILER(GCC, >=, 7) + +// nvcc does not implement __builtin_nansf128 +# if _CCCL_CUDA_COMPILER(NVCC) +# undef _CCCL_BUILTIN_NANSF128 +# endif // _CCCL_CUDA_COMPILER(NVCC) +#endif // _CCCL_HAS_FLOAT128() + +#if _CCCL_CHECK_BUILTIN(builtin_memcmp) || _CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC, >=, 19, 28) +# define _CCCL_BUILTIN_MEMCMP(...) __builtin_memcmp(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_memcmp) || _CCCL_COMPILER(GCC) || _CCCL_COMPILER(MSVC, >=, 19, 28) + +#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG) +# undef _CCCL_BUILTIN_MEMCMP +#endif // _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG) + +#if _CCCL_CHECK_BUILTIN(builtin_memmove) || _CCCL_COMPILER(GCC) +# define _CCCL_BUILTIN_MEMMOVE(...) __builtin_memmove(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_memmove) || _CCCL_COMPILER(GCC) + +#if _CCCL_CUDA_COMPILER(NVCC) +# undef _CCCL_BUILTIN_MEMMOVE +#endif // _CCCL_CUDA_COMPILER(NVCC) + +#if _CCCL_CHECK_BUILTIN(builtin_operator_new) && _CCCL_CHECK_BUILTIN(builtin_operator_delete) \ + && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_OPERATOR_DELETE(...) __builtin_operator_delete(__VA_ARGS__) +# define _CCCL_BUILTIN_OPERATOR_NEW(...) __builtin_operator_new(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(builtin_operator_new) && _CCCL_CHECK_BUILTIN(builtin_operator_delete) + +#if _CCCL_CHECK_BUILTIN(builtin_prefetch) || _CCCL_COMPILER(GCC) +# define _CCCL_BUILTIN_PREFETCH(...) NV_IF_TARGET(NV_IS_HOST, __builtin_prefetch(__VA_ARGS__);) +#else +# define _CCCL_BUILTIN_PREFETCH(...) +#endif // _CCCL_CHECK_BUILTIN(builtin_prefetch) + +#if _CCCL_HAS_BUILTIN(__decay) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_DECAY(...) __decay(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__decay) && clang-cuda + +#if _CCCL_CHECK_BUILTIN(has_nothrow_assign) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \ + || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_HAS_NOTHROW_ASSIGN(...) __has_nothrow_assign(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(has_nothrow_assign) && gcc >= 4.3 + +#if _CCCL_CHECK_BUILTIN(has_nothrow_constructor) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \ + || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_HAS_NOTHROW_CONSTRUCTOR(...) __has_nothrow_constructor(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(has_nothrow_constructor) && gcc >= 4.3 + +#if _CCCL_CHECK_BUILTIN(has_nothrow_copy) || _CCCL_COMPILER(GCC, >=, 4, 3) || _CCCL_COMPILER(MSVC) \ + || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_HAS_NOTHROW_COPY(...) __has_nothrow_copy(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(has_nothrow_copy) && gcc >= 4.3 + +#if _CCCL_HAS_BUILTIN(__integer_pack) +# define _CCCL_BUILTIN_INTEGER_PACK(...) __integer_pack(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__integer_pack) + +#if _CCCL_CHECK_BUILTIN(is_array) +# define _CCCL_BUILTIN_IS_ARRAY(...) __is_array(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_array) + +// clang prior to clang-19 gives wrong results for __is_array of _Tp[0] +#if _CCCL_COMPILER(CLANG, <, 19) +# undef _CCCL_BUILTIN_IS_ARRAY +#endif // clang < 19 + +#if _CCCL_CHECK_BUILTIN(is_assignable) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(GCC, >=, 9) +# define _CCCL_BUILTIN_IS_ASSIGNABLE(...) __is_assignable(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_assignable) && gcc >= 9.0 + +#if _CCCL_CHECK_BUILTIN(is_constructible) || _CCCL_COMPILER(GCC, >=, 8) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_IS_CONSTRUCTIBLE(...) __is_constructible(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_constructible) && gcc >= 8.0 + +#if _CCCL_CHECK_BUILTIN(is_convertible_to) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_IS_CONVERTIBLE_TO(...) __is_convertible_to(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_convertible_to) + +#if _CCCL_CHECK_BUILTIN(is_destructible) || _CCCL_COMPILER(MSVC) +# define _CCCL_BUILTIN_IS_DESTRUCTIBLE(...) __is_destructible(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_destructible) + +#if _CCCL_CHECK_BUILTIN(is_layout_compatible) || _CCCL_COMPILER(MSVC, >=, 19, 29) +# define _CCCL_BUILTIN_IS_LAYOUT_COMPATIBLE(...) __is_layout_compatible(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_layout_compatible) || _CCCL_COMPILER(MSVC, >=, 19, 29) + +#if _CCCL_CHECK_BUILTIN(is_lvalue_reference) +# define _CCCL_BUILTIN_IS_LVALUE_REFERENCE(...) __is_lvalue_reference(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_lvalue_reference) + +#if _CCCL_HAS_BUILTIN(__is_member_function_pointer) +# define _CCCL_BUILTIN_IS_MEMBER_FUNCTION_POINTER(...) __is_member_function_pointer(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_member_function_pointer) + +#if _CCCL_HAS_BUILTIN(__is_member_object_pointer) +# define _CCCL_BUILTIN_IS_MEMBER_OBJECT_POINTER(...) __is_member_object_pointer(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_member_object_pointer) + +#if _CCCL_HAS_BUILTIN(__is_member_pointer) +# define _CCCL_BUILTIN_IS_MEMBER_POINTER(...) __is_member_pointer(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_member_pointer) + +#if _CCCL_CHECK_BUILTIN(is_nothrow_assignable) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_IS_NOTHROW_ASSIGNABLE(...) __is_nothrow_assignable(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_nothrow_assignable) + +#if _CCCL_CHECK_BUILTIN(is_nothrow_constructible) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_IS_NOTHROW_CONSTRUCTIBLE(...) __is_nothrow_constructible(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_nothrow_constructible) + +#if _CCCL_CHECK_BUILTIN(is_nothrow_destructible) || _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_IS_NOTHROW_DESTRUCTIBLE(...) __is_nothrow_destructible(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_nothrow_destructible) + +#if _CCCL_CHECK_BUILTIN(is_object) +# define _CCCL_BUILTIN_IS_OBJECT(...) __is_object(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_object) + +// Disabled due to libstdc++ conflict +#if 0 // _CCCL_HAS_BUILTIN(__is_pointer) +# define _CCCL_BUILTIN_IS_POINTER(...) __is_pointer(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_pointer) + +#if _CCCL_CHECK_BUILTIN(is_pointer_interconvertible_base_of) || _CCCL_COMPILER(MSVC, >=, 19, 29) +# define _CCCL_BUILTIN_IS_POINTER_INTERCONVERTIBLE_BASE_OF(...) __is_pointer_interconvertible_base_of(__VA_ARGS__) +#endif // _CCCL_CHECK_BUILTIN(is_pointer_interconvertible_base_of) || _CCCL_COMPILER(MSVC, >=, 19, 29) + +#if _CCCL_HAS_BUILTIN(__is_reference) +# define _CCCL_BUILTIN_IS_REFERENCE(...) __is_reference(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_reference) + +// Disabled due to libstdc++ conflict +#if 0 // _CCCL_HAS_BUILTIN(__is_referenceable) +# define _CCCL_BUILTIN_IS_REFERENCEABLE(...) __is_referenceable(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_referenceable) + +#if _CCCL_HAS_BUILTIN(__is_rvalue_reference) +# define _CCCL_BUILTIN_IS_RVALUE_REFERENCE(...) __is_rvalue_reference(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_rvalue_reference) + +// Disabled due to libstdc++ conflict +#if 0 // _CCCL_HAS_BUILTIN(__is_scalar) +# define _CCCL_BUILTIN_IS_SCALAR(...) __is_scalar(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_scalar) + +#if _CCCL_CHECK_BUILTIN(make_integer_seq) || _CCCL_COMPILER(MSVC, >=, 19, 23) +# define _CCCL_BUILTIN_MAKE_INTEGER_SEQ(...) __make_integer_seq<__VA_ARGS__> +#endif // _CCCL_CHECK_BUILTIN(make_integer_seq) + +#if _CCCL_HAS_BUILTIN(__reference_constructs_from_temporary) +# define _CCCL_BUILTIN_REFERENCE_CONSTRUCTS_FROM_TEMPORARY(...) __reference_constructs_from_temporary(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__reference_constructs_from_temporary) + +#if _CCCL_HAS_BUILTIN(__reference_converts_from_temporary) +# define _CCCL_BUILTIN_REFERENCE_CONVERTS_FROM_TEMPORARY(...) __reference_converts_from_temporary(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__reference_converts_from_temporary) + +#if _CCCL_HAS_BUILTIN(__remove_const) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_CONST(...) __remove_const(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_const) + +#if _CCCL_HAS_BUILTIN(__remove_cv) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_CV(...) __remove_cv(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_cv) + +#if _CCCL_HAS_BUILTIN(__remove_cvref) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_CVREF(...) __remove_cvref(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_cvref) + +#if _CCCL_COMPILER(NVRTC, <, 12, 4) // NVRTC below 12.4 fails to properly compile that builtin +# undef _CCCL_BUILTIN_REMOVE_CVREF +#endif // _CCCL_COMPILER(NVRTC, <, 12, 4) + +#if _CCCL_HAS_BUILTIN(__remove_extent) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_EXTENT(...) __remove_extent(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_extent) + +#if _CCCL_HAS_BUILTIN(__remove_pointer) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_POINTER(...) __remove_pointer(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_pointer) + +#if _CCCL_HAS_BUILTIN(__remove_reference) +# define _CCCL_BUILTIN_REMOVE_REFERENCE_T(...) __remove_reference(__VA_ARGS__) +#elif _CCCL_HAS_BUILTIN(__remove_reference_t) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_REFERENCE_T(...) __remove_reference_t(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_reference_t) + +#if _CCCL_COMPILER(NVRTC, <, 12, 4) // NVRTC below 12.4 fails to properly compile cuda::std::move with that +# undef _CCCL_BUILTIN_REMOVE_REFERENCE_T +#endif // _CCCL_COMPILER(NVRTC, <, 12, 4) + +#if _CCCL_HAS_BUILTIN(__remove_volatile) && _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_BUILTIN_REMOVE_VOLATILE(...) __remove_volatile(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__remove_volatile) + +#if _CCCL_HAS_BUILTIN(__type_pack_element) +# define _CCCL_BUILTIN_TYPE_PACK_ELEMENT(...) __type_pack_element<__VA_ARGS__> +#endif // _CCCL_HAS_BUILTIN(__type_pack_element) + +#if _CCCL_HAS_BUILTIN(__is_complete_type) +# define _CCCL_BUILTIN_IS_COMPLETE_TYPE(...) __is_complete_type(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__is_complete_type) + +#if _CCCL_HAS_BUILTIN(__builtin_clear_padding) \ + && (_CCCL_HOST_COMPILATION() || !(_CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC))) +# define _CCCL_BUILTIN_CLEAR_PADDING(...) __builtin_clear_padding(__VA_ARGS__) +#endif // _CCCL_HAS_BUILTIN(__builtin_clear_padding) && (_CCCL_HOST_COMPILATION() || !(_CCCL_COMPILER(GCC) || + // _CCCL_COMPILER(NVHPC))) + +// NVCC prior to 12.2 have trouble with pack expansion into __type_pack_element in an alias template +#if _CCCL_CUDACC_BELOW(12, 2) +# undef _CCCL_BUILTIN_TYPE_PACK_ELEMENT +#endif // _CCCL_CUDACC_BELOW(12, 2) + +#if _CCCL_COMPILER(MSVC) // To use __builtin_FUNCSIG(), both MSVC and nvcc need to support it +# if _CCCL_COMPILER(MSVC, >=, 19, 35) && _CCCL_CUDACC_AT_LEAST(12, 3) +# define _CCCL_BUILTIN_PRETTY_FUNCTION() __builtin_FUNCSIG() +# else // ^^^ _CCCL_COMPILER(MSVC, >=, 19, 35) ^^^ / vvv _CCCL_COMPILER(MSVC, <, 19, 35) vvv +# define _CCCL_BUILTIN_PRETTY_FUNCTION() __FUNCSIG__ +# define _CCCL_BROKEN_MSVC_FUNCSIG +# endif // _CCCL_COMPILER(MSVC, <, 19, 35) +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_BUILTIN_PRETTY_FUNCTION() __PRETTY_FUNCTION__ +#endif // !_CCCL_COMPILER(MSVC) + +// GCC's builtin_strlen isn't reliable at constexpr time +// NVRTC does not expose builtin_strlen +#if !_CCCL_COMPILER(GCC) && !_CCCL_COMPILER(NVRTC) +# define _CCCL_BUILTIN_STRLEN(...) __builtin_strlen(__VA_ARGS__) +#endif + +// The new __nv_atomic builtins are available when __CUDACC_DEVICE_ATOMIC_BUILTINS__ is defined +#if defined(__CUDACC_DEVICE_ATOMIC_BUILTINS__) && _CCCL_PTX_ARCH() >= 600 && !_CCCL_COMPILER(MSVC) +# define _CCCL_HAS_NV_ATOMIC_BUILTINS() 1 +#else // ^^^ has intrinsics ^^^ / vvv no intrinsics +# define _CCCL_HAS_NV_ATOMIC_BUILTINS() 0 +#endif // no intrinsics + +#endif // __CCCL_BUILTIN_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/compiler.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/compiler.h new file mode 100644 index 00000000..da41e2f1 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/compiler.h @@ -0,0 +1,238 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_COMPILER_H +#define __CCCL_COMPILER_H + +#include + +// Utility to compare version numbers. To use: +// 1) Define a macro that makes a pair of (major, minor) numbers: +// #define MYPRODUCT_MAKE_VERSION(_MAJOR, _MINOR) (_MAJOR * 100 + _MINOR) +// 2) Define a macro that you will use to compare versions, e.g.: +// #define MYPRODUCT(...) _CCCL_VERSION_COMPARE(MYPRODUCT, MYPRODUCT_##__VA_ARGS__) +// Signatures: +// MYPRODUCT(_PROD) - is the product _PROD version non-zero? +// MYPRODUCT(_PROD, _OP, _MAJOR) - compare the product _PROD major version to _MAJOR using operator _OP +// MYPRODUCT(_PROD, _OP, _MAJOR, _MINOR) - compare the product _PROD version to _MAJOR._MINOR using operator _OP +// 3) Define the product version macros as a function-like macro that returns the version number or +// _CCCL_VERSION_INVALID() if the version cannot be determined, e. g.: +// #define MYPRODUCT_<_PROD>() (1, 2) +// or +// #define MYPRODUCT_<_PROD>() _CCCL_VERSION_INVALID() +#define _CCCL_VERSION_MAJOR_(_MAJOR, _MINOR) _MAJOR +#define _CCCL_VERSION_MAJOR(_PAIR) _CCCL_VERSION_MAJOR_ _PAIR +#define _CCCL_VERSION_INVALID() (-1, -1) +#define _CCCL_MAKE_VERSION(_PREFIX, _PAIR) (_CCCL_PP_EVAL(_CCCL_PP_CAT(_PREFIX, MAKE_VERSION), _CCCL_PP_EXPAND _PAIR)) +#define _CCCL_VERSION_IS_INVALID(_PAIR) (_CCCL_VERSION_MAJOR(_PAIR) == _CCCL_VERSION_MAJOR(_CCCL_VERSION_INVALID())) +#define _CCCL_VERSION_COMPARE_1(_PREFIX, _VER) (!_CCCL_VERSION_IS_INVALID(_VER())) +#define _CCCL_VERSION_COMPARE_3(_PREFIX, _VER, _OP, _MAJOR) \ + (!_CCCL_VERSION_IS_INVALID(_VER()) && (_CCCL_VERSION_MAJOR(_VER()) _OP _MAJOR)) +#define _CCCL_VERSION_COMPARE_4(_PREFIX, _VER, _OP, _MAJOR, _MINOR) \ + (!_CCCL_VERSION_IS_INVALID(_VER()) \ + && (_CCCL_MAKE_VERSION(_PREFIX, _VER()) _OP _CCCL_MAKE_VERSION(_PREFIX, (_MAJOR, _MINOR)))) +#define _CCCL_VERSION_SELECT_COUNT(_ARG1, _ARG2, _ARG3, _ARG4, _ARG5, ...) _ARG5 +#define _CCCL_VERSION_SELECT2(_ARGS) _CCCL_VERSION_SELECT_COUNT _ARGS +// MSVC traditonal preprocessor requires an extra level of indirection +#define _CCCL_VERSION_SELECT(...) \ + _CCCL_VERSION_SELECT2( \ + (__VA_ARGS__, \ + _CCCL_VERSION_COMPARE_4, \ + _CCCL_VERSION_COMPARE_3, \ + _CCCL_VERSION_COMPARE_BAD_ARG_COUNT, \ + _CCCL_VERSION_COMPARE_1, \ + _CCCL_VERSION_COMPARE_BAD_ARG_COUNT)) +#define _CCCL_VERSION_COMPARE(_PREFIX, ...) _CCCL_VERSION_SELECT(__VA_ARGS__)(_PREFIX, __VA_ARGS__) + +#define _CCCL_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 100 + (_MINOR)) +#define _CCCL_COMPILER(...) _CCCL_VERSION_COMPARE(_CCCL_COMPILER_, _CCCL_COMPILER_##__VA_ARGS__) + +#define _CCCL_COMPILER_NVHPC() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_CLANG() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_GCC() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_MSVC() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_MSVC2019() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_MSVC2022() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_MSVC2026() _CCCL_VERSION_INVALID() +#define _CCCL_COMPILER_NVRTC() _CCCL_VERSION_INVALID() + +// Determine the host compiler and its version +#if defined(__INTEL_COMPILER) +# ifndef CCCL_IGNORE_DEPRECATED_COMPILER +# warning \ + "The Intel C++ Compiler Classic (icc/icpc) is not supported by CCCL. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this message." +# endif // !CCCL_IGNORE_DEPRECATED_COMPILER +#elif defined(__NVCOMPILER) +# undef _CCCL_COMPILER_NVHPC +# define _CCCL_COMPILER_NVHPC() (__NVCOMPILER_MAJOR__, __NVCOMPILER_MINOR__) +#elif defined(__clang__) +# undef _CCCL_COMPILER_CLANG +# define _CCCL_COMPILER_CLANG() (__clang_major__, __clang_minor__) +#elif defined(__GNUC__) +# undef _CCCL_COMPILER_GCC +# define _CCCL_COMPILER_GCC() (__GNUC__, __GNUC_MINOR__) +#elif defined(_MSC_VER) +// see https://learn.microsoft.com/en-us/cpp/overview/compiler-versions?view=msvc-180#version-macros +# undef _CCCL_COMPILER_MSVC +# define _CCCL_COMPILER_MSVC() (_MSC_VER / 100, _MSC_VER % 100) +# if _CCCL_COMPILER(MSVC, <, 19, 20) +# ifndef CCCL_IGNORE_DEPRECATED_COMPILER +# error \ + "Visual Studio 2017 (MSC_VER < 1920) and older are not supported by CCCL. Define CCCL_IGNORE_DEPRECATED_COMPILER to suppress this error." +# endif +# endif // _CCCL_COMPILER(MSVC, <, 19, 20) +# if _CCCL_COMPILER(MSVC, >=, 19, 20) && _CCCL_COMPILER(MSVC, <, 19, 30) +# undef _CCCL_COMPILER_MSVC2019 +# define _CCCL_COMPILER_MSVC2019() _CCCL_COMPILER_MSVC() +# endif // _CCCL_COMPILER(MSVC, >=, 19, 20) && _CCCL_COMPILER(MSVC, <, 19, 30) +# if _CCCL_COMPILER(MSVC, >=, 19, 30) && _CCCL_COMPILER(MSVC, <, 19, 50) +# undef _CCCL_COMPILER_MSVC2022 +# define _CCCL_COMPILER_MSVC2022() _CCCL_COMPILER_MSVC() +# endif // _CCCL_COMPILER(MSVC, >=, 19, 30) && _CCCL_COMPILER(MSVC, <, 19, 50) +# if _CCCL_COMPILER(MSVC, >=, 19, 50) +# undef _CCCL_COMPILER_MSVC2026 +# define _CCCL_COMPILER_MSVC2026() _CCCL_COMPILER_MSVC() +# endif // _CCCL_COMPILER(MSVC, >=, 19, 45) +#elif defined(__CUDACC_RTC__) +# undef _CCCL_COMPILER_NVRTC +# define _CCCL_COMPILER_NVRTC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__) +#endif + +#if defined(__CUDACC__) || defined(_NVHPC_CUDA) +# define _CCCL_CUDA_COMPILATION() 1 +#else // ^^^ compiling .cu file ^^^ / vvv not compiling .cu file vvv +# define _CCCL_CUDA_COMPILATION() 0 +#endif // ^^^ not compiling .cu file ^^^ + +// The CUDA compiler version shares the implementation with the C++ compiler +#define _CCCL_CUDA_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) _CCCL_COMPILER_MAKE_VERSION(_MAJOR, _MINOR) +#define _CCCL_CUDA_COMPILER(...) _CCCL_VERSION_COMPARE(_CCCL_CUDA_COMPILER_, _CCCL_CUDA_COMPILER_##__VA_ARGS__) + +#define _CCCL_CUDA_COMPILER_NVCC() _CCCL_VERSION_INVALID() +#define _CCCL_CUDA_COMPILER_NVHPC() _CCCL_VERSION_INVALID() +#define _CCCL_CUDA_COMPILER_CLANG() _CCCL_VERSION_INVALID() +#define _CCCL_CUDA_COMPILER_NVRTC() _CCCL_VERSION_INVALID() + +// Determine the cuda compiler +#if _CCCL_CUDA_COMPILATION() +# if defined(__NVCC__) +# undef _CCCL_CUDA_COMPILER_NVCC +# define _CCCL_CUDA_COMPILER_NVCC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__) +# elif defined(_NVHPC_CUDA) +# undef _CCCL_CUDA_COMPILER_NVHPC +# define _CCCL_CUDA_COMPILER_NVHPC() _CCCL_COMPILER_NVHPC() +# elif defined(__CUDA__) && _CCCL_COMPILER(CLANG) +# undef _CCCL_CUDA_COMPILER_CLANG +# define _CCCL_CUDA_COMPILER_CLANG() _CCCL_COMPILER_CLANG() +# elif _CCCL_COMPILER(NVRTC) +# undef _CCCL_CUDA_COMPILER_NVRTC +# define _CCCL_CUDA_COMPILER_NVRTC() _CCCL_COMPILER_NVRTC() +# endif // ^^^ _CCCL_COMPILER(NVRTC) ^^^ +#endif // _CCCL_CUDA_COMPILATION() + +// Determine if we are compiling host code, this includes both CUDA and C++ compilation +// nvc++ does not define __CUDA_ARCH__, but it compiles both host and device code at the same time +#if !defined(__CUDA_ARCH__) +# define _CCCL_HOST_COMPILATION() 1 +#else // ^^^ compiling host code ^^^ / vvv not compiling host code vvv +# define _CCCL_HOST_COMPILATION() 0 +#endif // ^^^ not compiling host code ^^^ + +#if (_CCCL_CUDA_COMPILATION() && defined(__CUDA_ARCH__)) || _CCCL_CUDA_COMPILER(NVHPC) +# define _CCCL_DEVICE_COMPILATION() 1 +#else // ^^^ compiling device code ^^^ / vvv not compiling device code vvv +# define _CCCL_DEVICE_COMPILATION() 0 +#endif // ^^^ not compiling device code ^^^ + +#if defined(__CUDACC_TILE__) && _CCCL_CUDA_COMPILER(NVCC, >, 13, 3) +# define _CCCL_TILE_COMPILATION() 1 +#else // ^^^ compiling .cu file in tile mode ^^^ / vvv not compiling in tile mode vvv +# define _CCCL_TILE_COMPILATION() 0 +#endif // ^^^ not compiling .cu file ^^^ + +#define _CCCL_CUDACC_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 1000 + (_MINOR) * 10) + +// clang-cuda does not define __CUDACC_VER_MAJOR__ and friends. They are instead retrieved from the CUDA_VERSION macro +// defined in "cuda.h". clang-cuda automatically pre-includes "__clang_cuda_runtime_wrapper.h" which includes "cuda.h" +#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(NVHPC) || _CCCL_CUDA_COMPILER(NVRTC) +# define _CCCL_CUDACC() (__CUDACC_VER_MAJOR__, __CUDACC_VER_MINOR__) +#elif _CCCL_CUDA_COMPILER(CLANG) +# define _CCCL_CUDACC() (CUDA_VERSION / 1000, (CUDA_VERSION % 1000) / 10) +#endif // ^^^ has cuda compiler ^^^ + +#if !defined(_CCCL_CUDACC) || !_CCCL_CUDA_COMPILATION() +# undef _CCCL_CUDACC +# define _CCCL_CUDACC() _CCCL_VERSION_INVALID() +#endif // !_CCCL_CUDACC || !_CCCL_CUDA_COMPILATION() + +#define _CCCL_CUDACC_EQUAL(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, ==, __VA_ARGS__) +#define _CCCL_CUDACC_BELOW(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, <, __VA_ARGS__) +#define _CCCL_CUDACC_AT_LEAST(...) _CCCL_VERSION_COMPARE(_CCCL_CUDACC_, _CCCL_CUDACC, >=, __VA_ARGS__) + +#if _CCCL_CUDA_COMPILATION() && _CCCL_CUDACC_BELOW(12) && !defined(CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12) +# error "CUDA versions below 12 are not supported." \ +"Define CCCL_IGNORE_DEPRECATED_CUDA_BELOW_12 to suppress this message." +#endif + +// Define the pragma for the host compiler +#if _CCCL_COMPILER(MSVC) +# define _CCCL_PRAGMA(_ARG) __pragma(_ARG) +#else +# define _CCCL_PRAGMA(_ARG) _Pragma(_CCCL_TO_STRING(_ARG)) +#endif // _CCCL_COMPILER(MSVC) + +// Define the proper object format for NVHPC and NVRTC +#if (_CCCL_COMPILER(NVHPC) && defined(__linux__)) || _CCCL_COMPILER(NVRTC) +# ifndef __ELF__ +# define __ELF__ +# endif // !__ELF__ +#endif // _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC) + +#if _CCCL_DEVICE_COMPILATION() +# define _CCCL_PRAGMA_UNROLL(_N) _CCCL_PRAGMA(unroll _N) +# define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA(unroll) +#elif _CCCL_COMPILER(NVHPC) || _CCCL_COMPILER(NVRTC) || _CCCL_COMPILER(CLANG) +# define _CCCL_PRAGMA_UNROLL(_N) _CCCL_PRAGMA(unroll _N) +# define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA(unroll) +#elif _CCCL_COMPILER(GCC, >=, 8) +// gcc supports only #pragma GCC unroll, but that causes problems when compiling with nvcc. So, we use #pragma unroll +// when compiling device code, and #pragma GCC unroll when compiling host code, but we need to suppress the warning +// about the unknown pragma for nvcc. +// #pragma GCC unroll does not support full unrolling, so we use the maximum value that it supports. +# define _CCCL_PRAGMA_UNROLL(_N) \ + _CCCL_BEGIN_NV_DIAG_SUPPRESS(1675) _CCCL_PRAGMA(GCC unroll _N) _CCCL_END_NV_DIAG_SUPPRESS() +# define _CCCL_PRAGMA_UNROLL_FULL() _CCCL_PRAGMA_UNROLL(65534) +#else // ^^^ has pragma unroll support ^^^ / vvv no pragma unroll support vvv +# define _CCCL_PRAGMA_UNROLL(_N) +# define _CCCL_PRAGMA_UNROLL_FULL() +#endif // ^^^ no pragma unroll support ^^^ + +#define _CCCL_PRAGMA_NOUNROLL() _CCCL_PRAGMA_UNROLL(1) + +#if _CCCL_COMPILER(MSVC) +# define _CCCL_WARNING(_MSG) _CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " _MSG)) +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_WARNING(_MSG) _CCCL_PRAGMA(GCC warning _MSG) +#endif // !_CCCL_COMPILER(MSVC) + +// Freestanding environment detection +// NVRTC is treated as freestanding since it has no access to the host standard library +#if defined(_CCCL_ENABLE_FREESTANDING) || _CCCL_COMPILER(NVRTC) +# define _CCCL_FREESTANDING() 1 +# define _CCCL_HOSTED() 0 +# define _CCCL_HOSTJIT() (!_CCCL_COMPILER(NVRTC)) +# define _CCCL_NO_TYPEID +#else // ^^^ _CCCL_ENABLE_FREESTANDING || _CCCL_COMPILER(NVRTC) ^^^ / vvv Hosted environment vvv +# define _CCCL_FREESTANDING() 0 +# define _CCCL_HOSTED() 1 +# define _CCCL_HOSTJIT() 0 +#endif // Hosted environment + +#endif // __CCCL_COMPILER_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_capabilities.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_capabilities.h new file mode 100644 index 00000000..2182bf5f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_capabilities.h @@ -0,0 +1,118 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_CUDA_CAPABILITIES +#define __CCCL_CUDA_CAPABILITIES + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +/// In device code, _CCCL_PTX_ARCH() expands to the PTX version for which we are compiling. +/// In host code, _CCCL_PTX_ARCH()'s value is implementation defined. +#if !defined(__CUDA_ARCH__) +# define _CCCL_PTX_ARCH() 0 +#else +# define _CCCL_PTX_ARCH() __CUDA_ARCH__ +#endif + +#ifdef _CCCL_DOXYGEN_INVOKED // Only parse this during doxygen passes: +//! When this macro is defined, Programmatic Dependent Launch (PDL) is disabled across CCCL +# define CCCL_DISABLE_PDL +#endif // _CCCL_DOXYGEN_INVOKED + +#ifdef CCCL_DISABLE_PDL +# define _CCCL_HAS_PDL() 0 +#else // CCCL_DISABLE_PDL +# define _CCCL_HAS_PDL() 1 +#endif // CCCL_DISABLE_PDL + +#if _CCCL_HAS_PDL() +// Waits for the previous kernel to complete (when it reaches its final membar). Should be put before the first global +// memory access in a kernel. +# define _CCCL_PDL_GRID_DEPENDENCY_SYNC() NV_IF_TARGET(NV_PROVIDES_SM_90, ::cudaGridDependencySynchronize();) +// Allows the subsequent kernel in the same stream to launch. Can be put anywhere in a kernel. +// Heuristic(ahendriksen): put it after the last load. +# define _CCCL_PDL_TRIGGER_NEXT_LAUNCH() NV_IF_TARGET(NV_PROVIDES_SM_90, ::cudaTriggerProgrammaticLaunchCompletion();) +#else // _CCCL_HAS_PDL() +# define _CCCL_PDL_GRID_DEPENDENCY_SYNC() +# define _CCCL_PDL_TRIGGER_NEXT_LAUNCH() +#endif // _CCCL_HAS_PDL() + +// Check whether the relocatable device code (RDC) is being generated. +#if defined(__CUDACC_RDC__) || defined(__CLANG_RDC__) || defined(_NVHPC_RDC) +# define _CCCL_HAS_RDC() 1 +#else // ^^^ has RDC ^^^ / vvv no RDC vvv +# define _CCCL_HAS_RDC() 0 +#endif // ^^^ no RDC ^^^ + +// Check whether extensible whole program is being compiled. +#if defined(__CUDACC_EWP__) +# define _CCCL_HAS_EWP() 1 +#else // ^^^ has EWP ^^^ / vvv no EWP vvv +# define _CCCL_HAS_EWP() 0 +#endif // ^^^ no EWP ^^^ + +// Control whether device runtime APIs can be used, because they require libcudadevrt to be linked. Defaults to true +// when RDC or EWP are enabled. Can be disabled by defining CCCL_DISABLE_DEVICE_RUNTIME. +#if (_CCCL_HAS_RDC() || _CCCL_HAS_EWP()) && !defined(CCCL_DISABLE_DEVICE_RUNTIME) +# define _CCCL_HAS_DEVICE_RUNTIME() 1 +#else // ^^^ has device runtime ^^^ / vvv no device runtime vvv +# define _CCCL_HAS_DEVICE_RUNTIME() 0 +#endif // ^^^ no device runtime ^^^ + +// Some functions can be called from host or device code and launch kernels inside. Thus, they use CUDA Dynamic +// Parallelism (CDP) and require compiling with Relocatable Device Code (RDC) or extensible whole program (EWP) and link +// with device runtime library. CDP is unsupported with clang-cuda below 22. +// TODO(bgruber): remove CUB_DISABLE_CDP in CCCL 4.0 +#if _CCCL_HAS_DEVICE_RUNTIME() && !defined(CCCL_DISABLE_CDP) && !defined(CUB_DISABLE_CDP) \ + && !_CCCL_CUDA_COMPILER(CLANG, <, 22) +// We have CDP, so host and device APIs can call kernels +# define _CCCL_HAS_CDP() 1 +#else // ^^^ has CDP ^^^ / vvv no CDP vvv +// We don't have CDP, only host APIs can call kernels +# define _CCCL_HAS_CDP() 0 +#endif // ^^^ no CDP ^^^ + +// When RDC is enabled, __launch_bounds__ cannot be used reliably. See #902. +#if !_CCCL_HAS_RDC() && !defined(CCCL_DISABLE_LAUNCH_BOUNDS) +# define _CCCL_LAUNCH_BOUNDS(...) __launch_bounds__(__VA_ARGS__) +#else // ^^^ has launch bounds attribute ^^^ / vvv no launch bounds attribute vvv +# define _CCCL_LAUNCH_BOUNDS(...) +#endif // ^^^ no launch bounds attribute ^^^ + +// __block_size__ attribute is available for nvcc and nvrtc 12.9+ for hopper+ architectures. For older nvcc and nvrtc, +// we can fallback to __cluster_dims__ attribute only specifying the ncta per cluster. +// This attribute should be used only for cluster launches. +#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 9) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 9)) && _CCCL_PTX_ARCH() >= 900 +# define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) __block_size__(_NTID, _NCTA_PER_CLUSTER) +#elif (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(NVRTC)) && _CCCL_PTX_ARCH() >= 900 +# define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) __cluster_dims__ _NCTA_PER_CLUSTER +#else // ^^ has __block_size__ attribute ^^^ / vvv no __block_size__ attribute vvv +# define _CCCL_BLOCK_SIZE(_NTID, _NCTA_PER_CLUSTER) +#endif // ^^^ no __block_size__ attribute ^^^ + +#if _CCCL_HAS_CDP() +# ifdef CUDA_FORCE_CDP1_IF_SUPPORTED +# error "CUDA Dynamic Parallelism 1 is no longer supported. Please undefine CUDA_FORCE_CDP1_IF_SUPPORTED." +# endif // CUDA_FORCE_CDP1_IF_SUPPORTED +#endif // _CCCL_HAS_CDP() + +#endif // __CCCL_CUDA_CAPABILITIES diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_toolkit.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_toolkit.h new file mode 100644 index 00000000..e29e1057 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/cuda_toolkit.h @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_CUDA_TOOLKIT_H +#define __CCCL_CUDA_TOOLKIT_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_CUDA_COMPILATION() || __has_include() +# define _CCCL_HAS_CTK() 1 +#else // ^^^ has cuda toolkit ^^^ / vvv no cuda toolkit vvv +# define _CCCL_HAS_CTK() 0 +#endif // ^^^ no cuda toolkit ^^^ + +// CUDA compilers preinclude cuda_runtime.h, so we need to include it here to get the CUDART_VERSION macro +#if _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILATION() +# include +#endif // _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILATION() + +// Check compatibility of the CUDA compiler and CUDA toolkit headers +// Some users might want to use a newer version of the CTK than the compiler ships. Enable that on their own peril +#ifndef CCCL_DISABLE_CTK_COMPATIBILITY_CHECK +# if _CCCL_CUDA_COMPILATION() +# if !_CCCL_CUDACC_EQUAL((CUDART_VERSION / 1000), (CUDART_VERSION % 1000) / 10) +# error "CUDA compiler and CUDA toolkit headers are incompatible, please check your include paths" +# endif // !_CCCL_CUDACC_EQUAL((CUDART_VERSION / 1000), (CUDART_VERSION % 1000) / 10) +# endif // _CCCL_CUDA_COMPILATION() +#endif // CCCL_DISABLE_CTK_COMPATIBILITY_CHECK + +#if _CCCL_HAS_CTK() +# define _CCCL_CTK() (CUDART_VERSION / 1000, (CUDART_VERSION % 1000) / 10) +#else // ^^^ has cuda toolkit ^^^ / vvv no cuda toolkit vvv +# define _CCCL_CTK() _CCCL_VERSION_INVALID() +#endif // ^^^ no cuda toolkit ^^^ + +#define _CCCL_CTK_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 1000 + (_MINOR) * 10) +#define _CCCL_CTK_BELOW(...) _CCCL_VERSION_COMPARE(_CCCL_CTK_, _CCCL_CTK, <, __VA_ARGS__) +#define _CCCL_CTK_AT_LEAST(...) _CCCL_VERSION_COMPARE(_CCCL_CTK_, _CCCL_CTK, >=, __VA_ARGS__) + +#endif // __CCCL_CUDA_TOOLKIT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/deprecated.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/deprecated.h new file mode 100644 index 00000000..a8e47095 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/deprecated.h @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_DEPRECATED_H +#define __CCCL_DEPRECATED_H + +#include +#include +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// Check for deprecation opt outs +#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_DIALECT) +# if !defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) +# define CCCL_IGNORE_DEPRECATED_CPP_DIALECT +# endif +#endif // suppress all dialect deprecation warnings +#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_14) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) +# if !defined(CCCL_IGNORE_DEPRECATED_CPP_14) +# define CCCL_IGNORE_DEPRECATED_CPP_14 +# endif +#endif // suppress all c++14 dialect deprecation warnings +#if defined(LIBCUDACXX_IGNORE_DEPRECATED_CPP_11) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) \ + || defined(CCCL_IGNORE_DEPRECATED_CPP_14) +# if !defined(CCCL_IGNORE_DEPRECATED_CPP_11) +# define CCCL_IGNORE_DEPRECATED_CPP_11 +# endif +#endif // suppress all c++11 dialect deprecation warnings +#if defined(LIBCUDACXX_IGNORE_DEPRECATED_COMPILER) || defined(THRUST_IGNORE_DEPRECATED_COMPILER) \ + || defined(CUB_IGNORE_DEPRECATED_COMPILER) || defined(CCCL_IGNORE_DEPRECATED_CPP_DIALECT) \ + || defined(CCCL_IGNORE_DEPRECATED_CPP_14) || defined(CCCL_IGNORE_DEPRECATED_CPP_11) +# if !defined(CCCL_IGNORE_DEPRECATED_COMPILER) +# define CCCL_IGNORE_DEPRECATED_COMPILER +# endif +#endif // suppress all compiler deprecation warnings +#if defined(LIBCUDACXX_IGNORE_DEPRECATED_API) || defined(THRUST_IGNORE_DEPRECATED_API) \ + || defined(CUB_IGNORE_DEPRECATED_API) +# if !defined(CCCL_IGNORE_DEPRECATED_API) +# define CCCL_IGNORE_DEPRECATED_API +# endif +#endif // suppress all API deprecation warnings + +#if defined(CCCL_IGNORE_DEPRECATED_API) || defined(_LIBCUDACXX_DISABLE_DEPRECATION_WARNINGS) +//! deprecated [Since 2.8] +# define CCCL_DEPRECATED +//! deprecated [Since 2.8] +# define CCCL_DEPRECATED_BECAUSE(MSG) +#elif _CCCL_HAS_ATTRIBUTE(deprecated) +//! deprecated [Since 2.8] +# define CCCL_DEPRECATED __attribute__((deprecated)) +//! deprecated [Since 2.8] +# define CCCL_DEPRECATED_BECAUSE(MSG) __attribute__((deprecated(MSG))) +#else // ^^^ attribute deprecated ^^^ / vvv standard deprecated attribute vvv +//! deprecated [Since 2.8] +# define CCCL_DEPRECATED [[deprecated]] +//! deprecated [Since 2.8] +# define CCCL_DEPRECATED_BECAUSE(MSG) [[deprecated(MSG)]] +#endif // ^^^ standard deprecated attribute ^^^ + +#if _CCCL_STD_VER >= 2020 +# define _CCCL_DEPRECATED_IN_CXX20 CCCL_DEPRECATED +#else // ^^^ _CCCL_STD_VER >= 2020 ^^^ / vvv _CCCL_STD_VER < 2020 vvv +# define _CCCL_DEPRECATED_IN_CXX20 +#endif // ^^^ _CCCL_STD_VER < 2020 ^^^ + +#if _CCCL_STD_VER >= 2023 +# define _CCCL_DEPRECATED_IN_CXX23 CCCL_DEPRECATED +#else // ^^^ _CCCL_STD_VER >= 2023 ^^^ / vvv _CCCL_STD_VER < 2023 vvv +# define _CCCL_DEPRECATED_IN_CXX23 +#endif // ^^^ _CCCL_STD_VER < 2023 ^^^ + +#endif // __CCCL_DEPRECATED_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/diagnostic.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/diagnostic.h new file mode 100644 index 00000000..a5daafb2 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/diagnostic.h @@ -0,0 +1,145 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_DIAGNOSTIC_H +#define __CCCL_DIAGNOSTIC_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// Enable us to selectively silence host compiler warnings +#if _CCCL_COMPILER(CLANG) +# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(clang diagnostic push) +# define _CCCL_DIAG_POP _CCCL_PRAGMA(clang diagnostic pop) +# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) _CCCL_PRAGMA(clang diagnostic ignored _WARNING) +# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) +#elif _CCCL_COMPILER(GCC) +# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(GCC diagnostic push) +# define _CCCL_DIAG_POP _CCCL_PRAGMA(GCC diagnostic pop) +# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) +# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) _CCCL_PRAGMA(GCC diagnostic ignored _WARNING) +# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) +#elif _CCCL_COMPILER(NVHPC) +# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(diagnostic push) +# define _CCCL_DIAG_POP _CCCL_PRAGMA(diagnostic pop) +# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) +# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) _CCCL_PRAGMA(diag_suppress _WARNING) +# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_DIAG_PUSH _CCCL_PRAGMA(warning(push)) +# define _CCCL_DIAG_POP _CCCL_PRAGMA(warning(pop)) +# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) +# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) _CCCL_PRAGMA(warning(disable : _WARNING)) +#else +# define _CCCL_DIAG_PUSH +# define _CCCL_DIAG_POP +# define _CCCL_DIAG_SUPPRESS_CLANG(_WARNING) +# define _CCCL_DIAG_SUPPRESS_GCC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_NVHPC(_WARNING) +# define _CCCL_DIAG_SUPPRESS_MSVC(_WARNING) +#endif + +// Enable us to selectively silence cuda compiler warnings +#if _CCCL_CUDA_COMPILER(NVCC) || _CCCL_COMPILER(NVRTC) +# if defined(__NVCC_DIAG_PRAGMA_SUPPORT__) +# define _CCCL_NV_DIAG_PUSH() _CCCL_PRAGMA(nv_diagnostic push) +# define _CCCL_NV_DIAG_POP() _CCCL_PRAGMA(nv_diagnostic pop) +# define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) _CCCL_PRAGMA(nv_diag_suppress _WARNING) +# define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) \ + _CCCL_NV_DIAG_PUSH() _CCCL_PP_FOR_EACH(_CCCL_DIAG_SUPPRESS_NVCC, __VA_ARGS__) +# define _CCCL_END_NV_DIAG_SUPPRESS() _CCCL_NV_DIAG_POP() +# else // ^^^ __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^ / vvv !__NVCC_DIAG_PRAGMA_SUPPORT__ vvv +# define _CCCL_NV_DIAG_PUSH() _CCCL_PRAGMA(diagnostic push) +# define _CCCL_NV_DIAG_POP() _CCCL_PRAGMA(diagnostic pop) +# define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) _CCCL_PRAGMA(diag_suppress _WARNING) +# define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) \ + _CCCL_NV_DIAG_PUSH() _CCCL_PP_FOR_EACH(_CCCL_DIAG_SUPPRESS_NVCC, __VA_ARGS__) +# define _CCCL_END_NV_DIAG_SUPPRESS() _CCCL_NV_DIAG_POP() +# endif // !__NVCC_DIAG_PRAGMA_SUPPORT__ +#else // ^^^ _CCCL_CUDA_COMPILER(NVCC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVCC) vvv +# define _CCCL_NV_DIAG_PUSH() +# define _CCCL_NV_DIAG_POP() +# define _CCCL_DIAG_SUPPRESS_NVCC(_WARNING) +# define _CCCL_BEGIN_NV_DIAG_SUPPRESS(...) +# define _CCCL_END_NV_DIAG_SUPPRESS() +#endif // !_CCCL_CUDA_COMPILER(NVCC) + +// Convenient shortcuts to silence common warnings +#if _CCCL_COMPILER(CLANG) +# define _CCCL_SUPPRESS_DEPRECATED_PUSH \ + _CCCL_DIAG_PUSH \ + _CCCL_DIAG_SUPPRESS_CLANG("-Wdeprecated") \ + _CCCL_DIAG_SUPPRESS_CLANG("-Wdeprecated-declarations") \ + _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199) +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP +#elif _CCCL_COMPILER(GCC) +# define _CCCL_SUPPRESS_DEPRECATED_PUSH \ + _CCCL_DIAG_PUSH \ + _CCCL_DIAG_SUPPRESS_GCC("-Wdeprecated") \ + _CCCL_DIAG_SUPPRESS_GCC("-Wdeprecated-declarations") \ + _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199) +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP +#elif _CCCL_COMPILER(NVHPC) +# define _CCCL_SUPPRESS_DEPRECATED_PUSH \ + _CCCL_DIAG_PUSH \ + _CCCL_DIAG_SUPPRESS_NVHPC(deprecated_entity) \ + _CCCL_DIAG_SUPPRESS_NVHPC(deprecated_entity_with_custom_message) \ + _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199) +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_SUPPRESS_DEPRECATED_PUSH \ + _CCCL_DIAG_PUSH \ + _CCCL_DIAG_SUPPRESS_MSVC(4996) \ + _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444) +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() _CCCL_DIAG_POP +#elif _CCCL_COMPILER(NVRTC) +# if _CCCL_COMPILER(NVRTC, >=, 13, 3) && defined(__NVCC_DIAG_PRAGMA_SUPPORT__) +# define _CCCL_SUPPRESS_DEPRECATED_PUSH _CCCL_NV_DIAG_PUSH() +// NVRTC 13.3 does not honor nv_diag_suppress when it is emitted in the same macro expansion as +// nv_diagnostic push. Keep the suppression in a separate source-level macro invocation. +// See https://github.com/NVIDIA/cccl/issues/9170 and nvbug 6239043. +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG _Pragma("nv_diag_suppress 1444,20199") +# else // ^^^ NVRTC >= 13.3 with __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^ +# define _CCCL_SUPPRESS_DEPRECATED_PUSH _CCCL_BEGIN_NV_DIAG_SUPPRESS(1444, 20199) +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +# endif // ^^^ NVRTC >= 13.3 with __NVCC_DIAG_PRAGMA_SUPPORT__ ^^^ +# define _CCCL_SUPPRESS_DEPRECATED_POP _CCCL_NV_DIAG_POP() +#else // unknown compiler +# define _CCCL_SUPPRESS_DEPRECATED_PUSH +# define _CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +# define _CCCL_SUPPRESS_DEPRECATED_POP +#endif // unknown compiler + +#if _CCCL_COMPILER(MSVC) +# define _CCCL_HAS_PRAGMA_MSVC_WARNING +# if !defined(_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING) +# define _CCCL_USE_PRAGMA_MSVC_WARNING +# endif // !_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING +#endif // !_CCCL_COMPILER(MSVC) + +#endif // __CCCL_DIAGNOSTIC_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/dialect.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/dialect.h new file mode 100644 index 00000000..989490bb --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/dialect.h @@ -0,0 +1,230 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_DIALECT_H +#define __CCCL_DIALECT_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +// Determine the C++ standard dialect +/////////////////////////////////////////////////////////////////////////////// +#if _CCCL_COMPILER(MSVC) +# if _MSVC_LANG <= 201103L +# define _CCCL_STD_VER 2011 +# elif _MSVC_LANG <= 201402L +# define _CCCL_STD_VER 2014 +# elif _MSVC_LANG <= 201703L +# define _CCCL_STD_VER 2017 +# elif _MSVC_LANG <= 202002L +# define _CCCL_STD_VER 2020 +# else +# define _CCCL_STD_VER 2023 // current year, or date of c++2b ratification +# endif +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# if __cplusplus <= 199711L +# define _CCCL_STD_VER 2003 +# elif __cplusplus <= 201103L +# define _CCCL_STD_VER 2011 +# elif __cplusplus <= 201402L +# define _CCCL_STD_VER 2014 +# elif __cplusplus <= 201703L +# define _CCCL_STD_VER 2017 +# elif __cplusplus <= 202002L +# define _CCCL_STD_VER 2020 +# elif __cplusplus <= 202302L +# define _CCCL_STD_VER 2023 +# else +# define _CCCL_STD_VER 2024 // current year, or date of c++2c ratification +# endif +#endif // !_CCCL_COMPILER(MSVC) + +/////////////////////////////////////////////////////////////////////////////// +// Conditionally enable constexpr per standard dialect +/////////////////////////////////////////////////////////////////////////////// + +#if _CCCL_STD_VER >= 2020 +# define _CCCL_CONSTEXPR_CXX20 constexpr +#else // ^^^ C++20 ^^^ / vvv C++17 vvv +# define _CCCL_CONSTEXPR_CXX20 +#endif // _CCCL_STD_VER <= 2017 + +#if _CCCL_STD_VER >= 2023 +# define _CCCL_CONSTEXPR_CXX23 constexpr +#else // ^^^ C++23 ^^^ / vvv C++20 vvv +# define _CCCL_CONSTEXPR_CXX23 +#endif // _CCCL_STD_VER <= 2020 + +/////////////////////////////////////////////////////////////////////////////// +// Detect whether we can use some language features based on standard dialect +/////////////////////////////////////////////////////////////////////////////// + +// concepts are only available from C++20 onwards +#if _CCCL_STD_VER <= 2017 || __cpp_concepts < 201907L +# define _CCCL_HAS_CONCEPTS() 0 +#else // ^^^ no concepts ^^^ / vvv has concepts vvv +# define _CCCL_HAS_CONCEPTS() 1 +#endif // ^^^ has concepts ^^^ + +// Three way comparison is only available from C++20 onwards +#if _CCCL_STD_VER <= 2017 || __cpp_impl_three_way_comparison < 201907L +# define _CCCL_NO_THREE_WAY_COMPARISON +#endif // _CCCL_STD_VER <= 2017 || __cpp_impl_three_way_comparison < 201907L + +// Some compilers turn on pack indexing in pre-C++26 code. We want to use it if it is +// available. +#if __cpp_pack_indexing >= 202311L && !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_COMPILER(CLANG, <, 20) +# define _CCCL_HAS_PACK_INDEXING() 1 +#else // ^^^ has pack indexing ^^^ / vvv no pack indexing vvv +# define _CCCL_HAS_PACK_INDEXING() 0 +#endif // no pack indexing + +#if _CCCL_STD_VER <= 2017 || __cpp_consteval < 201811L +# define _CCCL_NO_CONSTEVAL +# define _CCCL_CONSTEVAL constexpr +#else +# define _CCCL_CONSTEVAL consteval +#endif + +/////////////////////////////////////////////////////////////////////////////// +// Conditionally use certain language features depending on availability +/////////////////////////////////////////////////////////////////////////////// + +// We need to treat host and device separately +#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) +# define _CCCL_GLOBAL_CONSTANT _CCCL_DEVICE constexpr +#else // ^^^ _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) ^^^ / + // vvv !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) vvv +# define _CCCL_GLOBAL_CONSTANT inline constexpr +#endif // !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) + +#if _CCCL_STD_VER >= 2020 && __cpp_constinit >= 201907L +# define _CCCL_CONSTINIT constinit +#else // ^^^ has constinit ^^^ / vvv no constinit vvv +# define _CCCL_CONSTINIT _CCCL_REQUIRE_CONSTANT_INITIALIZATION +#endif // ^^^ no constinit ^^^ + +// nvcc and nvrtc don't implement multiarg operator[] even in C++23 mode +#if __cpp_multidimensional_subscript >= 202110L && !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) +# define _CCCL_HAS_MULTIARG_OPERATOR_BRACKETS() 1 +#else // ^^^ has multiarg operator[] ^^^ / vvv no multiarg operator[] vvv +# define _CCCL_HAS_MULTIARG_OPERATOR_BRACKETS() 0 +#endif // ^^^ no mutiarg operator[] ^^^ + +// clang 16+, gcc 13+ and nvc++ 25.9+ backport the static subscript operator back to c++17. +#if __cpp_multidimensional_subscript >= 202211L \ + || ((_CCCL_COMPILER(CLANG, >=, 16) || _CCCL_COMPILER(GCC, >=, 13) \ + || (_CCCL_COMPILER(NVHPC, >=, 25, 9) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 12))) \ + && (!_CCCL_CUDA_COMPILATION() || _CCCL_CUDA_COMPILER(CLANG))) +# define _CCCL_HAS_STATIC_SUBSCRIPT_OPERATOR() 1 +#else // ^^^ has static operator[] ^^^ / vvv no static operator[] vvv +# define _CCCL_HAS_STATIC_SUBSCRIPT_OPERATOR() 0 +#endif // ^^^ no static operator[] ^^^ + +// nvcc 13+, clang 16+ and gcc 13+ backport the static call operator back to c++17. +#if __cpp_static_call_operator >= 202207L \ + || ((_CCCL_COMPILER(CLANG, >=, 16) || _CCCL_COMPILER(GCC, >=, 13) \ + || (_CCCL_COMPILER(NVHPC, >=, 26, 1) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 13))) \ + && (!_CCCL_CUDA_COMPILATION() || _CCCL_CUDA_COMPILER(NVCC, >=, 13, 0) || _CCCL_CUDA_COMPILER(CLANG))) +# define _CCCL_HAS_STATIC_CALL_OPERATOR() 1 +#else // ^^^ has static operator() ^^^ / vvv no static operator() vvv +# define _CCCL_HAS_STATIC_CALL_OPERATOR() 0 +#endif // ^^^ no static operator() ^^^ + +// if consteval requires C++23, but most compilers support it even in C++20 mode while emitting some warnings. Those are +// silenced in prologue/epilogue. nvcc is happy about using it in C++20 since 13.0, but only when compiling host code. +// nvc++ requires libstdc++ at least 12 to support if consteval. +#if _CCCL_STD_VER == 2020 \ + && (_CCCL_COMPILER(GCC, >=, 12) || _CCCL_COMPILER(CLANG) \ + || (_CCCL_COMPILER(NVHPC) && _CCCL_HOST_STD_LIB(LIBSTDCXX, >=, 12))) +# define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 1 +#else +# define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 0 +#endif + +// nvcc before 13 doesn't support if consteval at all. Since 13, it accepts if consteval in host code (clang doesn't +// work) and since 13.1 it works in device code, too. +#if _CCCL_CUDA_COMPILER(NVCC, <, 13) || (_CCCL_CUDA_COMPILER(NVCC, <, 13, 1) && _CCCL_DEVICE_COMPILATION()) \ + || (_CCCL_CUDA_COMPILER(NVCC) && _CCCL_COMPILER(CLANG)) +# undef _CCCL_HAS_IF_CONSTEVAL_IN_CXX20 +# define _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() 0 +#endif // ^^^ disable if consteval in c++20 for nvcc ^^^ + +#if __cpp_if_consteval >= 202106L || _CCCL_HAS_IF_CONSTEVAL_IN_CXX20() +# define _CCCL_IF_CONSTEVAL if consteval +# define _CCCL_IF_CONSTEVAL_DEFAULT _CCCL_IF_CONSTEVAL +# define _CCCL_IF_NOT_CONSTEVAL if !consteval +# define _CCCL_IF_NOT_CONSTEVAL_DEFAULT _CCCL_IF_NOT_CONSTEVAL +#elif defined(_CCCL_BUILTIN_IS_CONSTANT_EVALUATED) +# if _CCCL_HOST_COMPILATION() && _CCCL_COMPILER(GCC) +# define _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() _CCCL_DIAG_PUSH _CCCL_DIAG_SUPPRESS_GCC("-Wtautological-compare") +# define _CCCL_END_IF_CONSTEVAL_SUPPRESS() _CCCL_DIAG_POP +# else // ^^^ _CCCL_HOST_COMPILATION() && _CCCL_COMPILER(GCC) ^^^ / + // vvv !_CCCL_HOST_COMPILATION() || ! _CCCL_COMPILER(GCC) vvv +# define _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() +# define _CCCL_END_IF_CONSTEVAL_SUPPRESS() +# endif // ^^^ !_CCCL_HOST_COMPILATION() || ! _CCCL_COMPILER(GCC) ^^^ + +# define _CCCL_IF_CONSTEVAL \ + _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() if (_CCCL_BUILTIN_IS_CONSTANT_EVALUATED()) _CCCL_END_IF_CONSTEVAL_SUPPRESS() +# define _CCCL_IF_CONSTEVAL_DEFAULT _CCCL_IF_CONSTEVAL +# define _CCCL_IF_NOT_CONSTEVAL \ + _CCCL_BEGIN_IF_CONSTEVAL_SUPPRESS() if (!_CCCL_BUILTIN_IS_CONSTANT_EVALUATED()) _CCCL_END_IF_CONSTEVAL_SUPPRESS() +# define _CCCL_IF_NOT_CONSTEVAL_DEFAULT _CCCL_IF_NOT_CONSTEVAL +#else // ^^^ has is constant evaluated ^^^ / vvv no is constant evaluated vvv +# define _CCCL_IF_CONSTEVAL if constexpr (false) +# define _CCCL_IF_CONSTEVAL_DEFAULT if constexpr (true) +# define _CCCL_IF_NOT_CONSTEVAL if constexpr (true) +# define _CCCL_IF_NOT_CONSTEVAL_DEFAULT if constexpr (false) +#endif // ^^^ no is constant evaluated ^^^ + +#if _CCCL_STD_VER >= 2020 && __cpp_char8_t >= 201811L +# define _CCCL_HAS_CHAR8_T() 1 +#else // ^^^ has char8_t ^^^ / vvv no char8_t vvv +# define _CCCL_HAS_CHAR8_T() 0 +#endif // ^^^ no char8_t ^^^ + +// We currently do not support any of the STL wchar facilities +#define _CCCL_HAS_WCHAR_T() 0 + +// Fixme: replace the condition with (!_CCCL_DEVICE_COMPILATION()) +// FIXME: Enable this for clang-cuda in a followup +#if !_CCCL_CUDA_COMPILATION() && !defined(CCCL_DISABLE_LONG_DOUBLE_SUPPORT) +# define _CCCL_HAS_LONG_DOUBLE() 1 +#else // ^^^ has long double ^^^ / vvv no long double vvv +# define _CCCL_HAS_LONG_DOUBLE() 0 +#endif // ^^^ no long double ^^^ + +// clang-21+ and gcc-16+ allow structured bindings to introduce a pack since C++17. +#if __cpp_structured_bindings >= 202411L || _CCCL_COMPILER(CLANG, >=, 21) || _CCCL_COMPILER(GCC, >=, 16) +# define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 1 +#else // ^^^ has structured bindings with pack ^^^ / vvv no structured bindings with pack vvv +# define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 0 +#endif // ^^^ no structured bindings with pack ^^^ + +// nvcc doesn't implement structured bindings pack yet. +#if _CCCL_CUDA_COMPILER(NVCC) +# undef _CCCL_HAS_STRUCTURED_BINDINGS_PACK +# define _CCCL_HAS_STRUCTURED_BINDINGS_PACK() 0 +#endif // _CCCL_CUDA_COMPILER(NVCC) + +#endif // __CCCL_DIALECT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/epilogue.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/epilogue.h new file mode 100644 index 00000000..70a1c999 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/epilogue.h @@ -0,0 +1,390 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py. + +// NO include guards here (this file is included multiple times) + +#include +#include + +#if !defined(_CCCL_PROLOGUE_INCLUDED) +# error "cccl internal error: must be included before " +#endif +#undef _CCCL_PROLOGUE_INCLUDED + +_CCCL_NV_DIAG_POP() +_CCCL_DIAG_POP + +// __declspec modifiers + +#if defined(align) +# error \ + "cccl internal error: macro `align` was redefined between and " +#elif defined(_CCCL_POP_MACRO_align) +# pragma pop_macro("align") +# undef _CCCL_POP_MACRO_align +#endif + +#if defined(allocate) +# error \ + "cccl internal error: macro `allocate` was redefined between and " +#elif defined(_CCCL_POP_MACRO_allocate) +# pragma pop_macro("allocate") +# undef _CCCL_POP_MACRO_allocate +#endif + +#if defined(allocator) +# error \ + "cccl internal error: macro `allocator` was redefined between and " +#elif defined(_CCCL_POP_MACRO_allocator) +# pragma pop_macro("allocator") +# undef _CCCL_POP_MACRO_allocator +#endif + +#if defined(appdomain) +# error \ + "cccl internal error: macro `appdomain` was redefined between and " +#elif defined(_CCCL_POP_MACRO_appdomain) +# pragma pop_macro("appdomain") +# undef _CCCL_POP_MACRO_appdomain +#endif + +#if defined(code_seg) +# error \ + "cccl internal error: macro `code_seg` was redefined between and " +#elif defined(_CCCL_POP_MACRO_code_seg) +# pragma pop_macro("code_seg") +# undef _CCCL_POP_MACRO_code_seg +#endif + +#if defined(deprecated) +# error \ + "cccl internal error: macro `deprecated` was redefined between and " +#elif defined(_CCCL_POP_MACRO_deprecated) +# pragma pop_macro("deprecated") +# undef _CCCL_POP_MACRO_deprecated +#endif + +#if defined(dllimport) +# error \ + "cccl internal error: macro `dllimport` was redefined between and " +#elif defined(_CCCL_POP_MACRO_dllimport) +# pragma pop_macro("dllimport") +# undef _CCCL_POP_MACRO_dllimport +#endif + +#if defined(dllexport) +# error \ + "cccl internal error: macro `dllexport` was redefined between and " +#elif defined(_CCCL_POP_MACRO_dllexport) +# pragma pop_macro("dllexport") +# undef _CCCL_POP_MACRO_dllexport +#endif + +#if defined(empty_bases) +# error \ + "cccl internal error: macro `empty_bases` was redefined between and " +#elif defined(_CCCL_POP_MACRO_empty_bases) +# pragma pop_macro("empty_bases") +# undef _CCCL_POP_MACRO_empty_bases +#endif + +#if defined(hybrid_patchable) +# error \ + "cccl internal error: macro `hybrid_patchable` was redefined between and " +#elif defined(_CCCL_POP_MACRO_hybrid_patchable) +# pragma pop_macro("hybrid_patchable") +# undef _CCCL_POP_MACRO_hybrid_patchable +#endif + +#if defined(jitintrinsic) +# error \ + "cccl internal error: macro `jitintrinsic` was redefined between and " +#elif defined(_CCCL_POP_MACRO_jitintrinsic) +# pragma pop_macro("jitintrinsic") +# undef _CCCL_POP_MACRO_jitintrinsic +#endif + +#if defined(lifetimebound) +# error \ + "cccl internal error: macro `lifetimebound` was redefined between and " +#elif defined(_CCCL_POP_MACRO_lifetimebound) +# pragma pop_macro("lifetimebound") +# undef _CCCL_POP_MACRO_lifetimebound +#endif + +#if defined(naked) +# error \ + "cccl internal error: macro `naked` was redefined between and " +#elif defined(_CCCL_POP_MACRO_naked) +# pragma pop_macro("naked") +# undef _CCCL_POP_MACRO_naked +#endif + +#if defined(noalias) +# error \ + "cccl internal error: macro `noalias` was redefined between and " +#elif defined(_CCCL_POP_MACRO_noalias) +# pragma pop_macro("noalias") +# undef _CCCL_POP_MACRO_noalias +#endif + +#if defined(noinline) +# error \ + "cccl internal error: macro `noinline` was redefined between and " +#elif defined(_CCCL_POP_MACRO_noinline) +# pragma pop_macro("noinline") +# undef _CCCL_POP_MACRO_noinline +#endif + +#if defined(noreturn) +# error \ + "cccl internal error: macro `noreturn` was redefined between and " +#elif defined(_CCCL_POP_MACRO_noreturn) +# pragma pop_macro("noreturn") +# undef _CCCL_POP_MACRO_noreturn +#endif + +#if defined(nothrow) +# error \ + "cccl internal error: macro `nothrow` was redefined between and " +#elif defined(_CCCL_POP_MACRO_nothrow) +# pragma pop_macro("nothrow") +# undef _CCCL_POP_MACRO_nothrow +#endif + +#if defined(novtable) +# error \ + "cccl internal error: macro `novtable` was redefined between and " +#elif defined(_CCCL_POP_MACRO_novtable) +# pragma pop_macro("novtable") +# undef _CCCL_POP_MACRO_novtable +#endif + +#if defined(no_sanitize_address) +# error \ + "cccl internal error: macro `no_sanitize_address` was redefined between and " +#elif defined(_CCCL_POP_MACRO_no_sanitize_address) +# pragma pop_macro("no_sanitize_address") +# undef _CCCL_POP_MACRO_no_sanitize_address +#endif + +#if defined(process) +# error \ + "cccl internal error: macro `process` was redefined between and " +#elif defined(_CCCL_POP_MACRO_process) +# pragma pop_macro("process") +# undef _CCCL_POP_MACRO_process +#endif + +#if defined(property) +# error \ + "cccl internal error: macro `property` was redefined between and " +#elif defined(_CCCL_POP_MACRO_property) +# pragma pop_macro("property") +# undef _CCCL_POP_MACRO_property +#endif + +#if defined(restrict) +# error \ + "cccl internal error: macro `restrict` was redefined between and " +#elif defined(_CCCL_POP_MACRO_restrict) +# pragma pop_macro("restrict") +# undef _CCCL_POP_MACRO_restrict +#endif + +#if defined(safebuffers) +# error \ + "cccl internal error: macro `safebuffers` was redefined between and " +#elif defined(_CCCL_POP_MACRO_safebuffers) +# pragma pop_macro("safebuffers") +# undef _CCCL_POP_MACRO_safebuffers +#endif + +#if defined(selectany) +# error \ + "cccl internal error: macro `selectany` was redefined between and " +#elif defined(_CCCL_POP_MACRO_selectany) +# pragma pop_macro("selectany") +# undef _CCCL_POP_MACRO_selectany +#endif + +#if defined(spectre) +# error \ + "cccl internal error: macro `spectre` was redefined between and " +#elif defined(_CCCL_POP_MACRO_spectre) +# pragma pop_macro("spectre") +# undef _CCCL_POP_MACRO_spectre +#endif + +#if defined(thread) +# error \ + "cccl internal error: macro `thread` was redefined between and " +#elif defined(_CCCL_POP_MACRO_thread) +# pragma pop_macro("thread") +# undef _CCCL_POP_MACRO_thread +#endif + +#if defined(uuid) +# error \ + "cccl internal error: macro `uuid` was redefined between and " +#elif defined(_CCCL_POP_MACRO_uuid) +# pragma pop_macro("uuid") +# undef _CCCL_POP_MACRO_uuid +#endif + +// [[msvc::attribute]] attributes + +#if defined(msvc) +# error \ + "cccl internal error: macro `msvc` was redefined between and " +#elif defined(_CCCL_POP_MACRO_msvc) +# pragma pop_macro("msvc") +# undef _CCCL_POP_MACRO_msvc +#endif + +#if defined(flatten) +# error \ + "cccl internal error: macro `flatten` was redefined between and " +#elif defined(_CCCL_POP_MACRO_flatten) +# pragma pop_macro("flatten") +# undef _CCCL_POP_MACRO_flatten +#endif + +#if defined(forceinline) +# error \ + "cccl internal error: macro `forceinline` was redefined between and " +#elif defined(_CCCL_POP_MACRO_forceinline) +# pragma pop_macro("forceinline") +# undef _CCCL_POP_MACRO_forceinline +#endif + +#if defined(forceinline_calls) +# error \ + "cccl internal error: macro `forceinline_calls` was redefined between and " +#elif defined(_CCCL_POP_MACRO_forceinline_calls) +# pragma pop_macro("forceinline_calls") +# undef _CCCL_POP_MACRO_forceinline_calls +#endif + +#if defined(intrinsic) +# error \ + "cccl internal error: macro `intrinsic` was redefined between and " +#elif defined(_CCCL_POP_MACRO_intrinsic) +# pragma pop_macro("intrinsic") +# undef _CCCL_POP_MACRO_intrinsic +#endif + +#if defined(noinline) +# error \ + "cccl internal error: macro `noinline` was redefined between and " +#elif defined(_CCCL_POP_MACRO_noinline) +# pragma pop_macro("noinline") +# undef _CCCL_POP_MACRO_noinline +#endif + +#if defined(noinline_calls) +# error \ + "cccl internal error: macro `noinline_calls` was redefined between and " +#elif defined(_CCCL_POP_MACRO_noinline_calls) +# pragma pop_macro("noinline_calls") +# undef _CCCL_POP_MACRO_noinline_calls +#endif + +#if defined(no_tls_guard) +# error \ + "cccl internal error: macro `no_tls_guard` was redefined between and " +#elif defined(_CCCL_POP_MACRO_no_tls_guard) +# pragma pop_macro("no_tls_guard") +# undef _CCCL_POP_MACRO_no_tls_guard +#endif + +// Windows nasty macros + +#if defined(min) +# error \ + "cccl internal error: macro `min` was redefined between and " +#elif defined(_CCCL_POP_MACRO_min) +# pragma pop_macro("min") +# undef _CCCL_POP_MACRO_min +#endif + +#if defined(max) +# error \ + "cccl internal error: macro `max` was redefined between and " +#elif defined(_CCCL_POP_MACRO_max) +# pragma pop_macro("max") +# undef _CCCL_POP_MACRO_max +#endif + +#if defined(interface) +# error \ + "cccl internal error: macro `interface` was redefined between and " +#elif defined(_CCCL_POP_MACRO_interface) +# pragma pop_macro("interface") +# undef _CCCL_POP_MACRO_interface +#endif + +// sal.h on Windows + +#if defined(__valid) +# error \ + "cccl internal error: macro `__valid` was redefined between and " +#elif defined(_CCCL_POP_MACRO___valid) +# pragma pop_macro("__valid") +# undef _CCCL_POP_MACRO___valid +#endif + +#if defined(__callback) +# error \ + "cccl internal error: macro `__callback` was redefined between and " +#elif defined(_CCCL_POP_MACRO___callback) +# pragma pop_macro("__callback") +# undef _CCCL_POP_MACRO___callback +#endif + +// other macros + +#if defined(clang) +# error \ + "cccl internal error: macro `clang` was redefined between and " +#elif defined(_CCCL_POP_MACRO_clang) +# pragma pop_macro("clang") +# undef _CCCL_POP_MACRO_clang +#endif + +// sys/sysmacros.h on linux + +#if defined(major) +# error \ + "cccl internal error: macro `major` was redefined between and " +#elif defined(_CCCL_POP_MACRO_major) +# pragma pop_macro("major") +# undef _CCCL_POP_MACRO_major +#endif + +#if defined(minor) +# error \ + "cccl internal error: macro `minor` was redefined between and " +#elif defined(_CCCL_POP_MACRO_minor) +# pragma pop_macro("minor") +# undef _CCCL_POP_MACRO_minor +#endif + +#if defined(makedev) +# error \ + "cccl internal error: macro `makedev` was redefined between and " +#elif defined(_CCCL_POP_MACRO_makedev) +# pragma pop_macro("makedev") +# undef _CCCL_POP_MACRO_makedev +#endif + +// NO include guards here (this file is included multiple times) diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/exceptions.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/exceptions.h new file mode 100644 index 00000000..57edeaf5 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/exceptions.h @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_EXCEPTIONS_H +#define __CCCL_EXCEPTIONS_H + +#include +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if defined(CCCL_DISABLE_EXCEPTIONS) // Escape hatch for users to manually disable exceptions +# define _CCCL_HAS_EXCEPTIONS() 0 +#elif _CCCL_COMPILER(NVRTC) // NVRTC has no exceptions +# define _CCCL_HAS_EXCEPTIONS() 0 +#elif _CCCL_COMPILER(MSVC) // MSVC needs special checks for `_HAS_EXCEPTIONS` and `_CPPUNWIND` +# define _CCCL_HAS_EXCEPTIONS() ((_HAS_EXCEPTIONS != 0) && (_CPPUNWIND != 0)) // disabled with /EH +#else // other compilers use `__EXCEPTIONS` +# define _CCCL_HAS_EXCEPTIONS() (__EXCEPTIONS) // disabled with -fno-exceptions +#endif // has exceptions + +#if _CCCL_HAS_EXCEPTIONS() && __cpp_constexpr_exceptions >= 202411L +# define _CCCL_HAS_CONSTEXPR_EXCEPTIONS() 1 +#else // ^^^ has constexpr exceptions ^^^ / vvv no constexpr exceptions vvv +# define _CCCL_HAS_CONSTEXPR_EXCEPTIONS() 0 +#endif // ^^^ no constexpr exceptions ^^^ + +#endif // __CCCL_EXCEPTIONS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/execution_space.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/execution_space.h new file mode 100644 index 00000000..ed3ddd9b --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/execution_space.h @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_EXECUTION_SPACE_H +#define __CCCL_EXECUTION_SPACE_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#if _CCCL_CUDA_COMPILATION() +# define _CCCL_HOST __host__ +# define _CCCL_DEVICE __device__ +# define _CCCL_HOST_DEVICE __host__ __device__ +#else // ^^^ _CCCL_CUDA_COMPILATION ^^^ / vvv !_CCCL_CUDA_COMPILATION vvv +# define _CCCL_HOST +# define _CCCL_DEVICE +# define _CCCL_HOST_DEVICE +#endif // !_CCCL_CUDA_COMPILATION + +#if _CCCL_TILE_COMPILATION() +# define _CCCL_TILE __tile__ +#else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() vvv +# define _CCCL_TILE +#endif // ^^^ !_CCCL_TILE_COMPILATION() ^^^ + +// clang-cuda before version 22 requires __host__ __device__ annotations on deduction guides +#if _CCCL_CUDA_COMPILER(CLANG, <, 22) +# define _CCCL_DEDUCTION_GUIDE_ATTRIBUTES _CCCL_HOST_DEVICE +#else // ^^^ _CCCL_CUDA_COMPILER(CLANG, <, 22) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG, <, 22) vvv +# define _CCCL_DEDUCTION_GUIDE_ATTRIBUTES +#endif // ^^ !_CCCL_CUDA_COMPILER(CLANG, <, 22) ^^^ + +// Global variables of non builtin types are only device accessible if they are marked as `__device__` +#if _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) +# define _CCCL_GLOBAL_VARIABLE _CCCL_DEVICE +#else // ^^^ _CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC) ^^^ / + // vvv !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) vvv +# define _CCCL_GLOBAL_VARIABLE +#endif // ^^^ !_CCCL_DEVICE_COMPILATION() || _CCCL_CUDA_COMPILER(NVHPC) ^^^ + +#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) || _CCCL_CUDA_COMPILER(NVRTC) || _CCCL_CUDA_COMPILER(CLANG, >=, 20)) \ + && _CCCL_PTX_ARCH() >= 700 +# define _CCCL_HAS_GRID_CONSTANT() 1 +# define _CCCL_GRID_CONSTANT __grid_constant__ +#else // ^^^ has __grid_constant__ ^^^ / vvv no __grid_constant__ vvv +# define _CCCL_HAS_GRID_CONSTANT() 0 +# define _CCCL_GRID_CONSTANT +#endif // ^^^ no __grid_constant__ ^^^ + +#if !defined(_CCCL_EXEC_CHECK_DISABLE) +# if _CCCL_CUDA_COMPILER(NVCC) +# define _CCCL_EXEC_CHECK_DISABLE _CCCL_PRAGMA(nv_exec_check_disable) +# else +# define _CCCL_EXEC_CHECK_DISABLE +# endif // _CCCL_CUDA_COMPILER(NVCC) +#endif // !_CCCL_EXEC_CHECK_DISABLE + +#if _CCCL_CUDA_COMPILER(NVHPC) +# define _CCCL_TARGET_CONSTEXPR +#else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv +# define _CCCL_TARGET_CONSTEXPR constexpr +#endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^ + +//! @brief List of all known PTX architectures supported by this CCCL version. +#define _CCCL_KNOWN_CUDA_ARCH_LIST 50, 52, 53, 60, 61, 62, 70, 75, 80, 86, 87, 88, 89, 90, 100, 103, 110, 120, 121 + +//! @brief List of all known architecture specific architectures supported by this CCCL version. +#define _CCCL_KNOWN_CUDA_ARCH_SPECIFIC_LIST 90, 100, 103, 110, 120, 121 + +#endif // __CCCL_EXECUTION_SPACE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/extended_data_types.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/extended_data_types.h new file mode 100644 index 00000000..658036c2 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/extended_data_types.h @@ -0,0 +1,148 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_EXTENDED_DATA_TYPES_H +#define __CCCL_EXTENDED_DATA_TYPES_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#define _CCCL_HAS_INT128() 0 +#define _CCCL_HAS_NVFP4() 0 +#define _CCCL_HAS_NVFP6() 0 +#define _CCCL_HAS_NVFP8() 0 +#define _CCCL_HAS_NVFP16() 0 +#define _CCCL_HAS_NVBF16() 0 +#define _CCCL_HAS_FLOAT128() 0 + +#if _CCCL_TILE_COMPILATION() // TODO(miscco): Fix access to extended floating point types +# define CCCL_DISABLE_NVFP4_SUPPORT +# define CCCL_DISABLE_NVFP6_SUPPORT +# define CCCL_DISABLE_NVFP8_SUPPORT +# define CCCL_DISABLE_INT128_SUPPORT +# define CCCL_DISABLE_FLOAT128_SUPPORT +#endif // _CCCL_TILE_COMPILATION() + +#if !defined(CCCL_DISABLE_INT128_SUPPORT) && _CCCL_OS(LINUX) \ + && ((_CCCL_COMPILER(NVRTC) && defined(__CUDACC_RTC_INT128__)) || defined(__SIZEOF_INT128__)) +# undef _CCCL_HAS_INT128 +# define _CCCL_HAS_INT128() 1 +#endif + +#if __has_include() && (_CCCL_HAS_CTK() || defined(LIBCUDACXX_ENABLE_HOST_NVFP16)) \ + && !defined(CCCL_DISABLE_FP16_SUPPORT) +# undef _CCCL_HAS_NVFP16 +# define _CCCL_HAS_NVFP16() 1 +struct __half; +struct __half2; +#endif + +#if __has_include() && _CCCL_HAS_NVFP16() && !defined(CCCL_DISABLE_BF16_SUPPORT) +# undef _CCCL_HAS_NVBF16 +# define _CCCL_HAS_NVBF16() 1 +struct __nv_bfloat16; +struct __nv_bfloat162; +#endif + +#if __has_include() && _CCCL_HAS_NVFP16() && _CCCL_HAS_NVBF16() && !defined(CCCL_DISABLE_NVFP8_SUPPORT) +# undef _CCCL_HAS_NVFP8 +# define _CCCL_HAS_NVFP8() 1 +struct __nv_fp8_e5m2; +struct __nv_fp8x2_e5m2; +struct __nv_fp8x4_e5m2; + +struct __nv_fp8_e4m3; +struct __nv_fp8x2_e4m3; +struct __nv_fp8x4_e4m3; + +# if _CCCL_CTK_AT_LEAST(12, 8) +struct __nv_fp8_e8m0; +struct __nv_fp8x2_e8m0; +struct __nv_fp8x4_e8m0; +# endif // _CCCL_CTK_AT_LEAST(12, 8) +#endif + +#if __has_include() && _CCCL_HAS_NVFP8() && !_CCCL_CUDA_COMPILER(NVHPC) \ + && !defined(CCCL_DISABLE_NVFP6_SUPPORT) +# undef _CCCL_HAS_NVFP6 +# define _CCCL_HAS_NVFP6() 1 +struct __nv_fp6_e3m2; +struct __nv_fp6x2_e3m2; +struct __nv_fp6x4_e3m2; + +struct __nv_fp6_e2m3; +struct __nv_fp6x2_e2m3; +struct __nv_fp6x4_e2m3; +#endif + +#if __has_include() && _CCCL_HAS_NVFP6() && !defined(CCCL_DISABLE_NVFP4_SUPPORT) +# undef _CCCL_HAS_NVFP4 +# define _CCCL_HAS_NVFP4() 1 +struct __nv_fp4_e2m1; +struct __nv_fp4x2_e2m1; +struct __nv_fp4x4_e2m1; +#endif + +#define _CCCL_HAS_NVFP4_E2M1() _CCCL_HAS_NVFP4() +#define _CCCL_HAS_NVFP6_E2M3() _CCCL_HAS_NVFP6() +#define _CCCL_HAS_NVFP6_E3M2() _CCCL_HAS_NVFP6() +#define _CCCL_HAS_NVFP8_E4M3() _CCCL_HAS_NVFP8() +#define _CCCL_HAS_NVFP8_E5M2() _CCCL_HAS_NVFP8() +#define _CCCL_HAS_NVFP8_E8M0() (_CCCL_HAS_NVFP8() && _CCCL_CTK_AT_LEAST(12, 8)) + +/*********************************************************************************************************************** + * __float128 + **********************************************************************************************************************/ + +#if !defined(CCCL_DISABLE_FLOAT128_SUPPORT) && _CCCL_HAS_INT128() && _CCCL_OS(LINUX) && !_CCCL_HOST_ARCH(ARM64) \ + && !_CCCL_TILE_COMPILATION() +// Detect host compiler support +# if (defined(__CUDACC_RTC_FLOAT128__) || defined(__SIZEOF_FLOAT128__) || defined(__FLOAT128__)) +# if _CCCL_DEVICE_COMPILATION() +// Only NVCC and NVRTC 12.8+ on architectures at least SM100 supports __float128 on device +# if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 8)) && _CCCL_PTX_ARCH() >= 1000 +# undef _CCCL_HAS_FLOAT128 +# define _CCCL_HAS_FLOAT128() 1 +# endif // _CCCL_CUDA_COMPILER(NVCC) && _CCCL_PTX_ARCH() >= 1000 +# else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv +# undef _CCCL_HAS_FLOAT128 +# define _CCCL_HAS_FLOAT128() 1 +# endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^ +# endif // Host compiler support +#endif // !defined(CCCL_DISABLE_FLOAT128_SUPPORT) && _CCCL_HAS_INT128() && _CCCL_OS(LINUX) && !_CCCL_HOST_ARCH(ARM64) + +// gcc does not allow to use q/Q floating point literals when __STRICT_ANSI__ is defined. They may be allowed by +// -fext-numeric-literals, but there is no way to detect it in the preprocessor. The user is required to define +// CCCL_GCC_HAS_EXTENDED_NUMERIC_LITERALS in this case. Otherwise, we disable the __float128 support. +// +// Note: since GCC 13, we could use f128/F128 literals, but for values > DBL_MAX, the compilation with nvcc fails due to +// "floating constant is out of range". +#if _CCCL_HAS_FLOAT128() && _CCCL_COMPILER(GCC) && defined(__STRICT_ANSI__) \ + && !defined(CCCL_GCC_HAS_EXTENDED_NUMERIC_LITERALS) +# undef _CCCL_HAS_FLOAT128 +# define _CCCL_HAS_FLOAT128() 0 +#endif // _CCCL_HAS_FLOAT128() + +#endif // __CCCL_EXTENDED_DATA_TYPES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/host_std_lib.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/host_std_lib.h new file mode 100644 index 00000000..634100ea --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/host_std_lib.h @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_HOST_STD_LIB_H +#define __CCCL_HOST_STD_LIB_H + +#include +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#define _CCCL_HOST_STD_LIB_LIBSTDCXX() _CCCL_VERSION_INVALID() +#define _CCCL_HOST_STD_LIB_LIBCXX() _CCCL_VERSION_INVALID() +#define _CCCL_HOST_STD_LIB_STL() _CCCL_VERSION_INVALID() + +// include a minimal header +#if __has_include() +# include +#elif __has_include() +# include +#endif // ^^^ __has_include() ^^^ + +#define _CCCL_HOST_STD_LIB_MAKE_VERSION(_MAJOR, _MINOR) ((_MAJOR) * 100 + (_MINOR)) +#define _CCCL_HOST_STD_LIB(...) _CCCL_VERSION_COMPARE(_CCCL_HOST_STD_LIB_, _CCCL_HOST_STD_LIB_##__VA_ARGS__) + +#if _CCCL_HOSTED() +# if defined(_MSVC_STL_VERSION) +# undef _CCCL_HOST_STD_LIB_STL +# define _CCCL_HOST_STD_LIB_STL() (_MSVC_STL_VERSION, 0) +# elif defined(__GLIBCXX__) +# undef _CCCL_HOST_STD_LIB_LIBSTDCXX +# define _CCCL_HOST_STD_LIB_LIBSTDCXX() (_GLIBCXX_RELEASE, 0) +# elif defined(_LIBCPP_VERSION) +# undef _CCCL_HOST_STD_LIB_LIBCXX +// since llvm-16, the version scheme has been changed from MMppp to MMmmpp +# if _LIBCPP_VERSION / 10000 < 2 +# define _CCCL_HOST_STD_LIB_LIBCXX() (_LIBCPP_VERSION / 1000, 0) +# else +# define _CCCL_HOST_STD_LIB_LIBCXX() (_LIBCPP_VERSION / 10000, (_LIBCPP_VERSION / 100) % 100) +# endif +# endif // ^^^ _LIBCPP_VERSION ^^^ +#endif // _CCCL_HOSTED() + +#define _CCCL_HAS_HOST_STD_LIB() \ + (_CCCL_HOST_STD_LIB(LIBSTDCXX) || _CCCL_HOST_STD_LIB(LIBCXX) || _CCCL_HOST_STD_LIB(STL)) + +#endif // __CCCL_HOST_STD_LIB_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/is_non_narrowing_convertible.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/is_non_narrowing_convertible.h new file mode 100644 index 00000000..101d95e6 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/is_non_narrowing_convertible.h @@ -0,0 +1,71 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_IS_NON_NARROWING_CONVERTIBLE_H +#define __CCCL_IS_NON_NARROWING_CONVERTIBLE_H + +#include + +//! There is compiler bug that results in incorrect results for the below `__is_non_narrowing_convertible` check. +//! This breaks some common functionality, so this *must* be included outside of a system header. See nvbug4867473. +#if defined(_CCCL_FORCE_SYSTEM_HEADER_GCC) || defined(_CCCL_FORCE_SYSTEM_HEADER_CLANG) \ + || defined(_CCCL_FORCE_SYSTEM_HEADER_MSVC) +# error \ + "This header must be included only within the . This most likely means a mix and match of different versions of CCCL." +#endif // system header detected + +namespace __cccl_internal +{ +#if _CCCL_CUDA_COMPILATION() +template +__host__ __device__ _Tp&& __cccl_declval(int); +template +__host__ __device__ _Tp __cccl_declval(long); +template +__host__ __device__ decltype(__cccl_internal::__cccl_declval<_Tp>(0)) __cccl_declval() noexcept; + +// This requires a type to be implicitly convertible (also non-arithmetic) +template +__host__ __device__ void __cccl_accepts_implicit_conversion(_Tp) noexcept; +#else // ^^^ CUDA compilation ^^^ / vvv no CUDA compilation +template +_Tp&& __cccl_declval(int); +template +_Tp __cccl_declval(long); +template +decltype(__cccl_internal::__cccl_declval<_Tp>(0)) __cccl_declval() noexcept; + +// This requires a type to be implicitly convertible (also non-arithmetic) +template +void __cccl_accepts_implicit_conversion(_Tp) noexcept; +#endif // no CUDA compilation + +template +using __cccl_void_t = void; + +template +struct __is_non_narrowing_convertible +{ + static constexpr bool value = false; +}; + +// This also prohibits narrowing conversion in case of arithmetic types +template +struct __is_non_narrowing_convertible<_Dest, + _Source, + __cccl_void_t( + __cccl_internal::__cccl_declval<_Source>())), + decltype(_Dest{__cccl_internal::__cccl_declval<_Source>()})>> +{ + static constexpr bool value = true; +}; +} // namespace __cccl_internal + +#endif // __CCCL_IS_NON_NARROWING_CONVERTIBLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/os.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/os.h new file mode 100644 index 00000000..2b7c8843 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/os.h @@ -0,0 +1,120 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_OS_H +#define __CCCL_OS_H + +// The header provides the following macros to determine the host architecture: +// +// _CCCL_OS(WINDOWS) +// _CCCL_OS(LINUX) +// _CCCL_OS(ANDROID) +// _CCCL_OS(QNX) + +// Determine the host compiler and its version +#if defined(_WIN32) || defined(_WIN64) /* _WIN64 for NVRTC */ +# define _CCCL_OS_WINDOWS_() 1 +#else +# define _CCCL_OS_WINDOWS_() 0 +#endif + +#if defined(__linux__) || defined(__LP64__) /* __LP64__ for NVRTC */ +# define _CCCL_OS_LINUX_() 1 +#else +# define _CCCL_OS_LINUX_() 0 +#endif + +#if defined(__ANDROID__) +# define _CCCL_OS_ANDROID_() 1 +#else +# define _CCCL_OS_ANDROID_() 0 +#endif + +#if defined(__QNX__) || defined(__QNXNTO__) +# define _CCCL_OS_QNX_() 1 +#else +# define _CCCL_OS_QNX_() 0 +#endif + +#if defined(__APPLE__) || defined(__APPLE_CC__) +# define _CCCL_OS_APPLE_() 1 +#else +# define _CCCL_OS_APPLE_() 0 +#endif + +#define _CCCL_OS(...) _CCCL_OS_##__VA_ARGS__##_() + +//! @def CCCL_OS(os) /* implementation defined */ +//! +//! @brief Detect the current operating system. +//! +//! @param os The name of the operating system to test. +//! +//! @note This macro is made available when including any libcu++ header. Users that wish to +//! include the smallest possible header for this macro should include ``. +//! +//! For supported operating systems, the macro expands to an implementation-defined true value +//! if the current operating system matches, or false otherwise. These values may be used in +//! boolean expressions (preprocessor or otherwise), but no other guarantees are made. +//! +//! Available values for `os` include: +//! +//! - ``WINDOWS``: Windows, either in 32-bit or 64-bit mode. +//! - ``LINUX``: Any kind of Linux installation. Note that other unix-based operating systems will +//! also match against this. +//! - ``ANDROID``: Android operating system. +//! - ``QNX``: QNX real-time operating system. +//! - ``APPLE``: macOS (Intel or Apple Silicon). +//! +//! Passing any other value will result in an undefined expansion, which may or may not be +//! diagnosed by the compiler. +//! +//! @note Some operating systems may satisfy multiple conditions. For example macOS and Android +//! satisfy both `APPLE`/`ANDROID` and `LINUX`. +//! +//! @par Example +//! @code +//! #define MY_OTHER_MACRO 1 +//! +//! // Expansion value can be used in ordinary macro conditionals +//! #if CCCL_OS(WINDOWS) && MY_OTHER_MACRO +//! // ... +//! #endif +//! +//! // Can be negated as usual +//! #if !CCCL_OS(QNX) +//! // ... +//! #endif +//! +//! #if CCCL_OS(APPLE) +//! // Will be visible only on macOS +//! #endif +//! +//! #if CCCL_OS(ANDROID) +//! // Will be visible only on Android +//! #endif +//! +//! #if CCCL_OS(LINUX) && !CCCL_OS(APPLE) && !CCCL_OS(ANDROID) +//! // Only visible on Linux +//! #endif +//! @endcode +//! +//! @return true if the specified OS is begin compiled for, false otherwise. +#ifdef _CCCL_DOXYGEN_INVOKED +# define CCCL_OS(os) /* implementation defined */ +#else +# define CCCL_OS(__os__) _CCCL_OS_##__os__##_() +#endif + +// Note: the public API is single-arg to constrain the API and allow for future expansion. The +// implementation is duplicated to guard against the OS targets being accidentally defined by +// the user. + +#endif // __CCCL_OS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/preprocessor.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/preprocessor.h new file mode 100644 index 00000000..a8f6cbbf --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/preprocessor.h @@ -0,0 +1,1366 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_PREPROCESSOR_H +#define __CCCL_PREPROCESSOR_H + +// Error when MSVC is used with the traditional preprocessor. +// We can't use `#pragma message` here because MSVC will encounter +// errors and exit before it processes pragma message directives. +#if defined(_MSC_VER) && !defined(__clang__) +# if (!defined(_MSVC_TRADITIONAL) || _MSVC_TRADITIONAL == 1) \ + && !defined(CCCL_IGNORE_MSVC_TRADITIONAL_PREPROCESSOR_WARNING) +# error \ +MSVC/cl.exe with traditional preprocessor is used. This may lead to unexpected compilation errors. Please \ +switch to the standard conforming preprocessor by passing `/Zc:preprocessor` to cl.exe. You can define \ +CCCL_IGNORE_MSVC_TRADITIONAL_PREPROCESSOR_WARNING to suppress this warning. +# endif // !defined(_MSVC_TRADITIONAL) || _MSVC_TRADITIONAL == 1 +#endif // defined(_MSC_VER) && !defined(__clang__) + +#ifdef __COUNTER__ +# define _CCCL_COUNTER() __COUNTER__ +#else +# define _CCCL_COUNTER() __LINE__ +#endif + +// Convert parameter to string +#define _CCCL_TO_STRING2(_STR) #_STR +#define _CCCL_TO_STRING(_STR) _CCCL_TO_STRING2(_STR) + +#define _CCCL_PP_FIRST(_FIRST, ...) _FIRST +#define _CCCL_PP_SECOND(_, _SECOND, ...) _SECOND +#define _CCCL_PP_THIRD(_1, _2, _THIRD) _THIRD + +#define _CCCL_PP_EXPAND(...) __VA_ARGS__ +#define _CCCL_PP_EAT(...) + +#define _CCCL_PP_CAT_(_Xp, ...) _Xp##__VA_ARGS__ +#define _CCCL_PP_CAT(_Xp, ...) _CCCL_PP_CAT_(_Xp, __VA_ARGS__) + +#define _CCCL_PP_CAT2_(_Xp, ...) _Xp##__VA_ARGS__ +#define _CCCL_PP_CAT2(_Xp, ...) _CCCL_PP_CAT2_(_Xp, __VA_ARGS__) + +#define _CCCL_PP_CAT3_(_Xp, ...) _Xp##__VA_ARGS__ +#define _CCCL_PP_CAT3(_Xp, ...) _CCCL_PP_CAT3_(_Xp, __VA_ARGS__) + +#define _CCCL_PP_CAT4_(_Xp, ...) _Xp##__VA_ARGS__ +#define _CCCL_PP_CAT4(_Xp, ...) _CCCL_PP_CAT4_(_Xp, __VA_ARGS__) + +#define _CCCL_PP_EVAL_(_Xp, _ARGS) _Xp _ARGS +#define _CCCL_PP_EVAL(_Xp, ...) _CCCL_PP_EVAL_(_Xp, (__VA_ARGS__)) + +#define _CCCL_PP_EVAL2_(_Xp, _ARGS) _Xp _ARGS +#define _CCCL_PP_EVAL2(_Xp, ...) _CCCL_PP_EVAL2_(_Xp, (__VA_ARGS__)) + +#define _CCCL_PP_CHECK(...) _CCCL_PP_EXPAND(_CCCL_PP_CHECK_N(__VA_ARGS__, 0, )) +#define _CCCL_PP_CHECK_N(_Xp, _Num, ...) _Num +#define _CCCL_PP_PROBE(_Xp) _Xp, 1, +#define _CCCL_PP_PROBE_N(_Xp, _Num) _Xp, _Num, + +#define _CCCL_PP_IS_PAREN(_Xp) _CCCL_PP_CHECK(_CCCL_PP_IS_PAREN_PROBE _Xp) +#define _CCCL_PP_IS_PAREN_PROBE(...) _CCCL_PP_PROBE(~) + +#define _CCCL_PP_IIF(_BIT) _CCCL_PP_CAT_(_CCCL_PP_IIF_, _BIT) +#define _CCCL_PP_IIF_0(_TRUE, ...) __VA_ARGS__ +#define _CCCL_PP_IIF_1(_TRUE, ...) _TRUE + +#define _CCCL_PP_LPAREN ( +#define _CCCL_PP_RPAREN ) + +#define _CCCL_PP_NOT(_BIT) _CCCL_PP_CAT_(_CCCL_PP_NOT_, _BIT) +#define _CCCL_PP_NOT_0 1 +#define _CCCL_PP_NOT_1 0 + +#define _CCCL_PP_EMPTY() +#define _CCCL_PP_COMMA() , +#define _CCCL_PP_LBRACE() { +#define _CCCL_PP_RBRACE() } +#define _CCCL_PP_COMMA_IIF(_Xp) _CCCL_PP_IIF(_Xp)(_CCCL_PP_COMMA, _CCCL_PP_EMPTY)() + +#define _CCCL_PP_CASE(_ARG) _CCCL_PP_PROBE_N(~, _ARG) +#define _CCCL_PP_SWITCH(_PREFIX, ...) \ + _CCCL_PP_CAT(_PREFIX##_CASE_, _CCCP_PP_CASE_LABEL_(_PREFIX, __VA_ARGS__))(__VA_ARGS__) +#define _CCCL_PP_SWITCH2(_PREFIX, ...) \ + _CCCL_PP_CAT(_PREFIX##_CASE_, _CCCP_PP_CASE_LABEL_(_PREFIX, __VA_ARGS__))(__VA_ARGS__) +#define _CCCP_PP_CASE_LABEL_(_PREFIX, ...) \ + _CCCL_PP_EVAL(_CCCL_PP_CHECK, _CCCL_PP_CAT(_PREFIX##_SWITCH_, _CCCL_PP_FIRST(__VA_ARGS__)), _CCCL_SWITCH_DEFAULT, ) + +#define _CCCL_PP_FOR_EACH(_Mp, ...) _CCCL_PP_FOR_EACH_N(_CCCL_PP_COUNT(__VA_ARGS__), _Mp, __VA_ARGS__) +#define _CCCL_PP_FOR_EACH_N(_Np, _Mp, ...) _CCCL_PP_CAT2(_CCCL_PP_FOR_EACH_, _Np)(_Mp, __VA_ARGS__) +#define _CCCL_PP_FOR_EACH_1(_Mp, _1) _Mp(_1) +#define _CCCL_PP_FOR_EACH_2(_Mp, _1, _2) _Mp(_1) _Mp(_2) +#define _CCCL_PP_FOR_EACH_3(_Mp, _1, _2, _3) _Mp(_1) _Mp(_2) _Mp(_3) +#define _CCCL_PP_FOR_EACH_4(_Mp, _1, _2, _3, _4) _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) +#define _CCCL_PP_FOR_EACH_5(_Mp, _1, _2, _3, _4, _5) _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) +#define _CCCL_PP_FOR_EACH_6(_Mp, _1, _2, _3, _4, _5, _6) _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) +#define _CCCL_PP_FOR_EACH_7(_Mp, _1, _2, _3, _4, _5, _6, _7) _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) +#define _CCCL_PP_FOR_EACH_8(_Mp, _1, _2, _3, _4, _5, _6, _7, _8) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) +#define _CCCL_PP_FOR_EACH_9(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) +#define _CCCL_PP_FOR_EACH_10(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) +#define _CCCL_PP_FOR_EACH_11(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) +#define _CCCL_PP_FOR_EACH_12(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) +#define _CCCL_PP_FOR_EACH_13(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) +#define _CCCL_PP_FOR_EACH_14(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) _Mp(_14) +#define _CCCL_PP_FOR_EACH_15(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) _Mp(_14) \ + _Mp(_15) +#define _CCCL_PP_FOR_EACH_16(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) _Mp(_14) \ + _Mp(_15) _Mp(_16) +#define _CCCL_PP_FOR_EACH_17(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) _Mp(_14) \ + _Mp(_15) _Mp(_16) _Mp(_17) +#define _CCCL_PP_FOR_EACH_18(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) _Mp(_14) \ + _Mp(_15) _Mp(_16) _Mp(_17) _Mp(_18) +#define _CCCL_PP_FOR_EACH_19(_Mp, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19) \ + _Mp(_1) _Mp(_2) _Mp(_3) _Mp(_4) _Mp(_5) _Mp(_6) _Mp(_7) _Mp(_8) _Mp(_9) _Mp(_10) _Mp(_11) _Mp(_12) _Mp(_13) _Mp(_14) \ + _Mp(_15) _Mp(_16) _Mp(_17) _Mp(_18) _Mp(_19) + +#define _CCCL_PP_PROBE_EMPTY_PROBE__CCCL_PP_PROBE_EMPTY _CCCL_PP_PROBE(~) + +#define _CCCL_PP_PROBE_EMPTY() +#define _CCCL_PP_IS_NOT_EMPTY(...) \ + _CCCL_PP_EVAL(_CCCL_PP_CHECK, _CCCL_PP_CAT(_CCCL_PP_PROBE_EMPTY_PROBE_, _CCCL_PP_PROBE_EMPTY __VA_ARGS__())) \ + /**/ + +#define _CCCL_PP_TAIL(_, ...) __VA_ARGS__ + +/////////////////////////////////////////////////////////////////////////////// + +// Count the number of arguments. There must be at least one argument and fewer +// than 126 arguments. +// clang-format off +#define _CCCL_PP_COUNT_IMPL( \ + _125, _124, _123, _122, _121, _120, _119, _118, _117, _116, _115, _114, _113, _112, _111, _110, \ + _109, _108, _107, _106, _105, _104, _103, _102, _101, _100, _99, _98, _97, _96, _95, _94, \ + _93, _92, _91, _90, _89, _88, _87, _86, _85, _84, _83, _82, _81, _80, _79, _78, \ + _77, _76, _75, _74, _73, _72, _71, _70, _69, _68, _67, _66, _65, _64, _63, _62, \ + _61, _60, _59, _58, _57, _56, _55, _54, _53, _52, _51, _50, _49, _48, _47, _46, \ + _45, _44, _43, _42, _41, _40, _39, _38, _37, _36, _35, _34, _33, _32, _31, _30, \ + _29, _28, _27, _26, _25, _24, _23, _22, _21, _20, _19, _18, _17, _16, _15, _14, \ + _13, _12, _11, _10, _9, _8, _7, _6, _5, _4, _3, _2, _1, _0, ...) _0 + +#define _CCCL_PP_COUNT(...) \ + _CCCL_PP_EXPAND(_CCCL_PP_COUNT_IMPL( __VA_ARGS__, \ + 125, 124, 123, 122, 121, 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, \ + 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 99, 98, 97, 96, 95, 94, \ + 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, \ + 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, \ + 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, \ + 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, \ + 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, \ + 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)) +// clang-format on + +/////////////////////////////////////////////////////////////////////////////// + +#define _CCCL_PP_INC(_X) _CCCL_PP_INC_IMPL0(_X) + +#define _CCCL_PP_INC_IMPL0(_X) _CCCL_PP_CAT(_CCCL_PP_INC_IMPL_TAG, _X) + +#define _CCCL_PP_INC_IMPL_TAG0 1 +#define _CCCL_PP_INC_IMPL_TAG1 2 +#define _CCCL_PP_INC_IMPL_TAG2 3 +#define _CCCL_PP_INC_IMPL_TAG3 4 +#define _CCCL_PP_INC_IMPL_TAG4 5 +#define _CCCL_PP_INC_IMPL_TAG5 6 +#define _CCCL_PP_INC_IMPL_TAG6 7 +#define _CCCL_PP_INC_IMPL_TAG7 8 +#define _CCCL_PP_INC_IMPL_TAG8 9 +#define _CCCL_PP_INC_IMPL_TAG9 10 +#define _CCCL_PP_INC_IMPL_TAG10 11 +#define _CCCL_PP_INC_IMPL_TAG11 12 +#define _CCCL_PP_INC_IMPL_TAG12 13 +#define _CCCL_PP_INC_IMPL_TAG13 14 +#define _CCCL_PP_INC_IMPL_TAG14 15 +#define _CCCL_PP_INC_IMPL_TAG15 16 +#define _CCCL_PP_INC_IMPL_TAG16 17 +#define _CCCL_PP_INC_IMPL_TAG17 18 +#define _CCCL_PP_INC_IMPL_TAG18 19 +#define _CCCL_PP_INC_IMPL_TAG19 20 +#define _CCCL_PP_INC_IMPL_TAG20 21 +#define _CCCL_PP_INC_IMPL_TAG21 22 +#define _CCCL_PP_INC_IMPL_TAG22 23 +#define _CCCL_PP_INC_IMPL_TAG23 24 +#define _CCCL_PP_INC_IMPL_TAG24 25 +#define _CCCL_PP_INC_IMPL_TAG25 26 +#define _CCCL_PP_INC_IMPL_TAG26 27 +#define _CCCL_PP_INC_IMPL_TAG27 28 +#define _CCCL_PP_INC_IMPL_TAG28 29 +#define _CCCL_PP_INC_IMPL_TAG29 30 +#define _CCCL_PP_INC_IMPL_TAG30 31 +#define _CCCL_PP_INC_IMPL_TAG31 32 +#define _CCCL_PP_INC_IMPL_TAG32 33 +#define _CCCL_PP_INC_IMPL_TAG33 34 +#define _CCCL_PP_INC_IMPL_TAG34 35 +#define _CCCL_PP_INC_IMPL_TAG35 36 +#define _CCCL_PP_INC_IMPL_TAG36 37 +#define _CCCL_PP_INC_IMPL_TAG37 38 +#define _CCCL_PP_INC_IMPL_TAG38 39 +#define _CCCL_PP_INC_IMPL_TAG39 40 +#define _CCCL_PP_INC_IMPL_TAG40 41 +#define _CCCL_PP_INC_IMPL_TAG41 42 +#define _CCCL_PP_INC_IMPL_TAG42 43 +#define _CCCL_PP_INC_IMPL_TAG43 44 +#define _CCCL_PP_INC_IMPL_TAG44 45 +#define _CCCL_PP_INC_IMPL_TAG45 46 +#define _CCCL_PP_INC_IMPL_TAG46 47 +#define _CCCL_PP_INC_IMPL_TAG47 48 +#define _CCCL_PP_INC_IMPL_TAG48 49 +#define _CCCL_PP_INC_IMPL_TAG49 50 +#define _CCCL_PP_INC_IMPL_TAG50 51 +#define _CCCL_PP_INC_IMPL_TAG51 52 +#define _CCCL_PP_INC_IMPL_TAG52 53 +#define _CCCL_PP_INC_IMPL_TAG53 54 +#define _CCCL_PP_INC_IMPL_TAG54 55 +#define _CCCL_PP_INC_IMPL_TAG55 56 +#define _CCCL_PP_INC_IMPL_TAG56 57 +#define _CCCL_PP_INC_IMPL_TAG57 58 +#define _CCCL_PP_INC_IMPL_TAG58 59 +#define _CCCL_PP_INC_IMPL_TAG59 60 +#define _CCCL_PP_INC_IMPL_TAG60 61 +#define _CCCL_PP_INC_IMPL_TAG61 62 +#define _CCCL_PP_INC_IMPL_TAG62 63 +#define _CCCL_PP_INC_IMPL_TAG63 64 +#define _CCCL_PP_INC_IMPL_TAG64 65 +#define _CCCL_PP_INC_IMPL_TAG65 66 +#define _CCCL_PP_INC_IMPL_TAG66 67 +#define _CCCL_PP_INC_IMPL_TAG67 68 +#define _CCCL_PP_INC_IMPL_TAG68 69 +#define _CCCL_PP_INC_IMPL_TAG69 70 +#define _CCCL_PP_INC_IMPL_TAG70 71 +#define _CCCL_PP_INC_IMPL_TAG71 72 +#define _CCCL_PP_INC_IMPL_TAG72 73 +#define _CCCL_PP_INC_IMPL_TAG73 74 +#define _CCCL_PP_INC_IMPL_TAG74 75 +#define _CCCL_PP_INC_IMPL_TAG75 76 +#define _CCCL_PP_INC_IMPL_TAG76 77 +#define _CCCL_PP_INC_IMPL_TAG77 78 +#define _CCCL_PP_INC_IMPL_TAG78 79 +#define _CCCL_PP_INC_IMPL_TAG79 80 +#define _CCCL_PP_INC_IMPL_TAG80 81 +#define _CCCL_PP_INC_IMPL_TAG81 82 +#define _CCCL_PP_INC_IMPL_TAG82 83 +#define _CCCL_PP_INC_IMPL_TAG83 84 +#define _CCCL_PP_INC_IMPL_TAG84 85 +#define _CCCL_PP_INC_IMPL_TAG85 86 +#define _CCCL_PP_INC_IMPL_TAG86 87 +#define _CCCL_PP_INC_IMPL_TAG87 88 +#define _CCCL_PP_INC_IMPL_TAG88 89 +#define _CCCL_PP_INC_IMPL_TAG89 90 +#define _CCCL_PP_INC_IMPL_TAG90 91 +#define _CCCL_PP_INC_IMPL_TAG91 92 +#define _CCCL_PP_INC_IMPL_TAG92 93 +#define _CCCL_PP_INC_IMPL_TAG93 94 +#define _CCCL_PP_INC_IMPL_TAG94 95 +#define _CCCL_PP_INC_IMPL_TAG95 96 +#define _CCCL_PP_INC_IMPL_TAG96 97 +#define _CCCL_PP_INC_IMPL_TAG97 98 +#define _CCCL_PP_INC_IMPL_TAG98 99 +#define _CCCL_PP_INC_IMPL_TAG99 100 +#define _CCCL_PP_INC_IMPL_TAG100 101 +#define _CCCL_PP_INC_IMPL_TAG101 102 +#define _CCCL_PP_INC_IMPL_TAG102 103 +#define _CCCL_PP_INC_IMPL_TAG103 104 +#define _CCCL_PP_INC_IMPL_TAG104 105 +#define _CCCL_PP_INC_IMPL_TAG105 106 +#define _CCCL_PP_INC_IMPL_TAG106 107 +#define _CCCL_PP_INC_IMPL_TAG107 108 +#define _CCCL_PP_INC_IMPL_TAG108 109 +#define _CCCL_PP_INC_IMPL_TAG109 110 +#define _CCCL_PP_INC_IMPL_TAG110 111 +#define _CCCL_PP_INC_IMPL_TAG111 112 +#define _CCCL_PP_INC_IMPL_TAG112 113 +#define _CCCL_PP_INC_IMPL_TAG113 114 +#define _CCCL_PP_INC_IMPL_TAG114 115 +#define _CCCL_PP_INC_IMPL_TAG115 116 +#define _CCCL_PP_INC_IMPL_TAG116 117 +#define _CCCL_PP_INC_IMPL_TAG117 118 +#define _CCCL_PP_INC_IMPL_TAG118 119 +#define _CCCL_PP_INC_IMPL_TAG119 120 +#define _CCCL_PP_INC_IMPL_TAG120 121 +#define _CCCL_PP_INC_IMPL_TAG121 122 +#define _CCCL_PP_INC_IMPL_TAG122 123 +#define _CCCL_PP_INC_IMPL_TAG123 124 +#define _CCCL_PP_INC_IMPL_TAG124 125 +#define _CCCL_PP_INC_IMPL_TAG125 126 +#define _CCCL_PP_INC_IMPL_TAG126 127 +#define _CCCL_PP_INC_IMPL_TAG127 128 +#define _CCCL_PP_INC_IMPL_TAG128 129 +#define _CCCL_PP_INC_IMPL_TAG129 130 +#define _CCCL_PP_INC_IMPL_TAG130 131 +#define _CCCL_PP_INC_IMPL_TAG131 132 +#define _CCCL_PP_INC_IMPL_TAG132 133 +#define _CCCL_PP_INC_IMPL_TAG133 134 +#define _CCCL_PP_INC_IMPL_TAG134 135 +#define _CCCL_PP_INC_IMPL_TAG135 136 +#define _CCCL_PP_INC_IMPL_TAG136 137 +#define _CCCL_PP_INC_IMPL_TAG137 138 +#define _CCCL_PP_INC_IMPL_TAG138 139 +#define _CCCL_PP_INC_IMPL_TAG139 140 +#define _CCCL_PP_INC_IMPL_TAG140 141 +#define _CCCL_PP_INC_IMPL_TAG141 142 +#define _CCCL_PP_INC_IMPL_TAG142 143 +#define _CCCL_PP_INC_IMPL_TAG143 144 +#define _CCCL_PP_INC_IMPL_TAG144 145 +#define _CCCL_PP_INC_IMPL_TAG145 146 +#define _CCCL_PP_INC_IMPL_TAG146 147 +#define _CCCL_PP_INC_IMPL_TAG147 148 +#define _CCCL_PP_INC_IMPL_TAG148 149 +#define _CCCL_PP_INC_IMPL_TAG149 150 +#define _CCCL_PP_INC_IMPL_TAG150 151 +#define _CCCL_PP_INC_IMPL_TAG151 152 +#define _CCCL_PP_INC_IMPL_TAG152 153 +#define _CCCL_PP_INC_IMPL_TAG153 154 +#define _CCCL_PP_INC_IMPL_TAG154 155 +#define _CCCL_PP_INC_IMPL_TAG155 156 +#define _CCCL_PP_INC_IMPL_TAG156 157 +#define _CCCL_PP_INC_IMPL_TAG157 158 +#define _CCCL_PP_INC_IMPL_TAG158 159 +#define _CCCL_PP_INC_IMPL_TAG159 160 +#define _CCCL_PP_INC_IMPL_TAG160 161 +#define _CCCL_PP_INC_IMPL_TAG161 162 +#define _CCCL_PP_INC_IMPL_TAG162 163 +#define _CCCL_PP_INC_IMPL_TAG163 164 +#define _CCCL_PP_INC_IMPL_TAG164 165 +#define _CCCL_PP_INC_IMPL_TAG165 166 +#define _CCCL_PP_INC_IMPL_TAG166 167 +#define _CCCL_PP_INC_IMPL_TAG167 168 +#define _CCCL_PP_INC_IMPL_TAG168 169 +#define _CCCL_PP_INC_IMPL_TAG169 170 +#define _CCCL_PP_INC_IMPL_TAG170 171 +#define _CCCL_PP_INC_IMPL_TAG171 172 +#define _CCCL_PP_INC_IMPL_TAG172 173 +#define _CCCL_PP_INC_IMPL_TAG173 174 +#define _CCCL_PP_INC_IMPL_TAG174 175 +#define _CCCL_PP_INC_IMPL_TAG175 176 +#define _CCCL_PP_INC_IMPL_TAG176 177 +#define _CCCL_PP_INC_IMPL_TAG177 178 +#define _CCCL_PP_INC_IMPL_TAG178 179 +#define _CCCL_PP_INC_IMPL_TAG179 180 +#define _CCCL_PP_INC_IMPL_TAG180 181 +#define _CCCL_PP_INC_IMPL_TAG181 182 +#define _CCCL_PP_INC_IMPL_TAG182 183 +#define _CCCL_PP_INC_IMPL_TAG183 184 +#define _CCCL_PP_INC_IMPL_TAG184 185 +#define _CCCL_PP_INC_IMPL_TAG185 186 +#define _CCCL_PP_INC_IMPL_TAG186 187 +#define _CCCL_PP_INC_IMPL_TAG187 188 +#define _CCCL_PP_INC_IMPL_TAG188 189 +#define _CCCL_PP_INC_IMPL_TAG189 190 +#define _CCCL_PP_INC_IMPL_TAG190 191 +#define _CCCL_PP_INC_IMPL_TAG191 192 +#define _CCCL_PP_INC_IMPL_TAG192 193 +#define _CCCL_PP_INC_IMPL_TAG193 194 +#define _CCCL_PP_INC_IMPL_TAG194 195 +#define _CCCL_PP_INC_IMPL_TAG195 196 +#define _CCCL_PP_INC_IMPL_TAG196 197 +#define _CCCL_PP_INC_IMPL_TAG197 198 +#define _CCCL_PP_INC_IMPL_TAG198 199 +#define _CCCL_PP_INC_IMPL_TAG199 200 +#define _CCCL_PP_INC_IMPL_TAG200 201 +#define _CCCL_PP_INC_IMPL_TAG201 202 +#define _CCCL_PP_INC_IMPL_TAG202 203 +#define _CCCL_PP_INC_IMPL_TAG203 204 +#define _CCCL_PP_INC_IMPL_TAG204 205 +#define _CCCL_PP_INC_IMPL_TAG205 206 +#define _CCCL_PP_INC_IMPL_TAG206 207 +#define _CCCL_PP_INC_IMPL_TAG207 208 +#define _CCCL_PP_INC_IMPL_TAG208 209 +#define _CCCL_PP_INC_IMPL_TAG209 210 +#define _CCCL_PP_INC_IMPL_TAG210 211 +#define _CCCL_PP_INC_IMPL_TAG211 212 +#define _CCCL_PP_INC_IMPL_TAG212 213 +#define _CCCL_PP_INC_IMPL_TAG213 214 +#define _CCCL_PP_INC_IMPL_TAG214 215 +#define _CCCL_PP_INC_IMPL_TAG215 216 +#define _CCCL_PP_INC_IMPL_TAG216 217 +#define _CCCL_PP_INC_IMPL_TAG217 218 +#define _CCCL_PP_INC_IMPL_TAG218 219 +#define _CCCL_PP_INC_IMPL_TAG219 220 +#define _CCCL_PP_INC_IMPL_TAG220 221 +#define _CCCL_PP_INC_IMPL_TAG221 222 +#define _CCCL_PP_INC_IMPL_TAG222 223 +#define _CCCL_PP_INC_IMPL_TAG223 224 +#define _CCCL_PP_INC_IMPL_TAG224 225 +#define _CCCL_PP_INC_IMPL_TAG225 226 +#define _CCCL_PP_INC_IMPL_TAG226 227 +#define _CCCL_PP_INC_IMPL_TAG227 228 +#define _CCCL_PP_INC_IMPL_TAG228 229 +#define _CCCL_PP_INC_IMPL_TAG229 230 +#define _CCCL_PP_INC_IMPL_TAG230 231 +#define _CCCL_PP_INC_IMPL_TAG231 232 +#define _CCCL_PP_INC_IMPL_TAG232 233 +#define _CCCL_PP_INC_IMPL_TAG233 234 +#define _CCCL_PP_INC_IMPL_TAG234 235 +#define _CCCL_PP_INC_IMPL_TAG235 236 +#define _CCCL_PP_INC_IMPL_TAG236 237 +#define _CCCL_PP_INC_IMPL_TAG237 238 +#define _CCCL_PP_INC_IMPL_TAG238 239 +#define _CCCL_PP_INC_IMPL_TAG239 240 +#define _CCCL_PP_INC_IMPL_TAG240 241 +#define _CCCL_PP_INC_IMPL_TAG241 242 +#define _CCCL_PP_INC_IMPL_TAG242 243 +#define _CCCL_PP_INC_IMPL_TAG243 244 +#define _CCCL_PP_INC_IMPL_TAG244 245 +#define _CCCL_PP_INC_IMPL_TAG245 246 +#define _CCCL_PP_INC_IMPL_TAG246 247 +#define _CCCL_PP_INC_IMPL_TAG247 248 +#define _CCCL_PP_INC_IMPL_TAG248 249 +#define _CCCL_PP_INC_IMPL_TAG249 250 +#define _CCCL_PP_INC_IMPL_TAG250 251 +#define _CCCL_PP_INC_IMPL_TAG251 252 +#define _CCCL_PP_INC_IMPL_TAG252 253 +#define _CCCL_PP_INC_IMPL_TAG253 254 +#define _CCCL_PP_INC_IMPL_TAG254 255 +#define _CCCL_PP_INC_IMPL_TAG255 256 +#define _CCCL_PP_INC_IMPL_TAG256 257 + +#define _CCCL_PP_DEC(_X) _CCCL_PP_DEC_IMPL0(_X) + +#define _CCCL_PP_DEC_IMPL0(_X) _CCCL_PP_CAT(_CCCL_PP_DEC_IMPL_TAG, _X) + +#define _CCCL_PP_DEC_IMPL_TAG0 ~##~ // This will generate a syntax error +#define _CCCL_PP_DEC_IMPL_TAG1 0 +#define _CCCL_PP_DEC_IMPL_TAG2 1 +#define _CCCL_PP_DEC_IMPL_TAG3 2 +#define _CCCL_PP_DEC_IMPL_TAG4 3 +#define _CCCL_PP_DEC_IMPL_TAG5 4 +#define _CCCL_PP_DEC_IMPL_TAG6 5 +#define _CCCL_PP_DEC_IMPL_TAG7 6 +#define _CCCL_PP_DEC_IMPL_TAG8 7 +#define _CCCL_PP_DEC_IMPL_TAG9 8 +#define _CCCL_PP_DEC_IMPL_TAG10 9 +#define _CCCL_PP_DEC_IMPL_TAG11 10 +#define _CCCL_PP_DEC_IMPL_TAG12 11 +#define _CCCL_PP_DEC_IMPL_TAG13 12 +#define _CCCL_PP_DEC_IMPL_TAG14 13 +#define _CCCL_PP_DEC_IMPL_TAG15 14 +#define _CCCL_PP_DEC_IMPL_TAG16 15 +#define _CCCL_PP_DEC_IMPL_TAG17 16 +#define _CCCL_PP_DEC_IMPL_TAG18 17 +#define _CCCL_PP_DEC_IMPL_TAG19 18 +#define _CCCL_PP_DEC_IMPL_TAG20 19 +#define _CCCL_PP_DEC_IMPL_TAG21 20 +#define _CCCL_PP_DEC_IMPL_TAG22 21 +#define _CCCL_PP_DEC_IMPL_TAG23 22 +#define _CCCL_PP_DEC_IMPL_TAG24 23 +#define _CCCL_PP_DEC_IMPL_TAG25 24 +#define _CCCL_PP_DEC_IMPL_TAG26 25 +#define _CCCL_PP_DEC_IMPL_TAG27 26 +#define _CCCL_PP_DEC_IMPL_TAG28 27 +#define _CCCL_PP_DEC_IMPL_TAG29 28 +#define _CCCL_PP_DEC_IMPL_TAG30 29 +#define _CCCL_PP_DEC_IMPL_TAG31 30 +#define _CCCL_PP_DEC_IMPL_TAG32 31 +#define _CCCL_PP_DEC_IMPL_TAG33 32 +#define _CCCL_PP_DEC_IMPL_TAG34 33 +#define _CCCL_PP_DEC_IMPL_TAG35 34 +#define _CCCL_PP_DEC_IMPL_TAG36 35 +#define _CCCL_PP_DEC_IMPL_TAG37 36 +#define _CCCL_PP_DEC_IMPL_TAG38 37 +#define _CCCL_PP_DEC_IMPL_TAG39 38 +#define _CCCL_PP_DEC_IMPL_TAG40 39 +#define _CCCL_PP_DEC_IMPL_TAG41 40 +#define _CCCL_PP_DEC_IMPL_TAG42 41 +#define _CCCL_PP_DEC_IMPL_TAG43 42 +#define _CCCL_PP_DEC_IMPL_TAG44 43 +#define _CCCL_PP_DEC_IMPL_TAG45 44 +#define _CCCL_PP_DEC_IMPL_TAG46 45 +#define _CCCL_PP_DEC_IMPL_TAG47 46 +#define _CCCL_PP_DEC_IMPL_TAG48 47 +#define _CCCL_PP_DEC_IMPL_TAG49 48 +#define _CCCL_PP_DEC_IMPL_TAG50 49 +#define _CCCL_PP_DEC_IMPL_TAG51 50 +#define _CCCL_PP_DEC_IMPL_TAG52 51 +#define _CCCL_PP_DEC_IMPL_TAG53 52 +#define _CCCL_PP_DEC_IMPL_TAG54 53 +#define _CCCL_PP_DEC_IMPL_TAG55 54 +#define _CCCL_PP_DEC_IMPL_TAG56 55 +#define _CCCL_PP_DEC_IMPL_TAG57 56 +#define _CCCL_PP_DEC_IMPL_TAG58 57 +#define _CCCL_PP_DEC_IMPL_TAG59 58 +#define _CCCL_PP_DEC_IMPL_TAG60 59 +#define _CCCL_PP_DEC_IMPL_TAG61 60 +#define _CCCL_PP_DEC_IMPL_TAG62 61 +#define _CCCL_PP_DEC_IMPL_TAG63 62 +#define _CCCL_PP_DEC_IMPL_TAG64 63 +#define _CCCL_PP_DEC_IMPL_TAG65 64 +#define _CCCL_PP_DEC_IMPL_TAG66 65 +#define _CCCL_PP_DEC_IMPL_TAG67 66 +#define _CCCL_PP_DEC_IMPL_TAG68 67 +#define _CCCL_PP_DEC_IMPL_TAG69 68 +#define _CCCL_PP_DEC_IMPL_TAG70 69 +#define _CCCL_PP_DEC_IMPL_TAG71 70 +#define _CCCL_PP_DEC_IMPL_TAG72 71 +#define _CCCL_PP_DEC_IMPL_TAG73 72 +#define _CCCL_PP_DEC_IMPL_TAG74 73 +#define _CCCL_PP_DEC_IMPL_TAG75 74 +#define _CCCL_PP_DEC_IMPL_TAG76 75 +#define _CCCL_PP_DEC_IMPL_TAG77 76 +#define _CCCL_PP_DEC_IMPL_TAG78 77 +#define _CCCL_PP_DEC_IMPL_TAG79 78 +#define _CCCL_PP_DEC_IMPL_TAG80 79 +#define _CCCL_PP_DEC_IMPL_TAG81 80 +#define _CCCL_PP_DEC_IMPL_TAG82 81 +#define _CCCL_PP_DEC_IMPL_TAG83 82 +#define _CCCL_PP_DEC_IMPL_TAG84 83 +#define _CCCL_PP_DEC_IMPL_TAG85 84 +#define _CCCL_PP_DEC_IMPL_TAG86 85 +#define _CCCL_PP_DEC_IMPL_TAG87 86 +#define _CCCL_PP_DEC_IMPL_TAG88 87 +#define _CCCL_PP_DEC_IMPL_TAG89 88 +#define _CCCL_PP_DEC_IMPL_TAG90 89 +#define _CCCL_PP_DEC_IMPL_TAG91 90 +#define _CCCL_PP_DEC_IMPL_TAG92 91 +#define _CCCL_PP_DEC_IMPL_TAG93 92 +#define _CCCL_PP_DEC_IMPL_TAG94 93 +#define _CCCL_PP_DEC_IMPL_TAG95 94 +#define _CCCL_PP_DEC_IMPL_TAG96 95 +#define _CCCL_PP_DEC_IMPL_TAG97 96 +#define _CCCL_PP_DEC_IMPL_TAG98 97 +#define _CCCL_PP_DEC_IMPL_TAG99 98 +#define _CCCL_PP_DEC_IMPL_TAG100 99 +#define _CCCL_PP_DEC_IMPL_TAG101 100 +#define _CCCL_PP_DEC_IMPL_TAG102 101 +#define _CCCL_PP_DEC_IMPL_TAG103 102 +#define _CCCL_PP_DEC_IMPL_TAG104 103 +#define _CCCL_PP_DEC_IMPL_TAG105 104 +#define _CCCL_PP_DEC_IMPL_TAG106 105 +#define _CCCL_PP_DEC_IMPL_TAG107 106 +#define _CCCL_PP_DEC_IMPL_TAG108 107 +#define _CCCL_PP_DEC_IMPL_TAG109 108 +#define _CCCL_PP_DEC_IMPL_TAG110 109 +#define _CCCL_PP_DEC_IMPL_TAG111 110 +#define _CCCL_PP_DEC_IMPL_TAG112 111 +#define _CCCL_PP_DEC_IMPL_TAG113 112 +#define _CCCL_PP_DEC_IMPL_TAG114 113 +#define _CCCL_PP_DEC_IMPL_TAG115 114 +#define _CCCL_PP_DEC_IMPL_TAG116 115 +#define _CCCL_PP_DEC_IMPL_TAG117 116 +#define _CCCL_PP_DEC_IMPL_TAG118 117 +#define _CCCL_PP_DEC_IMPL_TAG119 118 +#define _CCCL_PP_DEC_IMPL_TAG120 119 +#define _CCCL_PP_DEC_IMPL_TAG121 120 +#define _CCCL_PP_DEC_IMPL_TAG122 121 +#define _CCCL_PP_DEC_IMPL_TAG123 122 +#define _CCCL_PP_DEC_IMPL_TAG124 123 +#define _CCCL_PP_DEC_IMPL_TAG125 124 +#define _CCCL_PP_DEC_IMPL_TAG126 125 +#define _CCCL_PP_DEC_IMPL_TAG127 126 +#define _CCCL_PP_DEC_IMPL_TAG128 127 +#define _CCCL_PP_DEC_IMPL_TAG129 128 +#define _CCCL_PP_DEC_IMPL_TAG130 129 +#define _CCCL_PP_DEC_IMPL_TAG131 130 +#define _CCCL_PP_DEC_IMPL_TAG132 131 +#define _CCCL_PP_DEC_IMPL_TAG133 132 +#define _CCCL_PP_DEC_IMPL_TAG134 133 +#define _CCCL_PP_DEC_IMPL_TAG135 134 +#define _CCCL_PP_DEC_IMPL_TAG136 135 +#define _CCCL_PP_DEC_IMPL_TAG137 136 +#define _CCCL_PP_DEC_IMPL_TAG138 137 +#define _CCCL_PP_DEC_IMPL_TAG139 138 +#define _CCCL_PP_DEC_IMPL_TAG140 139 +#define _CCCL_PP_DEC_IMPL_TAG141 140 +#define _CCCL_PP_DEC_IMPL_TAG142 141 +#define _CCCL_PP_DEC_IMPL_TAG143 142 +#define _CCCL_PP_DEC_IMPL_TAG144 143 +#define _CCCL_PP_DEC_IMPL_TAG145 144 +#define _CCCL_PP_DEC_IMPL_TAG146 145 +#define _CCCL_PP_DEC_IMPL_TAG147 146 +#define _CCCL_PP_DEC_IMPL_TAG148 147 +#define _CCCL_PP_DEC_IMPL_TAG149 148 +#define _CCCL_PP_DEC_IMPL_TAG150 149 +#define _CCCL_PP_DEC_IMPL_TAG151 150 +#define _CCCL_PP_DEC_IMPL_TAG152 151 +#define _CCCL_PP_DEC_IMPL_TAG153 152 +#define _CCCL_PP_DEC_IMPL_TAG154 153 +#define _CCCL_PP_DEC_IMPL_TAG155 154 +#define _CCCL_PP_DEC_IMPL_TAG156 155 +#define _CCCL_PP_DEC_IMPL_TAG157 156 +#define _CCCL_PP_DEC_IMPL_TAG158 157 +#define _CCCL_PP_DEC_IMPL_TAG159 158 +#define _CCCL_PP_DEC_IMPL_TAG160 159 +#define _CCCL_PP_DEC_IMPL_TAG161 160 +#define _CCCL_PP_DEC_IMPL_TAG162 161 +#define _CCCL_PP_DEC_IMPL_TAG163 162 +#define _CCCL_PP_DEC_IMPL_TAG164 163 +#define _CCCL_PP_DEC_IMPL_TAG165 164 +#define _CCCL_PP_DEC_IMPL_TAG166 165 +#define _CCCL_PP_DEC_IMPL_TAG167 166 +#define _CCCL_PP_DEC_IMPL_TAG168 167 +#define _CCCL_PP_DEC_IMPL_TAG169 168 +#define _CCCL_PP_DEC_IMPL_TAG170 169 +#define _CCCL_PP_DEC_IMPL_TAG171 170 +#define _CCCL_PP_DEC_IMPL_TAG172 171 +#define _CCCL_PP_DEC_IMPL_TAG173 172 +#define _CCCL_PP_DEC_IMPL_TAG174 173 +#define _CCCL_PP_DEC_IMPL_TAG175 174 +#define _CCCL_PP_DEC_IMPL_TAG176 175 +#define _CCCL_PP_DEC_IMPL_TAG177 176 +#define _CCCL_PP_DEC_IMPL_TAG178 177 +#define _CCCL_PP_DEC_IMPL_TAG179 178 +#define _CCCL_PP_DEC_IMPL_TAG180 179 +#define _CCCL_PP_DEC_IMPL_TAG181 180 +#define _CCCL_PP_DEC_IMPL_TAG182 181 +#define _CCCL_PP_DEC_IMPL_TAG183 182 +#define _CCCL_PP_DEC_IMPL_TAG184 183 +#define _CCCL_PP_DEC_IMPL_TAG185 184 +#define _CCCL_PP_DEC_IMPL_TAG186 185 +#define _CCCL_PP_DEC_IMPL_TAG187 186 +#define _CCCL_PP_DEC_IMPL_TAG188 187 +#define _CCCL_PP_DEC_IMPL_TAG189 188 +#define _CCCL_PP_DEC_IMPL_TAG190 189 +#define _CCCL_PP_DEC_IMPL_TAG191 190 +#define _CCCL_PP_DEC_IMPL_TAG192 191 +#define _CCCL_PP_DEC_IMPL_TAG193 192 +#define _CCCL_PP_DEC_IMPL_TAG194 193 +#define _CCCL_PP_DEC_IMPL_TAG195 194 +#define _CCCL_PP_DEC_IMPL_TAG196 195 +#define _CCCL_PP_DEC_IMPL_TAG197 196 +#define _CCCL_PP_DEC_IMPL_TAG198 197 +#define _CCCL_PP_DEC_IMPL_TAG199 198 +#define _CCCL_PP_DEC_IMPL_TAG200 199 +#define _CCCL_PP_DEC_IMPL_TAG201 200 +#define _CCCL_PP_DEC_IMPL_TAG202 201 +#define _CCCL_PP_DEC_IMPL_TAG203 202 +#define _CCCL_PP_DEC_IMPL_TAG204 203 +#define _CCCL_PP_DEC_IMPL_TAG205 204 +#define _CCCL_PP_DEC_IMPL_TAG206 205 +#define _CCCL_PP_DEC_IMPL_TAG207 206 +#define _CCCL_PP_DEC_IMPL_TAG208 207 +#define _CCCL_PP_DEC_IMPL_TAG209 208 +#define _CCCL_PP_DEC_IMPL_TAG210 209 +#define _CCCL_PP_DEC_IMPL_TAG211 210 +#define _CCCL_PP_DEC_IMPL_TAG212 211 +#define _CCCL_PP_DEC_IMPL_TAG213 212 +#define _CCCL_PP_DEC_IMPL_TAG214 213 +#define _CCCL_PP_DEC_IMPL_TAG215 214 +#define _CCCL_PP_DEC_IMPL_TAG216 215 +#define _CCCL_PP_DEC_IMPL_TAG217 216 +#define _CCCL_PP_DEC_IMPL_TAG218 217 +#define _CCCL_PP_DEC_IMPL_TAG219 218 +#define _CCCL_PP_DEC_IMPL_TAG220 219 +#define _CCCL_PP_DEC_IMPL_TAG221 220 +#define _CCCL_PP_DEC_IMPL_TAG222 221 +#define _CCCL_PP_DEC_IMPL_TAG223 222 +#define _CCCL_PP_DEC_IMPL_TAG224 223 +#define _CCCL_PP_DEC_IMPL_TAG225 224 +#define _CCCL_PP_DEC_IMPL_TAG226 225 +#define _CCCL_PP_DEC_IMPL_TAG227 226 +#define _CCCL_PP_DEC_IMPL_TAG228 227 +#define _CCCL_PP_DEC_IMPL_TAG229 228 +#define _CCCL_PP_DEC_IMPL_TAG230 229 +#define _CCCL_PP_DEC_IMPL_TAG231 230 +#define _CCCL_PP_DEC_IMPL_TAG232 231 +#define _CCCL_PP_DEC_IMPL_TAG233 232 +#define _CCCL_PP_DEC_IMPL_TAG234 233 +#define _CCCL_PP_DEC_IMPL_TAG235 234 +#define _CCCL_PP_DEC_IMPL_TAG236 235 +#define _CCCL_PP_DEC_IMPL_TAG237 236 +#define _CCCL_PP_DEC_IMPL_TAG238 237 +#define _CCCL_PP_DEC_IMPL_TAG239 238 +#define _CCCL_PP_DEC_IMPL_TAG240 239 +#define _CCCL_PP_DEC_IMPL_TAG241 240 +#define _CCCL_PP_DEC_IMPL_TAG242 241 +#define _CCCL_PP_DEC_IMPL_TAG243 242 +#define _CCCL_PP_DEC_IMPL_TAG244 243 +#define _CCCL_PP_DEC_IMPL_TAG245 244 +#define _CCCL_PP_DEC_IMPL_TAG246 245 +#define _CCCL_PP_DEC_IMPL_TAG247 246 +#define _CCCL_PP_DEC_IMPL_TAG248 247 +#define _CCCL_PP_DEC_IMPL_TAG249 248 +#define _CCCL_PP_DEC_IMPL_TAG250 249 +#define _CCCL_PP_DEC_IMPL_TAG251 250 +#define _CCCL_PP_DEC_IMPL_TAG252 251 +#define _CCCL_PP_DEC_IMPL_TAG253 252 +#define _CCCL_PP_DEC_IMPL_TAG254 253 +#define _CCCL_PP_DEC_IMPL_TAG255 254 +#define _CCCL_PP_DEC_IMPL_TAG256 255 +#define _CCCL_PP_DEC_IMPL_TAG257 256 + +//////////////////////////////////////////////////////////////////////////////// + +// _CCCL_PP_REPEAT(COUNT, MACRO, STATE, INCREMENT) +// +// Expands to: MACRO(STATE) MACRO(INCREMENT(STATE)) ... MACRO(INCREMENT(INCREMENT(INCREMENT(...)))) +// STATE defaults to 0, INCREMENT defaults to _CCCL_PP_INC +#define _CCCL_PP_REPEAT_AUX1(_N, _M) _CCCL_PP_CAT(_CCCL_PP_REPEAT, _N)(_M, 0, _CCCL_PP_INC) +#define _CCCL_PP_REPEAT_AUX2(_N, _M, _S) _CCCL_PP_CAT(_CCCL_PP_REPEAT, _N)(_M, _S, _CCCL_PP_INC) +#define _CCCL_PP_REPEAT_AUX3(_N, _M, _S, _F) _CCCL_PP_CAT(_CCCL_PP_REPEAT, _N)(_M, _S, _F) + +#define _CCCL_PP_REPEAT_AUX(_C, _N, ...) _CCCL_PP_CAT(_CCCL_PP_REPEAT_AUX, _C)(_N, __VA_ARGS__) +#define _CCCL_PP_REPEAT(_N, ...) _CCCL_PP_REPEAT_AUX(_CCCL_PP_COUNT(__VA_ARGS__), _N, __VA_ARGS__) + +#define _CCCL_PP_REPEAT0(_M, _S, _F) +#define _CCCL_PP_REPEAT1(_M, _S, _F) _M(_S) +#define _CCCL_PP_REPEAT2(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT1(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT3(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT2(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT4(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT3(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT5(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT4(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT6(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT5(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT7(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT6(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT8(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT7(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT9(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT8(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT10(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT9(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT11(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT10(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT12(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT11(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT13(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT12(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT14(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT13(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT15(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT14(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT16(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT15(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT17(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT16(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT18(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT17(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT19(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT18(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT20(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT19(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT21(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT20(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT22(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT21(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT23(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT22(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT24(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT23(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT25(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT24(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT26(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT25(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT27(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT26(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT28(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT27(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT29(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT28(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT30(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT29(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT31(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT30(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT32(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT31(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT33(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT32(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT34(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT33(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT35(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT34(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT36(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT35(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT37(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT36(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT38(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT37(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT39(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT38(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT40(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT39(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT41(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT40(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT42(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT41(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT43(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT42(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT44(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT43(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT45(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT44(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT46(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT45(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT47(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT46(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT48(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT47(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT49(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT48(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT50(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT49(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT51(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT50(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT52(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT51(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT53(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT52(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT54(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT53(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT55(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT54(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT56(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT55(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT57(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT56(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT58(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT57(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT59(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT58(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT60(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT59(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT61(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT60(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT62(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT61(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT63(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT62(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT64(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT63(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT65(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT64(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT66(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT65(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT67(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT66(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT68(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT67(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT69(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT68(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT70(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT69(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT71(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT70(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT72(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT71(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT73(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT72(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT74(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT73(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT75(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT74(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT76(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT75(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT77(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT76(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT78(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT77(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT79(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT78(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT80(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT79(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT81(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT80(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT82(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT81(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT83(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT82(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT84(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT83(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT85(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT84(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT86(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT85(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT87(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT86(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT88(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT87(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT89(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT88(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT90(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT89(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT91(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT90(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT92(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT91(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT93(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT92(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT94(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT93(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT95(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT94(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT96(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT95(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT97(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT96(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT98(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT97(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT99(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT98(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT100(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT99(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT101(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT100(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT102(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT101(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT103(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT102(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT104(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT103(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT105(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT104(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT106(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT105(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT107(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT106(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT108(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT107(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT109(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT108(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT110(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT109(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT111(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT110(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT112(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT111(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT113(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT112(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT114(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT113(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT115(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT114(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT116(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT115(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT117(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT116(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT118(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT117(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT119(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT118(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT120(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT119(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT121(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT120(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT122(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT121(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT123(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT122(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT124(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT123(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT125(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT124(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT126(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT125(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT127(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT126(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT128(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT127(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT129(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT128(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT130(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT129(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT131(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT130(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT132(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT131(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT133(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT132(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT134(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT133(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT135(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT134(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT136(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT135(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT137(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT136(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT138(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT137(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT139(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT138(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT140(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT139(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT141(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT140(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT142(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT141(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT143(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT142(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT144(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT143(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT145(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT144(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT146(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT145(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT147(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT146(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT148(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT147(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT149(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT148(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT150(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT149(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT151(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT150(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT152(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT151(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT153(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT152(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT154(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT153(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT155(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT154(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT156(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT155(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT157(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT156(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT158(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT157(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT159(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT158(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT160(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT159(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT161(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT160(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT162(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT161(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT163(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT162(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT164(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT163(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT165(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT164(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT166(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT165(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT167(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT166(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT168(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT167(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT169(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT168(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT170(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT169(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT171(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT170(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT172(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT171(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT173(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT172(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT174(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT173(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT175(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT174(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT176(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT175(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT177(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT176(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT178(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT177(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT179(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT178(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT180(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT179(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT181(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT180(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT182(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT181(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT183(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT182(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT184(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT183(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT185(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT184(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT186(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT185(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT187(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT186(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT188(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT187(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT189(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT188(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT190(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT189(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT191(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT190(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT192(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT191(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT193(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT192(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT194(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT193(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT195(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT194(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT196(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT195(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT197(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT196(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT198(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT197(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT199(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT198(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT200(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT199(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT201(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT200(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT202(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT201(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT203(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT202(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT204(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT203(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT205(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT204(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT206(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT205(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT207(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT206(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT208(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT207(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT209(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT208(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT210(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT209(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT211(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT210(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT212(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT211(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT213(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT212(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT214(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT213(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT215(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT214(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT216(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT215(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT217(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT216(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT218(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT217(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT219(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT218(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT220(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT219(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT221(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT220(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT222(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT221(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT223(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT222(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT224(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT223(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT225(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT224(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT226(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT225(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT227(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT226(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT228(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT227(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT229(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT228(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT230(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT229(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT231(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT230(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT232(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT231(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT233(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT232(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT234(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT233(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT235(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT234(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT236(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT235(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT237(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT236(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT238(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT237(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT239(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT238(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT240(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT239(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT241(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT240(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT242(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT241(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT243(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT242(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT244(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT243(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT245(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT244(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT246(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT245(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT247(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT246(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT248(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT247(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT249(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT248(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT250(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT249(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT251(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT250(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT252(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT251(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT253(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT252(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT254(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT253(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT255(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT254(_M, _F(_S), _F) +#define _CCCL_PP_REPEAT256(_M, _S, _F) _M(_S) _CCCL_PP_REPEAT255(_M, _F(_S), _F) + +//////////////////////////////////////////////////////////////////////////////// + +// _CCCL_PP_REPEAT_REVERSE(COUNT, MACRO, STATE, INCREMENT) +// +// Expands to: MACRO(INCREMENT(INCREMENT(INCREMENT(...)))) ... MACRO(INCREMENT(STATE)) MACRO(STATE) +// STATE defaults to 0, INCREMENT defaults to _CCCL_PP_INC +#define _CCCL_PP_REPEAT_REVERSE_AUX1(_N, _M) _CCCL_PP_CAT(_CCCL_PP_REPEAT_REVERSE, _N)(_M, 0, _CCCL_PP_INC) +#define _CCCL_PP_REPEAT_REVERSE_AUX2(_N, _M, _S) _CCCL_PP_CAT(_CCCL_PP_REPEAT_REVERSE, _N)(_M, _S, _CCCL_PP_INC) +#define _CCCL_PP_REPEAT_REVERSE_AUX3(_N, _M, _S, _F) _CCCL_PP_CAT(_CCCL_PP_REPEAT_REVERSE, _N)(_M, _S, _F) + +#define _CCCL_PP_REPEAT_REVERSE_AUX(_C, _N, ...) _CCCL_PP_CAT(_CCCL_PP_REPEAT_REVERSE_AUX, _C)(_N, __VA_ARGS__) +#define _CCCL_PP_REPEAT_REVERSE(_N, ...) _CCCL_PP_REPEAT_REVERSE_AUX(_CCCL_PP_COUNT(__VA_ARGS__), _N, __VA_ARGS__) + +#define _CCCL_PP_REPEAT_REVERSE0(_M, _S, _F) +#define _CCCL_PP_REPEAT_REVERSE1(_M, _S, _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE2(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE1(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE3(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE2(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE4(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE3(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE5(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE4(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE6(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE5(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE7(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE6(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE8(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE7(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE9(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE8(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE10(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE9(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE11(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE10(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE12(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE11(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE13(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE12(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE14(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE13(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE15(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE14(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE16(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE15(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE17(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE16(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE18(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE17(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE19(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE18(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE20(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE19(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE21(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE20(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE22(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE21(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE23(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE22(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE24(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE23(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE25(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE24(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE26(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE25(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE27(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE26(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE28(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE27(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE29(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE28(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE30(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE29(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE31(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE30(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE32(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE31(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE33(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE32(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE34(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE33(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE35(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE34(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE36(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE35(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE37(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE36(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE38(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE37(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE39(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE38(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE40(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE39(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE41(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE40(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE42(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE41(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE43(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE42(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE44(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE43(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE45(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE44(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE46(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE45(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE47(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE46(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE48(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE47(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE49(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE48(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE50(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE49(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE51(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE50(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE52(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE51(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE53(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE52(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE54(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE53(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE55(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE54(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE56(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE55(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE57(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE56(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE58(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE57(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE59(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE58(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE60(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE59(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE61(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE60(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE62(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE61(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE63(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE62(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE64(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE63(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE65(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE64(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE66(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE65(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE67(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE66(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE68(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE67(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE69(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE68(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE70(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE69(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE71(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE70(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE72(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE71(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE73(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE72(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE74(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE73(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE75(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE74(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE76(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE75(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE77(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE76(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE78(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE77(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE79(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE78(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE80(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE79(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE81(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE80(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE82(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE81(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE83(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE82(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE84(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE83(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE85(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE84(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE86(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE85(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE87(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE86(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE88(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE87(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE89(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE88(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE90(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE89(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE91(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE90(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE92(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE91(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE93(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE92(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE94(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE93(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE95(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE94(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE96(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE95(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE97(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE96(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE98(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE97(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE99(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE98(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE100(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE99(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE101(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE100(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE102(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE101(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE103(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE102(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE104(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE103(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE105(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE104(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE106(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE105(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE107(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE106(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE108(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE107(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE109(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE108(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE110(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE109(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE111(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE110(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE112(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE111(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE113(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE112(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE114(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE113(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE115(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE114(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE116(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE115(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE117(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE116(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE118(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE117(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE119(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE118(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE120(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE119(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE121(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE120(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE122(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE121(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE123(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE122(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE124(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE123(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE125(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE124(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE126(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE125(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE127(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE126(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE128(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE127(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE129(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE128(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE130(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE129(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE131(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE130(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE132(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE131(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE133(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE132(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE134(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE133(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE135(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE134(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE136(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE135(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE137(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE136(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE138(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE137(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE139(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE138(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE140(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE139(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE141(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE140(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE142(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE141(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE143(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE142(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE144(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE143(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE145(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE144(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE146(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE145(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE147(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE146(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE148(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE147(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE149(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE148(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE150(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE149(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE151(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE150(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE152(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE151(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE153(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE152(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE154(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE153(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE155(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE154(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE156(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE155(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE157(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE156(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE158(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE157(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE159(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE158(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE160(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE159(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE161(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE160(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE162(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE161(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE163(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE162(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE164(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE163(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE165(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE164(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE166(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE165(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE167(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE166(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE168(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE167(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE169(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE168(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE170(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE169(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE171(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE170(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE172(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE171(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE173(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE172(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE174(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE173(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE175(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE174(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE176(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE175(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE177(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE176(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE178(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE177(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE179(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE178(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE180(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE179(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE181(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE180(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE182(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE181(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE183(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE182(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE184(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE183(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE185(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE184(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE186(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE185(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE187(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE186(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE188(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE187(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE189(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE188(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE190(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE189(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE191(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE190(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE192(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE191(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE193(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE192(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE194(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE193(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE195(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE194(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE196(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE195(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE197(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE196(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE198(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE197(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE199(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE198(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE200(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE199(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE201(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE200(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE202(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE201(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE203(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE202(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE204(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE203(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE205(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE204(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE206(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE205(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE207(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE206(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE208(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE207(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE209(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE208(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE210(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE209(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE211(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE210(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE212(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE211(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE213(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE212(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE214(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE213(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE215(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE214(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE216(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE215(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE217(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE216(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE218(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE217(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE219(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE218(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE220(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE219(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE221(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE220(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE222(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE221(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE223(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE222(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE224(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE223(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE225(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE224(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE226(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE225(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE227(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE226(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE228(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE227(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE229(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE228(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE230(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE229(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE231(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE230(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE232(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE231(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE233(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE232(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE234(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE233(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE235(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE234(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE236(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE235(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE237(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE236(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE238(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE237(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE239(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE238(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE240(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE239(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE241(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE240(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE242(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE241(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE243(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE242(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE244(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE243(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE245(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE244(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE246(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE245(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE247(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE246(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE248(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE247(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE249(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE248(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE250(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE249(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE251(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE250(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE252(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE251(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE253(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE252(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE254(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE253(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE255(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE254(_M, _F(_S), _F) _M(_S) +#define _CCCL_PP_REPEAT_REVERSE256(_M, _S, _F) _CCCL_PP_REPEAT_REVERSE255(_M, _F(_S), _F) _M(_S) + +#define _CCCL_PP_SPLICE_WITH_IMPL1(_SEP, _P1) _P1 +#define _CCCL_PP_SPLICE_WITH_IMPL2(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL1(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL3(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL2(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL4(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL3(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL5(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL4(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL6(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL5(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL7(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL6(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL8(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL7(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL9(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL8(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL10(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL9(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL11(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL10(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL12(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL11(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL13(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL12(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL14(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL13(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL15(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL14(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL16(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL15(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL17(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL16(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL18(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL17(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL19(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL18(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL20(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL19(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL21(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL20(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL22(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL21(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL23(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL22(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL24(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL23(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL25(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL24(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL26(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL25(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL27(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL26(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL28(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL27(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL29(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL28(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL30(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL29(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL31(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL30(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL32(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL31(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL33(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL32(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL34(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL33(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL35(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL34(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL36(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL35(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL37(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL36(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL38(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL37(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL39(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL38(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL40(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL39(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL41(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL40(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL42(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL41(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL43(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL42(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL44(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL43(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL45(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL44(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL46(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL45(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL47(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL46(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL48(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL47(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL49(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL48(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL50(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL49(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL51(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL50(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL52(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL51(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL53(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL52(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL54(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL53(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL55(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL54(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL56(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL55(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL57(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL56(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL58(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL57(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL59(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL58(_SEP, __VA_ARGS__)) +#define _CCCL_PP_SPLICE_WITH_IMPL60(_SEP, _P1, ...) \ + _CCCL_PP_CAT(_P1##_SEP, _CCCL_PP_SPLICE_WITH_IMPL59(_SEP, __VA_ARGS__)) + +#define _CCCL_PP_SPLICE_WITH_IMPL_DISPATCH(_NUM) _CCCL_PP_SPLICE_WITH_IMPL##_NUM + +// Splices a pack of arguments into a single token, separated by _SEP +// E.g., _CCCL_PP_SPLICE_WITH(_, A, B, C) will evaluate to A_B_C +#define _CCCL_PP_SPLICE_WITH(_SEP, ...) \ + _CCCL_PP_EXPAND(_CCCL_PP_EVAL(_CCCL_PP_SPLICE_WITH_IMPL_DISPATCH, _CCCL_PP_COUNT(__VA_ARGS__))(_SEP, __VA_ARGS__)) + +#endif // __CCCL_PREPROCESSOR_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/prologue.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/prologue.h new file mode 100644 index 00000000..da21a9cc --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/prologue.h @@ -0,0 +1,348 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py. + +// NO include guards here (this file is included multiple times) + +#if defined(_CCCL_PROLOGUE_INCLUDED) +# error \ + "cccl internal error: must be included before next is reincluded" +#endif +#define _CCCL_PROLOGUE_INCLUDED() 1 + +#include +#include +#include + +// __declspec modifiers + +#if defined(align) +# pragma push_macro("align") +# undef align +# define _CCCL_POP_MACRO_align +#endif // defined(align) + +#if defined(allocate) +# pragma push_macro("allocate") +# undef allocate +# define _CCCL_POP_MACRO_allocate +#endif // defined(allocate) + +#if defined(allocator) +# pragma push_macro("allocator") +# undef allocator +# define _CCCL_POP_MACRO_allocator +#endif // defined(allocator) + +#if defined(appdomain) +# pragma push_macro("appdomain") +# undef appdomain +# define _CCCL_POP_MACRO_appdomain +#endif // defined(appdomain) + +#if defined(code_seg) +# pragma push_macro("code_seg") +# undef code_seg +# define _CCCL_POP_MACRO_code_seg +#endif // defined(code_seg) + +#if defined(deprecated) +# pragma push_macro("deprecated") +# undef deprecated +# define _CCCL_POP_MACRO_deprecated +#endif // defined(deprecated) + +#if defined(dllimport) +# pragma push_macro("dllimport") +# undef dllimport +# define _CCCL_POP_MACRO_dllimport +#endif // defined(dllimport) + +#if defined(dllexport) +# pragma push_macro("dllexport") +# undef dllexport +# define _CCCL_POP_MACRO_dllexport +#endif // defined(dllexport) + +#if defined(empty_bases) +# pragma push_macro("empty_bases") +# undef empty_bases +# define _CCCL_POP_MACRO_empty_bases +#endif // defined(empty_bases) + +#if defined(hybrid_patchable) +# pragma push_macro("hybrid_patchable") +# undef hybrid_patchable +# define _CCCL_POP_MACRO_hybrid_patchable +#endif // defined(hybrid_patchable) + +#if defined(jitintrinsic) +# pragma push_macro("jitintrinsic") +# undef jitintrinsic +# define _CCCL_POP_MACRO_jitintrinsic +#endif // defined(jitintrinsic) + +#if defined(lifetimebound) +# pragma push_macro("lifetimebound") +# undef lifetimebound +# define _CCCL_POP_MACRO_lifetimebound +#endif // defined(lifetimebound) + +#if defined(naked) +# pragma push_macro("naked") +# undef naked +# define _CCCL_POP_MACRO_naked +#endif // defined(naked) + +#if defined(noalias) +# pragma push_macro("noalias") +# undef noalias +# define _CCCL_POP_MACRO_noalias +#endif // defined(noalias) + +#if defined(noinline) +# pragma push_macro("noinline") +# undef noinline +# define _CCCL_POP_MACRO_noinline +#endif // defined(noinline) + +#if defined(noreturn) +# pragma push_macro("noreturn") +# undef noreturn +# define _CCCL_POP_MACRO_noreturn +#endif // defined(noreturn) + +#if defined(nothrow) +# pragma push_macro("nothrow") +# undef nothrow +# define _CCCL_POP_MACRO_nothrow +#endif // defined(nothrow) + +#if defined(novtable) +# pragma push_macro("novtable") +# undef novtable +# define _CCCL_POP_MACRO_novtable +#endif // defined(novtable) + +#if defined(no_sanitize_address) +# pragma push_macro("no_sanitize_address") +# undef no_sanitize_address +# define _CCCL_POP_MACRO_no_sanitize_address +#endif // defined(no_sanitize_address) + +#if defined(process) +# pragma push_macro("process") +# undef process +# define _CCCL_POP_MACRO_process +#endif // defined(process) + +#if defined(property) +# pragma push_macro("property") +# undef property +# define _CCCL_POP_MACRO_property +#endif // defined(property) + +#if defined(restrict) +# pragma push_macro("restrict") +# undef restrict +# define _CCCL_POP_MACRO_restrict +#endif // defined(restrict) + +#if defined(safebuffers) +# pragma push_macro("safebuffers") +# undef safebuffers +# define _CCCL_POP_MACRO_safebuffers +#endif // defined(safebuffers) + +#if defined(selectany) +# pragma push_macro("selectany") +# undef selectany +# define _CCCL_POP_MACRO_selectany +#endif // defined(selectany) + +#if defined(spectre) +# pragma push_macro("spectre") +# undef spectre +# define _CCCL_POP_MACRO_spectre +#endif // defined(spectre) + +#if defined(thread) +# pragma push_macro("thread") +# undef thread +# define _CCCL_POP_MACRO_thread +#endif // defined(thread) + +#if defined(uuid) +# pragma push_macro("uuid") +# undef uuid +# define _CCCL_POP_MACRO_uuid +#endif // defined(uuid) + +// [[msvc::attribute]] attributes + +#if defined(msvc) +# pragma push_macro("msvc") +# undef msvc +# define _CCCL_POP_MACRO_msvc +#endif // defined(msvc) + +#if defined(flatten) +# pragma push_macro("flatten") +# undef flatten +# define _CCCL_POP_MACRO_flatten +#endif // defined(flatten) + +#if defined(forceinline) +# pragma push_macro("forceinline") +# undef forceinline +# define _CCCL_POP_MACRO_forceinline +#endif // defined(forceinline) + +#if defined(forceinline_calls) +# pragma push_macro("forceinline_calls") +# undef forceinline_calls +# define _CCCL_POP_MACRO_forceinline_calls +#endif // defined(forceinline_calls) + +#if defined(intrinsic) +# pragma push_macro("intrinsic") +# undef intrinsic +# define _CCCL_POP_MACRO_intrinsic +#endif // defined(intrinsic) + +#if defined(noinline) +# pragma push_macro("noinline") +# undef noinline +# define _CCCL_POP_MACRO_noinline +#endif // defined(noinline) + +#if defined(noinline_calls) +# pragma push_macro("noinline_calls") +# undef noinline_calls +# define _CCCL_POP_MACRO_noinline_calls +#endif // defined(noinline_calls) + +#if defined(no_tls_guard) +# pragma push_macro("no_tls_guard") +# undef no_tls_guard +# define _CCCL_POP_MACRO_no_tls_guard +#endif // defined(no_tls_guard) + +// Windows nasty macros + +#if defined(min) +# pragma push_macro("min") +# undef min +# define _CCCL_POP_MACRO_min +#endif // defined(min) + +#if defined(max) +# pragma push_macro("max") +# undef max +# define _CCCL_POP_MACRO_max +#endif // defined(max) + +#if defined(interface) +# pragma push_macro("interface") +# undef interface +# define _CCCL_POP_MACRO_interface +#endif // defined(interface) + +// sal.h on Windows + +#if defined(__valid) +# pragma push_macro("__valid") +# undef __valid +# define _CCCL_POP_MACRO___valid +#endif // defined(__valid) + +#if defined(__callback) +# pragma push_macro("__callback") +# undef __callback +# define _CCCL_POP_MACRO___callback +#endif // defined(__callback) + +// other macros + +#if defined(clang) +# pragma push_macro("clang") +# undef clang +# define _CCCL_POP_MACRO_clang +#endif // defined(clang) + +// sys/sysmacros.h on linux + +#if defined(major) +# pragma push_macro("major") +# undef major +# define _CCCL_POP_MACRO_major +#endif // defined(major) + +#if defined(minor) +# pragma push_macro("minor") +# undef minor +# define _CCCL_POP_MACRO_minor +#endif // defined(minor) + +#if defined(makedev) +# pragma push_macro("makedev") +# undef makedev +# define _CCCL_POP_MACRO_makedev +#endif // defined(makedev) + +_CCCL_DIAG_PUSH +_CCCL_NV_DIAG_PUSH() + +// disable some msvc warnings +// https://github.com/microsoft/STL/blob/master/stl/inc/yvals_core.h#L353 +// warning C4100: 'quack': unreferenced formal parameter +// warning C4127: conditional expression is constant +// warning C4180: qualifier applied to function type has no meaning; ignored +// warning C4197: 'purr': top-level volatile in cast is ignored +// warning C4324: 'roar': structure was padded due to alignment specifier +// warning C4455: literal suffix identifiers that do not start with an underscore are reserved +// warning C4503: 'hum': decorated name length exceeded, name was truncated +// warning C4522: 'woof' : multiple assignment operators specified +// warning C4668: 'meow' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' +// warning C4800: 'boo': forcing value to bool 'true' or 'false' (performance warning) +// warning C4996: 'meow': was declared deprecated +_CCCL_DIAG_SUPPRESS_MSVC(4100 4127 4180 4197 4296 4324 4455 4503 4522 4668 4800 4996) + +// Suppress compiler warnings about C++ extensions. + +#if _CCCL_COMPILER(GCC, >=, 12) +_CCCL_DIAG_SUPPRESS_GCC("-Wc++20-extensions") +_CCCL_DIAG_SUPPRESS_GCC("-Wc++23-extensions") +#endif // _CCCL_COMPILER(GCC, >=, 12) +#if _CCCL_COMPILER(GCC, >=, 14) +_CCCL_DIAG_SUPPRESS_GCC("-Wc++26-extensions") +#endif // _CCCL_COMPILER(GCC, >=, 14) + +_CCCL_DIAG_SUPPRESS_CLANG("-Wc++20-extensions") +#if _CCCL_COMPILER(CLANG, >=, 17) +_CCCL_DIAG_SUPPRESS_CLANG("-Wc++23-extensions") +_CCCL_DIAG_SUPPRESS_CLANG("-Wc++26-extensions") +#else // ^^^ _CCCL_COMPILER(CLANG, >=, 17) ^^^ / vvv _CCCL_COMPILER(CLANG, <, 17) vvv +_CCCL_DIAG_SUPPRESS_CLANG("-Wc++2b-extensions") +#endif // ^^^ _CCCL_COMPILER(CLANG, <, 17) ^^^ + +// Suppress `if consteval`-related warnings. + +_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_nonstandard) +_CCCL_DIAG_SUPPRESS_NVHPC(is_constant_evaluated_in_nonconstexpr_context) +_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_in_nonconstexpr_function) + +_CCCL_DIAG_SUPPRESS_NVCC(3215) // "if consteval" and "if not consteval" are not standard in this mode +_CCCL_DIAG_SUPPRESS_NVCC(3206) // "if consteval" and "if not consteval" are meaningless in a non-constexpr function +_CCCL_DIAG_SUPPRESS_NVCC(3060) // call to __builtin_is_constant_evaluated appearing in a non-constexpr function always + // produces "false" + +// NO include guards here (this file is included multiple times) diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/ptx_isa.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/ptx_isa.h new file mode 100644 index 00000000..209ea812 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/ptx_isa.h @@ -0,0 +1,369 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_PTX_ISA_H_ +#define __CCCL_PTX_ISA_H_ + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include // __CUDA_MINIMUM_ARCH__ and friends + +/* + * Targeting macros + * + * Information from: + * https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#release-notes + */ + +// The first define is for future major versions of CUDACC. +// We make sure that these get the highest known PTX ISA version. +// For clang cuda check +// https://github.com/llvm/llvm-project/blob/release/.x/clang/lib/Driver/ToolChains/Cuda.cpp getNVPTXTargetFeatures +#if _CCCL_CUDACC_AT_LEAST(14, 0) && !_CCCL_CUDA_COMPILER(CLANG) +# define __cccl_ptx_isa 940ULL +// PTX ISA 9.4 is available from CUDA 13.4 +#elif _CCCL_CUDACC_AT_LEAST(13, 4) && !_CCCL_CUDA_COMPILER(CLANG) +# define __cccl_ptx_isa 940ULL +// PTX ISA 9.3 is available from CUDA 13.3 +#elif _CCCL_CUDACC_AT_LEAST(13, 3) && !_CCCL_CUDA_COMPILER(CLANG) +# define __cccl_ptx_isa 930ULL +// PTX ISA 9.2 is available from CUDA 13.2 +#elif _CCCL_CUDACC_AT_LEAST(13, 2) && !_CCCL_CUDA_COMPILER(CLANG) +# define __cccl_ptx_isa 920ULL +// PTX ISA 9.1 is available from CUDA 13.1 +#elif _CCCL_CUDACC_AT_LEAST(13, 1) && !_CCCL_CUDA_COMPILER(CLANG) +# define __cccl_ptx_isa 910ULL +// PTX ISA 9.0 is available from CUDA 13.0, driver r580 +#elif _CCCL_CUDACC_AT_LEAST(13, 0) && !_CCCL_CUDA_COMPILER(CLANG) +# define __cccl_ptx_isa 900ULL +// PTX ISA 8.8 is available from CUDA 12.9, driver r575 +#elif _CCCL_CUDACC_AT_LEAST(12, 9) && !_CCCL_CUDA_COMPILER(CLANG, <, 22) +# define __cccl_ptx_isa 880ULL +// PTX ISA 8.7 is available from CUDA 12.8, driver r570 +#elif _CCCL_CUDACC_AT_LEAST(12, 8) && !_CCCL_CUDA_COMPILER(CLANG, <, 20) +# define __cccl_ptx_isa 870ULL +// PTX ISA 8.5 is available from CUDA 12.5, driver r555 +#elif _CCCL_CUDACC_AT_LEAST(12, 5) && !_CCCL_CUDA_COMPILER(CLANG, <, 19) +# define __cccl_ptx_isa 850ULL +// PTX ISA 8.4 is available from CUDA 12.4, driver r550 +#elif _CCCL_CUDACC_AT_LEAST(12, 4) && !_CCCL_CUDA_COMPILER(CLANG, <, 19) +# define __cccl_ptx_isa 840ULL +// PTX ISA 8.3 is available from CUDA 12.3, driver r545 +#elif _CCCL_CUDACC_AT_LEAST(12, 3) && !_CCCL_CUDA_COMPILER(CLANG, <, 18) +# define __cccl_ptx_isa 830ULL +// PTX ISA 8.2 is available from CUDA 12.2, driver r535 +#elif _CCCL_CUDACC_AT_LEAST(12, 2) && !_CCCL_CUDA_COMPILER(CLANG, <, 18) +# define __cccl_ptx_isa 820ULL +// PTX ISA 8.1 is available from CUDA 12.1, driver r530 +#elif _CCCL_CUDACC_AT_LEAST(12, 1) && !_CCCL_CUDA_COMPILER(CLANG, <, 17) +# define __cccl_ptx_isa 810ULL +// PTX ISA 8.0 is available from CUDA 12.0, driver r525 +#elif _CCCL_CUDACC_AT_LEAST(12, 0) && !_CCCL_CUDA_COMPILER(CLANG, <, 17) +# define __cccl_ptx_isa 800ULL +// PTX ISA 7.8 is available from CUDA 11.8, driver r520 +#elif _CCCL_CUDACC_AT_LEAST(11, 8) && !_CCCL_CUDA_COMPILER(CLANG, <, 16) +# define __cccl_ptx_isa 780ULL +// PTX ISA 7.7 is available from CUDA 11.7, driver r515 +#elif _CCCL_CUDACC_AT_LEAST(11, 7) && !_CCCL_CUDA_COMPILER(CLANG, <, 16) +# define __cccl_ptx_isa 770ULL +// PTX ISA 7.6 is available from CUDA 11.6, driver r510 +#elif _CCCL_CUDACC_AT_LEAST(11, 6) && !_CCCL_CUDA_COMPILER(CLANG, <, 16) +# define __cccl_ptx_isa 760ULL +// PTX ISA 7.5 is available from CUDA 11.5, driver r495 +#elif _CCCL_CUDACC_AT_LEAST(11, 5) && !_CCCL_CUDA_COMPILER(CLANG, <, 14) +# define __cccl_ptx_isa 750ULL +// PTX ISA 7.4 is available from CUDA 11.4, driver r470 +#elif _CCCL_CUDACC_AT_LEAST(11, 4) && !_CCCL_CUDA_COMPILER(CLANG, <, 14) +# define __cccl_ptx_isa 740ULL +// PTX ISA 7.3 is available from CUDA 11.3, driver r465 +#elif _CCCL_CUDACC_AT_LEAST(11, 3) && !_CCCL_CUDA_COMPILER(CLANG, <, 14) +# define __cccl_ptx_isa 730ULL +// PTX ISA 7.2 is available from CUDA 11.2, driver r460 +#elif _CCCL_CUDACC_AT_LEAST(11, 2) && !_CCCL_CUDA_COMPILER(CLANG, <, 13) +# define __cccl_ptx_isa 720ULL +// PTX ISA 7.1 is available from CUDA 11.1, driver r455 +#elif _CCCL_CUDACC_AT_LEAST(11, 1) && !_CCCL_CUDA_COMPILER(CLANG, <, 13) +# define __cccl_ptx_isa 710ULL +// PTX ISA 7.0 is available from CUDA 11.0, driver r445 +#elif _CCCL_CUDACC_AT_LEAST(11, 0) && !_CCCL_CUDA_COMPILER(CLANG, <, 12) +# define __cccl_ptx_isa 700ULL +// Fallback case. Define the ISA version to be zero. This ensures that the macro is always defined. +#else +# define __cccl_ptx_isa 0ULL +#endif + +// We define certain feature test macros depending on availability. When +// __CUDA_MINIMUM_ARCH__ is not available, we define the following features +// depending on PTX ISA. This permits checking for the feature in host code. +// When __CUDA_MINIMUM_ARCH__ is available, we only enable the feature when the +// hardware supports it. +#if __cccl_ptx_isa >= 800 +# if (!defined(__CUDA_MINIMUM_ARCH__)) || (defined(__CUDA_MINIMUM_ARCH__) && 900 <= __CUDA_MINIMUM_ARCH__) +# define __cccl_lib_local_barrier_arrive_tx +# define __cccl_lib_experimental_ctk12_cp_async_exposure +# endif +#endif // __cccl_ptx_isa >= 800 + +// NVRTC ships a built-in copy of , so including CCCL's version of this header will omit the +// content since the header guards are already defined. To make older NVRTC versions have a few newer feature macros +// required for the PTX tests, we define them here outside the header guards. +// TODO(bgruber): limit this workaround to NVRTC versions older than the first one shipping those macros +#if _CCCL_COMPILER(NVRTC) + +// missing SM_88 +# if !defined(NV_PROVIDES_SM_88) +# define _NV_TARGET_VAL_SM_88 880 +# define NV_PROVIDES_SM_88 __NV_PROVIDES_SM_88 +# define NV_IS_EXACTLY_SM_88 __NV_IS_EXACTLY_SM_88 +# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_88) +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_88 1 +# define _NV_TARGET___NV_IS_EXACTLY_SM_88 1 +# else +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_88 0 +# define _NV_TARGET___NV_IS_EXACTLY_SM_88 0 +# endif +# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_88) +# define _NV_TARGET___NV_PROVIDES_SM_88 1 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_88 1 +# else +# define _NV_TARGET___NV_PROVIDES_SM_88 0 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_88 0 +# endif +# endif // !NV_PROVIDES_SM_88 + +// missing SM_90a +# ifndef NV_HAS_FEATURE_SM_90a +# define NV_HAS_FEATURE_SM_90a __NV_HAS_FEATURE_SM_90a +# if defined(__CUDA_ARCH_FEAT_SM90_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 900)) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_90a 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_90a 0 +# endif +# endif // NV_HAS_FEATURE_SM_90a + +// missing SM_100 +# ifndef NV_PROVIDES_SM_100 +# define _NV_TARGET_VAL_SM_100 1000 +# define NV_PROVIDES_SM_100 __NV_PROVIDES_SM_100 +# define NV_IS_EXACTLY_SM_100 __NV_IS_EXACTLY_SM_100 +# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_100) +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_100 1 +# define _NV_TARGET___NV_IS_EXACTLY_SM_100 1 +# else +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_100 0 +# define _NV_TARGET___NV_IS_EXACTLY_SM_100 0 +# endif +# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_100) +# define _NV_TARGET___NV_PROVIDES_SM_100 1 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_100 1 +# else +# define _NV_TARGET___NV_PROVIDES_SM_100 0 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_100 0 +# endif +# endif // !NV_PROVIDES_SM_100 + +// missing SM_100a +# ifndef NV_HAS_FEATURE_SM_100a +# define NV_HAS_FEATURE_SM_100a __NV_HAS_FEATURE_SM_100a +# if defined(__CUDA_ARCH_FEAT_SM100_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1000)) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100a 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100a 0 +# endif +# endif // !NV_HAS_FEATURE_SM_100a + +// missing SM_103 +# ifndef NV_PROVIDES_SM_103 +# define _NV_TARGET_VAL_SM_103 1030 +# define NV_PROVIDES_SM_103 __NV_PROVIDES_SM_103 +# define NV_IS_EXACTLY_SM_103 __NV_IS_EXACTLY_SM_103 +# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_103) +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_103 1 +# define _NV_TARGET___NV_IS_EXACTLY_SM_103 1 +# else +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_103 0 +# define _NV_TARGET___NV_IS_EXACTLY_SM_103 0 +# endif +# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_103) +# define _NV_TARGET___NV_PROVIDES_SM_103 1 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_103 1 +# else +# define _NV_TARGET___NV_PROVIDES_SM_103 0 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_103 0 +# endif +# endif // !NV_PROVIDES_SM_103 + +// missing SM_103 +# ifndef NV_HAS_FEATURE_SM_103a +# define NV_HAS_FEATURE_SM_103a __NV_HAS_FEATURE_SM_103a +# if defined(__CUDA_ARCH_FEAT_SM103_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1030)) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103a 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103a 0 +# endif +# endif // !NV_HAS_FEATURE_SM_103a + +// missing SM_110 +# ifndef NV_PROVIDES_SM_110 +# define _NV_TARGET_VAL_SM_110 1100 +# define NV_PROVIDES_SM_110 __NV_PROVIDES_SM_110 +# define NV_IS_EXACTLY_SM_110 __NV_IS_EXACTLY_SM_110 +# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_110) +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_110 1 +# define _NV_TARGET___NV_IS_EXACTLY_SM_110 1 +# else +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_110 0 +# define _NV_TARGET___NV_IS_EXACTLY_SM_110 0 +# endif +# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_110) +# define _NV_TARGET___NV_PROVIDES_SM_110 1 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_110 1 +# else +# define _NV_TARGET___NV_PROVIDES_SM_110 0 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_110 0 +# endif +# endif // !NV_PROVIDES_SM_110 + +// missing SM_110a +# ifndef NV_HAS_FEATURE_SM_110a +# define NV_HAS_FEATURE_SM_110a __NV_HAS_FEATURE_SM_110a +# if defined(__CUDA_ARCH_FEAT_SM110_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1100)) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110a 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110a 0 +# endif +# endif // NV_HAS_FEATURE_SM_110a + +// missing SM_120 +# ifndef NV_PROVIDES_SM_120 +# define _NV_TARGET_VAL_SM_120 1200 +# define NV_PROVIDES_SM_120 __NV_PROVIDES_SM_120 +# define NV_IS_EXACTLY_SM_120 __NV_IS_EXACTLY_SM_120 +# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_120) +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_120 1 +# define _NV_TARGET___NV_IS_EXACTLY_SM_120 1 +# else +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_120 0 +# define _NV_TARGET___NV_IS_EXACTLY_SM_120 0 +# endif +# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_120) +# define _NV_TARGET___NV_PROVIDES_SM_120 1 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_120 1 +# else +# define _NV_TARGET___NV_PROVIDES_SM_120 0 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_120 0 +# endif +# endif // !NV_PROVIDES_SM_120 + +// missing SM_120a +# ifndef NV_HAS_FEATURE_SM_120a +# define NV_HAS_FEATURE_SM_120a __NV_HAS_FEATURE_SM_120a +# if defined(__CUDA_ARCH_FEAT_SM120_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1200)) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120a 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120a 0 +# endif +# endif // _CCCL_COMPILER(NVRTC) + +// missing SM_121 +# if !defined(NV_PROVIDES_SM_121) +# define _NV_TARGET_VAL_SM_121 1210 +# define NV_PROVIDES_SM_121 __NV_PROVIDES_SM_121 +# define NV_IS_EXACTLY_SM_121 __NV_IS_EXACTLY_SM_121 +# if (__CUDA_ARCH__ == _NV_TARGET_VAL_SM_121) +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_121 1 +# define _NV_TARGET___NV_IS_EXACTLY_SM_121 1 +# else +# define _NV_TARGET_BOOL___NV_IS_EXACTLY_SM_121 0 +# define _NV_TARGET___NV_IS_EXACTLY_SM_121 0 +# endif +# if (__CUDA_ARCH__ >= _NV_TARGET_VAL_SM_121) +# define _NV_TARGET___NV_PROVIDES_SM_121 1 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_121 1 +# else +# define _NV_TARGET___NV_PROVIDES_SM_121 0 +# define _NV_TARGET_BOOL___NV_PROVIDES_SM_121 0 +# endif +# endif // !NV_PROVIDES_SM_121 + +// missing SM_121a +# ifndef NV_HAS_FEATURE_SM_121a +# define NV_HAS_FEATURE_SM_121a __NV_HAS_FEATURE_SM_121a +# if defined(__CUDA_ARCH_FEAT_SM121_ALL) || (defined(__CUDA_ARCH_SPECIFIC__) && (__CUDA_ARCH_SPECIFIC__ == 1210)) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121a 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121a 0 +# endif +# endif // NV_HAS_FEATURE_SM_121a + +//---------------------------------------------------------------------------------------------------------------------- +// family-specific SM versions + +// missing SM_100f +# ifndef NV_HAS_FEATURE_SM_100f +# define NV_HAS_FEATURE_SM_100f __NV_HAS_FEATURE_SM_100f +# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1000) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100f 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_100f 0 +# endif +# endif // NV_HAS_FEATURE_SM_100 + +// missing SM_103f +# ifndef NV_HAS_FEATURE_SM_103f +# define NV_HAS_FEATURE_SM_103f __NV_HAS_FEATURE_SM_103f +# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1030) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103f 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_103f 0 +# endif +# endif // NV_HAS_FEATURE_SM_103f + +// missing SM_110f +# ifndef NV_HAS_FEATURE_SM_110f +# define NV_HAS_FEATURE_SM_110f __NV_HAS_FEATURE_SM_110f +# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1100) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110f 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_110f 0 +# endif +# endif // NV_HAS_FEATURE_SM_110f + +// missing SM_120f +# ifndef NV_HAS_FEATURE_SM_120f +# define NV_HAS_FEATURE_SM_120f __NV_HAS_FEATURE_SM_120f +# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1200) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120f 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_120f 0 +# endif +# endif // NV_HAS_FEATURE_SM_120f + +// missing SM_121f +# ifndef NV_HAS_FEATURE_SM_121f +# define NV_HAS_FEATURE_SM_121f __NV_HAS_FEATURE_SM_121f +# if defined(__CUDA_ARCH_FAMILY_SPECIFIC__) && (__CUDA_ARCH_FAMILY_SPECIFIC__ == 1210) +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121f 1 +# else +# define _NV_TARGET_BOOL___NV_HAS_FEATURE_SM_121f 0 +# endif +# endif // NV_HAS_FEATURE_SM_121f + +#endif // _CCCL_COMPILER(NVRTC) +#endif // __CCCL_PTX_ISA_H_ diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/rtti.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/rtti.h new file mode 100644 index 00000000..14ce5dc5 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/rtti.h @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_RTTI_H +#define __CCCL_RTTI_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +// NOTE: some compilers support the `typeid` feature but not the `dynamic_cast` +// feature. This is why we have separate macros for each. + +#ifndef _CCCL_NO_RTTI +# if defined(CCCL_DISABLE_RTTI) // Escape hatch for users to manually disable RTTI +# define _CCCL_NO_RTTI +# elif defined(__CUDA_ARCH__) +# define _CCCL_NO_RTTI // No RTTI in CUDA device code +# elif _CCCL_COMPILER(NVRTC) +# define _CCCL_NO_RTTI +# elif _CCCL_COMPILER(MSVC) +# if _CPPRTTI == 0 +# define _CCCL_NO_RTTI +# endif +# elif _CCCL_COMPILER(CLANG) +# if !_CCCL_HAS_FEATURE(cxx_rtti) +# define _CCCL_NO_RTTI +# endif +# else +# if __GXX_RTTI == 0 && __cpp_rtti == 0 +# define _CCCL_NO_RTTI +# endif +# endif +#endif // !_CCCL_NO_RTTI + +#ifndef _CCCL_NO_TYPEID +# if defined(CCCL_DISABLE_RTTI) // CCCL_DISABLE_RTTI disables typeid also +# define _CCCL_NO_TYPEID +# elif defined(__CUDA_ARCH__) +# define _CCCL_NO_TYPEID // No typeid in CUDA device code +# elif _CCCL_COMPILER(NVRTC) +# define _CCCL_NO_TYPEID +# elif _CCCL_COMPILER(MSVC) +// No-op, MSVC always supports typeid even when RTTI is disabled +# elif _CCCL_COMPILER(CLANG) +# if !_CCCL_HAS_FEATURE(cxx_rtti) +# define _CCCL_NO_TYPEID +# endif +# else +# if __GXX_RTTI == 0 && __cpp_rtti == 0 +# define _CCCL_NO_TYPEID +# endif +# endif +#endif // !_CCCL_NO_TYPEID + +#endif // __CCCL_RTTI_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/sequence_access.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/sequence_access.h new file mode 100644 index 00000000..574b44a8 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/sequence_access.h @@ -0,0 +1,83 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_SEQUENCE_ACCESS_H +#define __CCCL_SEQUENCE_ACCESS_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// We need to define hidden friends for {cr,r,}{begin,end} of our containers as we will otherwise encounter ambigouities +#define _CCCL_SYNTHESIZE_SEQUENCE_ACCESS(_ClassName, _ConstIter) \ + [[nodiscard]] _CCCL_API friend iterator begin(_ClassName& __sequence) noexcept(noexcept(__sequence.begin())) \ + { \ + return __sequence.begin(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstIter begin(const _ClassName& __sequence) noexcept(noexcept(__sequence.begin())) \ + { \ + return __sequence.begin(); \ + } \ + [[nodiscard]] _CCCL_API friend iterator end(_ClassName& __sequence) noexcept(noexcept(__sequence.end())) \ + { \ + return __sequence.end(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstIter end(const _ClassName& __sequence) noexcept(noexcept(__sequence.end())) \ + { \ + return __sequence.end(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstIter cbegin(const _ClassName& __sequence) noexcept( \ + noexcept(__sequence.begin())) \ + { \ + return __sequence.begin(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstIter cend(const _ClassName& __sequence) noexcept(noexcept(__sequence.end())) \ + { \ + return __sequence.end(); \ + } +#define _CCCL_SYNTHESIZE_SEQUENCE_REVERSE_ACCESS(_ClassName, _ConstRevIter) \ + [[nodiscard]] _CCCL_API friend reverse_iterator rbegin(_ClassName& __sequence) noexcept( \ + noexcept(__sequence.rbegin())) \ + { \ + return __sequence.rbegin(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstRevIter rbegin(const _ClassName& __sequence) noexcept( \ + noexcept(__sequence.rbegin())) \ + { \ + return __sequence.rbegin(); \ + } \ + [[nodiscard]] _CCCL_API friend reverse_iterator rend(_ClassName& __sequence) noexcept(noexcept(__sequence.rend())) \ + { \ + return __sequence.rend(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstRevIter rend(const _ClassName& __sequence) noexcept( \ + noexcept(__sequence.rend())) \ + { \ + return __sequence.rend(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstRevIter crbegin(const _ClassName& __sequence) noexcept( \ + noexcept(__sequence.rbegin())) \ + { \ + return __sequence.rbegin(); \ + } \ + [[nodiscard]] _CCCL_API friend _ConstRevIter crend(const _ClassName& __sequence) noexcept( \ + noexcept(__sequence.rend())) \ + { \ + return __sequence.rend(); \ + } + +#endif // __CCCL_SEQUENCE_ACCESS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/system_header.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/system_header.h new file mode 100644 index 00000000..3d2d1ac6 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/system_header.h @@ -0,0 +1,38 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_SYSTEM_HEADER_H +#define __CCCL_SYSTEM_HEADER_H + +#include +#include // IWYU pragma: export + +// Enforce that cccl headers are treated as system headers +#if _CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC) +# define _CCCL_FORCE_SYSTEM_HEADER_GCC +#elif _CCCL_COMPILER(CLANG) +# define _CCCL_FORCE_SYSTEM_HEADER_CLANG +#elif _CCCL_COMPILER(MSVC) +# define _CCCL_FORCE_SYSTEM_HEADER_MSVC +#endif // other compilers + +// Potentially enable that cccl headers are treated as system headers +#if !defined(_CCCL_NO_SYSTEM_HEADER) && !(_CCCL_COMPILER(MSVC) && defined(_LIBCUDACXX_DISABLE_PRAGMA_MSVC_WARNING)) \ + && !_CCCL_COMPILER(NVRTC) && !defined(_LIBCUDACXX_DISABLE_PRAGMA_GCC_SYSTEM_HEADER) +# if _CCCL_COMPILER(GCC) || _CCCL_COMPILER(NVHPC) +# define _CCCL_IMPLICIT_SYSTEM_HEADER_GCC +# elif _CCCL_COMPILER(CLANG) +# define _CCCL_IMPLICIT_SYSTEM_HEADER_CLANG +# elif _CCCL_COMPILER(MSVC) +# define _CCCL_IMPLICIT_SYSTEM_HEADER_MSVC +# endif // other compilers +#endif // Use system header + +#endif // __CCCL_SYSTEM_HEADER_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/unreachable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/unreachable.h new file mode 100644 index 00000000..2c2dba1a --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/unreachable.h @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_UNREACHABLE_H +#define __CCCL_UNREACHABLE_H + +#include +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_COMPILER(MSVC) && !_CCCL_DEVICE_COMPILATION() +# define _CCCL_UNREACHABLE() __assume(0) +#else +# define _CCCL_UNREACHABLE() __builtin_unreachable() +#endif + +#endif // __CCCL_UNREACHABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/version.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/version.h new file mode 100644 index 00000000..781c7125 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/version.h @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// +// This file is somewhat automatically generated. Disable clang-format. +// clang-format off + + +#ifndef __CCCL_VERSION_H +#define __CCCL_VERSION_H + +#define CCCL_VERSION 3005000 +#define CCCL_MAJOR_VERSION (CCCL_VERSION / 1000000) +#define CCCL_MINOR_VERSION (((CCCL_VERSION / 1000) % 1000)) +#define CCCL_PATCH_VERSION (CCCL_VERSION % 1000) + +#if CCCL_PATCH_VERSION > 99 +# error "CCCL patch version cannot be greater than 99 for compatibility with Thrust/CUB's MMMmmmpp format." +#endif + +#endif // __CCCL_VERSION_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/visibility.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/visibility.h new file mode 100644 index 00000000..3cfba491 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cccl/visibility.h @@ -0,0 +1,198 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef __CCCL_VISIBILITY_H +#define __CCCL_VISIBILITY_H + +#ifndef _CUDA__CCCL_CONFIG +# error "<__cccl/visibility.h> should only be included in from " +#endif // _CUDA__CCCL_CONFIG + +#include +#include + +// We want to ensure that all warning emitting from this header are suppressed +#if defined(_CCCL_FORCE_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_FORCE_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_FORCE_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +// For unknown reasons, nvc++ need to selectively disable this warning +// We do not want to use our usual macro because that would have push / pop semantics +#if _CCCL_COMPILER(NVHPC) +# pragma nv_diag_suppress 1407 +#endif // _CCCL_COMPILER(NVHPC) + +// Enable us to hide kernels +#if _CCCL_OS(WINDOWS) || _CCCL_COMPILER(NVRTC) +# define _CCCL_VISIBILITY_HIDDEN +#else // ^^^ _CCCL_COMPILER(NVRTC) ^^^ / vvv _CCCL_COMPILER(NVRTC) vvv +# define _CCCL_VISIBILITY_HIDDEN __attribute__((__visibility__("hidden"))) +#endif // !_CCCL_COMPILER(NVRTC) + +#if _CCCL_COMPILER(NVRTC) +# define _CCCL_VISIBILITY_DEFAULT +#elif _CCCL_OS(WINDOWS) +# define _CCCL_VISIBILITY_DEFAULT __declspec(dllimport) +#else // ^^^ _CCCL_COMPILER(NVRTC) ^^^ / vvv !_CCCL_COMPILER(NVRTC) vvv +# define _CCCL_VISIBILITY_DEFAULT __attribute__((__visibility__("default"))) +#endif // !_CCCL_COMPILER(NVRTC) + +#if _CCCL_COMPILER(NVRTC) +# define _CCCL_VISIBILITY_EXPORT +#elif _CCCL_OS(WINDOWS) +# define _CCCL_VISIBILITY_EXPORT __declspec(dllexport) +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_VISIBILITY_EXPORT _CCCL_VISIBILITY_DEFAULT +#endif // !_CCCL_COMPILER(MSVC) + +#if _CCCL_OS(WINDOWS) || _CCCL_COMPILER(NVRTC) +# define _CCCL_TYPE_VISIBILITY_DEFAULT +# define _CCCL_TYPE_VISIBILITY_HIDDEN +#elif _CCCL_HAS_ATTRIBUTE(__type_visibility__) +# define _CCCL_TYPE_VISIBILITY_DEFAULT __attribute__((__type_visibility__("default"))) +# define _CCCL_TYPE_VISIBILITY_HIDDEN __attribute__((__type_visibility__("hidden"))) +#else // ^^^ _CCCL_HAS_ATTRIBUTE(__type_visibility__) ^^^ / vvv !_CCCL_HAS_ATTRIBUTE(__type_visibility__) vvv +# define _CCCL_TYPE_VISIBILITY_DEFAULT _CCCL_VISIBILITY_DEFAULT +# define _CCCL_TYPE_VISIBILITY_HIDDEN _CCCL_VISIBILITY_HIDDEN +#endif // !_CCCL_COMPILER(NVRTC) + +#if _CCCL_COMPILER(MSVC) +# define _CCCL_FORCEINLINE __forceinline +# define _CCCL_FORCEINLINE_LAMBDA +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define _CCCL_FORCEINLINE __inline__ __attribute__((__always_inline__)) +# define _CCCL_FORCEINLINE_LAMBDA __attribute__((__always_inline__)) +#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^ + +#if _CCCL_COMPILER(NVRTC) +# define _CCCL_NOINLINE __attribute__((noinline)) +#elif _CCCL_OS(WINDOWS) +# define _CCCL_NOINLINE __declspec(noinline) +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv _CCCL_COMPILER(MSVC) vvv +// We can't use __noinline__ here because of CTK defining this macro. +# define _CCCL_NOINLINE __attribute__((noinline)) +#endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^ + +#if _CCCL_DEVICE_COMPILATION() +# define _CCCL_NOINLINE_DEVICE _CCCL_NOINLINE +#else // ^^^ _CCCL_DEVICE_COMPILATION() ^^^ / vvv !_CCCL_DEVICE_COMPILATION() vvv +# define _CCCL_NOINLINE_DEVICE +#endif // ^^^ !_CCCL_DEVICE_COMPILATION() ^^^ + +#if _CCCL_HAS_ATTRIBUTE(__exclude_from_explicit_instantiation__) +# define _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION __attribute__((__exclude_from_explicit_instantiation__)) +#else // ^^^ exclude_from_explicit_instantiation ^^^ / vvv !exclude_from_explicit_instantiation vvv +// NVCC complains mightily about being unable to inline functions if we use _CCCL_FORCEINLINE here +# define _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION +#endif // !exclude_from_explicit_instantiation + +#if _CCCL_COMPILER(NVHPC) // NVHPC has issues with visibility attributes on symbols with internal linkage +# define _CCCL_HIDE_FROM_ABI inline +#else // ^^^ _CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_COMPILER(NVHPC) vvv +# define _CCCL_HIDE_FROM_ABI _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION inline +#endif // !_CCCL_COMPILER(NVHPC) + +// Note: we will allow the user to redefine _CCCL_KERNEL_ATTRIBUTES until CCCL 4.0, since they may have +// redefined CUB_DETAIL_KERNEL_ATTRIBUTES or THRUST_DETAIL_KERNEL_ATTRIBUTES. +#if !defined(_CCCL_KERNEL_ATTRIBUTES) +# define _CCCL_KERNEL_ATTRIBUTES __global__ _CCCL_VISIBILITY_HIDDEN +#endif // !_CCCL_KERNEL_ATTRIBUTES + +#if defined(CUB_DETAIL_KERNEL_ATTRIBUTES) || defined(THRUST_DETAIL_KERNEL_ATTRIBUTES) +# error \ + "Redefining CCCL's kernel attributes via CUB_DETAIL_KERNEL_ATTRIBUTES or THRUST_DETAIL_KERNEL_ATTRIBUTES is not allowed. If you absolutely rely on this, you can override them by defining _CCCL_KERNEL_ATTRIBUTES, but this will be disallowed in CCCL 4.0." +#endif // !_CCCL_KERNEL_ATTRIBUTES + +//! @brief \c _CCCL_HIDE_FROM_ABI and \c _CCCL_FORCEINLINE cannot be used together because +//! they both try to add `inline` to the function declaration. The following macros slice +//! the function attributes differently to avoid this problem: +//! - \c _CCCL_API declares the function host/device and hides the symbol from the ABI +//! - \c _CCCL_NODEBUG_API does the same while also hiding the function from +//! debuggers and marking the function as \c inline. +//! - \c _CCCL_TRIVIAL_API does the same as \c _CCCL_NODEBUG_API while also force-inlining +//! the function. +#if _CCCL_COMPILER(NVHPC) // NVHPC has issues with visibility attributes on symbols with internal linkage +# define _CCCL_API _CCCL_HOST_DEVICE +# define _CCCL_HOST_DEVICE_API _CCCL_HOST_DEVICE +# define _CCCL_HOST_API _CCCL_HOST +# define _CCCL_DEVICE_API _CCCL_DEVICE +# define _CCCL_TILE_API _CCCL_TILE +#else // ^^^ _CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_COMPILER(NVHPC) vvv +# define _CCCL_API _CCCL_TILE _CCCL_HOST_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION +# define _CCCL_HOST_DEVICE_API _CCCL_HOST_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION +# define _CCCL_HOST_API _CCCL_HOST _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION +# define _CCCL_DEVICE_API _CCCL_DEVICE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION +# define _CCCL_TILE_API _CCCL_TILE _CCCL_VISIBILITY_HIDDEN _CCCL_EXCLUDE_FROM_EXPLICIT_INSTANTIATION +#endif // !_CCCL_COMPILER(NVHPC) + +//! @brief \c _CCCL_NODEBUG_API marks a function's visibility as hidden and causes +//! debuggers to skip it. This is useful for functions like \c cuda::std::move that +//! debuggers should not step into. If a \c _CCCL_NODEBUG_API function \c F calls a normal +//! function \c G, stepping into \c F in a debugger will skip over \c F and step directly +//! into \c G. In a stacktrace, \c F will still be shone, but you will not be able to +//! set the debugger's active frame to \c F. +#define _CCCL_NODEBUG_API _CCCL_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline +#define _CCCL_NODEBUG_HOST_API _CCCL_HOST_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline +#define _CCCL_NODEBUG_DEVICE_API _CCCL_DEVICE_API _CCCL_ARTIFICIAL _CCCL_NODEBUG inline + +//! @brief \c _CCCL_TRIVIAL_API force-inlines a function, marks its visibility as hidden, +//! and causes debuggers to skip it. This is useful for trivial internal functions that do +//! dispatching or other plumbing work. It is particularly useful in the definition of +//! customization point objects. +#define _CCCL_TRIVIAL_API _CCCL_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE +#define _CCCL_TRIVIAL_HOST_API _CCCL_HOST_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE +#define _CCCL_TRIVIAL_DEVICE_API _CCCL_DEVICE_API _CCCL_ARTIFICIAL _CCCL_NODEBUG _CCCL_FORCEINLINE + +// Some functions have their addresses appear in public types (e.g., in +// `cuda::__overrides_for` specializations). If the function is declared +// `__attribute__((visibility("hidden")))`, and if the address appears, say, in the type +// of a member of a class that is declared `__attribute__((visibility("default")))`, GCC +// complains bitterly. So we avoid declaring those functions `hidden`. Instead of the +// typical `_CCCL_API` macro, we use `_CCCL_PUBLIC_API` for those functions. +#if _CCCL_OS(WINDOWS) +# define _CCCL_PUBLIC_API _CCCL_HOST_DEVICE +# define _CCCL_PUBLIC_HOST_API _CCCL_HOST +# define _CCCL_PUBLIC_DEVICE_API _CCCL_DEVICE +#else // ^^^ _CCCL_OS(WINDOWS) ^^^ / vvv !_CCCL_OS(WINDOWS) vvv +# define _CCCL_PUBLIC_API _CCCL_HOST_DEVICE _CCCL_VISIBILITY_DEFAULT +# define _CCCL_PUBLIC_HOST_API _CCCL_HOST _CCCL_VISIBILITY_DEFAULT +# define _CCCL_PUBLIC_DEVICE_API _CCCL_DEVICE _CCCL_VISIBILITY_DEFAULT +#endif // !_CCCL_OS(WINDOWS) + +#ifdef _CCCL_DOXYGEN_INVOKED // Only for documentation +//! If defined, usage of CUDA Dynamic Parallelism is disabled and APIs launching kernels can only be called from the +//! host +# define CCCL_DISABLE_CDP +#endif // _CCCL_DOXYGEN_INVOKED + +#if _CCCL_HAS_CDP() +// We have CDP, so host and device APIs can call kernels +# define _CCCL_CDP_API _CCCL_API +#else // ^^^ _CCCL_HAS_CDP() ^^^ / vvv !_CCCL_HAS_CDP() vvv +// We don't have CDP, only host APIs can call kernels +# define _CCCL_CDP_API _CCCL_HOST_API +#endif // ^^^ !_CCCL_HAS_CDP() ^^^ + +//! _LIBCUDACXX_HIDE_FROM_ABI is for backwards compatibility for external projects. +//! _CCCL_API and its variants are the preferred way to declare functions +//! that should be hidden from the ABI. +//! Defined here to suppress any warnings from the definition +#define _LIBCUDACXX_HIDE_FROM_ABI _CCCL_API inline + +#endif // __CCCL_VISIBILITY_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/arithmetic.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/arithmetic.h new file mode 100644 index 00000000..403b9031 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/arithmetic.h @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_ARITHMETIC_H +#define _CUDA_STD___CONCEPTS_ARITHMETIC_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// [concepts.arithmetic], arithmetic concepts + +template +_CCCL_CONCEPT integral = is_integral_v<_Tp>; + +template +_CCCL_CONCEPT signed_integral = integral<_Tp> && is_signed_v<_Tp>; + +template +_CCCL_CONCEPT unsigned_integral = integral<_Tp> && !signed_integral<_Tp>; + +template +_CCCL_CONCEPT floating_point = is_floating_point_v<_Tp>; + +template +_CCCL_CONCEPT __cccl_signed_integer = __cccl_is_signed_integer_v<_Tp>; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_ARITHMETIC_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/assignable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/assignable.h new file mode 100644 index 00000000..84994a88 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/assignable.h @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_ASSIGNABLE_H +#define _CUDA_STD___CONCEPTS_ASSIGNABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.assignable] + +template +concept assignable_from = + is_lvalue_reference_v<_Lhs> && common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>> + && requires(_Lhs __lhs, _Rhs&& __rhs) { + { __lhs = ::cuda::std::forward<_Rhs>(__rhs) } -> same_as<_Lhs>; + }; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __assignable_from_, + requires(_Lhs __lhs, + _Rhs&& __rhs)(requires(is_lvalue_reference_v<_Lhs>), + requires(common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>>), + requires(same_as<_Lhs, decltype(__lhs = ::cuda::std::forward<_Rhs>(__rhs))>))); + +template +_CCCL_CONCEPT assignable_from = _CCCL_FRAGMENT(__assignable_from_, _Lhs, _Rhs); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_ASSIGNABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/boolean_testable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/boolean_testable.h new file mode 100644 index 00000000..7b3df15a --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/boolean_testable.h @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_BOOLEAN_TESTABLE_H +#define _CUDA_STD___CONCEPTS_BOOLEAN_TESTABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concepts.booleantestable] + +template +concept __boolean_testable_impl = convertible_to<_Tp, bool>; + +template +concept __boolean_testable = __boolean_testable_impl<_Tp> && requires(_Tp&& __t) { + { !::cuda::std::forward<_Tp>(__t) } -> __boolean_testable_impl; +}; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT __boolean_testable_impl = convertible_to<_Tp, bool>; + +template +_CCCL_CONCEPT_FRAGMENT( + __boolean_testable_, + requires(_Tp&& __t)(requires(__boolean_testable_impl<_Tp>), + requires(__boolean_testable_impl(__t))>))); + +template +_CCCL_CONCEPT __boolean_testable = _CCCL_FRAGMENT(__boolean_testable_, _Tp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_BOOLEAN_TESTABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/class_or_enum.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/class_or_enum.h new file mode 100644 index 00000000..526b1e09 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/class_or_enum.h @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_CLASS_OR_ENUM_H +#define _CUDA_STD___CONCEPTS_CLASS_OR_ENUM_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +_CCCL_CONCEPT __class_or_enum = is_class_v<_Tp> || is_union_v<_Tp> || is_enum_v<_Tp>; + +// Work around Clang bug https://llvm.org/PR52970 +// TODO: remove this workaround once libc++ no longer has to support Clang 13 (it was fixed in Clang 14). +template +_CCCL_CONCEPT __workaround_52970 = is_class_v> || is_union_v>; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_CLASS_OR_ENUM_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/common_reference_with.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/common_reference_with.h new file mode 100644 index 00000000..d680e346 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/common_reference_with.h @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_COMMON_REFERENCE_WITH_H +#define _CUDA_STD___CONCEPTS_COMMON_REFERENCE_WITH_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.commonref] + +template +concept common_reference_with = + same_as, common_reference_t<_Up, _Tp>> + && convertible_to<_Tp, common_reference_t<_Tp, _Up>> && convertible_to<_Up, common_reference_t<_Tp, _Up>>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT(__common_reference_exists_, + requires()(typename(common_reference_t<_Tp, _Up>), typename(common_reference_t<_Up, _Tp>))); + +template +_CCCL_CONCEPT _Common_reference_exists = _CCCL_FRAGMENT(__common_reference_exists_, _Tp, _Up); + +template +_CCCL_CONCEPT_FRAGMENT( + __common_reference_with_, + requires()(requires(_Common_reference_exists<_Tp, _Up>), + requires(same_as, common_reference_t<_Up, _Tp>>), + requires(convertible_to<_Tp, common_reference_t<_Tp, _Up>>), + requires(convertible_to<_Up, common_reference_t<_Tp, _Up>>))); + +template +_CCCL_CONCEPT common_reference_with = _CCCL_FRAGMENT(__common_reference_with_, _Tp, _Up); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_COMMON_REFERENCE_WITH_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/concept_macros.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/concept_macros.h new file mode 100644 index 00000000..c821e022 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/concept_macros.h @@ -0,0 +1,389 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (c) Facebook, Inc. and its affiliates. +// Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA___CONCEPTS_CONCEPT_MACROS_H +#define _CUDA___CONCEPTS_CONCEPT_MACROS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +//////////////////////////////////////////////////////////////////////////////// +// _CCCL_TEMPLATE +// Usage: +// _CCCL_TEMPLATE(typename A, typename _Bp) +// _CCCL_REQUIRES( Concept1 _CCCL_AND Concept2<_Bp>) +// void foo(A a, _Bp b) +// {} + +// Barebones enable if implementation to use outside of cuda::std +template +struct __cccl_select +{}; + +template <> +struct __cccl_select +{ + template + using type = _Tp; +}; + +template +using __cccl_enable_if_t = typename __cccl_select<_Bp>::template type<_Tp>; + +template +using __cccl_requires_t = typename __cccl_select<_Bp>::template type<_Tp>; + +#if _CCCL_HAS_CONCEPTS() || defined(_CCCL_DOXYGEN_INVOKED) +# define _CCCL_TEMPLATE(...) template <__VA_ARGS__> +# define _CCCL_REQUIRES(...) requires __VA_ARGS__ +# define _CCCL_AND && +# define _CCCL_TRAILING_REQUIRES_IMPL_(...) requires __VA_ARGS__ +# define _CCCL_TRAILING_REQUIRES(...) ->__VA_ARGS__ _CCCL_TRAILING_REQUIRES_IMPL_ +# define _CCCL_CONCEPT concept +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv +# define _CCCL_TEMPLATE(...) template <__VA_ARGS__ +# define _CCCL_REQUIRES(...) , bool __cccl_true_ = true, __cccl_enable_if_t < __VA_ARGS__ && __cccl_true_, int > = 0 > +# define _CCCL_AND &&__cccl_true_, int > = 0, __cccl_enable_if_t < +# define _CCCL_TRAILING_REQUIRES(...) ->__cccl_requires_t < __VA_ARGS__ _CCCL_TRAILING_REQUIRES_IMPL_ +# define _CCCL_TRAILING_REQUIRES_IMPL_(...) , __VA_ARGS__ > +# define _CCCL_CONCEPT inline constexpr bool +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +// The following concepts emulation macros need variable template support + +template +struct __cccl_tag; + +template +_CCCL_API constexpr bool __cccl_is_true() +{ + return true; +} + +#if _CCCL_COMPILER(MSVC) +template +_CCCL_API inline __cccl_enable_if_t<_Bp> __cccl_requires() +{} +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +template = 0> +inline constexpr int __cccl_requires = 0; +#endif // !_CCCL_COMPILER(MSVC) + +template +extern _Tp __cccl_make_dependent; + +template +using __cccl_requires_expr_impl = decltype(__cccl_make_dependent<_Impl, _Args...>); + +template +_CCCL_API constexpr void __cccl_unused(_Tp&&) noexcept +{} + +// So that we can refer to the ::cuda::std namespace below +_CCCL_BEGIN_NAMESPACE_CUDA_STD +_CCCL_END_NAMESPACE_CUDA_STD + +// We put an alias for ::cuda::std here because of a bug in nvcc <12.2 +// where a requirement such as: +// +// { expression } -> ::concept +// +// where ::concept is a fully qualified name, would not compile. The +// ::cuda::std macro is fully qualified. +namespace __cccl_unqualified_cuda_std = ::cuda::std; // NOLINT(misc-unused-alias-decls) + +#if _CCCL_CUDACC_BELOW(12, 2) +# define _CCCL_CONCEPT_VSTD __cccl_unqualified_cuda_std // must not be fully qualified +#else +# define _CCCL_CONCEPT_VSTD ::cuda::std +#endif + +// GCC < 14 can't mangle noexcept expressions. See +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70790. +#if _CCCL_COMPILER(GCC, <, 14) +# define _CCCL_HAS_NOEXCEPT_MANGLING() 0 +#else +# define _CCCL_HAS_NOEXCEPT_MANGLING() 1 +#endif + +// We use this macro to ignore the result of required expressions. It is needed because +// gcc < 10 complains about ignored [[nodiscard]] expressions when emulating concepts. +#if _CCCL_COMPILER(GCC, <, 10) +# define _CCCL_CONCEPT_IGNORE_RESULT_(...) static_cast(__VA_ARGS__) +#else +# define _CCCL_CONCEPT_IGNORE_RESULT_(...) __VA_ARGS__ +#endif + +// The "0" or "1" suffixes indicate whether _REQ is parenthesized or not. +#define _CCCL_CONCEPT_REQUIREMENT_0(_REQ) _CCCL_PP_SWITCH(_CCCL_CONCEPT_REQUIREMENT, _REQ) +#define _CCCL_CONCEPT_REQUIREMENT_1(_REQ) _CCCL_CONCEPT_IGNORE_RESULT_ _REQ + +// Permissible requirements are of the form (where ... indicates that the pattern can +// contain commas): +// +// - EXPR +// - (EXPR...) +// - noexcept(EXPR...) +// - requires(BOOL-EXPR...) +// - typename(TYPE...) +// - _Same_as(TYPE...) EXPR... +// - _Satisfies(CONCEPT...) EXPR... +// +// The last 4 are handled below: +#define _CCCL_CONCEPT_REQUIREMENT_SWITCH_requires _CCCL_PP_CASE(_CCCL_SWITCH_REQUIRES) +#define _CCCL_CONCEPT_REQUIREMENT_SWITCH_noexcept _CCCL_PP_CASE(_CCCL_SWITCH_NOEXCEPT) +#define _CCCL_CONCEPT_REQUIREMENT_SWITCH_typename _CCCL_PP_CASE(_CCCL_SWITCH_TYPENAME) +#define _CCCL_CONCEPT_REQUIREMENT_SWITCH__Same_as _CCCL_PP_CASE(_CCCL_SWITCH_SAME_AS) +#define _CCCL_CONCEPT_REQUIREMENT_SWITCH__Satisfies _CCCL_PP_CASE(_CCCL_SWITCH_SATISFIES) + +// Converts "requires(ARGS...)" to "ARGS..." +#define _CCCL_CONCEPT_EAT_REQUIRES_(...) _CCCL_PP_CAT(_CCCL_CONCEPT_EAT_REQUIRES_, __VA_ARGS__) +#define _CCCL_CONCEPT_EAT_REQUIRES_requires(...) __VA_ARGS__ + +// Converts "noexcept(ARGS...)" to "ARGS..." +#define _CCCL_CONCEPT_EAT_NOEXCEPT_(...) _CCCL_PP_CAT(_CCCL_CONCEPT_EAT_NOEXCEPT_, __VA_ARGS__) +#define _CCCL_CONCEPT_EAT_NOEXCEPT_noexcept(...) __VA_ARGS__ + +// Converts "typename(TYPE...)" to "TYPE..." +#define _CCCL_CONCEPT_EAT_TYPENAME_(_REQ) _CCCL_PP_CAT2(_CCCL_CONCEPT_EAT_TYPENAME_, _REQ) +#define _CCCL_CONCEPT_EAT_TYPENAME_typename(...) __VA_ARGS__ + +// Converts "[typename]opt TYPE..." to "typename TYPE..." +#define _CCCL_CONCEPT_TRY_ADD_TYPENAME_(...) _CCCL_PP_SWITCH2(_CCCL_CONCEPT_TRY_ADD_TYPENAME, __VA_ARGS__) +#define _CCCL_CONCEPT_TRY_ADD_TYPENAME_SWITCH_typename _CCCL_PP_CASE(_CCCL_SWITCH_TYPENAME) +#define _CCCL_CONCEPT_TRY_ADD_TYPENAME_CASE__CCCL_SWITCH_DEFAULT(...) typename __VA_ARGS__ +#define _CCCL_CONCEPT_TRY_ADD_TYPENAME_CASE__CCCL_SWITCH_TYPENAME(...) __VA_ARGS__ + +// Converts "_Same_as(TYPE) EXPR..." to "EXPR..." +#define _CCCL_CONCEPT_EAT_SAME_AS_(...) _CCCL_PP_CAT(_CCCL_CONCEPT_EAT_SAME_AS_, __VA_ARGS__) +#define _CCCL_CONCEPT_EAT_SAME_AS__Same_as(...) + +// Converts "_Same_as(TYPE) EXPR..." to "TYPE" (The ridiculous concatenation of _CCCL with +// _PP_EXPAND(__VA_ARGS__) is the only way to get MSVC's broken preprocessor to do macro +// expansion here.) +#define _CCCL_CONCEPT_GET_TYPE_FROM_SAME_AS_(...) \ + _CCCL_PP_CAT(_CCCL, _CCCL_PP_EVAL(_CCCL_PP_FIRST, _CCCL_PP_CAT(_CCCL_CONCEPT_GET_TYPE_FROM_SAME_AS_, __VA_ARGS__))) +#define _CCCL_CONCEPT_GET_TYPE_FROM_SAME_AS__Same_as(...) _PP_EXPAND(__VA_ARGS__), + +// Converts "_Satisfies(TYPE) EXPR..." to "EXPR..." +#define _CCCL_CONCEPT_EAT_SATISFIES_(...) _CCCL_PP_CAT(_CCCL_CONCEPT_EAT_SATISFIES_, __VA_ARGS__) +#define _CCCL_CONCEPT_EAT_SATISFIES__Satisfies(...) + +// Converts "_Satisfies(TYPE) EXPR..." to "TYPE" (The ridiculous concatenation of _CCCL +// with _PP_EXPAND(__VA_ARGS__) is the only way to get MSVC's broken preprocessor to do macro +// expansion here.) +#define _CCCL_CONCEPT_GET_CONCEPT_FROM_SATISFIES_(...) \ + _CCCL_PP_CAT(_CCCL, \ + _CCCL_PP_EVAL(_CCCL_PP_FIRST, _CCCL_PP_CAT(_CCCL_CONCEPT_GET_CONCEPT_FROM_SATISFIES_, __VA_ARGS__))) +#define _CCCL_CONCEPT_GET_CONCEPT_FROM_SATISFIES__Satisfies(...) _PP_EXPAND(__VA_ARGS__), + +// Here are the implementations of the internal macros, first for when concepts +// are available, and then for when they're not. +#if _CCCL_HAS_CONCEPTS() || defined(_CCCL_DOXYGEN_INVOKED) + +// "_CCCL_CONCEPT_FRAGMENT(NAME, ARGS...)(REQS...)" expands into +// "concept NAME = requires(ARGS...) { _CCCL_CONCEPT_REQUIREMENT_(REQS)... }" +# define _CCCL_CONCEPT_FRAGMENT(_NAME, ...) concept _NAME = _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_##__VA_ARGS__ +# define _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_requires(...) requires(__VA_ARGS__) _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_ +# define _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_(...) {_CCCL_PP_FOR_EACH(_CCCL_CONCEPT_REQUIREMENT_, __VA_ARGS__)} + +// Converts "EXPR" to "_CCCL_CONCEPT_REQUIREMENT_0(EXPR)", and +// "(EXPR)" to "_CCCL_CONCEPT_REQUIREMENT_1((EXPR))" +# define _CCCL_CONCEPT_REQUIREMENT_(_REQ) \ + _CCCL_PP_CAT(_CCCL_CONCEPT_REQUIREMENT_, _CCCL_PP_IS_PAREN(_REQ)) \ + (_REQ); + +// The following macros handle the various special forms of requirements: +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_DEFAULT(_REQ) _REQ +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_REQUIRES(_REQ) requires _CCCL_CONCEPT_EAT_REQUIRES_(_REQ) +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_NOEXCEPT(_REQ) \ + _CCCL_PP_EXPAND({ _CCCL_CONCEPT_EAT_NOEXCEPT_(_REQ) } noexcept) +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_TYPENAME(_REQ) \ + _CCCL_CONCEPT_TRY_ADD_TYPENAME_(_CCCL_CONCEPT_EAT_TYPENAME_(_REQ)) +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_SAME_AS(_REQ) \ + {_CCCL_CONCEPT_EAT_SAME_AS_(_REQ)}->_CCCL_CONCEPT_VSTD::same_as<_CCCL_CONCEPT_GET_TYPE_FROM_SAME_AS_(_REQ)> +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_SATISFIES(_REQ) \ + {_CCCL_CONCEPT_EAT_SATISFIES_(_REQ)}->_CCCL_CONCEPT_GET_CONCEPT_FROM_SATISFIES_(_REQ) + +# define _CCCL_FRAGMENT(_NAME, ...) _NAME<__VA_ARGS__> + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// "_CCCL_CONCEPT_FRAGMENT(Foo, ARGS...)(REQS...)" expands into: +// +// _CCCL_API inline auto Foo_CCCL_CONCEPT_FRAGMENT_impl_(ARGS...) +// -> __cccl_enable_if_t< +// ::__cccl_is_true()> +// {} +// +// template +// _CCCL_API inline auto Foo_CCCL_CONCEPT_FRAGMENT_(::__cccl_tag*, +// decltype(&Foo_CCCL_CONCEPT_FRAGMENT_impl_)) +// -> char(&)[1]; +// +// template +// _CCCL_API inline auto Foo_CCCL_CONCEPT_FRAGMENT_(...) +// -> char(&)[2] +// +# define _CCCL_CONCEPT_FRAGMENT(_NAME, ...) \ + _CCCL_API inline auto _NAME##_CCCL_CONCEPT_FRAGMENT_impl_ _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_##__VA_ARGS__> {} \ + template \ + _CCCL_API inline auto _NAME##_CCCL_CONCEPT_FRAGMENT_( \ + ::__cccl_tag<_As...>*, decltype(&_NAME##_CCCL_CONCEPT_FRAGMENT_impl_<_As...>)) -> char (&)[1]; \ + _CCCL_API inline auto _NAME##_CCCL_CONCEPT_FRAGMENT_(...) -> char (&)[2] +# define _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_requires(...) \ + (__VA_ARGS__)->__cccl_enable_if_t < _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_IMPL_ +# define _CCCL_CONCEPT_FRAGMENT_REQUIREMENTS_IMPL_(...) \ + ::__cccl_is_true() + +// Called with each individual requirement in the list of requirements +# define _CCCL_CONCEPT_REQUIREMENT_(_REQ) \ + void(), _CCCL_PP_CAT(_CCCL_CONCEPT_REQUIREMENT_, _CCCL_PP_IS_PAREN(_REQ))(_REQ), + +// The following macros handle the various special forms of requirements: +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_DEFAULT(_REQ) _CCCL_CONCEPT_IGNORE_RESULT_(_REQ) +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_REQUIRES(_REQ) \ + ::__cccl_requires<_CCCL_CONCEPT_EAT_REQUIRES_(_REQ)> +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_NOEXCEPT(_REQ) _CCCL_CONCEPT_NOEXCEPT_REQUIREMENT_(_REQ) +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_TYPENAME(_REQ) \ + static_cast<::__cccl_tag<_CCCL_CONCEPT_EAT_TYPENAME_(_REQ)>*>(nullptr) +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_SAME_AS(_REQ) \ + ::__cccl_requires<::cuda::std::same_as<_CCCL_CONCEPT_SAME_AS_REQUIREMENT_(_REQ)>> +# define _CCCL_CONCEPT_REQUIREMENT_CASE__CCCL_SWITCH_SATISFIES(_REQ) \ + ::__cccl_requires < _CCCL_CONCEPT_GET_CONCEPT_FROM_SATISFIES_(_REQ) < decltype(_CCCL_CONCEPT_EAT_SATISFIES_(_REQ)) \ + >> + +// Converts "_Same_as(TYPE) EXPR..." to "TYPE, decltype(EXPR...)" +# define _CCCL_CONCEPT_SAME_AS_REQUIREMENT_(_REQ) \ + _CCCL_CONCEPT_GET_TYPE_FROM_SAME_AS_(_REQ), decltype(_CCCL_CONCEPT_EAT_SAME_AS_(_REQ)) + +# if _CCCL_HAS_NOEXCEPT_MANGLING() +// Converts "noexcept(EXPR)" to "::__cccl_requires" +# define _CCCL_CONCEPT_NOEXCEPT_REQUIREMENT_(_REQ) ::__cccl_requires<_REQ> +# else +// If the compiler cannot mangle noexcept expressions, just check that the expression is +// well-formed. This converts "noexcept(EXPR)" to "static_cast(EXPR)" +# define _CCCL_CONCEPT_NOEXCEPT_REQUIREMENT_(_REQ) _CCCL_CONCEPT_IGNORE_RESULT_(_CCCL_CONCEPT_EAT_NOEXCEPT_(_REQ)) +# endif + +// "_CCCL_FRAGMENT(Foo, Args...)" expands to +// "(1 == sizeof(Foo_CCCL_CONCEPT_FRAGMENT_(static_cast<::__cccl_tag*>(nullptr), nullptr)))" +# define _CCCL_FRAGMENT(_NAME, ...) \ + (1 == sizeof(_NAME##_CCCL_CONCEPT_FRAGMENT_(static_cast<::__cccl_tag<__VA_ARGS__>*>(nullptr), nullptr))) + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +//////////////////////////////////////////////////////////////////////////////// +// _CCCL_REQUIRES_EXPR +// Usage: +// template +// _CCCL_CONCEPT equality_comparable = +// _CCCL_REQUIRES_EXPR((T), T const& lhs, T const& rhs) ( +// lhs == rhs, +// lhs != rhs +// ); +// +// Can only be used as the last requirement in a concept definition. +#if _CCCL_HAS_CONCEPTS() || defined(_CCCL_DOXYGEN_INVOKED) + +# define _CCCL_REQUIRES_EXPR(_TY, ...) requires(__VA_ARGS__) _CCCL_REQUIRES_EXPR_IMPL_ +# define _CCCL_REQUIRES_EXPR_IMPL_(...) {_CCCL_PP_FOR_EACH(_CCCL_CONCEPT_REQUIREMENT_, __VA_ARGS__)} + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +# define _CCCL_REQUIRES_EXPR(_TY, ...) _CCCL_REQUIRES_EXPR_IMPL(_TY, _CCCL_REQUIRES_EXPR_ID(_TY), __VA_ARGS__) +# define _CCCL_REQUIRES_EXPR_IMPL(_TY, _ID, ...) \ + ::__cccl_requires_expr_impl< \ + struct _CCCL_PP_CAT(__cccl_requires_expr_detail_, _ID) _CCCL_REQUIRES_EXPR_TPARAM_REFS \ + _TY>::__cccl_is_satisfied(static_cast<::__cccl_tag*>(nullptr), 0); \ + struct _CCCL_PP_CAT(__cccl_requires_expr_detail_, _ID) \ + { \ + using __cccl_self_t = _CCCL_PP_CAT(__cccl_requires_expr_detail_, _ID); \ + template \ + _CCCL_API inline static auto __cccl_well_formed(__VA_ARGS__) _CCCL_REQUIRES_EXPR_REQUIREMENTS_ + +// Expands "T1, T2, variadic T3" to ", class T1, class T2, class... T3" +# define _CCCL_REQUIRES_EXPR_TPARAM_DEFNS(...) _CCCL_PP_FOR_EACH(_CCCL_REQUIRES_EXPR_TPARAM_DEFN, __VA_ARGS__) + +// Expands "TY" to ", class TY" and "variadic TY" to ", class... TY" +# define _CCCL_REQUIRES_EXPR_TPARAM_DEFN(_TY) , _CCCL_PP_SWITCH2(_CCCL_REQUIRES_EXPR_TPARAM_DEFN, _TY) +# define _CCCL_REQUIRES_EXPR_TPARAM_DEFN_SWITCH_variadic _CCCL_PP_CASE(_CCCL_SWITCH_VARIADIC) +# define _CCCL_REQUIRES_EXPR_TPARAM_DEFN_CASE__CCCL_SWITCH_DEFAULT(_TY) class _TY +# define _CCCL_REQUIRES_EXPR_TPARAM_DEFN_CASE__CCCL_SWITCH_VARIADIC(_TY) \ + class... _CCCL_PP_CAT(_CCCL_REQUIRES_EXPR_EAT_VARIADIC_, _TY) + +// Expands "T1, T2, variadic T3" to ", T1, T2, T3..." +# define _CCCL_REQUIRES_EXPR_TPARAM_REFS(...) _CCCL_PP_FOR_EACH(_CCCL_REQUIRES_EXPR_TPARAM_REF, __VA_ARGS__) + +// Expands "TY" to ", TY" and "variadic TY" to ", TY..." +# define _CCCL_REQUIRES_EXPR_TPARAM_REF(_TY) , _CCCL_PP_SWITCH2(_CCCL_REQUIRES_EXPR_TPARAM_REF, _TY) +# define _CCCL_REQUIRES_EXPR_TPARAM_REF_SWITCH_variadic _CCCL_PP_CASE(_CCCL_SWITCH_VARIADIC) +# define _CCCL_REQUIRES_EXPR_TPARAM_REF_CASE__CCCL_SWITCH_DEFAULT(_TY) _TY +# define _CCCL_REQUIRES_EXPR_TPARAM_REF_CASE__CCCL_SWITCH_VARIADIC(_TY) \ + _CCCL_PP_CAT(_CCCL_REQUIRES_EXPR_EAT_VARIADIC_, _TY)... + +// NVRTC does not support __COUNTER__ so we need a better way of defining unique identifiers +# if _CCCL_COMPILER(NVRTC) + +// Expands ((Ty...), Ty...) into _CCCL_REQUIRES_EXPR_ID_NO_PAREN(Ty...) +# define _CCCL_REQUIRES_EXPR_ID(_TY, ...) _CCCL_REQUIRES_EXPR_ID_NO_PAREN _TY + +// Expands "T1, T2, variadic T3" to "T1_T2_T3_##__LINE__" +# define _CCCL_REQUIRES_EXPR_ID_NO_PAREN(...) \ + _CCCL_REQUIRES_EXPR_ID_CONCAT_ALL(_CCCL_PP_FOR_EACH(_CCCL_REQUIRES_EXPR_ID_IMPL, __VA_ARGS__), _CCCL_COUNTER()) + +// Expands "T1, T2, T3" to "T1T2T3" +# define _CCCL_REQUIRES_EXPR_ID_CONCAT_ALL_IMPL(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, ...) \ + _0##_1##_2##_3##_4##_5##_6##_7##_8##_9 +# define _CCCL_REQUIRES_EXPR_ID_CONCAT_ALL(...) \ + _CCCL_PP_EVAL(_CCCL_REQUIRES_EXPR_ID_CONCAT_ALL_IMPL, __VA_ARGS__, , , , , , , , , ) + +// Expands "TY" to "TY" and "variadic TY" to "TY" +# define _CCCL_REQUIRES_EXPR_ID_IMPL(_TY) , _CCCL_PP_SWITCH2(_CCCL_REQUIRES_EXPR_ID_IMPL, _TY) +# define _CCCL_REQUIRES_EXPR_ID_IMPL_SWITCH_variadic _CCCL_PP_CASE(_CCCL_SWITCH_VARIADIC) +# define _CCCL_REQUIRES_EXPR_ID_IMPL_CASE__CCCL_SWITCH_DEFAULT(_TY) _TY +# define _CCCL_REQUIRES_EXPR_ID_IMPL_CASE__CCCL_SWITCH_VARIADIC(_TY) \ + _CCCL_PP_CAT(_CCCL_REQUIRES_EXPR_EAT_VARIADIC_, _TY) + +# else // ^^^ _CCCL_COMPILER(NVRTC) ^^^^/ vvv !_CCCL_COMPILER(NVRTC) +# define _CCCL_REQUIRES_EXPR_ID(...) _CCCL_COUNTER() +# endif // !_CCCL_COMPILER(NVRTC) + +# define _CCCL_REQUIRES_EXPR_EAT_VARIADIC_variadic + +# define _CCCL_REQUIRES_EXPR_REQUIREMENTS_(...) \ + ->decltype(_CCCL_PP_FOR_EACH(_CCCL_CONCEPT_REQUIREMENT_, __VA_ARGS__) void()) {} \ + template )> \ + _CCCL_API static constexpr bool __cccl_is_satisfied(::__cccl_tag<_Args...>*, int) \ + { \ + return true; \ + } \ + _CCCL_API static constexpr bool __cccl_is_satisfied(void*, long) \ + { \ + return false; \ + } \ + } +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +#include + +#endif //_CUDA___CONCEPTS_CONCEPT_MACROS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/constructible.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/constructible.h new file mode 100644 index 00000000..8eb98baf --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/constructible.h @@ -0,0 +1,174 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_CONSTRUCTIBLE_H +#define _CUDA_STD___CONCEPTS_CONSTRUCTIBLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.constructible] +template +concept constructible_from = destructible<_Tp> && is_constructible_v<_Tp, _Args...>; + +// [concept.default.init] +template +concept __default_initializable = requires { ::new _Tp; }; + +template +concept default_initializable = constructible_from<_Tp> && requires { _Tp{}; } && __default_initializable<_Tp>; + +// [concept.moveconstructible] +template +concept move_constructible = constructible_from<_Tp, _Tp> && convertible_to<_Tp, _Tp>; + +// [concept.copyconstructible] +template +concept copy_constructible = + move_constructible<_Tp> && constructible_from<_Tp, _Tp&> && convertible_to<_Tp&, _Tp> + && constructible_from<_Tp, const _Tp&> && convertible_to && constructible_from<_Tp, const _Tp> + && convertible_to; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT(__constructible_from_, + requires()(requires(destructible<_Tp>), requires(is_constructible_v<_Tp, _Args...>))); + +template +_CCCL_CONCEPT constructible_from = _CCCL_FRAGMENT(__constructible_from_, _Tp, _Args...); + +template +_CCCL_CONCEPT_FRAGMENT(__default_initializable_, requires()((::new _Tp))); + +template +_CCCL_CONCEPT __default_initializable = _CCCL_FRAGMENT(__default_initializable_, _Tp); + +template +_CCCL_CONCEPT_FRAGMENT(_Default_initializable_, + requires(_Tp = _Tp{})(requires(constructible_from<_Tp>), requires(__default_initializable<_Tp>))); + +template +_CCCL_CONCEPT default_initializable = _CCCL_FRAGMENT(_Default_initializable_, _Tp); + +// [concept.moveconstructible] +template +_CCCL_CONCEPT_FRAGMENT(__move_constructible_, + requires()(requires(constructible_from<_Tp, _Tp>), requires(convertible_to<_Tp, _Tp>))); + +template +_CCCL_CONCEPT move_constructible = _CCCL_FRAGMENT(__move_constructible_, _Tp); + +// [concept.copyconstructible] +template +_CCCL_CONCEPT_FRAGMENT( + __copy_constructible_, + requires()( + requires(move_constructible<_Tp>), + requires(constructible_from<_Tp, add_lvalue_reference_t<_Tp>>&& convertible_to, _Tp>), + requires(constructible_from<_Tp, const add_lvalue_reference_t<_Tp>>&& + convertible_to, _Tp>), + requires(constructible_from<_Tp, const _Tp>&& convertible_to))); + +template +_CCCL_CONCEPT copy_constructible = _CCCL_FRAGMENT(__copy_constructible_, _Tp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +//! The code below provides the following concepts in the ::cuda:: namespace: +//! +//! - `__list_initializable_from` +//! - `__nothrow_list_initializable_from` +//! - `__initializable_from` +//! - `__nothrow_initializable_from` +//! - `__emplaceable_from` +//! - `__nothrow_emplaceable_from` + +_CCCL_BEGIN_NAMESPACE_CUDA + +// constructible_from using list initialization syntax. +template +_CCCL_CONCEPT __list_initializable_from = + _CCCL_REQUIRES_EXPR((_Tp, variadic _Args), _Args&&... __args)(_Tp{static_cast<_Args&&>(__args)...}); + +template +_CCCL_CONCEPT __nothrow_list_initializable_from = + _CCCL_REQUIRES_EXPR((_Tp, variadic _Args), _Args&&... __args)(noexcept(_Tp{static_cast<_Args&&>(__args)...})); + +//! Constructible from arguments using either direct non-list initialization or direct +//! list initialization. +template +_CCCL_CONCEPT __initializable_from = + ::cuda::std::constructible_from<_Tp, _Args...> || __list_initializable_from<_Tp, _Args...>; + +template +_CCCL_CONCEPT __nothrow_initializable_from = + __initializable_from<_Tp, _Args...> + && (::cuda::std::constructible_from<_Tp, _Args...> + ? ::cuda::std::is_nothrow_constructible_v<_Tp, _Args...> + : __nothrow_list_initializable_from<_Tp, _Args...>); + +#if !_CCCL_COMPILER(MSVC) && !_CCCL_CUDA_COMPILER(NVCC, <, 12, 9) + +//! Constructible with direct non-list initialization syntax from the result of +//! a function call expression (often useful for immovable types). +template +_CCCL_CONCEPT __emplaceable_from = _CCCL_REQUIRES_EXPR((_Tp, _Fn, variadic _Args), _Fn&& __fn, _Args&&... __args)( + _Tp(static_cast<_Fn&&>(__fn)(static_cast<_Args&&>(__args)...))); + +template +_CCCL_CONCEPT __nothrow_emplaceable_from = + _CCCL_REQUIRES_EXPR((_Tp, _Fn, variadic _Args), _Fn&& __fn, _Args&&... __args)( + noexcept(_Tp(static_cast<_Fn&&>(__fn)(static_cast<_Args&&>(__args)...)))); + +#else // ^^^ !_CCCL_COMPILER(MSVC) ^^^ / vvv _CCCL_COMPILER(MSVC) vvv + +//! Constructible with direct non-list initialization syntax from the result of +//! a function call expression (often useful for immovable types). MSVC cannot +//! use the above formulation because it has poor support for deferred materialization +//! of temporary object (aka, guaranteed copy elision). +template +_CCCL_CONCEPT __emplaceable_from = _CCCL_REQUIRES_EXPR((_Tp, _Fn, variadic _Args), _Fn&& __fn, _Args&&... __args)( + _Same_as(_Tp) static_cast<_Fn&&>(__fn)(static_cast<_Args&&>(__args)...)); + +template +_CCCL_CONCEPT __nothrow_emplaceable_from = + __emplaceable_from<_Tp, _Fn, _Args...> && ::cuda::std::__is_nothrow_callable_v<_Fn, _Args...>; + +#endif // ^^^ _CCCL_COMPILER(MSVC) ^^^ + +_CCCL_END_NAMESPACE_CUDA + +#include + +#endif // _CUDA_STD___CONCEPTS_CONSTRUCTIBLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/convertible_to.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/convertible_to.h new file mode 100644 index 00000000..4e41115a --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/convertible_to.h @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_CONVERTIBLE_TO_H +#define _CUDA_STD___CONCEPTS_CONVERTIBLE_TO_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// [concept.convertible] + +#if _CCCL_HAS_CONCEPTS() + +template +concept convertible_to = is_convertible_v<_From, _To> && requires { static_cast<_To>(::cuda::std::declval<_From>()); }; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +# if _CCCL_COMPILER(MSVC) +_CCCL_BEGIN_NV_DIAG_SUPPRESS(1211) // nonstandard cast to array type ignored +# endif // _CCCL_COMPILER(MSVC) +_CCCL_BEGIN_NV_DIAG_SUPPRESS(171) // invalid type conversion, e.g. [with _From=int **, _To=const int *const *] + +// We cannot put this conversion check with the other constraint, as types with deleted operator will break here +template +_CCCL_CONCEPT_FRAGMENT(__test_conversion_, requires()(static_cast<_To>(::cuda::std::declval<_From>()))); + +template +_CCCL_CONCEPT __test_conversion = _CCCL_FRAGMENT(__test_conversion_, _From, _To); + +template +_CCCL_CONCEPT_FRAGMENT(__convertible_to_, + requires()(requires(is_convertible_v<_From, _To>), requires(__test_conversion<_From, _To>))); + +template +_CCCL_CONCEPT convertible_to = _CCCL_FRAGMENT(__convertible_to_, _From, _To); + +# if _CCCL_COMPILER(MSVC) +_CCCL_END_NV_DIAG_SUPPRESS() // nonstandard cast to array type ignored +# endif // _CCCL_COMPILER(MSVC) +_CCCL_END_NV_DIAG_SUPPRESS() // invalid type conversion, e.g. [with _From=int **, _To=const int *const *] + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_CONVERTIBLE_TO_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/copyable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/copyable.h new file mode 100644 index 00000000..b1a08668 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/copyable.h @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_COPYABLE_H +#define _CUDA_STD___CONCEPTS_COPYABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concepts.object] + +template +concept copyable = copy_constructible<_Tp> && movable<_Tp> && assignable_from<_Tp&, _Tp&> + && assignable_from<_Tp&, const _Tp&> && assignable_from<_Tp&, const _Tp>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __copyable_, + requires()(requires(copy_constructible<_Tp>), + requires(movable<_Tp>), + requires(assignable_from<_Tp&, _Tp&>), + requires(assignable_from<_Tp&, const _Tp&>), + requires(assignable_from<_Tp&, const _Tp>))); + +template +_CCCL_CONCEPT copyable = _CCCL_FRAGMENT(__copyable_, _Tp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_COPYABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/derived_from.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/derived_from.h new file mode 100644 index 00000000..562e35d0 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/derived_from.h @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_DERIVED_FROM_H +#define _CUDA_STD___CONCEPTS_DERIVED_FROM_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.derived] + +template +concept derived_from = is_base_of_v<_Bp, _Dp> && is_convertible_v; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __derived_from_, + requires()(requires(is_base_of_v<_Bp, _Dp>), + requires(is_convertible_v, add_pointer_t>))); + +template +_CCCL_CONCEPT derived_from = _CCCL_FRAGMENT(__derived_from_, _Dp, _Bp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_DERIVED_FROM_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/destructible.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/destructible.h new file mode 100644 index 00000000..9299f48c --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/destructible.h @@ -0,0 +1,76 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_DESTRUCTIBLE_H +#define _CUDA_STD___CONCEPTS_DESTRUCTIBLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_COMPILER(MSVC) + +template +_CCCL_CONCEPT destructible = __is_nothrow_destructible(_Tp); + +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv + +template +inline constexpr bool __destructible_impl = false; + +template +inline constexpr bool __destructible_impl<_Tp, + enable_if_t>, +# if _CCCL_COMPILER(GCC) + enable_if_t>> +# else // ^^^ _CCCL_COMPILER(GCC) ^^^ / vvv !_CCCL_COMPILER(GCC) vvv + void_t().~_Tp())>> +# endif // !_CCCL_COMPILER(GCC) + = noexcept(::cuda::std::declval<_Tp>().~_Tp()); + +template +inline constexpr bool __destructible = __destructible_impl<_Tp>; + +template +inline constexpr bool __destructible<_Tp&> = true; + +template +inline constexpr bool __destructible<_Tp&&> = true; + +template +inline constexpr bool __destructible<_Tp[_Nm]> = __destructible<_Tp>; + +template +_CCCL_CONCEPT destructible = __destructible<_Tp>; + +#endif // !_CCCL_COMPILER(MSVC) + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_DESTRUCTIBLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/equality_comparable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/equality_comparable.h new file mode 100644 index 00000000..fb4c7a65 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/equality_comparable.h @@ -0,0 +1,98 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_EQUALITY_COMPARABLE_H +#define _CUDA_STD___CONCEPTS_EQUALITY_COMPARABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.equalitycomparable] + +template +concept __weakly_equality_comparable_with = + requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) { + { __t == __u } -> __boolean_testable; + { __t != __u } -> __boolean_testable; + { __u == __t } -> __boolean_testable; + { __u != __t } -> __boolean_testable; + }; + +template +concept equality_comparable = __weakly_equality_comparable_with<_Tp, _Tp>; + +template +concept equality_comparable_with = + equality_comparable<_Tp> && equality_comparable<_Up> + && common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> + && equality_comparable, __make_const_lvalue_ref<_Up>>> + && __weakly_equality_comparable_with<_Tp, _Up>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT _With_lvalue_reference = _CCCL_REQUIRES_EXPR((_Tp))(typename(__make_const_lvalue_ref<_Tp>)); + +template +_CCCL_CONCEPT_FRAGMENT( + __weakly_equality_comparable_with_, + requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u)( + requires(_With_lvalue_reference<_Tp>), + requires(_With_lvalue_reference<_Up>), + _Satisfies(__boolean_testable) __t == __u, + _Satisfies(__boolean_testable) __t != __u, + _Satisfies(__boolean_testable) __u == __t, + _Satisfies(__boolean_testable) __u != __t)); + +template +_CCCL_CONCEPT __weakly_equality_comparable_with = _CCCL_FRAGMENT(__weakly_equality_comparable_with_, _Tp, _Up); + +template +_CCCL_CONCEPT equality_comparable = __weakly_equality_comparable_with<_Tp, _Tp>; + +template +_CCCL_CONCEPT_FRAGMENT( + __equality_comparable_with_, + requires()( + requires(equality_comparable<_Tp>), + requires(equality_comparable<_Up>), + requires(common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>>), + requires(equality_comparable, __make_const_lvalue_ref<_Up>>>), + requires(__weakly_equality_comparable_with<_Tp, _Up>))); + +template +_CCCL_CONCEPT equality_comparable_with = _CCCL_FRAGMENT(__equality_comparable_with_, _Tp, _Up); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_EQUALITY_COMPARABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/invocable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/invocable.h new file mode 100644 index 00000000..889147f9 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/invocable.h @@ -0,0 +1,80 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_INVOCABLE_H +#define _CUDA_STD___CONCEPTS_INVOCABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.invocable] + +template +concept invocable = requires(_Fn&& __fn, _Args&&... __args) { + ::cuda::std::invoke(::cuda::std::forward<_Fn>(__fn), ::cuda::std::forward<_Args>(__args)...); // not required to be + // equality preserving +}; + +// [concept.regular.invocable] + +template +concept regular_invocable = invocable<_Fn, _Args...>; + +template +concept __invoke_constructible = requires(_Fun&& __fun, _Args&&... __args) { + static_cast>>( + ::cuda::std::invoke(::cuda::std::forward<_Fun>(__fun), ::cuda::std::forward<_Args>(__args)...)); +}; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT(_Invocable_, + requires(_Fn&& __fn, _Args&&... __args)((::cuda::std::invoke( + ::cuda::std::forward<_Fn>(__fn), ::cuda::std::forward<_Args>(__args)...)))); + +template +_CCCL_CONCEPT invocable = _CCCL_FRAGMENT(_Invocable_, _Fn, _Args...); + +template +_CCCL_CONCEPT regular_invocable = invocable<_Fn, _Args...>; + +template +_CCCL_CONCEPT_FRAGMENT( + __invoke_constructible_, + requires(_Fun&& __fun, _Args&&... __args)((static_cast>>( + ::cuda::std::invoke(::cuda::std::forward<_Fun>(__fun), ::cuda::std::forward<_Args>(__args)...))))); +template +_CCCL_CONCEPT __invoke_constructible = _CCCL_FRAGMENT(__invoke_constructible_, _Fun, _Args...); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_INVOCABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/movable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/movable.h new file mode 100644 index 00000000..fab06e6e --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/movable.h @@ -0,0 +1,58 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_MOVABLE_H +#define _CUDA_STD___CONCEPTS_MOVABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +template +concept movable = is_object_v<_Tp> && move_constructible<_Tp> && assignable_from<_Tp&, _Tp> && swappable<_Tp>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [concepts.object] +template +_CCCL_CONCEPT_FRAGMENT( + _Movable_, + requires()(requires(is_object_v<_Tp>), + requires(move_constructible<_Tp>), + requires(assignable_from<_Tp&, _Tp>), + requires(swappable<_Tp>))); + +template +_CCCL_CONCEPT movable = _CCCL_FRAGMENT(_Movable_, _Tp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_MOVABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/predicate.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/predicate.h new file mode 100644 index 00000000..a481790c --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/predicate.h @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_PREDICATE_H +#define _CUDA_STD___CONCEPTS_PREDICATE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +template +concept predicate = regular_invocable<_Fn, _Args...> && __boolean_testable>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [concept.predicate] +template +_CCCL_CONCEPT_FRAGMENT( + _Predicate_, + requires()(requires(regular_invocable<_Fn, _Args...>), requires(__boolean_testable>))); + +template +_CCCL_CONCEPT predicate = _CCCL_FRAGMENT(_Predicate_, _Fn, _Args...); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_PREDICATE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/regular.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/regular.h new file mode 100644 index 00000000..c912eb73 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/regular.h @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_REGULAR_H +#define _CUDA_STD___CONCEPTS_REGULAR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.object] + +template +concept regular = semiregular<_Tp> && equality_comparable<_Tp>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [concept.object] + +template +_CCCL_CONCEPT_FRAGMENT(__regular_, requires()(requires(semiregular<_Tp>), requires(equality_comparable<_Tp>))); + +template +_CCCL_CONCEPT regular = _CCCL_FRAGMENT(__regular_, _Tp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_REGULAR_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/relation.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/relation.h new file mode 100644 index 00000000..fc2862d0 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/relation.h @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_RELATION_H +#define _CUDA_STD___CONCEPTS_RELATION_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.relation] + +template +concept relation = + predicate<_Rp, _Tp, _Tp> && predicate<_Rp, _Up, _Up> && predicate<_Rp, _Tp, _Up> && predicate<_Rp, _Up, _Tp>; + +// [concept.equiv] + +template +concept equivalence_relation = relation<_Rp, _Tp, _Up>; + +// [concept.strictweakorder] + +template +concept strict_weak_order = relation<_Rp, _Tp, _Up>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __relation_, + requires()(requires(predicate<_Rp, _Tp, _Tp>), + requires(predicate<_Rp, _Up, _Up>), + requires(predicate<_Rp, _Tp, _Up>), + requires(predicate<_Rp, _Up, _Tp>))); + +template +_CCCL_CONCEPT relation = _CCCL_FRAGMENT(__relation_, _Rp, _Tp, _Up); + +// [concept.equiv] + +template +_CCCL_CONCEPT equivalence_relation = relation<_Rp, _Tp, _Up>; + +// [concept.strictweakorder] + +template +_CCCL_CONCEPT strict_weak_order = relation<_Rp, _Tp, _Up>; + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_RELATION_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/same_as.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/same_as.h new file mode 100644 index 00000000..30e09397 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/same_as.h @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_SAME_AS_H +#define _CUDA_STD___CONCEPTS_SAME_AS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// [concept.same] + +template +_CCCL_CONCEPT same_as = is_same_v<_Tp, _Up> && is_same_v<_Up, _Tp>; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_SAME_AS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/semiregular.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/semiregular.h new file mode 100644 index 00000000..091c9561 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/semiregular.h @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_SEMIREGULAR_H +#define _CUDA_STD___CONCEPTS_SEMIREGULAR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.object] + +template +concept semiregular = copyable<_Tp> && default_initializable<_Tp>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [concept.object] + +template +_CCCL_CONCEPT_FRAGMENT(__semiregular_, requires()(requires(copyable<_Tp>), requires(default_initializable<_Tp>))); + +template +_CCCL_CONCEPT semiregular = _CCCL_FRAGMENT(__semiregular_, _Tp); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_SEMIREGULAR_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/swappable.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/swappable.h new file mode 100644 index 00000000..76574c67 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/swappable.h @@ -0,0 +1,209 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_SWAPPABLE_H +#define _CUDA_STD___CONCEPTS_SWAPPABLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if _CCCL_COMPILER(MSVC) +_CCCL_BEGIN_NV_DIAG_SUPPRESS(461) // nonstandard cast to array type ignored +#endif // _CCCL_COMPILER(MSVC) + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES + +// [concept.swappable] + +_CCCL_BEGIN_NAMESPACE_CPO(__swap) + +template +void swap(_Tp&, _Tp&) = delete; + +#if _CCCL_HAS_CONCEPTS() +template +concept __unqualified_swappable_with = + (__class_or_enum> || __class_or_enum>) + && requires(_Tp&& __t, _Up&& __u) { swap(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Up>(__u)); }; + +template +concept __exchangeable = + !__unqualified_swappable_with<_Tp&, _Tp&> && move_constructible<_Tp> && assignable_from<_Tp&, _Tp>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __unqualified_swappable_with_, + requires(_Tp&& __t, _Up&& __u)((swap(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Up>(__u))))); + +template +_CCCL_CONCEPT __unqualified_swappable_with = _CCCL_FRAGMENT(__unqualified_swappable_with_, _Tp, _Up); + +template +_CCCL_CONCEPT_FRAGMENT(__exchangeable_, + requires()(requires(!__unqualified_swappable_with<_Tp&, _Tp&>), + requires(move_constructible<_Tp>), + requires(assignable_from<_Tp&, _Tp>))); + +template +_CCCL_CONCEPT __exchangeable = _CCCL_FRAGMENT(__exchangeable_, _Tp); +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +#if _CCCL_HAS_CONCEPTS() && !_CCCL_COMPILER(NVHPC) // nvbug4051640 +struct __fn; + +_CCCL_BEGIN_NV_DIAG_SUPPRESS(2642) +template +concept __swappable_arrays = + !__unqualified_swappable_with<_Tp (&)[_Size], _Up (&)[_Size]> && extent_v<_Tp> == extent_v<_Up> + && requires(_Tp (&__t)[_Size], _Up (&__u)[_Size], const __fn& __swap) { __swap(__t[0], __u[0]); }; +_CCCL_END_NV_DIAG_SUPPRESS() + +#else // ^^^ _CCCL_HAS_CONCEPTS() && !_CCCL_COMPILER(NVHPC) ^^^ / vvv !_CCCL_HAS_CONCEPTS() || _CCCL_COMPILER(NVHPC) vvv +template +inline constexpr bool __swappable_arrays = false; +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ || _CCCL_COMPILER(NVHPC) + +template +inline constexpr bool __noexcept_swappable_arrays = false; + +struct __fn +{ + // 2.1 `S` is `(void)swap(E1, E2)`* if `E1` or `E2` has class or enumeration type and... + // *The name `swap` is used here unqualified. + _CCCL_TEMPLATE(class _Tp, class _Up) + _CCCL_REQUIRES(__unqualified_swappable_with<_Tp, _Up>) + _CCCL_API constexpr void operator()(_Tp&& __t, _Up&& __u) const + noexcept(noexcept(swap(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Up>(__u)))) + { + swap(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Up>(__u)); + } + + // 2.2 Otherwise, if `E1` and `E2` are lvalues of array types with equal extent and... + _CCCL_TEMPLATE(class _Tp, class _Up, size_t _Size) + _CCCL_REQUIRES(__swappable_arrays<_Tp, _Up, _Size>) + _CCCL_API constexpr void operator()(_Tp (&__t)[_Size], _Up (&__u)[_Size]) const + noexcept(__noexcept_swappable_arrays<_Tp, _Up>) + { + // TODO(cjdb): replace with `::cuda::std::ranges::swap_ranges`. + for (size_t __i = 0; __i < _Size; ++__i) + { + (*this)(__t[__i], __u[__i]); + } + } + + // 2.3 Otherwise, if `E1` and `E2` are lvalues of the same type `T` that models... + _CCCL_TEMPLATE(class _Tp) + _CCCL_REQUIRES(__exchangeable<_Tp>) + _CCCL_API constexpr void operator()(_Tp& __x, _Tp& __y) const + noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_assignable_v<_Tp>) + { + __y = ::cuda::std::exchange(__x, ::cuda::std::move(__y)); + } +}; + +#if !_CCCL_HAS_CONCEPTS() || _CCCL_COMPILER(NVHPC) +template +_CCCL_CONCEPT_FRAGMENT( + __swappable_arrays_, + requires(_Tp (&__t)[_Size::value], _Up (&__u)[_Size::value], const __fn& __swap)( + requires(!__unqualified_swappable_with<_Tp (&)[_Size::value], _Up (&)[_Size::value]>), + requires(extent_v<_Tp> == extent_v<_Up>), + (__swap(__t[0], __u[0])))); + +template +inline constexpr bool __swappable_arrays<_Tp, _Up, _Size, void_t>> = + _CCCL_FRAGMENT(__swappable_arrays_, _Tp, _Up, ::cuda::std::integral_constant); +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ || _CCCL_COMPILER(NVHPC) + +template +inline constexpr bool __noexcept_swappable_arrays<_Tp, _Up, void_t>> = + noexcept(__swap::__fn{}(::cuda::std::declval<_Tp&>(), ::cuda::std::declval<_Up&>())); + +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto swap = __swap::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __swap_cpo = __swap::__fn; +} // namespace __cpo +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() +template +concept swappable = requires(_Tp& __a, _Tp& __b) { ::cuda::std::ranges::__swap_cpo{}(__a, __b); }; + +template +concept swappable_with = common_reference_with<_Tp, _Up> && requires(_Tp&& __t, _Up&& __u) { + ::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Tp>(__t)); + ::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Up>(__u), ::cuda::std::forward<_Up>(__u)); + ::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Up>(__u)); + ::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Up>(__u), ::cuda::std::forward<_Tp>(__t)); +}; +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv +template +_CCCL_CONCEPT_FRAGMENT(__swappable_, requires(_Tp& __a, _Tp& __b)((::cuda::std::ranges::__swap_cpo{}(__a, __b)))); + +template +_CCCL_CONCEPT swappable = _CCCL_FRAGMENT(__swappable_, _Tp); + +template +_CCCL_CONCEPT_FRAGMENT( + __swappable_with_, + requires(_Tp&& __t, _Up&& __u)( + requires(common_reference_with<_Tp, _Up>), + (::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Tp>(__t))), + (::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Up>(__u), ::cuda::std::forward<_Up>(__u))), + (::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Tp>(__t), ::cuda::std::forward<_Up>(__u))), + (::cuda::std::ranges::__swap_cpo{}(::cuda::std::forward<_Up>(__u), ::cuda::std::forward<_Tp>(__t))))); + +template +_CCCL_CONCEPT swappable_with = _CCCL_FRAGMENT(__swappable_with_, _Tp, _Up); +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#if _CCCL_COMPILER(MSVC) +_CCCL_END_NV_DIAG_SUPPRESS() // nonstandard cast to array type ignored +#endif // _CCCL_COMPILER(MSVC) + +#include + +#endif // _CUDA_STD___CONCEPTS_SWAPPABLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/totally_ordered.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/totally_ordered.h new file mode 100644 index 00000000..f7534699 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__concepts/totally_ordered.h @@ -0,0 +1,101 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CONCEPTS_TOTALLY_ORDERED_H +#define _CUDA_STD___CONCEPTS_TOTALLY_ORDERED_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [concept.totallyordered] + +template +concept __partially_ordered_with = requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) { + { __t < __u } -> __boolean_testable; + { __t > __u } -> __boolean_testable; + { __t <= __u } -> __boolean_testable; + { __t >= __u } -> __boolean_testable; + { __u < __t } -> __boolean_testable; + { __u > __t } -> __boolean_testable; + { __u <= __t } -> __boolean_testable; + { __u >= __t } -> __boolean_testable; +}; + +template +concept totally_ordered = equality_comparable<_Tp> && __partially_ordered_with<_Tp, _Tp>; + +template +concept totally_ordered_with = + totally_ordered<_Tp> && totally_ordered<_Up> && equality_comparable_with<_Tp, _Up> + && totally_ordered, __make_const_lvalue_ref<_Up>>> + && __partially_ordered_with<_Tp, _Up>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __partially_ordered_with_, + requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u)( + _Satisfies(__boolean_testable)(__t < __u), // + _Satisfies(__boolean_testable)(__t > __u), // + _Satisfies(__boolean_testable)(__t <= __u), // + _Satisfies(__boolean_testable)(__t >= __u), // + _Satisfies(__boolean_testable)(__u < __t), // + _Satisfies(__boolean_testable)(__u > __t), // + _Satisfies(__boolean_testable)(__u <= __t), // + _Satisfies(__boolean_testable)(__u >= __t))); + +template +_CCCL_CONCEPT __partially_ordered_with = _CCCL_FRAGMENT(__partially_ordered_with_, _Tp, _Up); + +template +_CCCL_CONCEPT_FRAGMENT(__totally_ordered_, + requires()(requires(equality_comparable<_Tp>), requires(__partially_ordered_with<_Tp, _Tp>))); + +template +_CCCL_CONCEPT totally_ordered = _CCCL_FRAGMENT(__totally_ordered_, _Tp); + +template +_CCCL_CONCEPT_FRAGMENT( + __totally_ordered_with_, + requires()(requires(totally_ordered<_Tp>), + requires(totally_ordered<_Up>), + requires(equality_comparable_with<_Tp, _Up>), + requires(totally_ordered, __make_const_lvalue_ref<_Up>>>), + requires(__partially_ordered_with<_Tp, _Up>))); + +template +_CCCL_CONCEPT totally_ordered_with = _CCCL_FRAGMENT(__totally_ordered_with_, _Tp, _Up); + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CONCEPTS_TOTALLY_ORDERED_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/byte.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/byte.h new file mode 100644 index 00000000..f5e90b57 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/byte.h @@ -0,0 +1,113 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CSTDDEF_BYTE_H +#define _CUDA_STD___CSTDDEF_BYTE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION + +enum class byte : unsigned char +{ +}; + +_CCCL_API constexpr byte operator|(byte __lhs, byte __rhs) noexcept +{ + return static_cast( + static_cast(static_cast(__lhs) | static_cast(__rhs))); +} + +_CCCL_API constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept +{ + return __lhs = __lhs | __rhs; +} + +_CCCL_API constexpr byte operator&(byte __lhs, byte __rhs) noexcept +{ + return static_cast( + static_cast(static_cast(__lhs) & static_cast(__rhs))); +} + +_CCCL_API constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept +{ + return __lhs = __lhs & __rhs; +} + +_CCCL_API constexpr byte operator^(byte __lhs, byte __rhs) noexcept +{ + return static_cast( + static_cast(static_cast(__lhs) ^ static_cast(__rhs))); +} + +_CCCL_API constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept +{ + return __lhs = __lhs ^ __rhs; +} + +_CCCL_API constexpr byte operator~(byte __b) noexcept +{ + return static_cast(static_cast(~static_cast(__b))); +} + +_CCCL_TEMPLATE(class _Integer) +_CCCL_REQUIRES(is_integral_v<_Integer>) +_CCCL_API constexpr byte& operator<<=(byte& __lhs, _Integer __shift) noexcept +{ + return __lhs = __lhs << __shift; +} + +_CCCL_TEMPLATE(class _Integer) +_CCCL_REQUIRES(is_integral_v<_Integer>) +_CCCL_API constexpr byte operator<<(byte __lhs, _Integer __shift) noexcept +{ + return static_cast(static_cast(static_cast(__lhs) << __shift)); +} + +_CCCL_TEMPLATE(class _Integer) +_CCCL_REQUIRES(is_integral_v<_Integer>) +_CCCL_API constexpr byte& operator>>=(byte& __lhs, _Integer __shift) noexcept +{ + return __lhs = __lhs >> __shift; +} + +_CCCL_TEMPLATE(class _Integer) +_CCCL_REQUIRES(is_integral_v<_Integer>) +_CCCL_API constexpr byte operator>>(byte __lhs, _Integer __shift) noexcept +{ + return static_cast(static_cast(static_cast(__lhs) >> __shift)); +} + +_CCCL_TEMPLATE(class _Integer) +_CCCL_REQUIRES(is_integral_v<_Integer>) +_CCCL_API constexpr _Integer to_integer(byte __b) noexcept +{ + return static_cast<_Integer>(__b); +} + +_CCCL_END_NAMESPACE_CUDA_STD_NOVERSION + +#include + +#endif // _CUDA_STD___CSTDDEF_BYTE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/types.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/types.h new file mode 100644 index 00000000..57208e29 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstddef/types.h @@ -0,0 +1,52 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CSTDDEF_TYPES_H +#define _CUDA_STD___CSTDDEF_TYPES_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_HOSTED() +# include +#else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv +# if !defined(offsetof) +# define offsetof(type, member) (::size_t) ((char*) &(((type*) 0)->member) - (char*) 0) +# endif // !offsetof +#endif // _CCCL_FREESTANDING() + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_FREESTANDING() +using max_align_t = long double; +#else // ^^^ _CCCL_FREESTANDING() ^^^ / vvv _CCCL_HOSTED() vvv +// Re-use the compiler's max_align_t where possible. +using ::max_align_t; +#endif // _CCCL_HOSTED() + +using nullptr_t = decltype(nullptr); +using ::ptrdiff_t; +using ::size_t; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CSTDDEF_TYPES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstring/memcpy.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstring/memcpy.h new file mode 100644 index 00000000..00bcc9e0 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__cstring/memcpy.h @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___CSTRING_MEMCPY +#define _CUDA_STD___CSTRING_MEMCPY + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#if _CCCL_HOSTED() +# include +#endif // _CCCL_HOSTED() + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +using ::size_t; + +// old compilers still trigger the name conflict +// nvcc 12.0 and 12.1 trigger segmentation fault +#if _CCCL_COMPILER(GCC, <=, 9) || _CCCL_CUDA_COMPILER(NVCC, <=, 12, 1) + +using ::memcpy; + +#else // ^^^ _CCCL_COMPILER(GCC, <=, 9) ^^^ / vvv _CCCL_COMPILER(GCC, >, 9) vvv + +// The template parameter is used to avoid name ambiguity when external code calls 'memcpy' without namespace +// qualification. Function templates have lower precedence than non-template functions for overload resolution. +template +_CCCL_API inline void* memcpy(void* __dest, const void* __src, size_t __count) noexcept +{ + _CCCL_ASSERT(::cuda::__is_valid_address_range(__src, __count), "memcpy: source range is invalid"); + _CCCL_ASSERT(::cuda::__is_valid_address_range(__dest, __count), "memcpy: destination range is invalid"); + _CCCL_ASSERT(!::cuda::__are_ptrs_overlapping(__src, __dest, __count), "memcpy: source and destination overlap"); + return ::memcpy(__dest, __src, __count); +} + +#endif // ^^^ _CCCL_COMPILER(GCC, <=, 9) ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___CSTRING_MEMCPY diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/exception_macros.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/exception_macros.h new file mode 100644 index 00000000..02c8edf4 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/exception_macros.h @@ -0,0 +1,126 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___EXCEPTION_EXCEPTION_MACROS_H +#define _CUDA_STD___EXCEPTION_EXCEPTION_MACROS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cccl_catch_any_lvalue +{ + template + _CCCL_API operator _Tp&() const noexcept; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +// The following macros are used to conditionally compile exception handling code. They +// are used in the same way as `try` and `catch`, but they allow for different behavior +// based on whether exceptions are enabled or not, and whether the code is being compiled +// for device or not. +// +// Usage: +// _CCCL_TRY +// { +// can_throw(); // Code that may throw an exception +// } +// _CCCL_CATCH (cuda_error& e) // Handle CUDA exceptions +// { +// printf("CUDA error: %s\n", e.what()); +// } +// _CCCL_CATCH_ALL // Handle any other exceptions +// { +// printf("unknown error\n"); +// } +// +// Notes: +// - the catch clause must always bind to a named variable + +// Expand to keywords only for host code when exceptions are enabled. nvc++ in CUDA mode traps when an exception is +// thrown in device code. +#if _CCCL_HAS_EXCEPTIONS() && _CCCL_HOST_COMPILATION() +# define _CCCL_TRY try +# define _CCCL_CATCH catch +# define _CCCL_CATCH_ALL catch (...) +# define _CCCL_CATCH_FALLTHROUGH + +// Even though nvc++ in CUDA mode replaces `throw` by `__trap()` call in device code, it instantiates the exception type +// which can introduce some host only symbols to the nvvm ir (for example snprintf). So we need to wrap it by the +// NV_IF_ELSE_TARGET macro. +# define _CCCL_THROW(_TYPE, ...) \ + do \ + { \ + NV_IF_ELSE_TARGET(NV_IS_HOST, (throw _TYPE(__VA_ARGS__);), (::cuda::std::terminate();)) \ + } while (0) +# define _CCCL_RETHROW throw +#else // ^^^ use exceptions ^^^ / vvv no exceptions vvv +# define _CCCL_TRY \ + if constexpr (true) \ + { +# define _CCCL_CATCH(...) \ + } \ + else if constexpr (false) \ + { \ + for (__VA_ARGS__ = ::cuda::std::__cccl_catch_any_lvalue{}; false;) +# define _CCCL_CATCH_ALL \ + } \ + else +# define _CCCL_CATCH_FALLTHROUGH \ + } \ + else \ + { \ + } + +# if _CCCL_HOSTJIT() +# define _CCCL_THROW(_TYPE, ...) \ + do \ + { \ + _CCCL_ASSERT(false, "An instance of class " #_TYPE " would be thrown."); \ + ::cuda::std::terminate(); \ + } while (0) +# else // ^^^ _CCCL_HOSTJIT() ^^^ / vvv !_CCCL_HOSTJIT() vvv +# define _CCCL_THROW(_TYPE, ...) \ + do \ + { \ + NV_IF_ELSE_TARGET(NV_IS_HOST, \ + ({ \ + ::fprintf(stderr, \ + "%s:%u: An instance of class %s would be thrown.\n what(): %s\nAborted\n", \ + __FILE__, \ + __LINE__, \ + #_TYPE, \ + (_TYPE(__VA_ARGS__)).what()); \ + ::fflush(stderr); \ + }), \ + ({ _CCCL_ASSERT(false, "An instance of class " #_TYPE " would be thrown."); })) \ + ::cuda::std::terminate(); \ + } while (0) +# endif // !_CCCL_HOSTJIT() +# define _CCCL_RETHROW ::cuda::std::terminate() +#endif // ^^^ no exceptions ^^^ + +#include + +#endif // _CUDA_STD___EXCEPTION_EXCEPTION_MACROS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/terminate.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/terminate.h new file mode 100644 index 00000000..6c31b483 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__exception/terminate.h @@ -0,0 +1,82 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___EXCEPTION_TERMINATE_H +#define _CUDA_STD___EXCEPTION_TERMINATE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_TILE_COMPILATION() +# include +#endif // !_CCCL_TILE_COMPILATION() + +#if _CCCL_HOSTED() +# include +#endif // _CCCL_HOSTED() + +#include + +_CCCL_DIAG_PUSH +_CCCL_DIAG_SUPPRESS_MSVC(4702) // unreachable code + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION // purposefully not using versioning namespace + +[[noreturn]] _CCCL_API inline void __cccl_terminate() noexcept +{ +#if _CCCL_TILE_COMPILATION() + NV_IF_ELSE_TARGET(NV_IS_HOST, (::exit(-1);), (assert(false);)) +#else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() + NV_IF_ELSE_TARGET(NV_IS_HOST, (::exit(-1);), (::__trap();)) +#endif // !_CCCL_TILE_COMPILATION() + _CCCL_UNREACHABLE(); +} + +#if 0 // Expose once atomic is universally available + +using terminate_handler = void (*)(); + +# ifdef __CUDA_ARCH__ +__device__ +# endif // __CUDA_ARCH__ + static _CCCL_CONSTINIT ::cuda::std::atomic + __cccl_terminate_handler{&__cccl_terminate}; + +_CCCL_API inline terminate_handler set_terminate(terminate_handler __func) noexcept +{ + return __cccl_terminate_handler.exchange(__func); +} +_CCCL_API inline terminate_handler get_terminate() noexcept +{ + return __cccl_terminate_handler.load(__func); +} + +#endif + +[[noreturn]] _CCCL_API inline void terminate() noexcept +{ + __cccl_terminate(); +} + +_CCCL_END_NAMESPACE_CUDA_STD_NOVERSION + +_CCCL_DIAG_POP + +#include + +#endif // _CUDA_STD___EXCEPTION_TERMINATE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/format.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/format.h new file mode 100644 index 00000000..c90924ac --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/format.h @@ -0,0 +1,157 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FLOATING_POINT_FORMAT_H +#define _CUDA_STD___FLOATING_POINT_FORMAT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +enum class __fp_format +{ + __binary16, // IEEE 754 binary16 + __binary32, // IEEE 754 binary32 + __binary64, // IEEE 754 binary64 + __binary128, // IEEE 754 binary128 + __bfloat16, // Google's 16-bit brain float + __fp80_x86, // x86 80-bit extended precision + __fp8_nv_e4m3, // NVIDIA's __nv_fp8_e4m3 + __fp8_nv_e5m2, // NVIDIA's __nv_fp8_e5m2 + __fp8_nv_e8m0, // NVIDIA's __nv_fp8_e8m0 + __fp6_nv_e2m3, // NVIDIA's __nv_fp6_e2m3 + __fp6_nv_e3m2, // NVIDIA's __nv_fp6_e3m2 + __fp4_nv_e2m1, // NVIDIA's __nv_fp4_e2m1 + + __invalid, +}; + +template +[[nodiscard]] _CCCL_API constexpr __fp_format __fp_format_of_v_impl() noexcept +{ + if constexpr (is_same_v<_Tp, float>) + { + return __fp_format::__binary32; + } + else if constexpr (is_same_v<_Tp, double>) + { + return __fp_format::__binary64; + } +#if _CCCL_HAS_LONG_DOUBLE() + else if constexpr (is_same_v<_Tp, long double>) + { +# if LDBL_MIN_EXP == -1021 && LDBL_MAX_EXP == 1024 && LDBL_MANT_DIG == 53 + return __fp_format::__binary64; +# elif LDBL_MIN_EXP == -16381 && LDBL_MAX_EXP == 16384 && LDBL_MANT_DIG == 64 + static_assert(sizeof(long double) == 16, + "When the long double format is x86 80-bit extended floating point, CCCL requires the size of long " + "double to be 16 bytes."); + return __fp_format::__fp80_x86; +# elif LDBL_MIN_EXP == -16381 && LDBL_MAX_EXP == 16384 && LDBL_MANT_DIG == 113 + return __fp_format::__binary128; +# else +# error "Unknown long double format. Define CCCL_DISABLE_LONG_DOUBLE to disable long double support in CCCL." +# endif + } +#endif // _CCCL_HAS_LONG_DOUBLE() +#if _CCCL_HAS_NVFP16() + else if constexpr (is_same_v<_Tp, __half>) + { + return __fp_format::__binary16; + } +#endif // _CCCL_HAS_NVFP16() +#if _CCCL_HAS_NVBF16() + else if constexpr (is_same_v<_Tp, __nv_bfloat16>) + { + return __fp_format::__bfloat16; + } +#endif // _CCCL_HAS_NVBF16() +#if _CCCL_HAS_NVFP8_E4M3() + else if constexpr (is_same_v<_Tp, __nv_fp8_e4m3>) + { + return __fp_format::__fp8_nv_e4m3; + } +#endif // _CCCL_HAS_NVFP8_E4M3() +#if _CCCL_HAS_NVFP8_E5M2() + else if constexpr (is_same_v<_Tp, __nv_fp8_e5m2>) + { + return __fp_format::__fp8_nv_e5m2; + } +#endif // _CCCL_HAS_NVFP8_E5M2() +#if _CCCL_HAS_NVFP8_E8M0() + else if constexpr (is_same_v<_Tp, __nv_fp8_e8m0>) + { + return __fp_format::__fp8_nv_e8m0; + } +#endif // _CCCL_HAS_NVFP8_E8M0() +#if _CCCL_HAS_NVFP6_E2M3() + else if constexpr (is_same_v<_Tp, __nv_fp6_e2m3>) + { + return __fp_format::__fp6_nv_e2m3; + } +#endif // _CCCL_HAS_NVFP6_E2M3() +#if _CCCL_HAS_NVFP6_E3M2() + else if constexpr (is_same_v<_Tp, __nv_fp6_e3m2>) + { + return __fp_format::__fp6_nv_e3m2; + } +#endif // _CCCL_HAS_NVFP6_E3M2() +#if _CCCL_HAS_NVFP4_E2M1() + else if constexpr (is_same_v<_Tp, __nv_fp4_e2m1>) + { + return __fp_format::__fp4_nv_e2m1; + } +#endif // _CCCL_HAS_NVFP4_E2M1() +#if _CCCL_HAS_FLOAT128() + else if constexpr (is_same_v<_Tp, __float128>) + { + return __fp_format::__binary128; + } +#endif // _CCCL_HAS_FLOAT128() + else + { + return __fp_format::__invalid; + } +} + +template +inline constexpr __fp_format __fp_format_of_v = ::cuda::std::__fp_format_of_v_impl<_Tp>(); + +template +inline constexpr __fp_format __fp_format_of_v = __fp_format_of_v<_Tp>; + +template +inline constexpr __fp_format __fp_format_of_v = __fp_format_of_v<_Tp>; + +template +inline constexpr __fp_format __fp_format_of_v = __fp_format_of_v<_Tp>; + +template <__fp_format _Fmt> +inline constexpr __fp_format __fp_format_of_v<__cccl_fp<_Fmt>> = _Fmt; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FLOATING_POINT_FORMAT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/properties.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/properties.h new file mode 100644 index 00000000..2e11a25d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/properties.h @@ -0,0 +1,229 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FLOATING_POINT_PROPERTIES_H +#define _CUDA_STD___FLOATING_POINT_PROPERTIES_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// __fp_is_signed_v + +template <__fp_format _Fmt> +inline constexpr bool __fp_is_signed_v = true; + +template <> +inline constexpr bool __fp_is_signed_v<__fp_format::__fp8_nv_e8m0> = false; + +// __fp_exp_nbits_v + +template <__fp_format _Fmt> +inline constexpr int __fp_exp_nbits_v = 0; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__binary16> = 5; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__binary32> = 8; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__binary64> = 11; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__binary128> = 15; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__bfloat16> = 8; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp80_x86> = 15; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp8_nv_e4m3> = 4; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp8_nv_e5m2> = 5; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp8_nv_e8m0> = 8; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp6_nv_e2m3> = 2; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp6_nv_e3m2> = 3; + +template <> +inline constexpr int __fp_exp_nbits_v<__fp_format::__fp4_nv_e2m1> = 2; + +// __fp_exp_bias_v + +template <__fp_format _Fmt> +inline constexpr int __fp_exp_bias_v = (1 << (__fp_exp_nbits_v<_Fmt> - 1)) - 1; + +// __fp_exp_min_v + +template <__fp_format _Fmt> +inline constexpr int __fp_exp_min_v = 1 - __fp_exp_bias_v<_Fmt>; + +template <> +inline constexpr int __fp_exp_min_v<__fp_format::__fp8_nv_e8m0> = -127; + +// __fp_exp_max_v + +template <__fp_format _Fmt> +inline constexpr int __fp_exp_max_v = (1 << __fp_exp_nbits_v<_Fmt>) -2 - __fp_exp_bias_v<_Fmt>; + +template <> +inline constexpr int __fp_exp_max_v<__fp_format::__fp8_nv_e4m3> = 8; + +template <> +inline constexpr int __fp_exp_max_v<__fp_format::__fp6_nv_e2m3> = 2; + +template <> +inline constexpr int __fp_exp_max_v<__fp_format::__fp6_nv_e3m2> = 4; + +template <> +inline constexpr int __fp_exp_max_v<__fp_format::__fp4_nv_e2m1> = 2; + +// __fp_mant_nbits_v + +template <__fp_format _Fmt> +inline constexpr int __fp_mant_nbits_v = 0; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__binary16> = 10; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__binary32> = 23; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__binary64> = 52; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__binary128> = 112; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__bfloat16> = 7; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp80_x86> = 64; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp8_nv_e4m3> = 3; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp8_nv_e5m2> = 2; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp8_nv_e8m0> = 0; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp6_nv_e2m3> = 3; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp6_nv_e3m2> = 2; + +template <> +inline constexpr int __fp_mant_nbits_v<__fp_format::__fp4_nv_e2m1> = 1; + +// __fp_has_implicit_bit_v + +template <__fp_format _Fmt> +inline constexpr bool __fp_has_implicit_bit_v = true; + +template <> +inline constexpr bool __fp_has_implicit_bit_v<__fp_format::__fp80_x86> = false; + +// __fp_digits_v + +template <__fp_format _Fmt> +inline constexpr int __fp_digits_v = __fp_mant_nbits_v<_Fmt> + static_cast(__fp_has_implicit_bit_v<_Fmt>); + +// __fp_has_denorm_v + +template <__fp_format _Fmt> +inline constexpr bool __fp_has_denorm_v = true; + +template <> +inline constexpr bool __fp_has_denorm_v<__fp_format::__fp8_nv_e8m0> = false; + +// __fp_has_inf_v + +template <__fp_format _Fmt> +inline constexpr bool __fp_has_inf_v = true; + +template <> +inline constexpr bool __fp_has_inf_v<__fp_format::__fp8_nv_e4m3> = false; + +template <> +inline constexpr bool __fp_has_inf_v<__fp_format::__fp8_nv_e8m0> = false; + +template <> +inline constexpr bool __fp_has_inf_v<__fp_format::__fp6_nv_e2m3> = false; + +template <> +inline constexpr bool __fp_has_inf_v<__fp_format::__fp6_nv_e3m2> = false; + +template <> +inline constexpr bool __fp_has_inf_v<__fp_format::__fp4_nv_e2m1> = false; + +// __fp_has_nan_v + +template <__fp_format _Fmt> +inline constexpr bool __fp_has_nan_v = true; + +template <> +inline constexpr bool __fp_has_nan_v<__fp_format::__fp6_nv_e2m3> = false; + +template <> +inline constexpr bool __fp_has_nan_v<__fp_format::__fp6_nv_e3m2> = false; + +template <> +inline constexpr bool __fp_has_nan_v<__fp_format::__fp4_nv_e2m1> = false; + +// __fp_has_nans_v + +template <__fp_format _Fmt> +inline constexpr bool __fp_has_nans_v = true; + +template <> +inline constexpr bool __fp_has_nans_v<__fp_format::__fp8_nv_e4m3> = false; + +template <> +inline constexpr bool __fp_has_nans_v<__fp_format::__fp8_nv_e8m0> = false; + +template <> +inline constexpr bool __fp_has_nans_v<__fp_format::__fp6_nv_e2m3> = false; + +template <> +inline constexpr bool __fp_has_nans_v<__fp_format::__fp6_nv_e3m2> = false; + +template <> +inline constexpr bool __fp_has_nans_v<__fp_format::__fp4_nv_e2m1> = false; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FLOATING_POINT_PROPERTIES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/storage.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/storage.h new file mode 100644 index 00000000..37af5ec4 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/storage.h @@ -0,0 +1,260 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FLOATING_POINT_STORAGE_H +#define _CUDA_STD___FLOATING_POINT_STORAGE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template <__fp_format _Fmt> +[[nodiscard]] _CCCL_API constexpr auto __fp_storage_type_impl() noexcept +{ + if constexpr (_Fmt == __fp_format::__fp8_nv_e4m3 || _Fmt == __fp_format::__fp8_nv_e5m2 + || _Fmt == __fp_format::__fp8_nv_e8m0 || _Fmt == __fp_format::__fp6_nv_e2m3 + || _Fmt == __fp_format::__fp6_nv_e3m2 || _Fmt == __fp_format::__fp4_nv_e2m1) + { + return uint8_t{}; + } + else if constexpr (_Fmt == __fp_format::__binary16 || _Fmt == __fp_format::__bfloat16) + { + return uint16_t{}; + } + else if constexpr (_Fmt == __fp_format::__binary32) + { + return uint32_t{}; + } + else if constexpr (_Fmt == __fp_format::__binary64) + { + return uint64_t{}; + } +#if _CCCL_HAS_INT128() + else if constexpr (_Fmt == __fp_format::__fp80_x86 || _Fmt == __fp_format::__binary128) + { + return __uint128_t{}; + } +#endif // _CCCL_HAS_INT128() + else + { + static_assert(__always_false_v, "Unsupported floating point format"); + } +} + +template <__fp_format _Fmt> +using __fp_storage_t = decltype(__fp_storage_type_impl<_Fmt>()); + +template +using __fp_storage_of_t = __fp_storage_t<__fp_format_of_v<_Tp>>; + +#if !_CCCL_TILE_COMPILATION() +template +struct __cccl_nvfp_manip_helper : _Tp +{ + using _Tp::__x; +}; +#endif // _CCCL_TILE_COMPILATION() + +template +[[nodiscard]] _CCCL_API constexpr _Tp __fp_from_storage(__fp_storage_of_t<_Tp> __v) noexcept +{ + if constexpr (__is_std_fp_v<_Tp> || __is_ext_compiler_fp_v<_Tp>) + { + return ::cuda::std::bit_cast<_Tp>(__v); + } + else if constexpr (__is_ext_cccl_fp_v<_Tp>) + { + _Tp __ret{}; + __ret.__storage_ = __v; + return __ret; + } +#if _CCCL_HAS_NVFP16() + else if constexpr (is_same_v<_Tp, __half>) + { +# if _CCCL_TILE_COMPILATION() + return ::cuda::std::bit_cast<_Tp>(__v); +# else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() + __cccl_nvfp_manip_helper<_Tp> __helper{}; + __helper.__x = __v; + return __helper; +# endif // !_CCCL_TILE_COMPILATION() + } +#endif // _CCCL_HAS_NVFP16() +#if _CCCL_HAS_NVBF16() + else if constexpr (is_same_v<_Tp, __nv_bfloat16>) + { +# if _CCCL_TILE_COMPILATION() + return ::cuda::std::bit_cast<_Tp>(__v); +# else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() + __cccl_nvfp_manip_helper<_Tp> __helper{}; + __helper.__x = __v; + return __helper; +# endif // !_CCCL_TILE_COMPILATION() + } +#endif // _CCCL_HAS_NVBF16() +#if _CCCL_HAS_NVFP8_E4M3() + else if constexpr (is_same_v<_Tp, __nv_fp8_e4m3>) + { + _Tp __ret{}; + __ret.__x = __v; + return __ret; + } +#endif // _CCCL_HAS_NVFP8_E4M3() +#if _CCCL_HAS_NVFP8_E5M2() + else if constexpr (is_same_v<_Tp, __nv_fp8_e5m2>) + { + _Tp __ret{}; + __ret.__x = __v; + return __ret; + } +#endif // _CCCL_HAS_NVFP8_E5M2() +#if _CCCL_HAS_NVFP8_E8M0() + else if constexpr (is_same_v<_Tp, __nv_fp8_e8m0>) + { + _Tp __ret{}; + __ret.__x = __v; + return __ret; + } +#endif // _CCCL_HAS_NVFP8_E8M0() +#if _CCCL_HAS_NVFP6_E2M3() + else if constexpr (is_same_v<_Tp, __nv_fp6_e2m3>) + { + _CCCL_ASSERT((__v & 0xc0u) == 0u, "Invalid __nv_fp6_e2m3 storage value"); + _Tp __ret{}; + __ret.__x = __v; + return __ret; + } +#endif // _CCCL_HAS_NVFP6_E2M3() +#if _CCCL_HAS_NVFP6_E3M2() + else if constexpr (is_same_v<_Tp, __nv_fp6_e3m2>) + { + _CCCL_ASSERT((__v & 0xc0u) == 0u, "Invalid __nv_fp6_e3m2 storage value"); + _Tp __ret{}; + __ret.__x = __v; + return __ret; + } +#endif // _CCCL_HAS_NVFP6_E3M2() +#if _CCCL_HAS_NVFP4_E2M1() + else if constexpr (is_same_v<_Tp, __nv_fp4_e2m1>) + { + _CCCL_ASSERT((__v & 0xf0u) == 0u, "Invalid __nv_fp4_e2m1 storage value"); + _Tp __ret{}; + __ret.__x = __v; + return __ret; + } +#endif // _CCCL_HAS_NVFP4_E2M1() + else + { + static_assert(__always_false_v<_Tp>, "Unsupported floating point format"); + } +} + +_CCCL_TEMPLATE(class _Tp, class _Up) +_CCCL_REQUIRES((!is_same_v<_Up, __fp_storage_of_t<_Tp>>) ) +_CCCL_API constexpr _Tp __fp_from_storage(const _Up& __v) noexcept = delete; + +template +[[nodiscard]] _CCCL_API constexpr __fp_storage_of_t<_Tp> __fp_get_storage(_Tp __v) noexcept +{ + if constexpr (__is_std_fp_v<_Tp> || __is_ext_compiler_fp_v<_Tp>) + { + return ::cuda::std::bit_cast<__fp_storage_of_t<_Tp>>(__v); + } + else if constexpr (__is_ext_cccl_fp_v<_Tp>) + { + return __v.__storage_; + } +#if _CCCL_HAS_NVFP16() + else if constexpr (is_same_v<_Tp, __half>) + { +# if _CCCL_TILE_COMPILATION() + return ::cuda::std::bit_cast<__fp_storage_of_t<_Tp>>(__v); +# else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() vvv + return __cccl_nvfp_manip_helper<_Tp>{__v}.__x; +# endif // !_CCCL_TILE_COMPILATION() + } +#endif // _CCCL_HAS_NVFP16() +#if _CCCL_HAS_NVBF16() + else if constexpr (is_same_v<_Tp, __nv_bfloat16>) + { +# if _CCCL_TILE_COMPILATION() + return ::cuda::std::bit_cast<__fp_storage_of_t<_Tp>>(__v); +# else // ^^^ _CCCL_TILE_COMPILATION() ^^^ / vvv !_CCCL_TILE_COMPILATION() vvv + return __cccl_nvfp_manip_helper<_Tp>{__v}.__x; +# endif // !_CCCL_TILE_COMPILATION() + } +#endif // _CCCL_HAS_NVBF16() + // Distinct extended floating-point types expose the same storage member. + // NOLINTBEGIN(bugprone-branch-clone) +#if _CCCL_HAS_NVFP8_E4M3() + else if constexpr (is_same_v<_Tp, __nv_fp8_e4m3>) + { + return __v.__x; + } +#endif // _CCCL_HAS_NVFP8_E4M3() +#if _CCCL_HAS_NVFP8_E5M2() + else if constexpr (is_same_v<_Tp, __nv_fp8_e5m2>) + { + return __v.__x; + } +#endif // _CCCL_HAS_NVFP8_E5M2() +#if _CCCL_HAS_NVFP8_E8M0() + else if constexpr (is_same_v<_Tp, __nv_fp8_e8m0>) + { + return __v.__x; + } +#endif // _CCCL_HAS_NVFP8_E8M0() +#if _CCCL_HAS_NVFP6_E2M3() + else if constexpr (is_same_v<_Tp, __nv_fp6_e2m3>) + { + return __v.__x; + } +#endif // _CCCL_HAS_NVFP6_E2M3() +#if _CCCL_HAS_NVFP6_E3M2() + else if constexpr (is_same_v<_Tp, __nv_fp6_e3m2>) + { + return __v.__x; + } +#endif // _CCCL_HAS_NVFP6_E3M2() +#if _CCCL_HAS_NVFP4_E2M1() + else if constexpr (is_same_v<_Tp, __nv_fp4_e2m1>) + { + return __v.__x; + } +#endif // _CCCL_HAS_NVFP4_E2M1() + // NOLINTEND(bugprone-branch-clone) + else + { + static_assert(__always_false_v<_Tp>, "Unsupported floating point format"); + } +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FLOATING_POINT_STORAGE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/traits.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/traits.h new file mode 100644 index 00000000..85ccd673 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__floating_point/traits.h @@ -0,0 +1,171 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FLOATING_POINT_TRAITS_H +#define _CUDA_STD___FLOATING_POINT_TRAITS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +// __is_std_fp_v + +template +inline constexpr bool __is_std_fp_v = false; + +template +inline constexpr bool __is_std_fp_v = __is_std_fp_v<_Tp>; + +template +inline constexpr bool __is_std_fp_v = __is_std_fp_v<_Tp>; + +template +inline constexpr bool __is_std_fp_v = __is_std_fp_v<_Tp>; + +template <> +inline constexpr bool __is_std_fp_v = true; + +template <> +inline constexpr bool __is_std_fp_v = true; + +template <> +inline constexpr bool __is_std_fp_v = true; + +// __is_ext_nv_fp_v + +template +inline constexpr bool __is_ext_nv_fp_v = false; + +template +inline constexpr bool __is_ext_nv_fp_v = __is_ext_nv_fp_v<_Tp>; + +template +inline constexpr bool __is_ext_nv_fp_v = __is_ext_nv_fp_v<_Tp>; + +template +inline constexpr bool __is_ext_nv_fp_v = __is_ext_nv_fp_v<_Tp>; + +#if _CCCL_HAS_NVFP16() +template <> +inline constexpr bool __is_ext_nv_fp_v<__half> = true; +#endif // _CCCL_HAS_NVFP16() + +#if _CCCL_HAS_NVBF16() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_bfloat16> = true; +#endif // _CCCL_HAS_NVBF16() + +#if _CCCL_HAS_NVFP8_E4M3() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_fp8_e4m3> = true; +#endif // _CCCL_HAS_NVFP8_E4M3() + +#if _CCCL_HAS_NVFP8_E5M2() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_fp8_e5m2> = true; +#endif // _CCCL_HAS_NVFP8_E5M2() + +#if _CCCL_HAS_NVFP8_E8M0() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_fp8_e8m0> = true; +#endif // _CCCL_HAS_NVFP8_E8M0() + +#if _CCCL_HAS_NVFP6_E2M3() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_fp6_e2m3> = true; +#endif // _CCCL_HAS_NVFP6_E2M3() + +#if _CCCL_HAS_NVFP6_E3M2() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_fp6_e3m2> = true; +#endif // _CCCL_HAS_NVFP6_E3M2() + +#if _CCCL_HAS_NVFP4_E2M1() +template <> +inline constexpr bool __is_ext_nv_fp_v<__nv_fp4_e2m1> = true; +#endif // _CCCL_HAS_NVFP4_E2M1() + +// __is_ext_compiler_fp_v + +template +inline constexpr bool __is_ext_compiler_fp_v = false; + +template +inline constexpr bool __is_ext_compiler_fp_v = __is_ext_compiler_fp_v<_Tp>; + +template +inline constexpr bool __is_ext_compiler_fp_v = __is_ext_compiler_fp_v<_Tp>; + +template +inline constexpr bool __is_ext_compiler_fp_v = __is_ext_compiler_fp_v<_Tp>; + +#if _CCCL_HAS_FLOAT128() +template <> +inline constexpr bool __is_ext_compiler_fp_v<__float128> = true; +#endif // _CCCL_HAS_FLOAT128() + +// __is_ext_cccl_fp_v + +template +inline constexpr bool __is_ext_cccl_fp_v = false; + +template +inline constexpr bool __is_ext_cccl_fp_v = __is_ext_cccl_fp_v<_Tp>; + +template +inline constexpr bool __is_ext_cccl_fp_v = __is_ext_cccl_fp_v<_Tp>; + +template +inline constexpr bool __is_ext_cccl_fp_v = __is_ext_cccl_fp_v<_Tp>; + +template <__fp_format _Fmt> +inline constexpr bool __is_ext_cccl_fp_v<__cccl_fp<_Fmt>> = true; + +// __is_ext_fp_v + +template +inline constexpr bool __is_ext_fp_v = __is_ext_nv_fp_v<_Tp> || __is_ext_compiler_fp_v<_Tp> || __is_ext_cccl_fp_v<_Tp>; + +// __is_fp_v (todo: use cuda::std::is_floating_point_v instead in the future) + +template +inline constexpr bool __is_fp_v = __is_std_fp_v<_Tp> || __is_ext_fp_v<_Tp>; + +// __fp_is_subset_v + +template <__fp_format _LhsFmt, __fp_format _RhsFmt> +inline constexpr bool __fp_is_subset_v = + (!__fp_is_signed_v<_LhsFmt> || __fp_is_signed_v<_RhsFmt>) + && __fp_exp_min_v<_LhsFmt> >= __fp_exp_min_v<_RhsFmt> && __fp_exp_max_v<_LhsFmt> <= __fp_exp_max_v<_RhsFmt> + && __fp_digits_v<_LhsFmt> <= __fp_digits_v<_RhsFmt> && (!__fp_has_denorm_v<_LhsFmt> || __fp_has_denorm_v<_RhsFmt>); + +// __fp_is_subset_of_v + +template +inline constexpr bool __fp_is_subset_of_v = __fp_is_subset_v<__fp_format_of_v<_Lhs>, __fp_format_of_v<_Rhs>>; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FLOATING_POINT_TRAITS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/binary_function.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/binary_function.h new file mode 100644 index 00000000..b81dfd42 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/binary_function.h @@ -0,0 +1,64 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_BINARY_FUNCTION_H +#define _CUDA_STD___FUNCTIONAL_BINARY_FUNCTION_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT CCCL_DEPRECATED binary_function +{ + using first_argument_type = _Arg1; + using second_argument_type = _Arg2; + using result_type = _Result; +}; + +#endif // defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) + +template +struct __binary_function_keep_layout_base +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using first_argument_type CCCL_DEPRECATED = _Arg1; + using second_argument_type CCCL_DEPRECATED = _Arg2; + using result_type CCCL_DEPRECATED = _Result; +#endif // _LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS +}; + +#if defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) +_CCCL_SUPPRESS_DEPRECATED_PUSH +_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +template +using __binary_function = binary_function<_Arg1, _Arg2, _Result>; +_CCCL_SUPPRESS_DEPRECATED_POP +#else +template +using __binary_function = __binary_function_keep_layout_base<_Arg1, _Arg2, _Result>; +#endif // !_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_BINARY_FUNCTION_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/identity.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/identity.h new file mode 100644 index 00000000..bc8e801f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/identity.h @@ -0,0 +1,57 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_IDENTITY_H +#define _CUDA_STD___FUNCTIONAL_IDENTITY_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +inline constexpr bool __is_identity_v = false; + +struct identity +{ + template + [[nodiscard]] _CCCL_API constexpr _Tp&& operator()(_Tp&& __t) const noexcept + { + return ::cuda::std::forward<_Tp>(__t); + } + + using is_transparent = void; +}; + +template <> +inline constexpr bool __is_identity_v = true; +template <> +inline constexpr bool __is_identity_v> = true; +template <> +inline constexpr bool __is_identity_v> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_IDENTITY_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/invoke.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/invoke.h new file mode 100644 index 00000000..0fc242a2 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/invoke.h @@ -0,0 +1,298 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_INVOKE_H +#define _CUDA_STD___FUNCTIONAL_INVOKE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __any +{ + _CCCL_API inline __any(...); +}; + +template +struct __member_pointer_class_type +{}; + +template +struct __member_pointer_class_type<_Ret _ClassType::*> +{ + using type = _ClassType; +}; + +template +using __member_pointer_class_type_t = typename __member_pointer_class_type<_DecayedFp>::type; + +template , + class _DecayA0 = decay_t<_A0>, + class _ClassT = __member_pointer_class_type_t<_DecayFp>> +using __enable_if_bullet1 = enable_if_t && is_base_of_v<_ClassT, _DecayA0>>; + +template , class _DecayA0 = decay_t<_A0>> +using __enable_if_bullet2 = + enable_if_t && __is_cuda_std_reference_wrapper_v<_DecayA0>>; + +template , + class _DecayA0 = decay_t<_A0>, + class _ClassT = __member_pointer_class_type_t<_DecayFp>> +using __enable_if_bullet3 = enable_if_t && !is_base_of_v<_ClassT, _DecayA0> + && !__is_cuda_std_reference_wrapper_v<_DecayA0>>; + +template , + class _DecayA0 = decay_t<_A0>, + class _ClassT = __member_pointer_class_type_t<_DecayFp>> +using __enable_if_bullet4 = enable_if_t && is_base_of_v<_ClassT, _DecayA0>>; + +template , class _DecayA0 = decay_t<_A0>> +using __enable_if_bullet5 = + enable_if_t && __is_cuda_std_reference_wrapper_v<_DecayA0>>; + +template , + class _DecayA0 = decay_t<_A0>, + class _ClassT = __member_pointer_class_type_t<_DecayFp>> +using __enable_if_bullet6 = enable_if_t && !is_base_of_v<_ClassT, _DecayA0> + && !__is_cuda_std_reference_wrapper_v<_DecayA0>>; + +// __invoke forward declarations + +// fall back - none of the bullets + +template +_CCCL_API inline __nat __invoke(__any, _Args&&... __args); + +// bullets 1, 2 and 3 + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr decltype((::cuda::std::declval<_A0>() + .*::cuda::std::declval<_Fp>())(::cuda::std::declval<_Args>()...)) +__invoke(_Fp&& __f, + _A0&& __a0, + _Args&&... __args) noexcept(noexcept((static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...))) +{ + return (static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...); +} + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr decltype((::cuda::std::declval<_A0>().get() + .*::cuda::std::declval<_Fp>())(::cuda::std::declval<_Args>()...)) +__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args) noexcept(noexcept((__a0.get().*__f)(static_cast<_Args&&>(__args)...))) +{ + return (__a0.get().*__f)(static_cast<_Args&&>(__args)...); +} + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr decltype(((*::cuda::std::declval<_A0>()) + .*::cuda::std::declval<_Fp>())(::cuda::std::declval<_Args>()...)) +__invoke(_Fp&& __f, + _A0&& __a0, + _Args&&... __args) noexcept(noexcept(((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...))) +{ + return ((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...); +} + +// bullets 4, 5 and 6 + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr decltype(::cuda::std::declval<_A0>().*::cuda::std::declval<_Fp>()) +__invoke(_Fp&& __f, _A0&& __a0) noexcept(noexcept(static_cast<_A0&&>(__a0).*__f)) +{ + return static_cast<_A0&&>(__a0).*__f; +} + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr decltype(::cuda::std::declval<_A0>().get().*::cuda::std::declval<_Fp>()) +__invoke(_Fp&& __f, _A0&& __a0) noexcept(noexcept(__a0.get().*__f)) +{ + return __a0.get().*__f; +} + +_CCCL_EXEC_CHECK_DISABLE +template > +_CCCL_API constexpr decltype((*::cuda::std::declval<_A0>()).*::cuda::std::declval<_Fp>()) +__invoke(_Fp&& __f, _A0&& __a0) noexcept(noexcept((*static_cast<_A0&&>(__a0)).*__f)) +{ + return (*static_cast<_A0&&>(__a0)).*__f; +} + +// bullet 7 + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_API constexpr decltype(::cuda::std::declval<_Fp>()(::cuda::std::declval<_Args>()...)) +__invoke(_Fp&& __f, _Args&&... __args) noexcept(noexcept(static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...))) +{ + return static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...); +} + +// __is_invocable +template +using __invoke_result_t = + decltype(::cuda::std::__invoke(::cuda::std::declval<_Fp>(), ::cuda::std::declval<_Args>()...)); + +template +_CCCL_CONCEPT __is_invocable = + _CCCL_REQUIRES_EXPR((_Fp, variadic _Args))(requires(!is_same_v<__nat, __invoke_result_t<_Fp, _Args...>>)); + +template +_CCCL_CONCEPT __is_invocable_r = _CCCL_REQUIRES_EXPR((_Ret, _Fp, variadic _Args))( + requires(__is_invocable<_Fp, _Args...>), + requires((is_void_v<_Ret> || __is_core_convertible<__invoke_result_t<_Fp, _Args...>, _Ret>::value))); + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT invoke_result // + : public enable_if<__is_invocable<_Fp, _Args...>, __invoke_result_t<_Fp, _Args...>> +{ +#if _CCCL_CUDA_COMPILER(NVCC) && defined(__CUDACC_EXTENDED_LAMBDA__) && !_CCCL_DEVICE_COMPILATION() +# if _CCCL_CUDACC_BELOW(12, 3) + static_assert(!__nv_is_extended_device_lambda_closure_type(remove_cvref_t<_Fp>), + "Attempt to use an extended __device__ lambda in a context " + "that requires querying its return type in host code. Use a " + "named function object, an extended __host__ __device__ lambda, or " + "cuda::proclaim_return_type instead."); +# else // ^^^ _CCCL_CUDACC_BELOW(12, 3) ^^^ / vvv _CCCL_CUDACC_AT_LEAST(12, 3) vvv + static_assert( + !__nv_is_extended_device_lambda_closure_type(remove_cvref_t<_Fp>) + || __nv_is_extended_host_device_lambda_closure_type(remove_cvref_t<_Fp>) + || __nv_is_extended_device_lambda_with_preserved_return_type(remove_cvref_t<_Fp>), + "Attempt to use an extended __device__ lambda in a context " + "that requires querying its return type in host code. Use a " + "named function object, an extended __host__ __device__ lambda, " + "cuda::proclaim_return_type, or an extended __device__ lambda " + "with a trailing return type instead ([] __device__ (...) -> RETURN_TYPE {...})."); +# endif // _CCCL_CUDACC_AT_LEAST(12, 3) +#endif +}; + +// is_invocable + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT is_invocable : bool_constant<__is_invocable<_Fn, _Args...>> +{}; + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT is_invocable_r : bool_constant<__is_invocable_r<_Ret, _Fn, _Args...>> +{}; + +template +inline constexpr bool is_invocable_v = __is_invocable<_Fn, _Args...>; + +template +inline constexpr bool is_invocable_r_v = __is_invocable_r<_Ret, _Fn, _Args...>; + +// is_nothrow_invocable + +template +_CCCL_API constexpr void __cccl_test_noexcept_conversion(_Tp) noexcept; + +template +inline constexpr bool __nothrow_invocable_r_imp = false; + +template +inline constexpr bool __nothrow_invocable_r_imp = + noexcept(::cuda::std::__cccl_test_noexcept_conversion<_Ret>( + ::cuda::std::__invoke(declval<_Fp>(), ::cuda::std::declval<_Args>()...))); + +template +inline constexpr bool __nothrow_invocable_r_imp = + noexcept(::cuda::std::__invoke(::cuda::std::declval<_Fp>(), ::cuda::std::declval<_Args>()...)); + +template +inline constexpr bool is_nothrow_invocable_v = + __nothrow_invocable_r_imp<__is_invocable<_Fp, _Args...>, true, void, _Fp, _Args...>; + +template +inline constexpr bool is_nothrow_invocable_r_v = + __nothrow_invocable_r_imp<__is_invocable_r<_Ret, _Fp, _Args...>, is_void_v<_Ret>, _Ret, _Fp, _Args...>; + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT is_nothrow_invocable : bool_constant> +{}; + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT +is_nothrow_invocable_r : bool_constant> +{}; + +// Not going directly through __invoke_result_t because we want the additional device lambda checks in invoke_result +template +using invoke_result_t = typename invoke_result<_Fn, _Args...>::type; + +template +_CCCL_API constexpr invoke_result_t<_Fn, _Args...> +invoke(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_v<_Fn, _Args...>) +{ + return ::cuda::std::__invoke(::cuda::std::forward<_Fn>(__f), ::cuda::std::forward<_Args>(__args)...); +} + +_CCCL_TEMPLATE(class _Ret, class _Fn, class... _Args) +_CCCL_REQUIRES(is_invocable_r_v<_Ret, _Fn, _Args...>) +_CCCL_API constexpr _Ret invoke_r(_Fn&& __f, _Args&&... __args) noexcept(is_nothrow_invocable_r_v<_Ret, _Fn, _Args...>) +{ + if constexpr (is_void_v<_Ret>) + { + ::cuda::std::__invoke(::cuda::std::forward<_Fn>(__f), ::cuda::std::forward<_Args>(__args)...); + } + else + { + return ::cuda::std::__invoke(::cuda::std::forward<_Fn>(__f), ::cuda::std::forward<_Args>(__args)...); + } +} + +/// The type of intermediate accumulator (according to P2322R6) +template +using __accumulator_t = decay_t>; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_INVOKE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/reference_wrapper.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/reference_wrapper.h new file mode 100644 index 00000000..ed56284d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/reference_wrapper.h @@ -0,0 +1,116 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_REFERENCE_WRAPPER_H +#define _CUDA_STD___FUNCTIONAL_REFERENCE_WRAPPER_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT reference_wrapper : public __weak_result_type<_Tp> +{ +public: + // types + using type = _Tp; + +private: + type* __f_{}; + + static _CCCL_API void __fun(_Tp&) noexcept; + static void __fun(_Tp&&) = delete; // NOLINT(modernize-use-equals-delete) + +public: + // NOLINTBEGIN(bugprone-forwarding-reference-overload) + template < + class _Up, + class = enable_if_t::value, decltype(__fun(::cuda::std::declval<_Up>()))>> + _CCCL_API constexpr reference_wrapper(_Up&& __u) noexcept(noexcept(__fun(::cuda::std::declval<_Up>()))) + { + type& __f = static_cast<_Up&&>(__u); + __f_ = ::cuda::std::addressof(__f); + } + // NOLINTEND(bugprone-forwarding-reference-overload) + + // access + _CCCL_API constexpr operator type&() const noexcept + { + return *__f_; + } + [[nodiscard]] _CCCL_API constexpr type& get() const noexcept + { + return *__f_; + } + + // invoke + template + _CCCL_API constexpr invoke_result_t operator()(_ArgTypes&&... __args) const + noexcept(is_nothrow_invocable_v<_Tp&, _ArgTypes...>) + { + return ::cuda::std::invoke(get(), ::cuda::std::forward<_ArgTypes>(__args)...); + } +}; + +template +_CCCL_DEDUCTION_GUIDE_ATTRIBUTES reference_wrapper(_Tp&) -> reference_wrapper<_Tp>; + +template +[[nodiscard]] _CCCL_API constexpr reference_wrapper<_Tp> ref(_Tp& __t) noexcept +{ + return reference_wrapper<_Tp>(__t); +} + +template +[[nodiscard]] _CCCL_API constexpr reference_wrapper<_Tp> ref(reference_wrapper<_Tp> __t) noexcept +{ + return __t; +} + +template +[[nodiscard]] _CCCL_API constexpr reference_wrapper cref(const _Tp& __t) noexcept +{ + return reference_wrapper(__t); +} + +template +[[nodiscard]] _CCCL_API constexpr reference_wrapper cref(reference_wrapper<_Tp> __t) noexcept +{ + return __t; +} + +template +void ref(const _Tp&&) = delete; +template +void cref(const _Tp&&) = delete; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_REFERENCE_WRAPPER_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unary_function.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unary_function.h new file mode 100644 index 00000000..e405b62c --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unary_function.h @@ -0,0 +1,63 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_UNARY_FUNCTION_H +#define _CUDA_STD___FUNCTIONAL_UNARY_FUNCTION_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT CCCL_DEPRECATED unary_function +{ + using argument_type = _Arg; + using result_type = _Result; +}; + +#endif // _LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION + +template +struct __unary_function_keep_layout_base +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using argument_type CCCL_DEPRECATED = _Arg; + using result_type CCCL_DEPRECATED = _Result; +#endif +}; + +#if defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) + +_CCCL_SUPPRESS_DEPRECATED_PUSH +_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +template +using __unary_function = unary_function<_Arg, _Result>; +_CCCL_SUPPRESS_DEPRECATED_POP + +#else +template +using __unary_function = __unary_function_keep_layout_base<_Arg, _Result>; +#endif // !_LIBCUDACXX_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_UNARY_FUNCTION_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unwrap_ref.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unwrap_ref.h new file mode 100644 index 00000000..d9b24ebb --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/unwrap_ref.h @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_UNWRAP_REF_H +#define _CUDA_STD___FUNCTIONAL_UNWRAP_REF_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct unwrap_reference +{ + using type _CCCL_NODEBUG_ALIAS = _Tp; +}; + +template +struct unwrap_reference> +{ + using type _CCCL_NODEBUG_ALIAS = _Tp&; +}; + +template +using unwrap_reference_t = typename unwrap_reference<_Tp>::type; + +template +struct unwrap_ref_decay : unwrap_reference> +{}; + +template +using unwrap_ref_decay_t = typename unwrap_ref_decay<_Tp>::type; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_UNWRAP_REF_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/weak_result_type.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/weak_result_type.h new file mode 100644 index 00000000..40ce6801 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__functional/weak_result_type.h @@ -0,0 +1,262 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FUNCTIONAL_WEAK_RESULT_TYPE_H +#define _CUDA_STD___FUNCTIONAL_WEAK_RESULT_TYPE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +_CCCL_CONCEPT __has_member_result_type = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::result_type)); + +// __weak_result_type + +template +struct __derives_from_unary_function +{ +private: + struct __two + { + char __lx; + char __lxx; + }; + static _CCCL_API inline __two __test(...); + template + static _CCCL_API inline __unary_function<_Ap, _Rp> __test(const volatile __unary_function<_Ap, _Rp>*); + +public: + static const bool value = !is_same_v; + using type = decltype(__test((_Tp*) nullptr)); +}; + +template +struct __derives_from_binary_function +{ +private: + struct __two + { + char __lx; + char __lxx; + }; + static __two _CCCL_API inline __test(...); + template + static _CCCL_API inline __binary_function<_A1, _A2, _Rp> __test(const volatile __binary_function<_A1, _A2, _Rp>*); + +public: + static const bool value = !is_same_v; + using type = decltype(__test((_Tp*) nullptr)); +}; + +template ::value> +struct __maybe_derive_from_unary_function // bool is true + : public __derives_from_unary_function<_Tp>::type +{}; + +template +struct __maybe_derive_from_unary_function<_Tp, false> +{}; + +template ::value> +struct __maybe_derive_from_binary_function // bool is true + : public __derives_from_binary_function<_Tp>::type +{}; + +template +struct __maybe_derive_from_binary_function<_Tp, false> +{}; + +template > +struct __weak_result_type_imp // bool is true + : public __maybe_derive_from_unary_function<_Tp> + , public __maybe_derive_from_binary_function<_Tp> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = typename _Tp::result_type; +#endif +}; + +template +struct __weak_result_type_imp<_Tp, false> + : public __maybe_derive_from_unary_function<_Tp> + , public __maybe_derive_from_binary_function<_Tp> +{}; + +template +struct __weak_result_type : public __weak_result_type_imp<_Tp> +{}; + +// 0 argument case + +template +struct __weak_result_type<_Rp()> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (&)()> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (*)()> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +// 1 argument case + +template +struct __weak_result_type<_Rp(_A1)> : public __unary_function<_A1, _Rp> +{}; + +template +struct __weak_result_type<_Rp (&)(_A1)> : public __unary_function<_A1, _Rp> +{}; + +template +struct __weak_result_type<_Rp (*)(_A1)> : public __unary_function<_A1, _Rp> +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)()> : public __unary_function<_Cp*, _Rp> +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)() const> : public __unary_function +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)() volatile> : public __unary_function +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)() const volatile> : public __unary_function +{}; + +// 2 argument case + +template +struct __weak_result_type<_Rp(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> +{}; + +template +struct __weak_result_type<_Rp (*)(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> +{}; + +template +struct __weak_result_type<_Rp (&)(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1)> : public __binary_function<_Cp*, _A1, _Rp> +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1) const> : public __binary_function +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1) volatile> : public __binary_function +{}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1) const volatile> : public __binary_function +{}; + +// 3 or more arguments + +template +struct __weak_result_type<_Rp(_A1, _A2, _A3, _A4...)> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (&)(_A1, _A2, _A3, _A4...)> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (*)(_A1, _A2, _A3, _A4...)> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...)> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) volatile> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +template +struct __weak_result_type<_Rp (_Cp::*)(_A1, _A2, _A3...) const volatile> +{ +#if _CCCL_STD_VER <= 2017 || defined(_LIBCUDACXX_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS) + using result_type _CCCL_NODEBUG_ALIAS CCCL_DEPRECATED = _Rp; +#endif +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FUNCTIONAL_WEAK_RESULT_TYPE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/array.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/array.h new file mode 100644 index 00000000..887c9d35 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/array.h @@ -0,0 +1,68 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_ARRAY_H +#define _CUDA_STD___FWD_ARRAY_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +// std:: forward declarations + +#if _CCCL_HAS_HOST_STD_LIB() +_CCCL_BEGIN_NAMESPACE_STD + +# if _CCCL_HOST_STD_LIB(STL) +template +class array; +# else // ^^^ _CCCL_HOST_STD_LIB(STL) ^^^ / vvv !_CCCL_HOST_STD_LIB(STL) vvv +template +struct array; +# endif // !_CCCL_HOST_STD_LIB(STL) + +_CCCL_END_NAMESPACE_STD +#endif // _CCCL_HAS_HOST_STD_LIB() + +// ::cuda::std:: forward declaration + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT array; + +template +inline constexpr bool __is_cuda_std_array_v = false; + +template +inline constexpr bool __is_cuda_std_array_v> = true; + +#if _CCCL_HAS_HOST_STD_LIB() +template +inline constexpr bool __is_std_array_v = false; + +template +inline constexpr bool __is_std_array_v<::std::array<_Tp, _Sz>> = true; +#endif // _CCCL_HAS_HOST_STD_LIB() + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_ARRAY_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/complex.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/complex.h new file mode 100644 index 00000000..363afb90 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/complex.h @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_COMPLEX_H +#define _CUDA_STD___FWD_COMPLEX_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +// std:: forward declarations + +#if _CCCL_HAS_HOST_STD_LIB() +_CCCL_BEGIN_NAMESPACE_STD + +template +class complex; + +_CCCL_END_NAMESPACE_STD +#endif // _CCCL_HAS_HOST_STD_LIB() + +// cuda::std:: forward declarations + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT complex; + +// __is_std_complex_v + +template +inline constexpr bool __is_std_complex_v = false; +#if _CCCL_HAS_HOST_STD_LIB() +template +inline constexpr bool __is_std_complex_v = __is_std_complex_v<_Tp>; +template +inline constexpr bool __is_std_complex_v = __is_std_complex_v<_Tp>; +template +inline constexpr bool __is_std_complex_v = __is_std_complex_v<_Tp>; +template +inline constexpr bool __is_std_complex_v<::std::complex<_Tp>> = true; +#endif // _CCCL_HAS_HOST_STD_LIB() + +// __is_cuda_std_complex_v + +template +inline constexpr bool __is_cuda_std_complex_v = false; +template +inline constexpr bool __is_cuda_std_complex_v = __is_cuda_std_complex_v<_Tp>; +template +inline constexpr bool __is_cuda_std_complex_v = __is_cuda_std_complex_v<_Tp>; +template +inline constexpr bool __is_cuda_std_complex_v = __is_cuda_std_complex_v<_Tp>; +template +inline constexpr bool __is_cuda_std_complex_v> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_COMPLEX_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/format.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/format.h new file mode 100644 index 00000000..e96561b1 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/format.h @@ -0,0 +1,120 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_FORMAT_H +#define _CUDA_STD___FWD_FORMAT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +#if __cpp_lib_format >= 201907L + +_CCCL_BEGIN_NAMESPACE_STD + +template +struct formatter; + +_CCCL_END_NAMESPACE_STD + +#endif // __cpp_lib_format >= 201907L + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT basic_format_parse_context; + +using format_parse_context = basic_format_parse_context; +#if _CCCL_HAS_WCHAR_T() +using wformat_parse_context = basic_format_parse_context; +#endif // _CCCL_HAS_WCHAR_T() + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT _CCCL_PREFERRED_NAME(format_parse_context) +#if _CCCL_HAS_WCHAR_T() + _CCCL_PREFERRED_NAME(wformat_parse_context) +#endif // _CCCL_HAS_WCHAR_T() + basic_format_parse_context; + +template +class __fmt_output_buffer; + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT basic_format_arg; + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT basic_format_context; + +using format_context = basic_format_context<__back_insert_iterator<__fmt_output_buffer>, char>; +#if _CCCL_HAS_WCHAR_T() +using wformat_context = basic_format_context<__back_insert_iterator<__fmt_output_buffer>, wchar_t>; +#endif // _CCCL_HAS_WCHAR_T() + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT _CCCL_PREFERRED_NAME(format_context) +#if _CCCL_HAS_WCHAR_T() + _CCCL_PREFERRED_NAME(wformat_context) +#endif // _CCCL_HAS_WCHAR_T() + basic_format_context; + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT basic_format_args; + +using format_args = basic_format_args; +#if _CCCL_HAS_WCHAR_T() +using wformat_args = basic_format_args; +#endif // _CCCL_HAS_WCHAR_T() + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT _CCCL_PREFERRED_NAME(format_args) +#if _CCCL_HAS_WCHAR_T() + _CCCL_PREFERRED_NAME(wformat_args) +#endif // _CCCL_HAS_WCHAR_T() + basic_format_args; + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT basic_format_string; + +template +using format_string = basic_format_string...>; + +#if _CCCL_HAS_WCHAR_T() +template +using wformat_string = basic_format_string...>; +#endif // _CCCL_HAS_WCHAR_T() + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT format_to_n_result; + +enum class range_format +{ + disabled, + map, + set, + sequence, + string, + debug_string, +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_FORMAT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/fp.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/fp.h new file mode 100644 index 00000000..7aee6d06 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/fp.h @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_FP_H +#define _CUDA_STD___FWD_FP_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +enum class __fp_format; + +template <__fp_format _Fmt> +class __cccl_fp; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_FP_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/get.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/get.h new file mode 100644 index 00000000..26f27725 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/get.h @@ -0,0 +1,132 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_GET_H +#define _CUDA_STD___FWD_GET_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +[[nodiscard]] _CCCL_API constexpr tuple_element_t<_Ip, tuple<_Tp...>>& get(tuple<_Tp...>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const tuple_element_t<_Ip, tuple<_Tp...>>& get(const tuple<_Tp...>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr tuple_element_t<_Ip, tuple<_Tp...>>&& get(tuple<_Tp...>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const tuple_element_t<_Ip, tuple<_Tp...>>&& get(const tuple<_Tp...>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr tuple_element_t<_Ip, pair<_T1, _T2>>& get(pair<_T1, _T2>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const tuple_element_t<_Ip, pair<_T1, _T2>>& get(const pair<_T1, _T2>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr tuple_element_t<_Ip, pair<_T1, _T2>>&& get(pair<_T1, _T2>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const tuple_element_t<_Ip, pair<_T1, _T2>>&& get(const pair<_T1, _T2>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr _Tp& get(array<_Tp, _Size>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const _Tp& get(const array<_Tp, _Size>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr _Tp&& get(array<_Tp, _Size>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const _Tp&& get(const array<_Tp, _Size>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr _Tp& get(complex<_Tp>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr _Tp&& get(complex<_Tp>&&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const _Tp& get(const complex<_Tp>&) noexcept; + +template +[[nodiscard]] _CCCL_API constexpr const _Tp&& get(const complex<_Tp>&&) noexcept; + +_CCCL_END_NAMESPACE_CUDA_STD + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES + +#if _CCCL_HAS_CONCEPTS() +template + requires((_Index == 0) && copyable<_Iter>) || (_Index == 1) +#else // ^^^ C++20 ^^^ / vvv C++17 vvv +template ) || (_Index == 1), int> = 0> +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ +_CCCL_API constexpr auto get(const subrange<_Iter, _Sent, _Kind>& __subrange); + +#if _CCCL_HAS_CONCEPTS() +template + requires(_Index < 2) +#else // ^^^ C++20 ^^^ / vvv C++17 vvv +template = 0> +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ +_CCCL_API constexpr auto get(subrange<_Iter, _Sent, _Kind>&& __subrange); + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +using ::cuda::std::ranges::get; + +// Explicitly rely on ADL, mostly for constructors of host STL types, where we cannot squeze using ::cuda::std::get; in +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr decltype(auto) __adl_get(_TupleLike&& __t) noexcept +{ + using ::cuda::std::get; + return get<_Ip>(::cuda::std::forward<_TupleLike>(__t)); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_GET_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/iterator.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/iterator.h new file mode 100644 index 00000000..7aeb940a --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/iterator.h @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_ITERATOR_H +#define _CUDA_STD___FWD_ITERATOR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT __back_insert_iterator; + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT iterator_traits; + +_LIBCUDACXX_BEGIN_HIDDEN_FRIEND_NAMESPACE + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT reverse_iterator; + +_LIBCUDACXX_END_HIDDEN_FRIEND_NAMESPACE(reverse_iterator) + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_ITERATOR_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/pair.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/pair.h new file mode 100644 index 00000000..e526d6b1 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/pair.h @@ -0,0 +1,53 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_PAIR_H +#define _CUDA_STD___FWD_PAIR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +// std:: forward declarations + +#if _CCCL_HAS_HOST_STD_LIB() +_CCCL_BEGIN_NAMESPACE_STD + +template +struct pair; + +_CCCL_END_NAMESPACE_STD +#endif // _CCCL_HAS_HOST_STD_LIB() + +// cuda::std:: forward declarations + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT pair; + +template +inline constexpr bool __is_cuda_std_pair = false; + +template +inline constexpr bool __is_cuda_std_pair> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_PAIR_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/reference_wrapper.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/reference_wrapper.h new file mode 100644 index 00000000..e95bf5ab --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/reference_wrapper.h @@ -0,0 +1,52 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_REFERENCE_WRAPPER_H +#define _CUDA_STD___FWD_REFERENCE_WRAPPER_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +// std:: forward declarations + +#if _CCCL_HAS_HOST_STD_LIB() +_CCCL_BEGIN_NAMESPACE_STD + +template +class reference_wrapper; + +_CCCL_END_NAMESPACE_STD +#endif // _CCCL_HAS_HOST_STD_LIB() + +// cuda::std:: forward declarations + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT reference_wrapper; + +template +inline constexpr bool __is_cuda_std_reference_wrapper_v = false; +template +inline constexpr bool __is_cuda_std_reference_wrapper_v> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_REFERENCE_WRAPPER_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/span.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/span.h new file mode 100644 index 00000000..a7d28727 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/span.h @@ -0,0 +1,45 @@ +// -*- C++ -*- +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_SPAN_H +#define _CUDA_STD___FWD_SPAN_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +inline constexpr size_t dynamic_extent = static_cast(-1); + +template +class span; + +template +inline constexpr bool __is_cuda_std_span_v = false; + +template +inline constexpr bool __is_cuda_std_span_v> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_SPAN_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/subrange.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/subrange.h new file mode 100644 index 00000000..ccd8aadc --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/subrange.h @@ -0,0 +1,65 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_SUBRANGE_H +#define _CUDA_STD___FWD_SUBRANGE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES + +enum class _CCCL_TYPE_VISIBILITY_DEFAULT subrange_kind : bool +{ + unsized, + sized +}; + +#if _CCCL_HAS_CONCEPTS() +template _Sent = _Iter, + subrange_kind _Kind = sized_sentinel_for<_Sent, _Iter> ? subrange_kind::sized : subrange_kind::unsized> + requires(_Kind == subrange_kind::sized || !sized_sentinel_for<_Sent, _Iter>) +class _CCCL_TYPE_VISIBILITY_DEFAULT subrange; +#else // ^^^ C++20 ^^^ / vvv C++17 vvv +template ? subrange_kind::sized : subrange_kind::unsized, + enable_if_t, int> = 0, + enable_if_t, int> = 0, + enable_if_t<(_Kind == subrange_kind::sized || !sized_sentinel_for<_Sent, _Iter>), int> = 0> +class _CCCL_TYPE_VISIBILITY_DEFAULT subrange; +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +inline constexpr bool __is_cuda_std_ranges_subrange_v = false; + +template +inline constexpr bool __is_cuda_std_ranges_subrange_v<::cuda::std::ranges::subrange<_Iter, _Sent, _Kind>> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_SUBRANGE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/tuple.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/tuple.h new file mode 100644 index 00000000..4720dc11 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__fwd/tuple.h @@ -0,0 +1,52 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___FWD_TUPLE_H +#define _CUDA_STD___FWD_TUPLE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#if _CCCL_HAS_HOST_STD_LIB() +_CCCL_BEGIN_NAMESPACE_STD + +template +class tuple; + +_CCCL_END_NAMESPACE_STD +#endif // _CCCL_HAS_HOST_STD_LIB() + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +class _CCCL_TYPE_VISIBILITY_DEFAULT tuple; + +template +inline constexpr bool __is_tuple_of_iterator_references_v = false; + +template +inline constexpr bool __is_cuda_std_tuple = false; + +template +inline constexpr bool __is_cuda_std_tuple> = true; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___FWD_TUPLE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/cstdio b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/cstdio new file mode 100644 index 00000000..87950e63 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/cstdio @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___HOST_STDLIB_CSTDIO +#define _CUDA_STD___HOST_STDLIB_CSTDIO + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_HOSTED() +# include +#endif // _CCCL_HOSTED() + +#endif // _CUDA_STD___HOST_STDLIB_CSTDIO diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/math.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/math.h new file mode 100644 index 00000000..d6396304 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/math.h @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___HOST_STDLIB_MATH_H +#define _CUDA_STD___HOST_STDLIB_MATH_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_HOSTED() +# include + +// Standard C++ library comes with it's own C++ compatible header. However, if the include paths are jumbled, +// it might happen that the original C is found first. This is a problem because C headers define many of the +// math functions as macros which would change our definitions. So, we check whether any of the functions are defined +// as a macro to distinguish the C++ copatibility header from the C header. +# if defined(fabs) || defined(fmod) || defined(remainder) || defined(remquo) || defined(fma) || defined(fmax) \ + || defined(fmin) || defined(fdim) || defined(exp) || defined(exp2) || defined(expm1) || defined(log) \ + || defined(log10) || defined(log2) || defined(log1p) || defined(pow) || defined(sqrt) || defined(cbrt) \ + || defined(hypot) || defined(sin) || defined(cos) || defined(tan) || defined(asin) || defined(acos) \ + || defined(atan) || defined(atan2) || defined(sinh) || defined(cosh) || defined(tanh) || defined(asinh) \ + || defined(acosh) || defined(atanh) || defined(erf) || defined(erfc) || defined(tgamma) || defined(lgamma) \ + || defined(ceil) || defined(floor) || defined(trunc) || defined(round) || defined(lround) || defined(llround) \ + || defined(nearbyint) || defined(rint) || defined(lrint) || defined(llrint) || defined(frexp) || defined(ldexp) \ + || defined(scalbn) || defined(scalbln) || defined(ilogb) || defined(logb) || defined(nextafter) \ + || defined(nexttoward) || defined(copysign) || defined(fpclassify) || defined(isfinite) || defined(isinf) \ + || defined(isnan) || defined(isnormal) || defined(signbit) || defined(isgreater) || defined(isgreaterequal) \ + || defined(isless) || defined(islessequal) || defined(islessgreater) || defined(isunordered) +# error \ + "libcu++ requires the C++ compatibility header, not the C header. Please, check your include paths." +# endif // math functions defined as macros + +#endif // _CCCL_HOSTED() + +#endif // _CUDA_STD___HOST_STDLIB_MATH_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/memory b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/memory new file mode 100644 index 00000000..119de005 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/memory @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___HOST_STDLIB_MEMORY +#define _CUDA_STD___HOST_STDLIB_MEMORY + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// When nvc++ uses CCCL components as part of its implementation of +// Standard C++ algorithms, a cycle of included files may result when CCCL code +// tries to use a standard algorithm. The THRUST_INCLUDING_ALGORITHMS_HEADER macro +// is defined only when CCCL is including an algorithms-related header, giving +// the compiler a chance to detect and break the cycle of includes. + +#if _CCCL_HOSTED() +# define THRUST_INCLUDING_ALGORITHMS_HEADER +# include +# undef THRUST_INCLUDING_ALGORITHMS_HEADER +#endif // _CCCL_HOSTED() + +#endif // _CUDA_STD___HOST_STDLIB_MEMORY diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/new b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/new new file mode 100644 index 00000000..cc2db623 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/new @@ -0,0 +1,29 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___HOST_STDLIB_NEW +#define _CUDA_STD___HOST_STDLIB_NEW + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// HostJiT also needs +#if !_CCCL_COMPILER(NVRTC) +# include +#endif // !_CCCL_COMPILER(NVRTC) + +#endif // _CUDA_STD___HOST_STDLIB_NEW diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/stdexcept b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/stdexcept new file mode 100644 index 00000000..053f822f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__host_stdlib/stdexcept @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___HOST_STDLIB_STDEXCEPT +#define _CUDA_STD___HOST_STDLIB_STDEXCEPT + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_HOSTED() +# include +#endif // _CCCL_HOSTED() + +#endif // _CUDA_STD___HOST_STDLIB_STDEXCEPT diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/atomic.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/atomic.h new file mode 100644 index 00000000..94d87d23 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/atomic.h @@ -0,0 +1,55 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_ATOMIC_H +#define _CUDA_STD___INTERNAL_ATOMIC_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#if _CCCL_CUDA_COMPILATION() +# define _CCCL_ATOMIC_ALWAYS_LOCK_FREE(size, ptr) (size <= 8) +#elif _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(GCC) +# define _CCCL_ATOMIC_ALWAYS_LOCK_FREE(...) __atomic_always_lock_free(__VA_ARGS__) +#endif // _CCCL_CUDA_COMPILER + +// Enable bypassing automatic storage checks in atomics when using CTK 12.2 and below and if NDEBUG is defined. +// A compiler bug prevents the safe use of `__is_local` and PTX spacep until after 13.0. +#ifndef _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE +# if _CCCL_CUDACC_BELOW(13, 1) && !defined(NDEBUG) +# define _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE +# endif // _CCCL_CUDACC_BELOW(13, 1) +#endif // _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE + +#define _CCCL_ATOMIC_FLAG_TYPE int + +// Clang provides 128b atomics as a builtin +#if defined(CCCL_ENABLE_EXPERIMENTAL_HOST_ATOMICS_128B) +# define _CCCL_HOST_128_ATOMICS_ENABLED() 1 +# define _CCCL_HOST_128_ATOMICS_MAYBE() 0 +// GCC does not provide 128b atomics, but they may be available as a library, this requires opt-in usage. +// See: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html "-mcx16" for more +#elif _CCCL_COMPILER(CLANG) || _CCCL_COMPILER(GCC) +# define _CCCL_HOST_128_ATOMICS_ENABLED() 0 +# define _CCCL_HOST_128_ATOMICS_MAYBE() 1 +#else +# define _CCCL_HOST_128_ATOMICS_ENABLED() 0 +# define _CCCL_HOST_128_ATOMICS_MAYBE() 0 +#endif + +#endif // _CUDA_STD___INTERNAL_ATOMIC_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/cpp_dialect.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/cpp_dialect.h new file mode 100644 index 00000000..e5d9f533 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/cpp_dialect.h @@ -0,0 +1,44 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_CPP_DIALECT_H +#define _CUDA_STD___INTERNAL_CPP_DIALECT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// Define LIBCUDACXX_COMPILER_DEPRECATION macro: +#if _CCCL_COMPILER(MSVC) || _CCCL_COMPILER(NVRTC) +# define LIBCUDACXX_COMP_DEPR_IMPL(msg) \ + _CCCL_PRAGMA(message(__FILE__ ":" _CCCL_TO_STRING(__LINE__) ": warning: " #msg)) +#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# define LIBCUDACXX_COMP_DEPR_IMPL(msg) _CCCL_PRAGMA(GCC warning #msg) +#endif // !_CCCL_COMPILER(MSVC) + +// clang-format off +#define LIBCUDACXX_DIALECT_DEPRECATION(REQ, CUR) \ + LIBCUDACXX_COMP_DEPR_IMPL( \ + libcu++ requires at least REQ. CUR is deprecated but still supported. CUR support will be removed in a \ + future release. Define CCCL_IGNORE_DEPRECATED_CPP_DIALECT to suppress this message.) +// clang-format on + +#ifndef CCCL_IGNORE_DEPRECATED_CPP_DIALECT +# if _CCCL_STD_VER < 2017 +# error libcu++ requires at least C++ 17. Define CCCL_IGNORE_DEPRECATED_CPP_DIALECT to suppress this message. +# endif // _CCCL_STD_VER < 2017 +#endif // CCCL_IGNORE_DEPRECATED_CPP_DIALECT + +#endif // _CUDA_STD___INTERNAL_CPP_DIALECT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/features.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/features.h new file mode 100644 index 00000000..c4e4662f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/features.h @@ -0,0 +1,127 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_FEATURES_H +#define _CUDA_STD___INTERNAL_FEATURES_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#define _LIBCUDACXX_HAS_CXX20_CHRONO_LITERALS() (!_CCCL_COMPILER(CLANG) || _CCCL_STD_VER >= 2020) +#define _LIBCUDACXX_HAS_MONOTONIC_CLOCK() 0 +#define _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() 0 + +#if _CCCL_CUDA_COMPILATION() || __cpp_aligned_new < 201606 +# define _LIBCUDACXX_HAS_ALIGNED_ALLOCATION() 0 +#else +# define _LIBCUDACXX_HAS_ALIGNED_ALLOCATION() 1 +#endif // !_CCCL_CUDA_COMPILATION() && __cpp_aligned_new >= 201606 + +// We need `is_constant_evaluated` for clang and gcc. MSVC also needs extensive rework +#if !defined(_CCCL_BUILTIN_IS_CONSTANT_EVALUATED) +# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0 +#elif _CCCL_COMPILER(NVRTC) +# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0 +#elif _CCCL_COMPILER(MSVC) +# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0 +#elif _CCCL_CUDA_COMPILER(CLANG) +# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 0 +#else +# define _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() 1 +#endif + +#if _LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() +# define _CCCL_CONSTEXPR_COMPLEX constexpr +#else +# define _CCCL_CONSTEXPR_COMPLEX +#endif // !_LIBCUDACXX_HAS_CONSTEXPR_COMPLEX_OPERATIONS() + +#ifndef _LIBCUDACXX_HAS_NO_INCOMPLETE_RANGES +# define _LIBCUDACXX_HAS_NO_INCOMPLETE_RANGES +#endif // _LIBCUDACXX_HAS_NO_INCOMPLETE_RANGES + +// libcu++ requires host device support for its tests. Until then restrict usage to at least 12.2 +#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2) +# define _LIBCUDACXX_HAS_NVFP16() 1 +#else +# define _LIBCUDACXX_HAS_NVFP16() 0 +#endif // _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2) + +// libcu++ requires host device support for its tests. Until then restrict usage to at least 12.2 +#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2) +# define _LIBCUDACXX_HAS_NVBF16() 1 +#else +# define _LIBCUDACXX_HAS_NVBF16() 0 +#endif // _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2) + +#if _CCCL_COMPILER(MSVC) +# define _CCCL_ALIGNAS_TYPE(x) alignas(x) +# define _CCCL_ALIGNAS(x) __declspec(align(x)) +#elif _CCCL_HAS_FEATURE(cxx_alignas) +# define _CCCL_ALIGNAS_TYPE(x) alignas(x) +# define _CCCL_ALIGNAS(x) alignas(x) +#else +# define _CCCL_ALIGNAS_TYPE(x) __attribute__((__aligned__(alignof(x)))) +# define _CCCL_ALIGNAS(x) __attribute__((__aligned__(x))) +#endif // !_CCCL_COMPILER(MSVC) && !_CCCL_HAS_FEATURE(cxx_alignas) + +// We can only expose constexpr allocations if the compiler supports it +// For now disable constexpr allocation support until we can actually use +#if 0 && __cpp_constexpr_dynamic_alloc >= 201907L && __cpp_lib_constexpr_dynamic_alloc >= 201907L \ + && _CCCL_STD_VER >= 2020 && !_CCCL_COMPILER(NVRTC) +# define _CCCL_HAS_CONSTEXPR_ALLOCATION +# define _CCCL_CONSTEXPR_CXX20_ALLOCATION constexpr +#else // ^^^ has constexpr allocations ^^^ / vvv no constexpr allocations vvv +# define _CCCL_CONSTEXPR_CXX20_ALLOCATION +#endif // ^^^ no constexpr allocations ^^^ + +// Enable removed C++17 features +#if defined(_LIBCUDACXX_ENABLE_CXX17_REMOVED_FEATURES) +# define _LIBCUDACXX_ENABLE_CXX17_REMOVED_BINDERS +#endif // _LIBCUDACXX_ENABLE_CXX17_REMOVED_FEATURES + +#ifndef _CCCL_DISABLE_ADDITIONAL_DIAGNOSTICS +# define _CCCL_DIAGNOSE_WARNING(_COND, _MSG) _CCCL_DIAGNOSE_IF(_COND, _MSG, "warning") +# define _CCCL_DIAGNOSE_ERROR(_COND, _MSG) _CCCL_DIAGNOSE_IF(_COND, _MSG, "error") +#else +# define _CCCL_DIAGNOSE_WARNING(_COND, _MSG) +# define _CCCL_DIAGNOSE_ERROR(_COND, _MSG) +#endif + +#define _CCCL_HAS_SIMD_F32X2_INTRINSICS() \ + (_CCCL_CUDACC_AT_LEAST(12, 8) && _CCCL_HAS_CTK() && !_CCCL_CUDA_COMPILER(CLANG)) +#define _CCCL_HAS_SIMD_F32X2_PTX() (__cccl_ptx_isa >= 860ULL) +#define _CCCL_HAS_SIMD_F32X2() \ + ((_CCCL_HAS_SIMD_F32X2_INTRINSICS() || _CCCL_HAS_SIMD_F32X2_PTX()) && !_CCCL_TILE_COMPILATION()) + +// nvcc >= 12.8 already optimizes 16-bit X2 min/max operations to SIMD instructions +#define _CCCL_HAS_SIMD_16BIT_MIN_MAX_COMPILER_OPTIMIZATION() _CCCL_CUDA_COMPILER(NVCC, >=, 12, 8) + +#define _CCCL_HAS_SIMD_8BIT_INTRINSICS() 0 // TODO(fbusato): CTK 13.2 produces non-optimal code for 8-bit SIMD instrs. +#define _CCCL_HAS_SIMD_8BIT_PTX() (__cccl_ptx_isa >= 920ULL) +#define _CCCL_HAS_SIMD_8BIT() \ + ((_CCCL_HAS_SIMD_8BIT_PTX() || _CCCL_HAS_SIMD_8BIT_INTRINSICS()) && !_CCCL_TILE_COMPILATION()) + +// Third party libraries + +#if (__has_include() || __has_include()) && \ + !_CCCL_COMPILER(NVRTC) && !defined(CCCL_DISABLE_DLPACK) +# define _CCCL_HAS_DLPACK() 1 +#else // ^^^ has dlpack ^^^ / vvv no dlpack vvv +# define _CCCL_HAS_DLPACK() 0 +#endif // ^^^ no dlpack ^^^ + +#endif // _CUDA_STD___INTERNAL_FEATURES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/namespaces.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/namespaces.h new file mode 100644 index 00000000..4876dd6f --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/namespaces.h @@ -0,0 +1,188 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_NAMESPACES_H +#define _CUDA_STD___INTERNAL_NAMESPACES_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +// During the header testing, we want to check if the code is wrapped by the prologue/epilogue +#if defined(_CCCL_HEADER_TEST) +# define _CCCL_PROLOGUE_INCLUDE_CHECK() \ + static_assert(_CCCL_PROLOGUE_INCLUDED(), "missing #include "); +#else // ^^^ defined(_CCCL_HEADER_TEST) ^^^ / vvv !defined(_CCCL_HEADER_TEST) vvv +# define _CCCL_PROLOGUE_INCLUDE_CHECK() +#endif // ^^^ !defined(_CCCL_HEADER_TEST) ^^^ + +#ifndef _LIBCUDACXX_ABI_NAMESPACE +# define _LIBCUDACXX_ABI_NAMESPACE _CCCL_PP_CAT(__, _LIBCUDACXX_CUDA_ABI_VERSION) +#endif // _LIBCUDACXX_ABI_NAMESPACE + +#define _CCCL_BEGIN_NAMESPACE_NOVERSION(_NS) \ + _CCCL_PROLOGUE_INCLUDE_CHECK() namespace _NS \ + { +#define _CCCL_END_NAMESPACE_NOVERSION(_NS) \ + } \ + _CCCL_PROLOGUE_INCLUDE_CHECK() +#define _CCCL_BEGIN_NAMESPACE(_NS) \ + _CCCL_BEGIN_NAMESPACE_NOVERSION(_NS) inline namespace _LIBCUDACXX_ABI_NAMESPACE \ + { +#define _CCCL_END_NAMESPACE(_NS) \ + } \ + _CCCL_END_NAMESPACE_NOVERSION(_NS) + +// Open a namespace for APIs that were version bumped in a minor release +// Version bump namespace should be removed from the APIs at the next major release +#define _CCCL_BEGIN_NAMESPACE_ABI_VER4_BUMP \ + static_assert(_LIBCUDACXX_CUDA_ABI_VERSION == 4, "Version bump should be removed"); \ + inline namespace __version_bump_ver4_ \ + { +#define _CCCL_END_NAMESPACE_ABI_VER4_BUMP \ + static_assert(_LIBCUDACXX_CUDA_ABI_VERSION == 4, "Version bump should be removed"); \ + } + +// Standard namespaces with or without versioning +#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION _CCCL_BEGIN_NAMESPACE_NOVERSION(cuda::std) +#define _CCCL_END_NAMESPACE_CUDA_STD_NOVERSION _CCCL_END_NAMESPACE_NOVERSION(cuda::std) +#define _CCCL_BEGIN_NAMESPACE_CUDA_STD _CCCL_BEGIN_NAMESPACE(cuda::std) +#define _CCCL_END_NAMESPACE_CUDA_STD _CCCL_END_NAMESPACE(cuda::std) + +// cuda specific namespaces +#define _CCCL_BEGIN_NAMESPACE_CUDA _CCCL_BEGIN_NAMESPACE(cuda) +#define _CCCL_END_NAMESPACE_CUDA _CCCL_END_NAMESPACE(cuda) +#define _CCCL_BEGIN_NAMESPACE_CUDA_MR _CCCL_BEGIN_NAMESPACE(cuda::mr) +#define _CCCL_END_NAMESPACE_CUDA_MR _CCCL_END_NAMESPACE(cuda::mr) +#define _CCCL_BEGIN_NAMESPACE_CUDA_DEVICE _CCCL_BEGIN_NAMESPACE(cuda::device) +#define _CCCL_END_NAMESPACE_CUDA_DEVICE _CCCL_END_NAMESPACE(cuda::device) +#define _CCCL_BEGIN_NAMESPACE_CUDA_PTX _CCCL_BEGIN_NAMESPACE(cuda::ptx) +#define _CCCL_END_NAMESPACE_CUDA_PTX _CCCL_END_NAMESPACE(cuda::ptx) +#define _CCCL_BEGIN_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL _CCCL_BEGIN_NAMESPACE(cuda::device::experimental) +#define _CCCL_END_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL _CCCL_END_NAMESPACE(cuda::device::experimental) +#define _CCCL_BEGIN_NAMESPACE_CUDA_DRIVER _CCCL_BEGIN_NAMESPACE(cuda::__driver) +#define _CCCL_END_NAMESPACE_CUDA_DRIVER _CCCL_END_NAMESPACE(cuda::__driver) + +// Namespaces related to +#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD _CCCL_BEGIN_NAMESPACE(cuda::std::simd) +#define _CCCL_END_NAMESPACE_CUDA_STD_SIMD _CCCL_END_NAMESPACE(cuda::std::simd) + +// Namespaces related to +#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES _CCCL_BEGIN_NAMESPACE(cuda::std::ranges) +#define _CCCL_END_NAMESPACE_CUDA_STD_RANGES _CCCL_END_NAMESPACE(cuda::std::ranges) +#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_VIEWS _CCCL_BEGIN_NAMESPACE(cuda::std::ranges::views) +#define _CCCL_END_NAMESPACE_CUDA_STD_VIEWS _CCCL_END_NAMESPACE(cuda::std::ranges::views) + +#define _CCCL_BEGIN_NAMESPACE_CPO(_CPO) \ + namespace _CPO \ + { +#define _CCCL_END_NAMESPACE_CPO } + +// Namespaces related to chrono / filesystem +#define _CCCL_BEGIN_NAMESPACE_FILESYSTEM \ + _CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION \ + inline namespace __fs \ + { \ + namespace filesystem \ + { \ + inline namespace _LIBCUDACXX_ABI_NAMESPACE \ + { +#define _CCCL_END_NAMESPACE_FILESYSTEM \ + } \ + } \ + } \ + _CCCL_END_NAMESPACE_CUDA_STD_NOVERSION + +// Shorthands for different qualifiers +// Namespaces related to execution +#define _CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION _CCCL_BEGIN_NAMESPACE(cuda::std::execution) +#define _CCCL_END_NAMESPACE_CUDA_STD_EXECUTION _CCCL_END_NAMESPACE(cuda::std::execution) + +#define _CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION _CCCL_BEGIN_NAMESPACE(cuda::execution) +#define _CCCL_END_NAMESPACE_CUDA_EXECUTION _CCCL_END_NAMESPACE(cuda::execution) + +#define _CCCL_BEGIN_NAMESPACE_CUDA_ARGUMENT _CCCL_BEGIN_NAMESPACE(cuda::args) +#define _CCCL_END_NAMESPACE_CUDA_ARGUMENT _CCCL_END_NAMESPACE(cuda::args) + +// Namespace to avoid name collisions with CPOs on clang-16 (see +// https://godbolt.org/z/9TadonrdM for example). MSVC's ancient parser also gets confused with +// __cccl_true in the main iter_move template. +#if _CCCL_COMPILER(CLANG, <=, 16) || _CCCL_COMPILER(MSVC) +# define _LIBCUDACXX_BEGIN_HIDDEN_FRIEND_NAMESPACE \ + namespace __hidden \ + { +# define _LIBCUDACXX_END_HIDDEN_FRIEND_NAMESPACE(_CLASS) \ + } \ + using __hidden::_CLASS; +#else // ^^^ _CCCL_COMPILER(CLANG, <=, 16) ^^^ / vvv _CCCL_COMPILER(CLANG, >, 16) vvv +# define _LIBCUDACXX_BEGIN_HIDDEN_FRIEND_NAMESPACE +# define _LIBCUDACXX_END_HIDDEN_FRIEND_NAMESPACE(_CLASS) +#endif // !_CCCL_COMPILER(CLANG, >, 16) + +#if defined(CCCL_DISABLE_ARCH_DEPENDENT_NAMESPACE) +# define _CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT +# define _CCCL_END_NAMESPACE_ARCH_DEPENDENT +#else // not defined(CCCL_DISABLE_ARCH_DEPENDENT_NAMESPACE) +# if _CCCL_CUDA_COMPILER(NVHPC) +# define _CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT \ + inline namespace _CCCL_PP_CAT(_CCCL_PP_SPLICE_WITH(_, _SM, NV_TARGET_SM_INTEGER_LIST), _NVHPC) \ + { +# define _CCCL_END_NAMESPACE_ARCH_DEPENDENT } +# else // ^^^ _CCCL_CUDA_COMPILER(NVHPC) ^^^ / vvv !_CCCL_CUDA_COMPILER(NVHPC) vvv +# define _CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT \ + inline namespace _CCCL_PP_SPLICE_WITH(_, _SM, __CUDA_ARCH_LIST__) \ + { +# define _CCCL_END_NAMESPACE_ARCH_DEPENDENT } +# endif // ^^^ !_CCCL_CUDA_COMPILER(NVHPC) ^^^ +#endif // not defined(CCCL_DISABLE_ARCH_DEPENDENT_NAMESPACE) + +// Host standard library namespaces +#if _CCCL_HOST_STD_LIB(LIBSTDCXX) +// We don't appy attributes on forward declarations, so we omit the _GLIBCXX_VISIBILITY(default) +# if _GLIBCXX_INLINE_VERSION +# define _CCCL_BEGIN_NAMESPACE_STD \ + _CCCL_PROLOGUE_INCLUDE_CHECK() namespace std \ + { \ + inline _GLIBCXX_BEGIN_NAMESPACE_VERSION +# define _CCCL_END_NAMESPACE_STD \ + _GLIBCXX_END_NAMESPACE_VERSION \ + } \ + _CCCL_PROLOGUE_INCLUDE_CHECK() +# else // ^^^ _GLIBCXX_INLINE_VERSION ^^^ / vvv !_GLIBCXX_INLINE_VERSION vvv +# define _CCCL_BEGIN_NAMESPACE_STD \ + _CCCL_PROLOGUE_INCLUDE_CHECK() namespace std \ + { +# define _CCCL_END_NAMESPACE_STD \ + } \ + _CCCL_PROLOGUE_INCLUDE_CHECK() +# endif // ^^^ !_GLIBCXX_INLINE_VERSION ^^^ +#elif _CCCL_HOST_STD_LIB(LIBCXX) +# define _CCCL_BEGIN_NAMESPACE_STD _CCCL_PROLOGUE_INCLUDE_CHECK() _LIBCPP_BEGIN_NAMESPACE_STD +# define _CCCL_END_NAMESPACE_STD _LIBCPP_END_NAMESPACE_STD _CCCL_PROLOGUE_INCLUDE_CHECK() +#elif _CCCL_HOST_STD_LIB(STL) +# define _CCCL_BEGIN_NAMESPACE_STD _CCCL_PROLOGUE_INCLUDE_CHECK() _STD_BEGIN +# define _CCCL_END_NAMESPACE_STD _STD_END _CCCL_PROLOGUE_INCLUDE_CHECK() +#else +# define _CCCL_BEGIN_NAMESPACE_STD \ + _CCCL_PROLOGUE_INCLUDE_CHECK() namespace std \ + { +# define _CCCL_END_NAMESPACE_STD \ + } \ + _CCCL_PROLOGUE_INCLUDE_CHECK() +#endif + +#endif // _CUDA_STD___INTERNAL_NAMESPACES_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/pstl_config.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/pstl_config.h new file mode 100644 index 00000000..1ba12811 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/pstl_config.h @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_PSTL_CONFIG_H +#define _CUDA_STD___INTERNAL_PSTL_CONFIG_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#define _CCCL_HAS_BACKEND_CUDA() _CCCL_CUDA_COMPILATION() && !_CCCL_COMPILER(NVRTC) +#define _CCCL_HAS_BACKEND_OMP() 0 +#define _CCCL_HAS_BACKEND_TBB() 0 + +#define _CCCL_HAS_PSTL_BACKEND() (_CCCL_HAS_BACKEND_CUDA() || _CCCL_HAS_BACKEND_OMP() || _CCCL_HAS_BACKEND_TBB()) + +#include + +#endif // _CUDA_STD___INTERNAL_PSTL_CONFIG_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/thread_api.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/thread_api.h new file mode 100644 index 00000000..1731bb0d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/thread_api.h @@ -0,0 +1,58 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_THREAD_API_H +#define _CUDA_STD___INTERNAL_THREAD_API_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +// Thread API +#ifndef _CCCL_HAS_THREAD_API_EXTERNAL +# if _CCCL_COMPILER(NVRTC) || defined(__EMSCRIPTEN__) +# define _CCCL_HAS_THREAD_API_EXTERNAL +# endif +#endif // _CCCL_HAS_THREAD_API_EXTERNAL + +#ifndef _CCCL_HAS_THREAD_API_CUDA +# if ((_CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)) || defined(__EMSCRIPTEN__) || _CCCL_HOSTJIT()) +# define _CCCL_HAS_THREAD_API_CUDA +# endif // ((_CCCL_DEVICE_COMPILATION() && !_CCCL_CUDA_COMPILER(NVHPC)) || defined(__EMSCRIPTEN__)) +#endif // _CCCL_HAS_THREAD_API_CUDA + +#ifndef _CCCL_HAS_THREAD_API_WIN32 +# if _CCCL_COMPILER(MSVC) && !defined(_CCCL_HAS_THREAD_API_CUDA) +# define _CCCL_HAS_THREAD_API_WIN32 +# endif // _CCCL_COMPILER(MSVC) && !defined(_CCCL_HAS_THREAD_API_CUDA) +#endif // _CCCL_HAS_THREAD_API_WIN32 + +#if !defined(_CCCL_HAS_THREAD_API_PTHREAD) && !defined(_CCCL_HAS_THREAD_API_WIN32) \ + && !defined(_CCCL_HAS_THREAD_API_EXTERNAL) +# if defined(__GNU__) || _CCCL_OS(LINUX) || _CCCL_OS(APPLE) || _CCCL_OS(QNX) \ + || (defined(__MINGW32__) && __has_include()) +# define _CCCL_HAS_THREAD_API_PTHREAD +# elif defined(_WIN32) +# define _CCCL_HAS_THREAD_API_WIN32 +# else +# define _CCCL_UNSUPPORTED_THREAD_API +# endif // _CCCL_HAS_THREAD_API +#endif + +#ifndef __STDCPP_THREADS__ +# define __STDCPP_THREADS__ 1 +#endif // __STDCPP_THREADS__ + +#endif // _CUDA_STD___INTERNAL_THREAD_API_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/version.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/version.h new file mode 100644 index 00000000..d7be937c --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__internal/version.h @@ -0,0 +1,52 @@ +//===---------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===---------------------------------------------------------------------===// + +#ifndef _CUDA_STD___INTERNAL_VERSION_H +#define _CUDA_STD___INTERNAL_VERSION_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include // IWYU pragma: export + +#define _LIBCUDACXX_CUDA_API_VERSION CCCL_VERSION +#define _LIBCUDACXX_CUDA_API_VERSION_MAJOR CCCL_MAJOR_VERSION +#define _LIBCUDACXX_CUDA_API_VERSION_MINOR CCCL_MINOR_VERSION +#define _LIBCUDACXX_CUDA_API_VERSION_PATCH CCCL_PATCH_VERSION + +#ifndef _LIBCUDACXX_CUDA_ABI_VERSION_LATEST +# define _LIBCUDACXX_CUDA_ABI_VERSION_LATEST 4 +#endif + +#ifdef _LIBCUDACXX_CUDA_ABI_VERSION +# if _LIBCUDACXX_CUDA_ABI_VERSION != 4 +# error Unsupported libcu++ ABI version requested. Only version 4 is allowed. +# endif +#else +# define _LIBCUDACXX_CUDA_ABI_VERSION _LIBCUDACXX_CUDA_ABI_VERSION_LATEST +#endif + +#if (_LIBCUDACXX_CUDA_ABI_VERSION < 4) && !defined(LIBCUDACXX_IGNORE_DEPRECATED_ABI) +# error "libcu++ ABIs older than version 4 are deprecated, define LIBCUDACXX_IGNORE_DEPRECATED_ABI to ignore" +#endif + +#ifdef _LIBCUDACXX_PIPELINE_ASSUMED_ABI_VERSION +# if _LIBCUDACXX_PIPELINE_ASSUMED_ABI_VERSION != _LIBCUDACXX_CUDA_ABI_VERSION +# error cuda_pipeline.h has assumed a different libcu++ ABI version than provided by this library. To fix this, please include a libcu++ header before including cuda_pipeline.h, or upgrade to a version of the toolkit this version of libcu++ shipped in. +# endif +#endif + +#endif // _CUDA_STD___INTERNAL_VERSION_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/access.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/access.h new file mode 100644 index 00000000..3059fe53 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/access.h @@ -0,0 +1,140 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_ACCESS_H +#define _CUDA_STD___ITERATOR_ACCESS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +namespace __begin +{ +struct __fn +{ + template + _CCCL_API constexpr _Tp* operator()(_Tp (&__array)[_Np]) const noexcept + { + return __array; + } + + template + _CCCL_API constexpr auto operator()(_Cp& __c) const noexcept(noexcept(__c.begin())) -> decltype(__c.begin()) + { + return __c.begin(); + } + + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(__c.begin())) -> decltype(__c.begin()) + { + return __c.begin(); + } +}; +} // namespace __begin + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto begin = __begin::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __begin_cpo = __begin::__fn; +} // namespace __cpo + +namespace __end +{ +struct __fn +{ + template + _CCCL_API constexpr _Tp* operator()(_Tp (&__array)[_Np]) const noexcept + { + return __array + _Np; + } + + template + _CCCL_API constexpr auto operator()(_Cp& __c) const noexcept(noexcept(__c.end())) -> decltype(__c.end()) + { + return __c.end(); + } + + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(__c.end())) -> decltype(__c.end()) + { + return __c.end(); + } +}; +} // namespace __end + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto end = __end::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __end_cpo = __end::__fn; +} // namespace __cpo + +namespace __cbegin +{ +struct __fn +{ + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(::cuda::std::__begin_cpo{}(__c))) + -> decltype(::cuda::std::__begin_cpo{}(__c)) + { + return ::cuda::std::__begin_cpo{}(__c); + } +}; +} // namespace __cbegin + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto cbegin = __cbegin::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __cbegin_cpo = __cbegin::__fn; +} // namespace __cpo + +namespace __cend +{ +struct __fn +{ + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(::cuda::std::__end_cpo{}(__c))) + -> decltype(::cuda::std::__end_cpo{}(__c)) + { + return ::cuda::std::__end_cpo{}(__c); + } +}; +} // namespace __cend + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto cend = __cend::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __cend_cpo = __cend::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_ACCESS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/advance.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/advance.h new file mode 100644 index 00000000..2458dbfb --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/advance.h @@ -0,0 +1,230 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_ADVANCE_H +#define _CUDA_STD___ITERATOR_ADVANCE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template ())), + class = enable_if_t>> +_CCCL_API constexpr void advance(_InputIter& __i, _Distance __orig_n) +{ + using _Difference = typename iterator_traits<_InputIter>::difference_type; + _Difference __n = static_cast<_Difference>(::cuda::std::__convert_to_integral(__orig_n)); + if constexpr (__has_random_access_traversal<_InputIter>) // To support pointers to incomplete types + { + __i += __n; + } + else if constexpr (__has_bidirectional_traversal<_InputIter>) + { + if (__n >= 0) + { + for (; __n > 0; --__n) + { + ++__i; + } + } + else + { + for (; __n < 0; ++__n) + { + --__i; + } + } + } + else + { + _CCCL_ASSERT(__n >= 0, "Attempt to advance(it, n) with negative n on a non-bidirectional iterator"); + for (; __n > 0; --__n) + { + ++__i; + } + } +} + +_CCCL_END_NAMESPACE_CUDA_STD + +// [range.iter.op.advance] + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES +_CCCL_BEGIN_NAMESPACE_CPO(__advance) +struct __fn +{ +private: + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API static constexpr auto __magnitude_geq(_Iter_difference __a, _Iter_difference __b) noexcept + { + return __a == 0 ? __b == 0 : // + __a > 0 ? __a >= __b + : __a <= __b; + } + +public: + // Preconditions: If `I` does not model `bidirectional_iterator`, `n` is not negative. + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(input_or_output_iterator<_Ip>) + _CCCL_API constexpr void operator()(_Ip& __i, iter_difference_t<_Ip> __n) const + { + _CCCL_ASSERT(__n >= 0 || bidirectional_iterator<_Ip>, "If `n < 0`, then `bidirectional_iterator` must be true."); + + // If `I` models `random_access_iterator`, equivalent to `i += n`. + if constexpr (random_access_iterator<_Ip>) + { + __i += __n; + return; + } + else if constexpr (bidirectional_iterator<_Ip>) + { + // Otherwise, if `n` is non-negative, increments `i` by `n`. + while (__n > 0) + { + --__n; + ++__i; + } + // Otherwise, decrements `i` by `-n`. + while (__n < 0) + { + ++__n; + --__i; + } + return; + } + else + { + // Otherwise, if `n` is non-negative, increments `i` by `n`. + while (__n > 0) + { + --__n; + ++__i; + } + return; + } + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip, class _Sp) + _CCCL_REQUIRES(input_or_output_iterator<_Ip> _CCCL_AND sentinel_for<_Sp, _Ip>) + _CCCL_API constexpr void operator()(_Ip& __i, _Sp __bound_sentinel) const + { + // If `I` and `S` model `assignable_from`, equivalent to `i = std::move(bound_sentinel)`. + if constexpr (assignable_from<_Ip&, _Sp>) + { + __i = ::cuda::std::move(__bound_sentinel); + } + // Otherwise, if `S` and `I` model `sized_sentinel_for`, + // equivalent to `ranges::advance(i, bound_sentinel - i)`. + else if constexpr (sized_sentinel_for<_Sp, _Ip>) + { + (*this)(__i, __bound_sentinel - __i); + } + // Otherwise, while `bool(i != bound_sentinel)` is true, increments `i`. + else + { + while (__i != __bound_sentinel) + { + ++__i; + } + } + } + + // Preconditions: + // * If `n > 0`, [i, bound_sentinel) denotes a range. + // * If `n == 0`, [i, bound_sentinel) or [bound_sentinel, i) denotes a range. + // * If `n < 0`, [bound_sentinel, i) denotes a range, `I` models `bidirectional_iterator`, + // and `I` and `S` model `same_as`. + // Returns: `n - M`, where `M` is the difference between the ending and starting position. + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip, class _Sp) + _CCCL_REQUIRES(input_or_output_iterator<_Ip> _CCCL_AND sentinel_for<_Sp, _Ip>) + _CCCL_API constexpr iter_difference_t<_Ip> operator()(_Ip& __i, iter_difference_t<_Ip> __n, _Sp __bound_sentinel) const + { + _CCCL_ASSERT((__n >= 0) || (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>), + "If `n < 0`, then `bidirectional_iterator && same_as` must be true."); + // If `S` and `I` model `sized_sentinel_for`: + if constexpr (sized_sentinel_for<_Sp, _Ip>) + { + // If |n| >= |bound_sentinel - i|, equivalent to `ranges::advance(i, bound_sentinel)`. + // __magnitude_geq(a, b) returns |a| >= |b|, assuming they have the same sign. + const auto __M = __bound_sentinel - __i; + if (__magnitude_geq(__n, __M)) + { + (*this)(__i, __bound_sentinel); + return __n - __M; + } + + // Otherwise, equivalent to `ranges::advance(i, n)`. + (*this)(__i, __n); + return 0; + } + else + { + // Otherwise, if `n` is non-negative, while `bool(i != bound_sentinel)` is true, increments `i` but at + // most `n` times. + while (__i != __bound_sentinel && __n > 0) + { + ++__i; + --__n; + } + + // Otherwise, while `bool(i != bound_sentinel)` is true, decrements `i` but at most `-n` times. + if constexpr (bidirectional_iterator<_Ip> && same_as<_Ip, _Sp>) + { + while (__i != __bound_sentinel && __n < 0) + { + --__i; + ++__n; + } + } + return __n; + } + } +}; +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto advance = __advance::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __advance_cpo = __advance::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +#include + +#endif // _CUDA_STD___ITERATOR_ADVANCE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/concepts.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/concepts.h new file mode 100644 index 00000000..a346d1cb --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/concepts.h @@ -0,0 +1,718 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_CONCEPTS_H +#define _CUDA_STD___ITERATOR_CONCEPTS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() + +// [iterator.concept.readable] +template +concept __indirectly_readable_impl = + requires(const _In __i) { + typename iter_value_t<_In>; + typename iter_reference_t<_In>; + typename iter_rvalue_reference_t<_In>; + { *__i } -> same_as>; + { ::cuda::std::ranges::__iter_move_cpo{}(__i) } -> same_as>; + } && common_reference_with&&, iter_value_t<_In>&> + && common_reference_with&&, iter_rvalue_reference_t<_In>&&> + && common_reference_with&&, const iter_value_t<_In>&>; + +template +concept indirectly_readable = __indirectly_readable_impl>; + +template +using iter_common_reference_t = common_reference_t, iter_value_t<_Tp>&>; + +// [iterator.concept.writable] +template +concept indirectly_writable = requires(_Out&& __o, _Tp&& __t) { + *__o = static_cast<_Tp &&>(__t); // not required to be equality-preserving + *static_cast<_Out &&>(__o) = static_cast<_Tp &&>(__t); // not required to be equality-preserving + const_cast &&>(*__o) = static_cast<_Tp &&>(__t); // not required to be + // equality-preserving + const_cast &&>(*static_cast<_Out &&>(__o)) = + static_cast<_Tp &&>(__t); // not required to be equality-preserving +}; + +// [iterator.concept.winc] +template +concept __integer_like = integral<_Tp> && !same_as<_Tp, bool>; + +template +concept __signed_integer_like = signed_integral<_Tp>; + +template +concept weakly_incrementable = + // TODO: remove this once the clang bug is fixed (bugs.llvm.org/PR48173). + !same_as<_Ip, bool> && // Currently, clang does not handle bool correctly. + movable<_Ip> && requires(_Ip __i) { + typename iter_difference_t<_Ip>; + requires __signed_integer_like>; + { ++__i } -> same_as<_Ip&>; // not required to be equality-preserving + __i++; // not required to be equality-preserving + }; + +// [iterator.concept.inc] +template +concept incrementable = regular<_Ip> && weakly_incrementable<_Ip> && requires(_Ip __i) { + { __i++ } -> same_as<_Ip>; +}; + +// [iterator.concept.iterator] +template +concept input_or_output_iterator = requires(_Ip __i) { + { *__i } -> __can_reference; +} && weakly_incrementable<_Ip>; + +// [iterator.concept.sentinel] +template +concept sentinel_for = semiregular<_Sp> && input_or_output_iterator<_Ip> && __weakly_equality_comparable_with<_Sp, _Ip>; + +template +inline constexpr bool disable_sized_sentinel_for = false; + +template +concept sized_sentinel_for = + sentinel_for<_Sp, _Ip> && !disable_sized_sentinel_for, remove_cv_t<_Ip>> + && requires(const _Ip& __i, const _Sp& __s) { + { __s - __i } -> same_as>; + { __i - __s } -> same_as>; + }; + +// [iterator.concept.input] +template +concept input_iterator = input_or_output_iterator<_Ip> && indirectly_readable<_Ip> && requires { + typename _ITER_CONCEPT<_Ip>; +} && derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>; + +// [iterator.concept.output] +template +concept output_iterator = + input_or_output_iterator<_Ip> && indirectly_writable<_Ip, _Tp> && requires(_Ip __it, _Tp&& __t) { + *__it++ = static_cast<_Tp &&>(__t); // not required to be equality-preserving + }; + +// [iterator.concept.forward] +template +concept forward_iterator = input_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, forward_iterator_tag> + && incrementable<_Ip> && sentinel_for<_Ip, _Ip>; + +// [iterator.concept.bidir] +template +concept __iter_can_decrement = requires(_Iter __iter) { + { --__iter } -> same_as<_Iter&>; + { __iter-- } -> same_as<_Iter>; +}; + +template +concept bidirectional_iterator = + forward_iterator<_Iter> && derived_from<_ITER_CONCEPT<_Iter>, bidirectional_iterator_tag> + && __iter_can_decrement<_Iter>; + +template +concept __iter_can_plus_equal = requires(_Iter __iter, const iter_difference_t<_Iter> __n) { + { __iter += __n } -> same_as<_Iter&>; +}; + +template +concept __iter_can_plus = requires(const _Iter __iter, const iter_difference_t<_Iter> __n) { + { __iter + __n } -> same_as<_Iter>; + { __n + __iter } -> same_as<_Iter>; +}; + +template +concept __iter_can_minus_equal = requires(_Iter __iter, const iter_difference_t<_Iter> __n) { + { __iter -= __n } -> same_as<_Iter&>; +}; + +template +concept __iter_can_minus = requires(const _Iter __iter, const iter_difference_t<_Iter> __n) { + { __iter - __n } -> same_as<_Iter>; +}; + +template +concept __iter_can_subscript = requires(const _Iter __iter, const iter_difference_t<_Iter> __n) { + { __iter[__n] } -> same_as>; +}; + +template +concept __random_access_operations = + __iter_can_plus_equal<_Iter> && __iter_can_plus<_Iter> && __iter_can_minus_equal<_Iter> && __iter_can_minus<_Iter> + && __iter_can_subscript<_Iter>; + +template +concept random_access_iterator = + bidirectional_iterator<_Iter> && derived_from<_ITER_CONCEPT<_Iter>, random_access_iterator_tag> + && totally_ordered<_Iter> && sized_sentinel_for<_Iter, _Iter> && __random_access_operations<_Iter>; + +template +concept contiguous_iterator = + random_access_iterator<_Ip> && derived_from<_ITER_CONCEPT<_Ip>, contiguous_iterator_tag> + && is_lvalue_reference_v> && same_as, remove_cvref_t>> + && requires(const _Ip& __i) { + { ::cuda::std::to_address(__i) } -> same_as>>; + }; + +template +concept __has_arrow = input_iterator<_Ip> && (is_pointer_v<_Ip> || requires(_Ip __i) { __i.operator->(); }); + +template +concept __has_const_arrow = (is_pointer_v<_Ip> || requires(const _Ip __i) { __i.operator->(); }); + +// [indirectcallable.indirectinvocable] +template +concept indirectly_unary_invocable = + indirectly_readable<_It> && copy_constructible<_Fp> && invocable<_Fp&, iter_value_t<_It>&> + && invocable<_Fp&, iter_reference_t<_It>> && invocable<_Fp&, iter_common_reference_t<_It>> + && common_reference_with&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>; + +template +concept indirectly_regular_unary_invocable = + indirectly_readable<_It> && copy_constructible<_Fp> && regular_invocable<_Fp&, iter_value_t<_It>&> + && regular_invocable<_Fp&, iter_reference_t<_It>> && regular_invocable<_Fp&, iter_common_reference_t<_It>> + && common_reference_with&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>; + +template +concept indirect_unary_predicate = + indirectly_readable<_It> && copy_constructible<_Fp> && predicate<_Fp&, iter_value_t<_It>&> + && predicate<_Fp&, iter_reference_t<_It>> && predicate<_Fp&, iter_common_reference_t<_It>>; + +template +concept indirect_binary_predicate = + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> + && predicate<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> + && predicate<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> + && predicate<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> + && predicate<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> + && predicate<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; + +template +concept indirect_equivalence_relation = + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> + && equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> + && equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> + && equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> + && equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> + && equivalence_relation<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; + +template +concept indirect_strict_weak_order = + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> + && strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> + && strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> + && strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> + && strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> + && strict_weak_order<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>; + +template + requires(indirectly_readable<_Its> && ...) && invocable<_Fp, iter_reference_t<_Its>...> +using indirect_result_t = invoke_result_t<_Fp, iter_reference_t<_Its>...>; + +template +concept indirectly_movable = indirectly_readable<_In> && indirectly_writable<_Out, iter_rvalue_reference_t<_In>>; + +template +concept indirectly_movable_storable = + indirectly_movable<_In, _Out> && indirectly_writable<_Out, iter_value_t<_In>> && movable> + && constructible_from, iter_rvalue_reference_t<_In>> + && assignable_from&, iter_rvalue_reference_t<_In>>; + +template +concept indirectly_copyable = indirectly_readable<_In> && indirectly_writable<_Out, iter_reference_t<_In>>; + +template +concept indirectly_copyable_storable = + indirectly_copyable<_In, _Out> && indirectly_writable<_Out, iter_value_t<_In>&> + && indirectly_writable<_Out, const iter_value_t<_In>&> && indirectly_writable<_Out, iter_value_t<_In>&&> + && indirectly_writable<_Out, const iter_value_t<_In>&&> && copyable> + && constructible_from, iter_reference_t<_In>> + && assignable_from&, iter_reference_t<_In>>; + +// Note: indirectly_swappable is located in iter_swap.h to prevent a dependency cycle +// (both iter_swap and indirectly_swappable require indirectly_readable). + +// Extension of indirectly_unary_invocable to binary operators +template +concept __indirectly_binary_invocable = + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> + && invocable<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> + && invocable<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> + && invocable<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> + && invocable<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> + && invocable<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>>; + +// Extension of indirectly_regular_unary_invocable to binary operators +template +concept __indirectly_regular_binary_invocable = + indirectly_readable<_It1> && indirectly_readable<_It2> && copy_constructible<_Fp> + && regular_invocable<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&> + && regular_invocable<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>> + && regular_invocable<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&> + && regular_invocable<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>> + && regular_invocable<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>> + && common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>>; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [iterator.concept.readable] +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_readable_impl_, + requires(const _In __i)( + typename(iter_value_t<_In>), + typename(iter_reference_t<_In>), + typename(iter_rvalue_reference_t<_In>), + requires(same_as, decltype(*__i)>), + requires(same_as, decltype(::cuda::std::ranges::__iter_move_cpo{}(__i))>), + requires(common_reference_with&&, iter_value_t<_In>&>), + requires(common_reference_with&&, iter_rvalue_reference_t<_In>&&>), + requires(common_reference_with&&, const iter_value_t<_In>&>))); + +template +_CCCL_CONCEPT indirectly_readable = _CCCL_FRAGMENT(__indirectly_readable_impl_, remove_cvref_t<_In>); + +template +using iter_common_reference_t = + enable_if_t, common_reference_t, iter_value_t<_Tp>&>>; + +// [iterator.concept.writable] +template +_CCCL_CONCEPT indirectly_writable = _CCCL_REQUIRES_EXPR((_Out, _Tp), _Out&& __o, _Tp&& __t)( + (*__o = static_cast<_Tp&&>(__t)), + (*static_cast<_Out&&>(__o) = static_cast<_Tp&&>(__t)), + (const_cast&&>(*__o) = static_cast<_Tp&&>(__t)), + (const_cast&&>(*static_cast<_Out&&>(__o)) = static_cast<_Tp&&>(__t))); + +// [iterator.concept.winc] +template +_CCCL_CONCEPT __integer_like = integral<_Tp> && !same_as<_Tp, bool>; + +template +_CCCL_CONCEPT __signed_integer_like = signed_integral<_Tp>; + +template +_CCCL_CONCEPT_FRAGMENT( + __weakly_incrementable_, + requires(_Ip __i)(typename(iter_difference_t<_Ip>), + requires(!same_as<_Ip, bool>), + requires(movable<_Ip>), + requires(__signed_integer_like>), + requires(same_as<_Ip&, decltype(++__i)>), + (__i++))); + +template +_CCCL_CONCEPT weakly_incrementable = _CCCL_FRAGMENT(__weakly_incrementable_, _Ip); + +// [iterator.concept.inc] +template +_CCCL_CONCEPT incrementable = _CCCL_REQUIRES_EXPR((_Ip), _Ip __i)( + requires(regular<_Ip>), + requires(weakly_incrementable<_Ip>), + // Requirement is unevaluated, not sure why clang-tidy complains + // NOLINTNEXTLINE(bugprone-pointer-arithmetic-on-polymorphic-object) + requires(same_as<_Ip, decltype(__i++)>)); + +// [iterator.concept.iterator] +template +_CCCL_CONCEPT_FRAGMENT( + __input_or_output_iterator_, + requires(_Ip __i)(requires(weakly_incrementable<_Ip>), requires(__can_reference))); + +template +_CCCL_CONCEPT input_or_output_iterator = _CCCL_FRAGMENT(__input_or_output_iterator_, _Ip); + +// [iterator.concept.sentinel] +template +_CCCL_CONCEPT_FRAGMENT(__sentinel_for_, + requires()(requires(semiregular<_Sp>), + requires(input_or_output_iterator<_Ip>), + requires(__weakly_equality_comparable_with<_Sp, _Ip>))); + +template +_CCCL_CONCEPT sentinel_for = _CCCL_FRAGMENT(__sentinel_for_, _Sp, _Ip); + +template +inline constexpr bool disable_sized_sentinel_for = false; + +template +_CCCL_CONCEPT_FRAGMENT( + __sized_sentinel_for_, + requires(const _Ip& __i, const _Sp& __s)( + requires(sentinel_for<_Sp, _Ip>), + requires(!disable_sized_sentinel_for, remove_cv_t<_Ip>>), + requires(same_as, decltype(__s - __i)>), + requires(same_as, decltype(__i - __s)>))); + +template +_CCCL_CONCEPT sized_sentinel_for = _CCCL_FRAGMENT(__sized_sentinel_for_, _Sp, _Ip); + +// [iterator.concept.input] +// NOTE: The ordering here is load bearing. MSVC has issues with finding iterator_traits +// We can work around this by checking other constraints first +template +_CCCL_CONCEPT_FRAGMENT( + __input_iterator_, + requires()(requires(input_or_output_iterator<_Ip>), + requires(indirectly_readable<_Ip>), + typename(_ITER_CONCEPT<_Ip>), + requires(derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>))); + +template +_CCCL_CONCEPT input_iterator = _CCCL_FRAGMENT(__input_iterator_, _Ip); + +// [iterator.concept.output] +template +_CCCL_CONCEPT_FRAGMENT(__output_iterator_, + requires(_Ip __it, _Tp&& __t)(requires(input_or_output_iterator<_Ip>), + requires(indirectly_writable<_Ip, _Tp>), + (*__it++ = static_cast<_Tp&&>(__t)))); + +template +_CCCL_CONCEPT output_iterator = _CCCL_FRAGMENT(__output_iterator_, _Ip, _Tp); + +// [iterator.concept.forward] +template +_CCCL_CONCEPT_FRAGMENT( + __forward_iterator_, + requires()(requires(input_iterator<_Ip>), + requires(derived_from<_ITER_CONCEPT<_Ip>, forward_iterator_tag>), + requires(incrementable<_Ip>), + requires(sentinel_for<_Ip, _Ip>))); + +template +_CCCL_CONCEPT forward_iterator = _CCCL_FRAGMENT(__forward_iterator_, _Ip); + +// [iterator.concept.bidir] +template +_CCCL_CONCEPT __iter_can_decrement = + _CCCL_REQUIRES_EXPR((_Iter), _Iter __iter)(_Same_as(_Iter&)(--__iter), _Same_as(_Iter) __iter--); + +template +_CCCL_CONCEPT bidirectional_iterator = _CCCL_REQUIRES_EXPR((_Iter))( + requires(forward_iterator<_Iter>), + requires(derived_from<_ITER_CONCEPT<_Iter>, bidirectional_iterator_tag>), + requires(__iter_can_decrement<_Iter>)); + +// [iterator.concept.random.access] +template +_CCCL_CONCEPT __iter_can_plus_equal = + _CCCL_REQUIRES_EXPR((_Iter), _Iter __iter, const iter_difference_t<_Iter> __n)(_Same_as(_Iter&) __iter += __n); + +template +_CCCL_CONCEPT __iter_can_plus = _CCCL_REQUIRES_EXPR((_Iter), const _Iter __iter, const iter_difference_t<_Iter> __n)( + _Same_as(_Iter) __iter + __n, _Same_as(_Iter) __n + __iter); + +template +_CCCL_CONCEPT __iter_can_minus_equal = + _CCCL_REQUIRES_EXPR((_Iter), _Iter __iter, const iter_difference_t<_Iter> __n)(_Same_as(_Iter&) __iter -= __n); + +template +_CCCL_CONCEPT __iter_can_minus = + _CCCL_REQUIRES_EXPR((_Iter), const _Iter __iter, const iter_difference_t<_Iter> __n)(_Same_as(_Iter) __iter - __n); + +template +_CCCL_CONCEPT __iter_can_subscript = _CCCL_REQUIRES_EXPR( + (_Iter), const _Iter __iter, const iter_difference_t<_Iter> __n)(_Same_as(iter_reference_t<_Iter>) __iter[__n]); + +template +_CCCL_CONCEPT __random_access_iterator_operations = _CCCL_REQUIRES_EXPR((_Iter))( + requires(__iter_can_plus_equal<_Iter>), + requires(__iter_can_plus<_Iter>), + requires(__iter_can_minus_equal<_Iter>), + requires(__iter_can_minus<_Iter>), + requires(__iter_can_subscript<_Iter>)); + +template +_CCCL_CONCEPT random_access_iterator = _CCCL_REQUIRES_EXPR((_Iter))( + requires(bidirectional_iterator<_Iter>), + requires(derived_from<_ITER_CONCEPT<_Iter>, random_access_iterator_tag>), + requires(totally_ordered<_Iter>), + requires(sized_sentinel_for<_Iter, _Iter>), + requires(__random_access_iterator_operations<_Iter>)); + +// [iterator.concept.contiguous] +template +_CCCL_CONCEPT_FRAGMENT( + __contiguous_iterator_, + requires(const _Ip& __i)( + requires(random_access_iterator<_Ip>), + requires(derived_from<_ITER_CONCEPT<_Ip>, contiguous_iterator_tag>), + requires(is_lvalue_reference_v>), + requires(same_as, remove_cvref_t>>), + requires(same_as>, decltype(::cuda::std::to_address(__i))>))); + +template +_CCCL_CONCEPT contiguous_iterator = _CCCL_FRAGMENT(__contiguous_iterator_, _Ip); + +template +_CCCL_CONCEPT_FRAGMENT(__has_arrow_, requires(_Ip __i)((__i.operator->()))); + +template +_CCCL_CONCEPT __has_arrow = input_iterator<_Ip> && (is_pointer_v<_Ip> || _CCCL_FRAGMENT(__has_arrow_, _Ip)); + +template +_CCCL_CONCEPT_FRAGMENT(__has_const_arrow_, requires(const _Ip __i)((__i.operator->()))); + +template +_CCCL_CONCEPT __has_const_arrow = (is_pointer_v<_Ip> || _CCCL_FRAGMENT(__has_const_arrow_, _Ip)); + +// [indirectcallable.indirectinvocable] +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_unary_invocable, + requires()( + requires(indirectly_readable<_It>), + requires(copy_constructible<_Fp>), + requires(invocable<_Fp&, iter_value_t<_It>&>), + requires(invocable<_Fp&, iter_reference_t<_It>>), + requires(invocable<_Fp&, iter_common_reference_t<_It>>), + requires( + common_reference_with&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>))); + +template +_CCCL_CONCEPT indirectly_unary_invocable = _CCCL_FRAGMENT(__indirectly_unary_invocable, _Fp, _It); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_regular_unary_invocable_, + requires()( + requires(indirectly_readable<_It>), + requires(copy_constructible<_Fp>), + requires(regular_invocable<_Fp&, iter_value_t<_It>&>), + requires(regular_invocable<_Fp&, iter_reference_t<_It>>), + requires(regular_invocable<_Fp&, iter_common_reference_t<_It>>), + requires( + common_reference_with&>, invoke_result_t<_Fp&, iter_reference_t<_It>>>))); + +template +_CCCL_CONCEPT indirectly_regular_unary_invocable = _CCCL_FRAGMENT(__indirectly_regular_unary_invocable_, _Fp, _It); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirect_unary_predicate_, + requires()(requires(indirectly_readable<_It>), + requires(copy_constructible<_Fp>), + requires(predicate<_Fp&, iter_value_t<_It>&>), + requires(predicate<_Fp&, iter_reference_t<_It>>), + requires(predicate<_Fp&, iter_common_reference_t<_It>>))); + +template +_CCCL_CONCEPT indirect_unary_predicate = _CCCL_FRAGMENT(__indirect_unary_predicate_, _Fp, _It); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirect_binary_predicate_, + requires()(requires(indirectly_readable<_It1>), + requires(indirectly_readable<_It2>), + requires(copy_constructible<_Fp>), + requires(predicate<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&>), + requires(predicate<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>), + requires(predicate<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>), + requires(predicate<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>), + requires(predicate<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>))); + +template +_CCCL_CONCEPT indirect_binary_predicate = _CCCL_FRAGMENT(__indirect_binary_predicate_, _Fp, _It1, _It2); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirect_equivalence_relation_, + requires()(requires(indirectly_readable<_It1>), + requires(indirectly_readable<_It2>), + requires(copy_constructible<_Fp>), + requires(equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&>), + requires(equivalence_relation<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>), + requires(equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>), + requires(equivalence_relation<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>), + requires(equivalence_relation<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>))); + +template +_CCCL_CONCEPT indirect_equivalence_relation = _CCCL_FRAGMENT(__indirect_equivalence_relation_, _Fp, _It1, _It2); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirect_strict_weak_order_, + requires()(requires(indirectly_readable<_It1>), + requires(indirectly_readable<_It2>), + requires(copy_constructible<_Fp>), + requires(strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&>), + requires(strict_weak_order<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>), + requires(strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>), + requires(strict_weak_order<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>), + requires(strict_weak_order<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>))); + +template +_CCCL_CONCEPT indirect_strict_weak_order = _CCCL_FRAGMENT(__indirect_strict_weak_order_, _Fp, _It1, _It2); + +template +using indirect_result_t = enable_if_t<(indirectly_readable<_Its> && ...) && invocable<_Fp, iter_reference_t<_Its>...>, + invoke_result_t<_Fp, iter_reference_t<_Its>...>>; + +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_movable_, + requires()(requires(indirectly_readable<_In>), requires(indirectly_writable<_Out, iter_rvalue_reference_t<_In>>))); + +template +_CCCL_CONCEPT indirectly_movable = _CCCL_FRAGMENT(__indirectly_movable_, _In, _Out); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_movable_storable_, + requires()(requires(indirectly_movable<_In, _Out>), + requires(indirectly_writable<_Out, iter_value_t<_In>>), + requires(movable>), + requires(constructible_from, iter_rvalue_reference_t<_In>>), + requires(assignable_from&, iter_rvalue_reference_t<_In>>))); + +template +_CCCL_CONCEPT indirectly_movable_storable = _CCCL_FRAGMENT(__indirectly_movable_storable_, _In, _Out); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_copyable_, + requires()(requires(indirectly_readable<_In>), requires(indirectly_writable<_Out, iter_reference_t<_In>>))); + +template +_CCCL_CONCEPT indirectly_copyable = _CCCL_FRAGMENT(__indirectly_copyable_, _In, _Out); + +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_copyable_storable_, + requires()(requires(indirectly_copyable<_In, _Out>), + requires(indirectly_writable<_Out, iter_value_t<_In>&>), + requires(indirectly_writable<_Out, const iter_value_t<_In>&>), + requires(indirectly_writable<_Out, iter_value_t<_In>&&>), + requires(indirectly_writable<_Out, const iter_value_t<_In>&&>), + requires(copyable>), + requires(constructible_from, iter_reference_t<_In>>), + requires(assignable_from&, iter_reference_t<_In>>))); + +template +_CCCL_CONCEPT indirectly_copyable_storable = _CCCL_FRAGMENT(__indirectly_copyable_storable_, _In, _Out); + +template +_CCCL_CONCEPT __indirectly_binary_invocable = _CCCL_REQUIRES_EXPR((_Fp, _It1, _It2))( + requires(indirectly_readable<_It1>), + requires(indirectly_readable<_It2>), + requires(copy_constructible<_Fp>), + requires(invocable<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&>), + requires(invocable<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>), + requires(invocable<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>), + requires(invocable<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>), + requires(invocable<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>>)); + +template +_CCCL_CONCEPT __indirectly_regular_binary_invocable = _CCCL_REQUIRES_EXPR((_Fp, _It1, _It2))( + requires(indirectly_readable<_It1>), + requires(indirectly_readable<_It2>), + requires(copy_constructible<_Fp>), + requires(regular_invocable<_Fp&, iter_value_t<_It1>&, iter_value_t<_It2>&>), + requires(regular_invocable<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>), + requires(regular_invocable<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>), + requires(regular_invocable<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>), + requires(regular_invocable<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_value_t<_It1>&, iter_reference_t<_It2>>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_value_t<_It2>&>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_reference_t<_It1>, iter_reference_t<_It2>>>), + requires(common_reference_with&, iter_value_t<_It2>&>, + invoke_result_t<_Fp&, iter_common_reference_t<_It1>, iter_common_reference_t<_It2>>>)); + +template +inline constexpr bool __has_iter_category = false; + +template +inline constexpr bool __has_iter_category<_Ip, void_t> = true; + +template +inline constexpr bool __has_iter_concept = false; + +template +inline constexpr bool __has_iter_concept<_Ip, void_t> = true; + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_CONCEPTS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/data.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/data.h new file mode 100644 index 00000000..a928fbfe --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/data.h @@ -0,0 +1,61 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_DATA_H +#define _CUDA_STD___ITERATOR_DATA_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr auto data(_Cont& __c) noexcept(noexcept(__c.data())) -> decltype(__c.data()) +{ + return __c.data(); +} + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr auto data(const _Cont& __c) noexcept(noexcept(__c.data())) -> decltype(__c.data()) +{ + return __c.data(); +} + +template +_CCCL_API constexpr _Tp* data(_Tp (&__array)[_Sz]) noexcept +{ + return __array; +} + +template +_CCCL_API constexpr const _Ep* data(initializer_list<_Ep> __il) noexcept +{ + return __il.begin(); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_DATA_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/distance.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/distance.h new file mode 100644 index 00000000..73ca5dfb --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/distance.h @@ -0,0 +1,130 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_DISTANCE_H +#define _CUDA_STD___ITERATOR_DISTANCE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_EXEC_CHECK_DISABLE +template +[[nodiscard]] _CCCL_API constexpr typename iterator_traits<_InputIter>::difference_type +distance(_InputIter __first, _InputIter __last) +{ + // Must clone branches because sized_sentinel_for may require the type to be complete + // NOLINTBEGIN(bugprone-branch-clone) + if constexpr (__has_random_access_traversal<_InputIter>) // To support pointers to incomplete types + { + return __last - __first; + } + else if constexpr (sized_sentinel_for<_InputIter, _InputIter>) + { + return __last - __first; + } + else + { + typename iterator_traits<_InputIter>::difference_type __r(0); + for (; __first != __last; ++__first) + { + ++__r; + } + return __r; + } + // NOLINTEND(bugprone-branch-clone) +} + +_CCCL_END_NAMESPACE_CUDA_STD + +// [range.iter.op.distance] + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES +_CCCL_BEGIN_NAMESPACE_CPO(__distance) +struct __fn +{ + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip, class _Sp) + _CCCL_REQUIRES((sentinel_for<_Sp, _Ip> && !sized_sentinel_for<_Sp, _Ip>) ) + [[nodiscard]] _CCCL_API constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const + { + iter_difference_t<_Ip> __n = 0; + while (__first != __last) + { + ++__first; + ++__n; + } + return __n; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip, class _Sp) + _CCCL_REQUIRES((sized_sentinel_for<_Sp, decay_t<_Ip>>) ) + [[nodiscard]] _CCCL_API constexpr iter_difference_t<_Ip> operator()(_Ip&& __first, _Sp __last) const + { + if constexpr (sized_sentinel_for<_Sp, remove_cvref_t<_Ip>>) + { + return __last - __first; + } + else + { + return __last - decay_t<_Ip>(__first); + } + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Rp) + _CCCL_REQUIRES((range<_Rp>) ) + [[nodiscard]] _CCCL_API constexpr range_difference_t<_Rp> operator()(_Rp&& __r) const + { + if constexpr (sized_range<_Rp>) + { + return static_cast>(::cuda::std::ranges::__size_cpo{}(__r)); + } + else + { + return operator()(::cuda::std::ranges::__begin_cpo{}(__r), ::cuda::std::ranges::__end_cpo{}(__r)); + } + } +}; +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto distance = __distance::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __distance_cpo = __distance::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +#include + +#endif // _CUDA_STD___ITERATOR_DISTANCE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/empty.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/empty.h new file mode 100644 index 00000000..dc6ddd10 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/empty.h @@ -0,0 +1,53 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_EMPTY_H +#define _CUDA_STD___ITERATOR_EMPTY_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +[[nodiscard]] _CCCL_API constexpr auto empty(const _Cont& __c) noexcept(noexcept(__c.empty())) -> decltype(__c.empty()) +{ + return __c.empty(); +} + +template +[[nodiscard]] _CCCL_API constexpr bool empty(const _Tp (&)[_Sz]) noexcept +{ + return false; +} + +template +[[nodiscard]] _CCCL_API constexpr bool empty(initializer_list<_Ep> __il) noexcept +{ + return __il.size() == 0; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_EMPTY_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/incrementable_traits.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/incrementable_traits.h new file mode 100644 index 00000000..3fda10ea --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/incrementable_traits.h @@ -0,0 +1,143 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_INCREMENTABLE_TRAITS_H +#define _CUDA_STD___ITERATOR_INCREMENTABLE_TRAITS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +_CCCL_CONCEPT __has_member_difference_type = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::difference_type)); + +template +inline constexpr bool __has_integral_minus_impl = false; + +// In C++17 we get issues trying to bind void* to a const& so special case it here +template +inline constexpr bool + __has_integral_minus_impl<_Tp, + enable_if_t>, + void_t() - ::cuda::std::declval())>> = + integral() - ::cuda::std::declval())>; + +template +_CCCL_CONCEPT __has_integral_minus = _CCCL_REQUIRES_EXPR((_Tp))(requires(__has_integral_minus_impl<_Tp>)); + +#if _CCCL_HAS_CONCEPTS() + +// [incrementable.traits] +template +struct incrementable_traits +{}; + +template + requires is_object_v<_Tp> +struct incrementable_traits<_Tp*> +{ + using difference_type = ptrdiff_t; +}; + +template +struct incrementable_traits : incrementable_traits<_Ip> +{}; + +template <__has_member_difference_type _Tp> +struct incrementable_traits<_Tp> +{ + using difference_type = typename _Tp::difference_type; +}; + +template <__has_integral_minus _Tp> + requires(!__has_member_difference_type<_Tp>) +struct incrementable_traits<_Tp> +{ + using difference_type = make_signed_t() - ::cuda::std::declval<_Tp>())>; +}; + +// Let `RI` be `remove_cvref_t`. The type `iter_difference_t` denotes +// `incrementable_traits::difference_type` if `iterator_traits` names a specialization +// generated from the primary template, and `iterator_traits::difference_type` otherwise. +template +using iter_difference_t = + typename __select_traits, incrementable_traits>>::difference_type; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [incrementable.traits] +template +struct incrementable_traits +{}; + +template +struct incrementable_traits<_Tp*, enable_if_t>> +{ + using difference_type = ptrdiff_t; +}; + +template +struct incrementable_traits : incrementable_traits<_Ip> +{}; + +template +struct incrementable_traits<_Tp, enable_if_t && !is_const_v<_Tp> && __has_member_difference_type<_Tp>>> +{ + using difference_type = typename _Tp::difference_type; +}; + +template +struct incrementable_traits< + _Tp, + enable_if_t && !is_const_v<_Tp> && !__has_member_difference_type<_Tp> && __has_integral_minus<_Tp>>> +{ + using difference_type = make_signed_t() - ::cuda::std::declval<_Tp>())>; +}; + +// Let `RI` be `remove_cvref_t`. The type `iter_difference_t` denotes +// `incrementable_traits::difference_type` if `iterator_traits` names a specialization +// generated from the primary template, and `iterator_traits::difference_type` otherwise. +template +using iter_difference_t = + typename __select_traits, incrementable_traits>>::difference_type; + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_INCREMENTABLE_TRAITS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_move.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_move.h new file mode 100644 index 00000000..110d49ba --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_move.h @@ -0,0 +1,165 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_ITER_MOVE_H +#define _CUDA_STD___ITERATOR_ITER_MOVE_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_DIAG_PUSH +_CCCL_DIAG_SUPPRESS_CLANG("-Wvoid-ptr-dereference") + +// [iterator.cust.move] + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES +_CCCL_BEGIN_NAMESPACE_CPO(__iter_move) + +_CCCL_API void iter_move(); + +#if _CCCL_HAS_CONCEPTS() +template +concept __unqualified_iter_move = + __class_or_enum> && requires(_Tp&& __t) { iter_move(::cuda::std::forward<_Tp>(__t)); }; + +template +concept __move_deref = !__unqualified_iter_move<_Tp> && requires(_Tp&& __t) { + *__t; + requires is_lvalue_reference_v; +}; + +template +concept __just_deref = !__unqualified_iter_move<_Tp> && !__move_deref<_Tp> && requires(_Tp&& __t) { + *__t; + requires(!is_lvalue_reference_v); +}; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT(__unqualified_iter_move_, + requires(_Tp&& __t)(requires(__class_or_enum>), + ((void) iter_move(::cuda::std::forward<_Tp>(__t))))); + +template +_CCCL_CONCEPT __unqualified_iter_move = _CCCL_FRAGMENT(__unqualified_iter_move_, _Tp); + +template +_CCCL_CONCEPT_FRAGMENT( + __move_deref_, + requires(_Tp&& __t)(requires(!__unqualified_iter_move<_Tp>), requires(is_lvalue_reference_v))); + +template +_CCCL_CONCEPT __move_deref = _CCCL_FRAGMENT(__move_deref_, _Tp); + +template +_CCCL_CONCEPT_FRAGMENT(__just_deref_, + requires(_Tp&& __t)(requires(!__unqualified_iter_move<_Tp>), + requires(!__move_deref<_Tp>), + requires(!is_lvalue_reference_v))); + +template +_CCCL_CONCEPT __just_deref = _CCCL_FRAGMENT(__just_deref_, _Tp); +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +// [iterator.cust.move] + +struct __fn +{ + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(__unqualified_iter_move<_Ip>) + [[nodiscard]] _CCCL_API constexpr decltype(auto) operator()(_Ip&& __i) const + noexcept(noexcept(iter_move(::cuda::std::forward<_Ip>(__i)))) + { + return iter_move(::cuda::std::forward<_Ip>(__i)); + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(__move_deref<_Ip>) + [[nodiscard]] _CCCL_API constexpr auto operator()(_Ip&& __i) const + noexcept(noexcept(::cuda::std::move(*::cuda::std::forward<_Ip>(__i)))) + -> decltype(::cuda::std::move(*::cuda::std::forward<_Ip>(__i))) + { + return ::cuda::std::move(*::cuda::std::forward<_Ip>(__i)); + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(__just_deref<_Ip>) + [[nodiscard]] _CCCL_API constexpr auto operator()(_Ip&& __i) const noexcept(noexcept(*::cuda::std::forward<_Ip>(__i))) + -> decltype(*::cuda::std::forward<_Ip>(__i)) + { + return *::cuda::std::forward<_Ip>(__i); + } +}; +_CCCL_END_NAMESPACE_CPO +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto iter_move = __iter_move::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __iter_move_cpo = __iter_move::__fn; +} // namespace __cpo +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_HAS_CONCEPTS() +template <__dereferenceable _Tp> + requires requires(_Tp& __t) { + { ::cuda::std::ranges::__iter_move_cpo{}(__t) } -> __can_reference; + } +using iter_rvalue_reference_t = decltype(::cuda::std::ranges::__iter_move_cpo{}(::cuda::std::declval<_Tp&>())); + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +template +_CCCL_CONCEPT_FRAGMENT( + __can_iter_rvalue_reference_t_, + requires(_Tp& __t)(requires(__dereferenceable<_Tp>), + requires(__can_reference))); + +template +_CCCL_CONCEPT __can_iter_rvalue_reference_t = _CCCL_FRAGMENT(__can_iter_rvalue_reference_t_, _Tp); + +template +using __iter_rvalue_reference_t = decltype(::cuda::std::ranges::__iter_move_cpo{}(::cuda::std::declval<_Tp&>())); + +template +using iter_rvalue_reference_t = enable_if_t<__can_iter_rvalue_reference_t<_Tp>, __iter_rvalue_reference_t<_Tp>>; +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +_CCCL_END_NAMESPACE_CUDA_STD + +_CCCL_DIAG_POP + +#include + +#endif // _CUDA_STD___ITERATOR_ITER_MOVE_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_swap.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_swap.h new file mode 100644 index 00000000..0dd9d254 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iter_swap.h @@ -0,0 +1,185 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES +// +//===----------------------------------------------------------------------===// +#ifndef _CUDA_STD___ITERATOR_ITER_SWAP_H +#define _CUDA_STD___ITERATOR_ITER_SWAP_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// [iter.cust.swap] +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES +_CCCL_BEGIN_NAMESPACE_CPO(__iter_swap) +template +void iter_swap(_I1, _I2) = delete; + +template +_CCCL_CONCEPT __unqualified_iter_swap = _CCCL_REQUIRES_EXPR((_T1, _T2), _T1&& __x, _T2&& __y)( + requires(__class_or_enum> || __class_or_enum>), + ((void) iter_swap(::cuda::std::forward<_T1>(__x), ::cuda::std::forward<_T2>(__y)))); + +#if _CCCL_HAS_NOEXCEPT_MANGLING() // older GCC cannot use noexcept inside a requires clause +template +_CCCL_CONCEPT __noexcept_unqualified_iter_swap = _CCCL_REQUIRES_EXPR((_T1, _T2), _T1&& __x, _T2&& __y)( + requires(__unqualified_iter_swap<_T1, _T2>), + noexcept(iter_swap(::cuda::std::forward<_T1>(__x), ::cuda::std::forward<_T2>(__y)))); +#else // ^^^ _CCCL_HAS_NOEXCEPT_MANGLING() ^^^ / vvv !_CCCL_HAS_NOEXCEPT_MANGLING() vvv +template > +inline constexpr bool __noexcept_unqualified_iter_swap = false; + +template +inline constexpr bool __noexcept_unqualified_iter_swap<_T1, _T2, true> = + noexcept(iter_swap(::cuda::std::declval<_T1>(), ::cuda::std::declval<_T2>())); +#endif // !_CCCL_HAS_NOEXCEPT_MANGLING() + +template +_CCCL_CONCEPT __readable_swappable = _CCCL_REQUIRES_EXPR((_T1, _T2))( + requires(!__unqualified_iter_swap<_T1, _T2>), + requires(indirectly_readable<_T1>), + requires(indirectly_readable<_T2>), + requires(__can_reference>), + requires(__can_reference>), + requires(swappable_with, iter_reference_t<_T2>>)); + +#if _CCCL_HAS_NOEXCEPT_MANGLING() // older GCC cannot use noexcept inside a requires clause +template +_CCCL_CONCEPT __noexcept_readable_swappable = _CCCL_REQUIRES_EXPR((_T1, _T2), _T1&& __x, _T2&& __y) // + (requires(__readable_swappable<_T1, _T2>), + noexcept(::cuda::std::ranges::__swap_cpo{}(*::cuda::std::forward<_T1>(__x), *::cuda::std::forward<_T2>(__y)))); +#else // ^^^ _CCCL_HAS_NOEXCEPT_MANGLING() ^^^ / vvv !_CCCL_HAS_NOEXCEPT_MANGLING() vvv +template > +inline constexpr bool __noexcept_readable_swappable = false; + +template +inline constexpr bool __noexcept_readable_swappable<_T1, _T2, true> = + noexcept(::cuda::std::ranges::__swap_cpo{}(*::cuda::std::declval<_T1>(), *::cuda::std::declval<_T2>())); +#endif // !_CCCL_HAS_NOEXCEPT_MANGLING() + +template +_CCCL_CONCEPT __movable_storable = _CCCL_REQUIRES_EXPR((_T1, _T2))( + requires(!__unqualified_iter_swap<_T1, _T2>), + requires(!__readable_swappable<_T1, _T2>), + requires(indirectly_movable_storable<_T1, _T2>), + requires(indirectly_movable_storable<_T2, _T1>)); + +#if _CCCL_HAS_NOEXCEPT_MANGLING() // older GCC cannot use noexcept inside a requires clause +template +_CCCL_CONCEPT __noexcept_movable_storable = + _CCCL_REQUIRES_EXPR((_T1, _T2), _T1&& __x, _T2&& __y, iter_value_t<_T2> __old)( + requires(__movable_storable<_T1, _T2>), + noexcept(iter_value_t<_T2>(::cuda::std::ranges::__iter_move_cpo{}(__y))), + noexcept(*__y = ::cuda::std::ranges::__iter_move_cpo{}(__x)), + noexcept(*::cuda::std::forward<_T1>(__x) = ::cuda::std::move(__old))); +#else // ^^^ _CCCL_HAS_NOEXCEPT_MANGLING() ^^^ / vvv !_CCCL_HAS_NOEXCEPT_MANGLING() vvv +template > +inline constexpr bool __noexcept_movable_storable = false; + +template +inline constexpr bool __noexcept_movable_storable<_T1, _T2, true> = + noexcept( + iter_value_t<_T2>(::cuda::std::ranges::__iter_move_cpo{}(::cuda::std::declval>()))) + && noexcept(*::cuda::std::declval>() = + ::cuda::std::ranges::__iter_move_cpo{}(::cuda::std::declval>())) + && noexcept(*::cuda::std::declval<_T1>() = ::cuda::std::declval>()); +#endif // !_CCCL_HAS_NOEXCEPT_MANGLING() + +struct __fn +{ + _CCCL_TEMPLATE(class _T1, class _T2) + _CCCL_REQUIRES(__unqualified_iter_swap<_T1, _T2>) + _CCCL_API constexpr void operator()(_T1&& __x, _T2&& __y) const noexcept(__noexcept_unqualified_iter_swap<_T1, _T2>) + { + (void) iter_swap(::cuda::std::forward<_T1>(__x), ::cuda::std::forward<_T2>(__y)); + } + + _CCCL_TEMPLATE(class _T1, class _T2) + _CCCL_REQUIRES(__readable_swappable<_T1, _T2>) + _CCCL_API constexpr void operator()(_T1&& __x, _T2&& __y) const noexcept(__noexcept_readable_swappable<_T1, _T2>) + { + ::cuda::std::ranges::__swap_cpo{}(*::cuda::std::forward<_T1>(__x), *::cuda::std::forward<_T2>(__y)); + } + + _CCCL_TEMPLATE(class _T1, class _T2) + _CCCL_REQUIRES(__movable_storable<_T2, _T1>) + _CCCL_API constexpr void operator()(_T1&& __x, _T2&& __y) const noexcept(__noexcept_movable_storable<_T1, _T2>) + { + iter_value_t<_T2> __old(::cuda::std::ranges::__iter_move_cpo{}(__y)); + *__y = ::cuda::std::ranges::__iter_move_cpo{}(__x); + *::cuda::std::forward<_T1>(__x) = ::cuda::std::move(__old); + } +}; +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto iter_swap = __iter_swap::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __iter_swap_cpo = __iter_swap::__fn; +} // namespace __cpo +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +_CCCL_BEGIN_NAMESPACE_CUDA_STD +#if _CCCL_HAS_CONCEPTS() +template +concept indirectly_swappable = + indirectly_readable<_I1> && indirectly_readable<_I2> && requires(const _I1 __i1, const _I2 __i2) { + ::cuda::std::ranges::__iter_swap_cpo{}(__i1, __i1); + ::cuda::std::ranges::__iter_swap_cpo{}(__i2, __i2); + ::cuda::std::ranges::__iter_swap_cpo{}(__i1, __i2); + ::cuda::std::ranges::__iter_swap_cpo{}(__i2, __i1); + }; +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv _CCCL_HAS_CONCEPTS() vvv +template +_CCCL_CONCEPT_FRAGMENT( + __indirectly_swappable_, + requires(const _I1 __i1, const _I2 __i2)( + requires(indirectly_readable<_I1>), + requires(indirectly_readable<_I2>), + (::cuda::std::ranges::__iter_swap_cpo{}(__i1, __i1)), + (::cuda::std::ranges::__iter_swap_cpo{}(__i2, __i2)), + (::cuda::std::ranges::__iter_swap_cpo{}(__i1, __i2)), + (::cuda::std::ranges::__iter_swap_cpo{}(__i2, __i1)))); + +template +_CCCL_CONCEPT indirectly_swappable = _CCCL_FRAGMENT(__indirectly_swappable_, _I1, _I2); +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +template +inline constexpr bool __noexcept_swappable = false; + +template +inline constexpr bool __noexcept_swappable<_I1, _I2, enable_if_t>> = + noexcept(::cuda::std::ranges::__iter_swap_cpo{}(::cuda::std::declval<_I1&>(), ::cuda::std::declval<_I2&>())); + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_ITER_SWAP_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator.h new file mode 100644 index 00000000..0b865235 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator.h @@ -0,0 +1,44 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_ITERATOR_H +#define _CUDA_STD___ITERATOR_ITERATOR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT CCCL_DEPRECATED iterator +{ + using value_type = _Tp; + using difference_type = _Distance; + using pointer = _Pointer; + using reference = _Reference; + using iterator_category = _Category; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_ITERATOR_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator_traits.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator_traits.h new file mode 100644 index 00000000..8dd0e366 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/iterator_traits.h @@ -0,0 +1,618 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_ITERATOR_TRAITS_H +#define _CUDA_STD___ITERATOR_ITERATOR_TRAITS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if _CCCL_HOSTED() +# if _CCCL_COMPILER(MSVC) +# include // for ::std::input_iterator_tag +# else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv +# include // for ::std::input_iterator_tag +# endif // !_CCCL_COMPILER(MSVC) + +# ifdef _GLIBCXX_DEBUG +# include +# endif // _GLIBCXX_DEBUG + +# if _CCCL_STD_VER >= 2020 +# include +template +struct __cccl_type_is_defined : ::cuda::std::false_type +{}; + +template +struct __cccl_type_is_defined<_Tp, ::cuda::std::void_t> : ::cuda::std::true_type +{}; + +// detect whether the used STL has contiguous_iterator_tag defined +namespace std +{ +struct __cccl_std_contiguous_iterator_tag_exists : __cccl_type_is_defined +{}; +} // namespace std + +# include +# endif // _CCCL_STD_VER >= 2020 + +#endif // _CCCL_HOSTED() + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +using __with_reference = _Tp&; + +template +_CCCL_CONCEPT __can_reference = _CCCL_REQUIRES_EXPR((_Tp))(typename(__with_reference<_Tp>)); + +// [iterator.traits] +#if _CCCL_HAS_CONCEPTS() +template +concept __dereferenceable = requires(_Tp& __t) { + { *__t } -> __can_reference; // not required to be equality-preserving +}; + +template <__dereferenceable _Tp> +using iter_reference_t = decltype(*::cuda::std::declval<_Tp&>()); + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ // vvv _CCCL_HAS_CONCEPTS() vvv + +_CCCL_DIAG_PUSH +_CCCL_DIAG_SUPPRESS_CLANG("-Wvoid-ptr-dereference") + +template +_CCCL_CONCEPT __dereferenceable = _CCCL_REQUIRES_EXPR((_Tp), _Tp& __t)(requires(__can_reference)); + +_CCCL_DIAG_POP + +template +using iter_reference_t = enable_if_t<__dereferenceable<_Tp>, decltype(*::cuda::std::declval<_Tp&>())>; +#endif // _CCCL_HAS_CONCEPTS() + +#if _CCCL_FREESTANDING() + +struct _CCCL_TYPE_VISIBILITY_DEFAULT input_iterator_tag +{}; +struct _CCCL_TYPE_VISIBILITY_DEFAULT output_iterator_tag +{}; +struct _CCCL_TYPE_VISIBILITY_DEFAULT forward_iterator_tag : public input_iterator_tag +{}; +struct _CCCL_TYPE_VISIBILITY_DEFAULT bidirectional_iterator_tag : public forward_iterator_tag +{}; +struct _CCCL_TYPE_VISIBILITY_DEFAULT random_access_iterator_tag : public bidirectional_iterator_tag +{}; +struct _CCCL_TYPE_VISIBILITY_DEFAULT contiguous_iterator_tag : public random_access_iterator_tag +{}; + +#else // ^^^ _CCCL_FREESTANDING() ^^^ / vvv _CCCL_HOSTED() vvv + +using input_iterator_tag = ::std::input_iterator_tag; +using output_iterator_tag = ::std::output_iterator_tag; +using forward_iterator_tag = ::std::forward_iterator_tag; +using bidirectional_iterator_tag = ::std::bidirectional_iterator_tag; +using random_access_iterator_tag = ::std::random_access_iterator_tag; + +# if _CCCL_STD_VER >= 2020 +struct _CCCL_TYPE_VISIBILITY_DEFAULT __contiguous_iterator_tag_backfill : public ::std::random_access_iterator_tag +{}; +using contiguous_iterator_tag = + _If<::std::__cccl_std_contiguous_iterator_tag_exists::value, + ::std::contiguous_iterator_tag, + __contiguous_iterator_tag_backfill>; +# else // ^^^ C++20 ^^^ / vvv C++17 vvv +struct _CCCL_TYPE_VISIBILITY_DEFAULT contiguous_iterator_tag : public random_access_iterator_tag +{}; +# endif // _CCCL_STD_VER <= 2017 + +#endif // _CCCL_HOSTED() + +template +struct __iter_traits_cache +{ + using type = __select_traits, remove_cvref_t<_Iter>>; +}; +template +using _ITER_TRAITS = typename __iter_traits_cache<_Iter>::type; + +#if _CCCL_HOSTED() +# if defined(_GLIBCXX_DEBUG) +_CCCL_TEMPLATE(class _Iter, class _Ty, class _Range) +_CCCL_REQUIRES(_IsSame<_Iter, ::__gnu_debug::_Safe_iterator<_Ty*, _Range>>::value) +_CCCL_API inline auto __iter_concept_fn(::__gnu_debug::_Safe_iterator<_Ty*, _Range>, __priority_tag<3>) + -> contiguous_iterator_tag; +# endif // _GLIBCXX_DEBUG +# if _CCCL_HOST_STD_LIB(LIBSTDCXX) +_CCCL_TEMPLATE(class _Iter, class _Ty, class _Range) +_CCCL_REQUIRES(_IsSame<_Iter, ::__gnu_cxx::__normal_iterator<_Ty*, _Range>>::value) +_CCCL_API inline auto __iter_concept_fn(::__gnu_cxx::__normal_iterator<_Ty*, _Range>, __priority_tag<3>) + -> contiguous_iterator_tag; +# endif // _CCCL_HOST_STD_LIB(LIBSTDCXX) +# if _CCCL_HOST_STD_LIB(LIBCXX) +_CCCL_TEMPLATE(class _Iter, class _Ty) +_CCCL_REQUIRES(_IsSame<_Iter, ::std::__wrap_iter<_Ty*>>::value) +_CCCL_API inline auto __iter_concept_fn(::std::__wrap_iter<_Ty*>, __priority_tag<3>) -> contiguous_iterator_tag; +# elif _CCCL_HOST_STD_LIB(STL) +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_Array_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_Array_const_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_Vector_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_Vector_const_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_String_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_String_const_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_String_view_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +_CCCL_TEMPLATE(class _Iter) +_CCCL_REQUIRES(_IsSame<_Iter, class _Iter::_Span_iterator>::value) +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<3>) -> contiguous_iterator_tag; +# endif // _CCCL_HOST_STD_LIB(STL) +#endif // _CCCL_HOSTED() + +_CCCL_TEMPLATE(class _Iter, class _Ty) +_CCCL_REQUIRES(_IsSame<_Iter, _Ty*>::value) +_CCCL_API inline auto __iter_concept_fn(_Ty*, __priority_tag<3>) -> contiguous_iterator_tag; + +template +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<2>) -> typename _ITER_TRAITS<_Iter>::iterator_concept; +template +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<1>) -> typename _ITER_TRAITS<_Iter>::iterator_category; +template +_CCCL_API inline auto __iter_concept_fn(_Iter, __priority_tag<0>) + -> enable_if_t<__is_primary_cccl_template<_Iter>::value && __is_primary_std_template<_Iter>::value, + random_access_iterator_tag>; + +template +using __iter_concept_t = + decltype(::cuda::std::__iter_concept_fn<_Iter>(::cuda::std::declval<_Iter>(), __priority_tag<3>{})); + +template +struct __iter_concept_cache +{}; + +template +struct __iter_concept_cache<_Iter, void_t<__iter_concept_t<_Iter>>> +{ + using type = __iter_concept_t<_Iter>; +}; + +template +using _ITER_CONCEPT = typename __iter_concept_cache<_Iter>::type; + +template +_CCCL_CONCEPT __has_member_reference = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::reference)); + +template +_CCCL_CONCEPT __has_member_pointer = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::pointer)); + +template +_CCCL_CONCEPT __has_member_iterator_category = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::iterator_category)); + +template +_CCCL_CONCEPT __has_member_iterator_concept = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::iterator_concept)); + +// The `cpp17-*-iterator` exposition-only concepts have very similar names to the `Cpp17*Iterator` named requirements +// from `[iterator.cpp17]`. To avoid confusion between the two, the exposition-only concepts have been banished to +// a "detail" namespace indicating they have a niche use-case. +namespace __iterator_traits_detail +{ +// [iterator.traits#concept:cpp17-iterator] +template +_CCCL_CONCEPT __cpp17_iterator = _CCCL_REQUIRES_EXPR((_Iter), _Iter __i)( + requires(copyable<_Iter>), + _Satisfies(__can_reference)(*__i), + _Same_as(_Iter&)(++__i), + _Satisfies(__can_reference)(*__i++)); + +// [iterator.traits#concept:cpp17-input-iterator] +template +_CCCL_CONCEPT __cpp17_input_iterator = _CCCL_REQUIRES_EXPR((_Iter), _Iter __i)( + requires(__cpp17_iterator<_Iter>), + requires(equality_comparable<_Iter>), + typename(typename incrementable_traits<_Iter>::difference_type), + typename(typename indirectly_readable_traits<_Iter>::value_type), + typename(common_reference_t&&, typename indirectly_readable_traits<_Iter>::value_type&>), + typename(common_reference_t::value_type&>), + requires(signed_integral::difference_type>)); + +// [iterator.traits#concept:cpp17-forward-iterator] +template +_CCCL_CONCEPT __cpp17_forward_iterator = _CCCL_REQUIRES_EXPR((_Iter), _Iter __i)( + requires(__cpp17_input_iterator<_Iter>), + requires(constructible_from<_Iter>), + requires(is_lvalue_reference_v>), + requires(same_as>, typename indirectly_readable_traits<_Iter>::value_type>), + requires(convertible_to), + _Same_as(iter_reference_t<_Iter>)(*__i++)); + +// [iterator.traits#concept:cpp17-bidirectional-iterator] +template +_CCCL_CONCEPT __cpp17_bidirectional_iterator = _CCCL_REQUIRES_EXPR((_Iter), _Iter __i)( + requires(__cpp17_forward_iterator<_Iter>), + _Same_as(_Iter&)(--__i), + requires(convertible_to), + _Same_as(iter_reference_t<_Iter>)(*__i--)); + +// [iterator.traits#concept:cpp17-random-access-iterator] +// Needs to be its own concept, because we need `typename incrementable_traits<_Iter>::difference_type` to be valid +template +_CCCL_CONCEPT __cpp17_random_access_iterator_operations = + _CCCL_REQUIRES_EXPR((_Iter), _Iter __i, typename incrementable_traits<_Iter>::difference_type __n)( + _Same_as(_Iter&)(__i += __n), + _Same_as(_Iter&)(__i -= __n), + _Same_as(_Iter)(__i + __n), + _Same_as(_Iter)(__n + __i), + _Same_as(_Iter)(__i - __n), + _Same_as(decltype(__n))(__i - __i), + requires(convertible_to>)); + +template +_CCCL_CONCEPT __cpp17_random_access_iterator = _CCCL_REQUIRES_EXPR((_Iter))( + requires(__cpp17_bidirectional_iterator<_Iter>), + requires(totally_ordered<_Iter>), + requires(__cpp17_random_access_iterator_operations<_Iter>)); +} // namespace __iterator_traits_detail + +// [iterator.traits]#3.1 +// If the qualified-id I::pointer is valid and denotes a type, then pointer names that type; +template +_CCCL_API auto __iterator_traits_deduce_member_pointer_or_void(int) -> typename _Iter::pointer; +// Otherwise, it names void. +template +_CCCL_API auto __iterator_traits_deduce_member_pointer_or_void(...) -> void; + +template +using __iterator_traits_member_pointer_or_void = + decltype(::cuda::std::__iterator_traits_deduce_member_pointer_or_void<_Iter>(0)); + +// [iterator.traits]#3.2 +// [iterator.traits]#3.2.1 +// If the qualified-id I::pointer is valid and denotes a type, then pointer names that type. +template +_CCCL_API auto __iterator_traits_deduce_member_pointer_or_arrow_or_void(int, __priority_tag<1>) -> + typename _Iter::pointer; + +// Otherwise, if decltype(declval().operator->()) is well-formed, then pointer names that type. +template +_CCCL_API auto __iterator_traits_deduce_member_pointer_or_arrow_or_void(int, __priority_tag<0>) + -> decltype(::cuda::std::declval<_Iter&>().operator->()); + +// Otherwise, pointer names void. +template +_CCCL_API auto __iterator_traits_deduce_member_pointer_or_arrow_or_void(...) -> void; + +template +using __iterator_traits_member_pointer_or_arrow_or_void = + decltype(::cuda::std::__iterator_traits_deduce_member_pointer_or_arrow_or_void<_Iter>(0, __priority_tag<1>{})); + +// [iterator.traits]#3.2.2 +// If the qualified-id `I::reference` is valid and denotes a type, `reference` names that type. +template +_CCCL_API auto __iterator_traits_deduce_member_reference(int) -> typename _Iter::reference; +// Otherwise, `reference` names `iter-reference-t`. +template +_CCCL_API auto __iterator_traits_deduce_member_reference(...) -> iter_reference_t<_Iter>; + +template +using __iterator_traits_member_reference = decltype(::cuda::std::__iterator_traits_deduce_member_reference<_Iter>(0)); + +// [iterator.traits]#3.2.3 +template +[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL auto __iterator_traits_deduce_iterator_category() noexcept +{ + if constexpr (__has_member_iterator_category<_Iter>) + { // If the qualified-id `I::iterator-category` is valid and denotes a type, `iterator-category` names that type. + return typename _Iter::iterator_category{}; + } + else if constexpr (__iterator_traits_detail::__cpp17_random_access_iterator<_Iter>) + { // Otherwise `random_access_iterator_tag` if `I` satisfies `cpp17-random-access-iterator`, + return random_access_iterator_tag{}; + } + else if constexpr (__iterator_traits_detail::__cpp17_bidirectional_iterator<_Iter>) + { // or otherwise `bidirectional_iterator_tag` if `I` satisfies `cpp17-bidirectional-iterator`, + return bidirectional_iterator_tag{}; + } + else if constexpr (__iterator_traits_detail::__cpp17_forward_iterator<_Iter>) + { // or otherwise `forward_iterator_tag` if `I` satisfies `cpp17-forward-iterator`, + return forward_iterator_tag{}; + } + else + { // or otherwise input_iterator_tag + return input_iterator_tag{}; + } +} + +template +using __iterator_traits_iterator_category = decltype(::cuda::std::__iterator_traits_deduce_iterator_category<_Iter>()); + +// [iterator.traits]#3.3 +// If the qualified-id `incrementable_traits::difference_type` is valid and denotes a type, then +// `difference_type` names that type; +template +_CCCL_API auto __iterator_traits_deduce_member_difference(int) -> typename incrementable_traits<_Iter>::difference_type; +// Otherwise, it names void. +template +_CCCL_API auto __iterator_traits_deduce_member_difference(...) -> void; + +template +using __iterator_traits_difference_type = decltype(::cuda::std::__iterator_traits_deduce_member_difference<_Iter>(0)); + +enum class __iterator_traits_selection +{ + __specialized_from_std, + __specifies_members, + __cpp17_input_iterator, + __cpp17_iterator, + __no_members, +}; + +// We need to consider if a user has specialized std::iterator_traits +template +_CCCL_CONCEPT __specialized_from_std = !__is_primary_std_template>::value; + +// If I has valid member types difference_type, value_type, reference, and iterator_category, +template +_CCCL_CONCEPT __specifies_members = _CCCL_REQUIRES_EXPR((_Iter))( + typename(typename _Iter::value_type), + typename(typename _Iter::difference_type), + typename(typename _Iter::reference), + typename(typename _Iter::iterator_category)); + +// [iterator.traits]#3.2.3 +template +[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL __iterator_traits_selection __select_iterator_traits_specialization() noexcept +{ + if constexpr (__specialized_from_std<_Iter>) + { // We need to consider if a user has specialized std::iterator_traits + return __iterator_traits_selection::__specialized_from_std; + } + if constexpr (__specifies_members<_Iter>) + { // If I has valid member types difference_type, value_type, reference, and iterator_category, + return __iterator_traits_selection::__specifies_members; + } + else if constexpr (__iterator_traits_detail::__cpp17_input_iterator<_Iter>) + { // Otherwise, if I satisfies the exposition-only concept cpp17-input-iterator, + return __iterator_traits_selection::__cpp17_input_iterator; + } + else if constexpr (__iterator_traits_detail::__cpp17_iterator<_Iter>) + { // Otherwise, if I satisfies the exposition-only concept cpp17-iterator, + return __iterator_traits_selection::__cpp17_iterator; + } + else + { // Otherwise, iterator_traits has no members by any of the above names. + return __iterator_traits_selection::__no_members; + } +} + +// [iterator.traits]#3 +template ()> +struct __iterator_traits; + +#if _CCCL_HOSTED() +// We need to properly accept specializations of `std::iterator_traits` +template +struct __iterator_traits<_Iter, __iterator_traits_selection::__specialized_from_std> + : public ::std::iterator_traits<_Iter> +{}; +#endif // _CCCL_HOSTED() + +// [iterator.traits]#3.1 +// If `I` has valid member types `difference-type`, `value-type`, `reference`, and +// `iterator-category`, then `iterator-traits` has the following publicly accessible members: +template +struct __iterator_traits<_Iter, __iterator_traits_selection::__specifies_members> +{ + using iterator_category = typename _Iter::iterator_category; + using value_type = typename _Iter::value_type; + using difference_type = typename _Iter::difference_type; + using pointer = __iterator_traits_member_pointer_or_void<_Iter>; + using reference = typename _Iter::reference; +}; + +// [iterator.traits]#3.2 +// Otherwise, if `I` satisfies the exposition-only concept `cpp17-input-iterator`, +// `iterator-traits` has the following publicly accessible members: +template +struct __iterator_traits<_Iter, __iterator_traits_selection::__cpp17_input_iterator> +{ + using iterator_category = __iterator_traits_iterator_category<_Iter>; + using value_type = typename indirectly_readable_traits<_Iter>::value_type; + using difference_type = typename incrementable_traits<_Iter>::difference_type; + using pointer = __iterator_traits_member_pointer_or_arrow_or_void<_Iter>; + using reference = __iterator_traits_member_reference<_Iter>; +}; + +// [iterator.traits]#3.3 +// Otherwise, if `I` satisfies the exposition-only concept `cpp17-iterator`, then +// `iterator_traits` has the following publicly accessible members: +template +struct __iterator_traits<_Iter, __iterator_traits_selection::__cpp17_iterator> +{ + using iterator_category = output_iterator_tag; + using value_type = void; + using difference_type = __iterator_traits_difference_type<_Iter>; + using pointer = void; + using reference = void; +}; + +// [iterator.traits]#3.4 +// Otherwise, `iterator_traits` has no members by any of the above names. +template +struct __iterator_traits<_Iter, __iterator_traits_selection::__no_members> +{}; + +template +struct _CCCL_TYPE_VISIBILITY_DEFAULT iterator_traits : __iterator_traits<_Iter> +{ + using __cccl_primary_template = iterator_traits; +}; + +// [iterator.traits]#5 +template +#if _CCCL_HAS_CONCEPTS() + requires is_object_v<_Tp> +#endif // _CCCL_HAS_CONCEPTS() +struct _CCCL_TYPE_VISIBILITY_DEFAULT iterator_traits<_Tp*> +{ + using difference_type = ptrdiff_t; + using value_type = remove_cv_t<_Tp>; + using pointer = _Tp*; + using reference = add_lvalue_reference_t<_Tp>; + using iterator_category = random_access_iterator_tag; + using iterator_concept = contiguous_iterator_tag; +}; + +template +_CCCL_CONCEPT __has_iterator_category_convertible_to = _CCCL_REQUIRES_EXPR((_Iter, _Tag)) // + (typename(typename iterator_traits<_Iter>::iterator_category), + requires(is_convertible_v::iterator_category, _Tag>)); + +template +_CCCL_CONCEPT __has_iterator_concept_convertible_to = _CCCL_REQUIRES_EXPR((_Iter, _Tag)) // + (typename(typename _Iter::iterator_concept), requires(is_convertible_v)); + +template +inline constexpr bool __has_input_traversal = + __has_iterator_category_convertible_to<_Iter, input_iterator_tag> + || __has_iterator_concept_convertible_to<_Iter, input_iterator_tag>; + +template +inline constexpr bool __has_forward_traversal = + __has_iterator_category_convertible_to<_Iter, forward_iterator_tag> + || __has_iterator_concept_convertible_to<_Iter, forward_iterator_tag>; + +template +inline constexpr bool __has_bidirectional_traversal = + __has_iterator_category_convertible_to<_Iter, bidirectional_iterator_tag> + || __has_iterator_concept_convertible_to<_Iter, bidirectional_iterator_tag>; + +template +inline constexpr bool __has_random_access_traversal = + __has_iterator_category_convertible_to<_Iter, random_access_iterator_tag> + || __has_iterator_concept_convertible_to<_Iter, random_access_iterator_tag>; + +// __has_contiguous_traversal determines if an iterator is known by +// libc++ to be contiguous, either because it advertises itself as such +// (in C++20) or because it is a pointer type or a known trivial wrapper +// around a (possibly fancy) pointer type, such as __wrap_iter. +// Such iterators receive special "contiguous" optimizations in +// std::copy and std::sort. +// +template +inline constexpr bool __has_contiguous_traversal = + __has_iterator_category_convertible_to<_Iter, contiguous_iterator_tag> + || __has_iterator_concept_convertible_to<_Iter, contiguous_iterator_tag>; + +// Any native pointer which is an iterator is also a contiguous iterator. +template +inline constexpr bool __has_contiguous_traversal<_Tp*> = true; + +template +using __iter_value_type = typename iterator_traits<_Iter>::value_type; + +template +using __iterator_category_type = typename iterator_traits<_Iter>::iterator_category; + +template +using __iterator_pointer_type = typename iterator_traits<_Iter>::pointer; + +template +using __iter_diff_t = typename iterator_traits<_Iter>::difference_type; + +template +using __iter_value_type = typename iterator_traits<_Iter>::value_type; + +// C++20 iterators do not play nicely with C++17 interfaces, because they commmonly use `iterator_concept` to +// communicate their iterator category. Deduce the actual iterator category from the maximum of `iterator_concept` and +// `iterator_category` +template +[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL auto __iterator_traits_category_or_concept() noexcept +{ + if constexpr (!__has_member_iterator_concept<_Iter>) + { + return __iterator_category_type<_Iter>{}; + } + else if constexpr (__has_contiguous_traversal<_Iter>) + { + return contiguous_iterator_tag{}; + } + else if constexpr (__has_random_access_traversal<_Iter>) + { + return random_access_iterator_tag{}; + } + else if constexpr (__has_bidirectional_traversal<_Iter>) + { + return bidirectional_iterator_tag{}; + } + else if constexpr (__has_forward_traversal<_Iter>) + { + return forward_iterator_tag{}; + } + else if constexpr (__has_input_traversal<_Iter>) + { + return input_iterator_tag{}; + } + else // if constexpr (__has_member_iterator_category<_Iter>) + { + return typename _Iter::iterator_category{}; + } +} + +template +using __iterator_traits_category_or_concept_t = decltype(::cuda::std::__iterator_traits_category_or_concept<_Iter>()); + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_ITERATOR_TRAITS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/next.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/next.h new file mode 100644 index 00000000..f29ed0de --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/next.h @@ -0,0 +1,104 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_NEXT_H +#define _CUDA_STD___ITERATOR_NEXT_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_TEMPLATE(class _InputIter) +_CCCL_REQUIRES(__has_input_traversal<_InputIter>) +[[nodiscard]] _CCCL_API constexpr _InputIter +next(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) +{ + _CCCL_ASSERT(__n >= 0 || __has_bidirectional_traversal<_InputIter>, + "Attempt to next(it, n) with negative n on a non-bidirectional iterator"); + + ::cuda::std::advance(__x, __n); + return __x; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +// [range.iter.op.next] + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES +_CCCL_BEGIN_NAMESPACE_CPO(__next) +struct __fn +{ + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(input_or_output_iterator<_Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x) const + { + ++__x; + return __x; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(input_or_output_iterator<_Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const + { + ::cuda::std::ranges::__advance_cpo{}(__x, __n); + return __x; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip, class _Sp) + _CCCL_REQUIRES(input_or_output_iterator<_Ip>&& sentinel_for<_Sp, _Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x, _Sp __bound_sentinel) const + { + ::cuda::std::ranges::__advance_cpo{}(__x, __bound_sentinel); + return __x; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip, class _Sp) + _CCCL_REQUIRES(input_or_output_iterator<_Ip>&& sentinel_for<_Sp, _Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n, _Sp __bound_sentinel) const + { + ::cuda::std::ranges::__advance_cpo{}(__x, __n, __bound_sentinel); + return __x; + } +}; +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto next = __next::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __next_cpo = __next::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +#include + +#endif // _CUDA_STD___ITERATOR_NEXT_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/prev.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/prev.h new file mode 100644 index 00000000..dc9e5219 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/prev.h @@ -0,0 +1,93 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_PREV_H +#define _CUDA_STD___ITERATOR_PREV_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +_CCCL_TEMPLATE(class _InputIter) +_CCCL_REQUIRES(__has_input_traversal<_InputIter>) +[[nodiscard]] _CCCL_API constexpr _InputIter +prev(_InputIter __x, typename iterator_traits<_InputIter>::difference_type __n = 1) +{ + _CCCL_ASSERT(__n <= 0 || __has_bidirectional_traversal<_InputIter>, "Attempt to prev(it, +n) on a non-bidi iterator"); + ::cuda::std::advance(__x, -__n); + return __x; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +// [range.iter.op.prev] + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES +_CCCL_BEGIN_NAMESPACE_CPO(__prev) +struct __fn +{ + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(bidirectional_iterator<_Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x) const + { + --__x; + return __x; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(bidirectional_iterator<_Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n) const + { + ::cuda::std::ranges::__advance_cpo{}(__x, -__n); + return __x; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Ip) + _CCCL_REQUIRES(bidirectional_iterator<_Ip>) + [[nodiscard]] _CCCL_API constexpr _Ip operator()(_Ip __x, iter_difference_t<_Ip> __n, _Ip __bound_iter) const + { + ::cuda::std::ranges::__advance_cpo{}(__x, -__n, __bound_iter); + return __x; + } +}; +_CCCL_END_NAMESPACE_CPO + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto prev = __prev::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __prev_cpo = __prev::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD_RANGES + +#include + +#endif // _CUDA_STD___ITERATOR_PREV_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/readable_traits.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/readable_traits.h new file mode 100644 index 00000000..17643d56 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/readable_traits.h @@ -0,0 +1,156 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_READABLE_TRAITS_H +#define _CUDA_STD___ITERATOR_READABLE_TRAITS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +_CCCL_CONCEPT __has_member_value_type = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::value_type)); + +template +_CCCL_CONCEPT __has_member_element_type = _CCCL_REQUIRES_EXPR((_Tp))(typename(typename _Tp::element_type)); + +template +struct __cond_value_type +{}; + +template +struct __cond_value_type<_Tp, enable_if_t>> +{ + using value_type = remove_cv_t<_Tp>; +}; + +#if _CCCL_HAS_CONCEPTS() + +// [readable.traits] +template +struct indirectly_readable_traits +{}; + +template + requires is_array_v<_Ip> +struct indirectly_readable_traits<_Ip> +{ + using value_type = remove_cv_t>; +}; + +template +struct indirectly_readable_traits : indirectly_readable_traits<_Ip> +{}; + +template +struct indirectly_readable_traits<_Tp*> : __cond_value_type<_Tp> +{}; + +template <__has_member_value_type _Tp> +struct indirectly_readable_traits<_Tp> : __cond_value_type +{}; + +template <__has_member_element_type _Tp> +struct indirectly_readable_traits<_Tp> : __cond_value_type +{}; + +template <__has_member_value_type _Tp> + requires __has_member_element_type<_Tp> +struct indirectly_readable_traits<_Tp> +{}; + +template <__has_member_value_type _Tp> + requires __has_member_element_type<_Tp> + && same_as, remove_cv_t> +struct indirectly_readable_traits<_Tp> : __cond_value_type +{}; + +#else // ^^^ _CCCL_HAS_CONCEPTS() ^^^ / vvv !_CCCL_HAS_CONCEPTS() vvv + +// [readable.traits] +template +struct indirectly_readable_traits +{}; + +template +struct indirectly_readable_traits<_Ip, enable_if_t && is_array_v<_Ip>>> +{ + using value_type = remove_cv_t>; +}; + +template +struct indirectly_readable_traits : indirectly_readable_traits<_Ip> +{}; + +template +struct indirectly_readable_traits<_Tp*> : __cond_value_type<_Tp> +{}; + +template +struct indirectly_readable_traits< + _Tp, + enable_if_t && __has_member_value_type<_Tp> && !__has_member_element_type<_Tp>>> + : __cond_value_type +{}; + +template +struct indirectly_readable_traits< + _Tp, + enable_if_t && !__has_member_value_type<_Tp> && __has_member_element_type<_Tp>>> + : __cond_value_type +{}; + +template +struct indirectly_readable_traits< + _Tp, + enable_if_t && __has_member_value_type<_Tp> && __has_member_element_type<_Tp> + && same_as, remove_cv_t>>> + : __cond_value_type +{}; + +#endif // ^^^ !_CCCL_HAS_CONCEPTS() ^^^ + +// Let `RI` be `remove_cvref_t`. The type `iter_value_t` denotes +// `indirectly_readable_traits::value_type` if `iterator_traits` names a specialization +// generated from the primary template, and `iterator_traits::value_type` otherwise. +template +using iter_value_t = + typename __select_traits, indirectly_readable_traits>>::value_type; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_READABLE_TRAITS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_access.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_access.h new file mode 100644 index 00000000..c0d9813d --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_access.h @@ -0,0 +1,154 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_REVERSE_ACCESS_H +#define _CUDA_STD___ITERATOR_REVERSE_ACCESS_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +namespace __rbegin +{ +struct __fn +{ + template + _CCCL_API constexpr reverse_iterator<_Tp*> operator()(_Tp (&__array)[_Np]) const noexcept + { + return reverse_iterator<_Tp*>(__array + _Np); + } + + template + _CCCL_API constexpr reverse_iterator operator()(initializer_list<_Ep> __il) const noexcept + { + return reverse_iterator(__il.end()); + } + + template + _CCCL_API constexpr auto operator()(_Cp& __c) const noexcept(noexcept(__c.rbegin())) -> decltype(__c.rbegin()) + { + return __c.rbegin(); + } + + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(__c.rbegin())) -> decltype(__c.rbegin()) + { + return __c.rbegin(); + } +}; +} // namespace __rbegin + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto rbegin = __rbegin::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __rbegin_cpo = __rbegin::__fn; +} // namespace __cpo + +namespace __rend +{ +struct __fn +{ + template + _CCCL_API constexpr reverse_iterator<_Tp*> operator()(_Tp (&__array)[_Np]) const noexcept + { + return reverse_iterator<_Tp*>(__array); + } + + template + _CCCL_API constexpr reverse_iterator operator()(initializer_list<_Ep> __il) const noexcept + { + return reverse_iterator(__il.begin()); + } + + template + _CCCL_API constexpr auto operator()(_Cp& __c) const noexcept(noexcept(__c.rend())) -> decltype(__c.rend()) + { + return __c.rend(); + } + + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(__c.rend())) -> decltype(__c.rend()) + { + return __c.rend(); + } +}; +} // namespace __rend + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto rend = __rend::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __rend_cpo = __rend::__fn; +} // namespace __cpo + +namespace __crbegin +{ +struct __fn +{ + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(::cuda::std::__rbegin_cpo{}(__c))) + -> decltype(::cuda::std::__rbegin_cpo{}(__c)) + { + return ::cuda::std::__rbegin_cpo{}(__c); + } +}; +} // namespace __crbegin + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto crbegin = __crbegin::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __crbegin_cpo = __crbegin::__fn; +} // namespace __cpo + +namespace __crend +{ +struct __fn +{ + template + _CCCL_API constexpr auto operator()(const _Cp& __c) const noexcept(noexcept(::cuda::std::__rend_cpo{}(__c))) + -> decltype(::cuda::std::__rend_cpo{}(__c)) + { + return ::cuda::std::__rend_cpo{}(__c); + } +}; +} // namespace __crend + +inline namespace __cpo +{ +_CCCL_GLOBAL_CONSTANT auto crend = __crend::__fn{}; + +// We want to avoid using the CPO internally because of __tile__ access +using __crend_cpo = __crend::__fn; +} // namespace __cpo + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ITERATOR_REVERSE_ACCESS_H diff --git a/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_iterator.h b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_iterator.h new file mode 100644 index 00000000..1f1793da --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/include/cuda/std/__iterator/reverse_iterator.h @@ -0,0 +1,375 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ITERATOR_REVERSE_ITERATOR_H +#define _CUDA_STD___ITERATOR_REVERSE_ITERATOR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() +# include +# include +#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +inline constexpr bool __noexcept_rev_iter_iter_move = false; + +template +inline constexpr bool __noexcept_rev_iter_iter_move<_Iter, void_t())>> = + is_nothrow_copy_constructible_v<_Iter> + && noexcept(::cuda::std::ranges::__iter_move_cpo{}(--::cuda::std::declval<_Iter&>())); + +template +inline constexpr bool __noexcept_rev_iter_iter_swap = false; + +template +inline constexpr bool __noexcept_rev_iter_iter_swap<_Iter, _Iter2, enable_if_t>> = + is_nothrow_copy_constructible_v<_Iter> && is_nothrow_copy_constructible_v<_Iter2> + && noexcept(::cuda::std::ranges::__iter_swap_cpo{}(--declval<_Iter&>(), --declval<_Iter2&>())); + +// MSVC has issues with `is_nothrow_convertible_v` sometimes, so do the noexcept expression +template +inline constexpr bool __noexcept_rev_iter_convertible = false; + +template +inline constexpr bool + __noexcept_rev_iter_convertible<_Iter, _Iter2, enable_if_t>> = + noexcept(_Iter2(::cuda::std::declval())); + +_LIBCUDACXX_BEGIN_HIDDEN_FRIEND_NAMESPACE + +_CCCL_SUPPRESS_DEPRECATED_PUSH +_CCCL_SUPPRESS_DEPRECATED_NVRTC_DIAG +template +class _CCCL_TYPE_VISIBILITY_DEFAULT reverse_iterator +{ +private: +#if _CCCL_STD_VER > 2017 + static_assert(__has_bidirectional_traversal<_Iter> || bidirectional_iterator<_Iter>, + "reverse_iterator requires It to be a bidirectional iterator."); +#endif // _CCCL_STD_VER > 2017 + +protected: + _Iter current; + +public: + using iterator_type = _Iter; + + using iterator_category = + _If<__has_random_access_traversal<_Iter>, random_access_iterator_tag, __iterator_traits_category_or_concept_t<_Iter>>; + using pointer = typename iterator_traits<_Iter>::pointer; + using iterator_concept = _If, random_access_iterator_tag, bidirectional_iterator_tag>; + using value_type = iter_value_t<_Iter>; + using difference_type = iter_difference_t<_Iter>; + using reference = iter_reference_t<_Iter>; + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _It2 = _Iter) + _CCCL_REQUIRES(is_default_constructible_v<_It2>) + _CCCL_API constexpr reverse_iterator() noexcept(is_nothrow_default_constructible_v<_It2>) + : current() + {} + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr explicit reverse_iterator(_Iter __x) noexcept(is_nothrow_copy_constructible_v<_Iter>) + : current(__x) + {} + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Up) + _CCCL_REQUIRES((!is_same_v<_Up, _Iter>) _CCCL_AND is_convertible_v<_Up const&, _Iter>) + _CCCL_API constexpr reverse_iterator(const reverse_iterator<_Up>& __u) noexcept( + __noexcept_rev_iter_convertible<_Up, _Iter>) + : current(__u.base()) + {} + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Up) + _CCCL_REQUIRES(( + !is_same_v<_Up, _Iter>) _CCCL_AND is_convertible_v<_Up const&, _Iter> _CCCL_AND is_assignable_v<_Iter&, _Up const&>) + _CCCL_API constexpr reverse_iterator& + operator=(const reverse_iterator<_Up>& __u) noexcept(is_nothrow_assignable_v<_Iter&, _Up const&>) + { + current = __u.base(); + return *this; + } + + _CCCL_EXEC_CHECK_DISABLE + [[nodiscard]] _CCCL_API constexpr _Iter base() const noexcept(is_nothrow_copy_constructible_v<_Iter>) + { + return current; + } + + _CCCL_EXEC_CHECK_DISABLE + [[nodiscard]] _CCCL_API constexpr reference operator*() const + { + return *::cuda::std::prev(current); + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Iter2 = _Iter) + _CCCL_REQUIRES(__has_const_arrow<_Iter2>) + _CCCL_API constexpr pointer operator->() const + { + if constexpr (is_pointer_v<_Iter>) + { + return ::cuda::std::prev(current); + } + else + { + return ::cuda::std::prev(current).operator->(); + } + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr reverse_iterator& operator++() + { + --current; + return *this; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr reverse_iterator operator++(int) + { + reverse_iterator __tmp{*this}; + --current; + return __tmp; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr reverse_iterator& operator--() + { + ++current; + return *this; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr reverse_iterator operator--(int) + { + reverse_iterator __tmp{*this}; + ++current; + return __tmp; + } + + _CCCL_EXEC_CHECK_DISABLE + [[nodiscard]] _CCCL_API constexpr reverse_iterator operator+(difference_type __n) const + { + return reverse_iterator{current - __n}; + } + + _CCCL_EXEC_CHECK_DISABLE + [[nodiscard]] _CCCL_API friend constexpr reverse_iterator operator+(difference_type __n, const reverse_iterator& __x) + { + return reverse_iterator{__x.base() - __n}; + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr reverse_iterator& operator+=(difference_type __n) + { + current -= __n; + return *this; + } + + _CCCL_EXEC_CHECK_DISABLE + [[nodiscard]] _CCCL_API constexpr reverse_iterator operator-(difference_type __n) const + { + return reverse_iterator{current + __n}; + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator-(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) + -> decltype(::cuda::std::declval() - ::cuda::std::declval()) + { + return __y.base() - __x.base(); + } + + _CCCL_EXEC_CHECK_DISABLE + _CCCL_API constexpr reverse_iterator& operator-=(difference_type __n) + { + current += __n; + return *this; + } + + _CCCL_EXEC_CHECK_DISABLE + [[nodiscard]] _CCCL_API constexpr reference operator[](difference_type __n) const + { + return *(*this + __n); + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr iter_rvalue_reference_t<_Iter2> + iter_move(const reverse_iterator& __i) noexcept(__noexcept_rev_iter_iter_move<_Iter2>) + { + auto __tmp = __i.base(); + return ::cuda::std::ranges::__iter_move_cpo{}(--__tmp); + } + + _CCCL_EXEC_CHECK_DISABLE + template + _CCCL_API friend constexpr auto iter_swap(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) noexcept( + __noexcept_rev_iter_iter_swap<_Iter, _Iter2>) _CCCL_TRAILING_REQUIRES(void)(indirectly_swappable<_Iter2, _Iter>) + { + auto __xtmp = __x.base(); + auto __ytmp = __y.base(); + return ::cuda::std::ranges::__iter_swap_cpo{}(--__xtmp, --__ytmp); + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator==(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) noexcept( + noexcept(bool(::cuda::std::declval() == ::cuda::std::declval()))) + -> decltype(static_cast(::cuda::std::declval() == ::cuda::std::declval())) + { + return __x.base() == __y.base(); + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator!=(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) noexcept( + noexcept(bool(::cuda::std::declval() != ::cuda::std::declval()))) + -> decltype(static_cast(::cuda::std::declval() != ::cuda::std::declval())) + { + return __x.base() != __y.base(); + } + +#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() + _CCCL_EXEC_CHECK_DISABLE + _CCCL_TEMPLATE(class _Iter2) + _CCCL_REQUIRES(three_way_comparable_with<_Iter, _Iter2>) + [[nodiscard]] _CCCL_API friend constexpr compare_three_way_result_t<_Iter, _Iter2> + operator<=>(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) + { + return __y.base() <=> __x.base(); + } +#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator<(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) + -> decltype(static_cast(::cuda::std::declval() > ::cuda::std::declval())) + { + return __x.base() > __y.base(); + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator>(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) + -> decltype(static_cast(::cuda::std::declval() < ::cuda::std::declval())) + { + return __x.base() < __y.base(); + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator>=(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) + -> decltype(static_cast(::cuda::std::declval() <= ::cuda::std::declval())) + { + return __x.base() <= __y.base(); + } + + _CCCL_EXEC_CHECK_DISABLE + template + [[nodiscard]] _CCCL_API friend constexpr auto + operator<=(const reverse_iterator& __x, const reverse_iterator<_Iter2>& __y) + -> decltype(static_cast(::cuda::std::declval() >= ::cuda::std::declval())) + { + return __x.base() >= __y.base(); + } +#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() +}; +_CCCL_SUPPRESS_DEPRECATED_POP + +_LIBCUDACXX_END_HIDDEN_FRIEND_NAMESPACE(reverse_iterator) + +template +inline constexpr bool disable_sized_sentinel_for, reverse_iterator<_Iter2>> = + !sized_sentinel_for<_Iter1, _Iter2>; + +template +[[nodiscard]] _CCCL_API constexpr reverse_iterator<_Iter> +make_reverse_iterator(_Iter __i) noexcept(is_nothrow_copy_constructible_v<_Iter>) +{ + return reverse_iterator<_Iter>{__i}; +} + +template