[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/std/__variant/bad_variant_access.h

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_BAD_VARIANT_ACCESS_H
+#define _CUDA_STD___VARIANT_BAD_VARIANT_ACCESS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__exception/terminate.h>
+
+#if _CCCL_HAS_EXCEPTIONS()
+
+#  if __cpp_lib_variant >= 201606L
+#    include <variant>
+#  else // ^^^ __cpp_lib_variant >= 201606L ^^^ / vvv __cpp_lib_variant < 201606L vvv
+#    include <exception>
+#  endif // ^^^ __cpp_lib_variant < 201606L ^^^
+#endif // _CCCL_HAS_EXCEPTIONS()
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HAS_EXCEPTIONS()
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_NOVERSION
+
+#  if __cpp_lib_variant >= 201606L
+
+using ::std::bad_variant_access;
+
+#  else // ^^^ __cpp_lib_variant >= 201606L ^^^ / vvv __cpp_lib_variant < 201606L vvv
+class _CCCL_TYPE_VISIBILITY_DEFAULT bad_variant_access : public ::std::exception
+{
+public:
+  const char* what() const noexcept override
+  {
+    return "bad access to cuda::std::variant";
+  }
+};
+#  endif // ^^^ __cpp_lib_variant < 201606L ^^^
+
+_CCCL_END_NAMESPACE_CUDA_STD_NOVERSION
+
+#endif // _CCCL_HAS_EXCEPTIONS()
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+[[noreturn]] _CCCL_API inline void __throw_bad_variant_access()
+{
+#if _CCCL_HAS_EXCEPTIONS()
+  NV_IF_ELSE_TARGET(NV_IS_HOST, (throw ::cuda::std::bad_variant_access();), (::cuda::std::terminate();))
+#else // ^^^ !_CCCL_HAS_EXCEPTIONS() ^^^ / vvv _CCCL_HAS_EXCEPTIONS() vvv
+  ::cuda::std::terminate();
+#endif // _CCCL_HAS_EXCEPTIONS()
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_BAD_VARIANT_ACCESS_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/comparison.h

@@ -0,0 +1,207 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_COMPARISON_H
+#define _CUDA_STD___VARIANT_COMPARISON_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__variant/variant.h>
+#include <cuda/std/__variant/visit.h>
+
+// [variant.syn]
+#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
+#  include <cuda/std/compare>
+#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class _Operator>
+struct __convert_to_bool
+{
+  template <class _T1, class _T2>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(_T1&& __t1, _T2&& __t2) const
+  {
+    static_assert(
+      is_convertible_v<decltype(_Operator{}(::cuda::std::forward<_T1>(__t1), ::cuda::std::forward<_T2>(__t2))), bool>,
+      "the relational operator does not return a type which is "
+      "implicitly convertible to bool");
+    return _Operator{}(::cuda::std::forward<_T1>(__t1), ::cuda::std::forward<_T2>(__t2));
+  }
+};
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool operator==(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  const auto __index_ = __lhs.index();
+  if (__index_ != __rhs.index())
+  {
+    return false;
+  }
+  if (__lhs.valueless_by_exception())
+  {
+    return true;
+  }
+
+  return __variant_binary_visitor::__visit(__index_, __convert_to_bool<equal_to<>>{}, __lhs, __rhs);
+}
+
+#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
+
+template <class... _Types>
+  requires(three_way_comparable<_Types> && ...)
+[[nodiscard]] _CCCL_API constexpr common_comparison_category_t<compare_three_way_result_t<_Types>...>
+operator<=>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  using __result_t = common_comparison_category_t<compare_three_way_result_t<_Types>...>;
+  if (__lhs.valueless_by_exception() && __rhs.valueless_by_exception())
+  {
+    return strong_ordering::equal;
+  }
+  if (__lhs.valueless_by_exception())
+  {
+    return strong_ordering::less;
+  }
+  if (__rhs.valueless_by_exception())
+  {
+    return strong_ordering::greater;
+  }
+  if (auto __c = __lhs.index() <=> __rhs.index(); __c != 0)
+  {
+    return __c;
+  }
+  auto __three_way = []<class _Type>(const _Type& __v, const _Type& __w) -> __result_t {
+    return __v <=> __w;
+  };
+  return __variant_binary_visitor::__visit(__lhs.index(), __three_way, __lhs, __rhs);
+}
+
+#endif // _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool operator!=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  if (__lhs.index() != __rhs.index())
+  {
+    return true;
+  }
+  if (__lhs.valueless_by_exception())
+  {
+    return false;
+  }
+  return __variant_binary_visitor::__visit(__lhs.index(), __convert_to_bool<not_equal_to<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool operator<(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  if (__rhs.valueless_by_exception())
+  {
+    return false;
+  }
+  if (__lhs.valueless_by_exception())
+  {
+    return true;
+  }
+  if (__lhs.index() < __rhs.index())
+  {
+    return true;
+  }
+  if (__lhs.index() > __rhs.index())
+  {
+    return false;
+  }
+  return __variant_binary_visitor::__visit(__lhs.index(), __convert_to_bool<less<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool operator>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  if (__lhs.valueless_by_exception())
+  {
+    return false;
+  }
+  if (__rhs.valueless_by_exception())
+  {
+    return true;
+  }
+  if (__lhs.index() > __rhs.index())
+  {
+    return true;
+  }
+  if (__lhs.index() < __rhs.index())
+  {
+    return false;
+  }
+  return __variant_binary_visitor::__visit(__lhs.index(), __convert_to_bool<greater<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool operator<=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  if (__lhs.valueless_by_exception())
+  {
+    return true;
+  }
+  if (__rhs.valueless_by_exception())
+  {
+    return false;
+  }
+  if (__lhs.index() < __rhs.index())
+  {
+    return true;
+  }
+  if (__lhs.index() > __rhs.index())
+  {
+    return false;
+  }
+  return __variant_binary_visitor::__visit(__lhs.index(), __convert_to_bool<less_equal<>>{}, __lhs, __rhs);
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool operator>=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs)
+{
+  if (__rhs.valueless_by_exception())
+  {
+    return true;
+  }
+  if (__lhs.valueless_by_exception())
+  {
+    return false;
+  }
+  if (__lhs.index() > __rhs.index())
+  {
+    return true;
+  }
+  if (__lhs.index() < __rhs.index())
+  {
+    return false;
+  }
+  return __variant_binary_visitor::__visit(__lhs.index(), __convert_to_bool<greater_equal<>>{}, __lhs, __rhs);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_COMPARISON_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/get.h

@@ -0,0 +1,192 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_GET_H
+#define _CUDA_STD___VARIANT_GET_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/get.h>
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__memory/addressof.h>
+#include <cuda/std/__tuple_dir/get.h>
+#include <cuda/std/__type_traits/add_pointer.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/move.h>
+#include <cuda/std/__variant/bad_variant_access.h>
+#include <cuda/std/__variant/variant.h>
+#include <cuda/std/__variant/variant_access.h>
+#include <cuda/std/__variant/variant_match.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool __holds_alternative(const variant<_Types...>& __v) noexcept
+{
+  return __v.index() == _Ip;
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr bool holds_alternative(const variant<_Types...>& __v) noexcept
+{
+  return ::cuda::std::__holds_alternative<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <size_t _Ip, class _Vp>
+[[nodiscard]] _CCCL_API constexpr auto&& __generic_get(_Vp&& __v)
+{
+  using __variant_detail::__access::__variant;
+  if (!::cuda::std::__holds_alternative<_Ip>(__v))
+  {
+    ::cuda::std::__throw_bad_variant_access();
+  }
+  return __variant::__get_alt<_Ip>(::cuda::std::forward<_Vp>(__v)).__value;
+}
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr variant_alternative_t<_Ip, variant<_Types...>>& get(variant<_Types...>& __v)
+{
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return ::cuda::std::__generic_get<_Ip>(__v);
+}
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr variant_alternative_t<_Ip, variant<_Types...>>&& get(variant<_Types...>&& __v)
+{
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return ::cuda::std::__generic_get<_Ip>(::cuda::std::move(__v));
+}
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
+get(const variant<_Types...>& __v)
+{
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return ::cuda::std::__generic_get<_Ip>(__v);
+}
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
+get(const variant<_Types...>&& __v)
+{
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return ::cuda::std::__generic_get<_Ip>(::cuda::std::move(__v));
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr _Tp& get(variant<_Types...>& __v)
+{
+  static_assert(!is_void_v<_Tp>);
+  return ::cuda::std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr _Tp&& get(variant<_Types...>&& __v)
+{
+  static_assert(!is_void_v<_Tp>);
+  return ::cuda::std::get<__find_exactly_one_t<_Tp, _Types...>::value>(::cuda::std::move(__v));
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr const _Tp& get(const variant<_Types...>& __v)
+{
+  static_assert(!is_void_v<_Tp>);
+  return ::cuda::std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr const _Tp&& get(const variant<_Types...>&& __v)
+{
+  static_assert(!is_void_v<_Tp>);
+  return ::cuda::std::get<__find_exactly_one_t<_Tp, _Types...>::value>(::cuda::std::move(__v));
+}
+
+template <size_t _Ip, class _Vp>
+[[nodiscard]] _CCCL_API constexpr auto* __generic_get_if(_Vp* __v) noexcept
+{
+  using __variant_detail::__access::__variant;
+  return __v && ::cuda::std::__holds_alternative<_Ip>(*__v)
+         ? ::cuda::std::addressof(__variant::__get_alt<_Ip>(*__v).__value)
+         : nullptr;
+}
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr add_pointer_t<variant_alternative_t<_Ip, variant<_Types...>>>
+get_if(variant<_Types...>* __v) noexcept
+{
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return ::cuda::std::__generic_get_if<_Ip>(__v);
+}
+
+template <size_t _Ip, class... _Types>
+[[nodiscard]] _CCCL_API constexpr add_pointer_t<const variant_alternative_t<_Ip, variant<_Types...>>>
+get_if(const variant<_Types...>* __v) noexcept
+{
+  static_assert(_Ip < sizeof...(_Types));
+  static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
+  return ::cuda::std::__generic_get_if<_Ip>(__v);
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr add_pointer_t<_Tp> get_if(variant<_Types...>* __v) noexcept
+{
+  static_assert(!is_void_v<_Tp>);
+  return ::cuda::std::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr add_pointer_t<const _Tp> get_if(const variant<_Types...>* __v) noexcept
+{
+  static_assert(!is_void_v<_Tp>);
+  return ::cuda::std::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+// __unchecked_get is the same as ::cuda::std::get, except, it is UB to use it
+// with the wrong type whereas ::cuda::std::get will throw or returning nullptr.
+// This makes it faster than ::cuda::std::get.
+template <size_t _Ip, class _Vp>
+[[nodiscard]] _CCCL_API constexpr auto&& __unchecked_get(_Vp&& __v) noexcept
+{
+  using __variant_detail::__access::__variant;
+  return __variant::__get_alt<_Ip>(::cuda::std::forward<_Vp>(__v)).__value;
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr auto&& __unchecked_get(const variant<_Types...>& __v) noexcept
+{
+  return ::cuda::std::__unchecked_get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+template <class _Tp, class... _Types>
+[[nodiscard]] _CCCL_API constexpr auto&& __unchecked_get(variant<_Types...>& __v) noexcept
+{
+  return ::cuda::std::__unchecked_get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_GET_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/hash.h

@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_HASH_H
+#define _CUDA_STD___VARIANT_HASH_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#ifndef __cuda_std__
+
+#  include <cuda/std/__functional/has.h>
+#  include <cuda/std/__fwd/variant.h>
+#  include <cuda/std/__type_traits/integral_constant.h>
+#  include <cuda/std/__type_traits/remove_const.h>
+#  include <cuda/std/__type_traits/type_list.h>
+#  include <cuda/std/__variant/variant.h>
+#  include <cuda/std/__variant/variant_access.h>
+#  include <cuda/std/cstdint>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class... _Types>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT hash<__enable_hash_helper<variant<_Types...>, remove_const_t<_Types>...>>
+{
+  using argument_type = variant<_Types...>;
+  using result_type   = size_t;
+
+  template <size_t _CurrentIndex>
+  [[nodiscard]] _CCCL_API constexpr static size_t
+  __hash(integral_constant<size_t, _CurrentIndex>, const size_t __index_, const argument_type& __v) noexcept
+  {
+    if (__index_ == _CurrentIndex)
+    {
+      using __value_type = remove_const_t<__type_index_c<_CurrentIndex, _Types...>>;
+      return hash<__value_type>{}(__access::__base::__get_alt<_CurrentIndex>(this->__as_base()).__value);
+    }
+    __hash(integral_constant<size_t, _CurrentIndex - 1>{}, __index_, __v);
+  }
+  [[nodiscard]] _CCCL_API constexpr static size_t
+  __hash(integral_constant<size_t, 0>, const size_t __index_, const argument_type& __v) noexcept
+  {
+    if (__index_ == 0)
+    {
+      using __value_type = remove_const_t<__type_index_c<0, _Types...>>;
+      return hash<__value_type>{}(__access::__base::__get_alt<0>(this->__as_base()).__value);
+    }
+    // We already checked that every variant has a value, so we should never reach this line
+    _CCCL_UNREACHABLE();
+  }
+
+  [[nodiscard]] _CCCL_API constexpr result_type operator()(const argument_type& __v) const
+  {
+    size_t __res = __v.valueless_by_exception()
+                   ? 299792458 // Random value chosen by the universe upon creation
+                   : __hash(integral_constant<size_t, sizeof...(_Types) - 1>{}, __v.index(), __v);
+    return ::cuda::std::__hash_combine(__res, hash<size_t>{}(__v.index()));
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // __cuda_std__
+
+#endif // _CUDA_STD___VARIANT_HASH_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/sfinae_helpers.h

@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2023-24 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_SFINAE_HELPERS_H
+#define _CUDA_STD___VARIANT_SFINAE_HELPERS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__tuple_dir/sfinae_helpers.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <bool _CanCopy>
+struct __variant_copy_base
+{};
+template <>
+struct __variant_copy_base<false>
+{
+  _CCCL_HIDE_FROM_ABI __variant_copy_base()                                      = default;
+  __variant_copy_base(__variant_copy_base const&)                                = delete;
+  _CCCL_HIDE_FROM_ABI __variant_copy_base(__variant_copy_base&&)                 = default;
+  _CCCL_HIDE_FROM_ABI __variant_copy_base& operator=(__variant_copy_base const&) = default;
+  _CCCL_HIDE_FROM_ABI __variant_copy_base& operator=(__variant_copy_base&&)      = default;
+};
+
+template <bool _CanCopy, bool _CanMove>
+struct __variant_move_base : __variant_copy_base<_CanCopy>
+{};
+template <bool _CanCopy>
+struct __variant_move_base<_CanCopy, false> : __variant_copy_base<_CanCopy>
+{
+  _CCCL_HIDE_FROM_ABI __variant_move_base()                                      = default;
+  _CCCL_HIDE_FROM_ABI __variant_move_base(__variant_move_base const&)            = default;
+  __variant_move_base(__variant_move_base&&)                                     = delete;
+  _CCCL_HIDE_FROM_ABI __variant_move_base& operator=(__variant_move_base const&) = default;
+  _CCCL_HIDE_FROM_ABI __variant_move_base& operator=(__variant_move_base&&)      = default;
+};
+
+template <bool _CanCopy, bool _CanMove, bool _CanCopyAssign>
+struct __variant_copy_assign_base : __variant_move_base<_CanCopy, _CanMove>
+{};
+template <bool _CanCopy, bool _CanMove>
+struct __variant_copy_assign_base<_CanCopy, _CanMove, false> : __variant_move_base<_CanCopy, _CanMove>
+{
+  _CCCL_HIDE_FROM_ABI __variant_copy_assign_base()                                        = default;
+  _CCCL_HIDE_FROM_ABI __variant_copy_assign_base(__variant_copy_assign_base const&)       = default;
+  _CCCL_HIDE_FROM_ABI __variant_copy_assign_base(__variant_copy_assign_base&&)            = default;
+  __variant_copy_assign_base& operator=(__variant_copy_assign_base const&)                = delete;
+  _CCCL_HIDE_FROM_ABI __variant_copy_assign_base& operator=(__variant_copy_assign_base&&) = default;
+};
+
+template <bool _CanCopy, bool _CanMove, bool _CanCopyAssign, bool _CanMoveAssign>
+struct __variant_move_assign_base : __variant_copy_assign_base<_CanCopy, _CanMove, _CanCopyAssign>
+{};
+template <bool _CanCopy, bool _CanMove, bool _CanCopyAssign>
+struct __variant_move_assign_base<_CanCopy, _CanMove, _CanCopyAssign, false>
+    : __variant_copy_assign_base<_CanCopy, _CanMove, _CanCopyAssign>
+{
+  _CCCL_HIDE_FROM_ABI __variant_move_assign_base()                                             = default;
+  _CCCL_HIDE_FROM_ABI __variant_move_assign_base(__variant_move_assign_base const&)            = default;
+  _CCCL_HIDE_FROM_ABI __variant_move_assign_base(__variant_move_assign_base&&)                 = default;
+  _CCCL_HIDE_FROM_ABI __variant_move_assign_base& operator=(__variant_move_assign_base const&) = default;
+  __variant_move_assign_base& operator=(__variant_move_assign_base&&)                          = delete;
+};
+
+template <bool _CanCopy, bool _CanMove, bool _CanCopyAssign, bool _CanMoveAssign>
+using __variant_base = __variant_move_assign_base<_CanCopy, _CanMove, _CanCopyAssign, _CanMoveAssign>;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_SFINAE_HELPERS_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant.h

@@ -0,0 +1,250 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_H
+#define _CUDA_STD___VARIANT_VARIANT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__type_traits/dependent_type.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/fold.h>
+#include <cuda/std/__type_traits/is_array.h>
+#include <cuda/std/__type_traits/is_copy_assignable.h>
+#include <cuda/std/__type_traits/is_copy_constructible.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+#include <cuda/std/__type_traits/is_move_assignable.h>
+#include <cuda/std/__type_traits/is_move_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
+#include <cuda/std/__type_traits/is_reference.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/in_place.h>
+#include <cuda/std/__variant/sfinae_helpers.h>
+#include <cuda/std/__variant/variant_access.h>
+#include <cuda/std/__variant/variant_base.h>
+#include <cuda/std/__variant/variant_constraints.h>
+#include <cuda/std/__variant/variant_visit.h>
+#include <cuda/std/initializer_list>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class... _Types>
+class _CCCL_TYPE_VISIBILITY_DEFAULT variant
+    : private __variant_base<__fold_and_v<is_copy_constructible_v<_Types>...>,
+                             __fold_and_v<is_move_constructible_v<_Types>...>,
+                             __fold_and_v<(is_copy_constructible_v<_Types> && is_copy_assignable_v<_Types>) ...>,
+                             __fold_and_v<(is_move_constructible_v<_Types> && is_move_assignable_v<_Types>) ...>>
+{
+  static_assert(0 < sizeof...(_Types), "variant must consist of at least one alternative.");
+
+  static_assert((!is_array_v<_Types> && ...), "variant can not have an array type as an alternative.");
+
+  static_assert((!is_reference_v<_Types> && ...), "variant can not have a reference type as an alternative.");
+
+  static_assert((!is_void_v<_Types> && ...), "variant can not have a void type as an alternative.");
+
+  using __first_type  = variant_alternative_t<0, variant>;
+  using __constraints = __variant_detail::__variant_constraints<_Types...>;
+
+public:
+  // Needs to be dependent to guard against incomplete types
+  template <bool _Dummy = true,
+            class       = enable_if_t<__dependent_type<is_default_constructible<__first_type>, _Dummy>::value>>
+  _CCCL_API constexpr variant() noexcept(is_nothrow_default_constructible_v<__first_type>)
+      : __impl_(in_place_index<0>)
+  {}
+
+  _CCCL_HIDE_FROM_ABI constexpr variant(const variant&) = default;
+  _CCCL_HIDE_FROM_ABI constexpr variant(variant&&)      = default;
+
+  template <class _Arg>
+  using __match_construct =
+    conditional_t<!is_same_v<remove_cvref_t<_Arg>, variant> && !__is_cuda_std_inplace_type_v<remove_cvref_t<_Arg>> //
+                    && !__is_cuda_std_inplace_index_v<remove_cvref_t<_Arg>>,
+                  typename __constraints::template __match_construct<_Arg>,
+                  __variant_detail::__invalid_variant_constraints>;
+
+  // CTAD fails if we do not SFINAE the empty variant away first
+  template <class _Arg,
+            class              = enable_if_t<sizeof...(_Types) != 0>,
+            class _Constraints = __match_construct<_Arg>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API constexpr variant(_Arg&& __arg) noexcept(_Constraints::__nothrow_constructible)
+      : __impl_(in_place_index<_Constraints::_Ip>, ::cuda::std::forward<_Arg>(__arg))
+  {}
+
+  template <size_t _Ip, class... _Args>
+  using __variadic_construct =
+    conditional_t<(_Ip < sizeof...(_Types)),
+                  typename __constraints::template __variadic_construct<_Ip, _Args...>,
+                  __variant_detail::__invalid_variant_constraints>;
+
+  template <size_t _Ip,
+            class... _Args,
+            class _Constraints = __variadic_construct<_Ip, _Args...>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API explicit constexpr variant(in_place_index_t<_Ip>,
+                                       _Args&&... __args) noexcept(_Constraints::__nothrow_constructible)
+      : __impl_(in_place_index<_Ip>, ::cuda::std::forward<_Args>(__args)...)
+  {}
+
+  template <size_t _Ip, class _Up, class... _Args>
+  using __variadic_ilist_construct =
+    conditional_t<(_Ip < sizeof...(_Types)),
+                  typename __constraints::template __variadic_ilist_construct<_Ip, _Up, _Args...>,
+                  __variant_detail::__invalid_variant_constraints>;
+
+  template <size_t _Ip,
+            class _Up,
+            class... _Args,
+            class _Constraints = __variadic_ilist_construct<_Ip, _Up, _Args...>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API explicit constexpr variant(in_place_index_t<_Ip>, initializer_list<_Up> __il, _Args&&... __args) noexcept(
+    _Constraints::__nothrow_constructible)
+      : __impl_(in_place_index<_Ip>, __il, ::cuda::std::forward<_Args>(__args)...)
+  {}
+
+  template <class _Tp,
+            class... _Args,
+            size_t _Ip         = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+            class _Constraints = __variadic_construct<_Ip, _Args...>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API explicit constexpr variant(in_place_type_t<_Tp>,
+                                       _Args&&... __args) noexcept(_Constraints::__nothrow_constructible)
+      : __impl_(in_place_index<_Ip>, ::cuda::std::forward<_Args>(__args)...)
+  {}
+
+  template <class _Tp,
+            class _Up,
+            class... _Args,
+            size_t _Ip         = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+            class _Constraints = __variadic_ilist_construct<_Ip, _Up, _Args...>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API explicit constexpr variant(in_place_type_t<_Tp>, initializer_list<_Up> __il, _Args&&... __args) noexcept(
+    _Constraints::__nothrow_constructible)
+      : __impl_(in_place_index<_Ip>, __il, ::cuda::std::forward<_Args>(__args)...)
+  {}
+
+  _CCCL_HIDE_FROM_ABI ~variant() = default;
+
+  _CCCL_HIDE_FROM_ABI constexpr variant& operator=(const variant&) = default;
+  _CCCL_HIDE_FROM_ABI constexpr variant& operator=(variant&&)      = default;
+
+  template <class _Arg>
+  using __match_assign =
+    conditional_t<!is_same_v<remove_cvref_t<_Arg>, variant>,
+                  typename __constraints::template __match_assign<_Arg>,
+                  __variant_detail::__invalid_variant_constraints>;
+
+  template <class _Arg, class _Constraints = __match_assign<_Arg>, class = enable_if_t<_Constraints::__assignable>>
+  _CCCL_API inline variant& operator=(_Arg&& __arg) noexcept(_Constraints::__nothrow_assignable)
+  {
+    __impl_.template __assign<_Constraints::_Ip>(::cuda::std::forward<_Arg>(__arg));
+    return *this;
+  }
+
+  template <size_t _Ip,
+            class... _Args,
+            class _Constraints = __variadic_construct<_Ip, _Args...>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API inline typename _Constraints::_Tp& emplace(_Args&&... __args)
+  {
+    return __impl_.template __emplace<_Ip>(::cuda::std::forward<_Args>(__args)...);
+  }
+
+  template <size_t _Ip,
+            class _Up,
+            class... _Args,
+            class _Constraints = __variadic_ilist_construct<_Ip, _Up, _Args...>,
+            class              = enable_if_t<_Constraints::__constructible>>
+  _CCCL_API inline typename _Constraints::_Tp& emplace(initializer_list<_Up> __il, _Args&&... __args)
+  {
+    return __impl_.template __emplace<_Ip>(__il, ::cuda::std::forward<_Args>(__args)...);
+  }
+
+  template <class _Tp,
+            class... _Args,
+            size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+            enable_if_t<is_constructible_v<_Tp, _Args...>, int> = 0>
+  _CCCL_API inline _Tp& emplace(_Args&&... __args)
+  {
+    return __impl_.template __emplace<_Ip>(::cuda::std::forward<_Args>(__args)...);
+  }
+
+  template <class _Tp,
+            class _Up,
+            class... _Args,
+            size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value,
+            enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>, int> = 0>
+  _CCCL_API inline _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args)
+  {
+    return __impl_.template __emplace<_Ip>(__il, ::cuda::std::forward<_Args>(__args)...);
+  }
+
+  [[nodiscard]] _CCCL_API constexpr bool valueless_by_exception() const noexcept
+  {
+    return __impl_.valueless_by_exception();
+  }
+
+  [[nodiscard]] _CCCL_API constexpr size_t index() const noexcept
+  {
+    return __impl_.index();
+  }
+
+  // Needs to be dependent to guard against incomplete types
+  template <bool _Dummy>
+  using __swap_constraint =
+    __dependent_type<typename __variant_detail::__variant_constraints<_Types...>::template __swappable<_Dummy>, _Dummy>;
+
+  template <bool _Dummy       = true,
+            class _Constraint = __swap_constraint<_Dummy>,
+            class             = enable_if_t<_Constraint::__is_swappable_v>>
+  _CCCL_API inline void swap(variant& __that) noexcept(_Constraint::__is_nothrow_swappable_v)
+  {
+    __impl_.__swap(__that.__impl_);
+  }
+
+  _CCCL_API static constexpr size_t __size() noexcept
+  {
+    return sizeof...(_Types);
+  }
+
+private:
+  __variant_detail::__impl<_Types...> __impl_;
+
+  friend struct __variant_detail::__access::__variant;
+  friend struct __variant_detail::__visitation::__variant;
+};
+
+template <class... _Types>
+_CCCL_API inline auto swap(variant<_Types...>& __lhs, variant<_Types...>& __rhs) noexcept(noexcept(__lhs.swap(__rhs)))
+  -> decltype(__lhs.swap(__rhs))
+{
+  return __lhs.swap(__rhs);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant_access.h

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_ACCESS_H
+#define _CUDA_STD___VARIANT_VARIANT_ACCESS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/in_place.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+namespace __variant_detail::__access
+{
+struct __union
+{
+  template <class _Vp>
+  [[nodiscard]] _CCCL_API static constexpr auto&& __get_alt(_Vp&& __v, in_place_index_t<0>) noexcept
+  {
+    return ::cuda::std::forward<_Vp>(__v).__head_;
+  }
+
+  template <class _Vp, size_t _Ip>
+  [[nodiscard]] _CCCL_API static constexpr auto&& __get_alt(_Vp&& __v, in_place_index_t<_Ip>) noexcept
+  {
+    return __get_alt(::cuda::std::forward<_Vp>(__v).__tail_, in_place_index<_Ip - 1>);
+  }
+};
+
+struct __base
+{
+  template <size_t _Ip, class _Vp>
+  [[nodiscard]] _CCCL_API static constexpr auto&& __get_alt(_Vp&& __v) noexcept
+  {
+    return __union::__get_alt(::cuda::std::forward<_Vp>(__v).__data_, in_place_index<_Ip>);
+  }
+};
+
+struct __variant
+{
+  template <size_t _Ip, class _Vp>
+  [[nodiscard]] _CCCL_API static constexpr auto&& __get_alt(_Vp&& __v) noexcept
+  {
+    return __base::__get_alt<_Ip>(::cuda::std::forward<_Vp>(__v).__impl_);
+  }
+};
+} // namespace __variant_detail::__access
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_ACCESS_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant_base.h

@@ -0,0 +1,683 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_BASE_H
+#define _CUDA_STD___VARIANT_VARIANT_BASE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__memory/addressof.h>
+#include <cuda/std/__memory/construct_at.h>
+#include <cuda/std/__type_traits/fold.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_move_assignable.h>
+#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/in_place.h>
+#include <cuda/std/__utility/move.h>
+#include <cuda/std/__utility/swap.h>
+#include <cuda/std/__variant/variant_access.h>
+#include <cuda/std/__variant/variant_traits.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+namespace __variant_detail
+{
+// NOTE: we need to define all special member functions, because NVCC has issues with host only types
+template <size_t _Index, class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT __alt
+{
+  using __value_type = _Tp;
+
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class... _Args>
+  _CCCL_API explicit constexpr __alt(in_place_t, _Args&&... __args)
+      : __value(::cuda::std::forward<_Args>(__args)...)
+  {}
+  _CCCL_EXEC_CHECK_DISABLE
+  constexpr __alt(const __alt&) = default;
+  _CCCL_EXEC_CHECK_DISABLE
+  constexpr __alt(__alt&&) = default;
+  _CCCL_EXEC_CHECK_DISABLE
+  constexpr __alt& operator=(const __alt&) = default;
+  _CCCL_EXEC_CHECK_DISABLE
+  constexpr __alt& operator=(__alt&&) = default;
+
+  _CCCL_EXEC_CHECK_DISABLE
+  ~__alt() = default;
+
+  __value_type __value;
+};
+
+template <_Trait _DestructibleTrait, size_t _Index, class... _Types>
+union _CCCL_TYPE_VISIBILITY_DEFAULT __union;
+
+template <_Trait _DestructibleTrait, size_t _Index>
+union _CCCL_TYPE_VISIBILITY_DEFAULT __union<_DestructibleTrait, _Index>
+{};
+
+#define _LIBCUDACXX_VARIANT_UNION_BODY(destructible_trait)                       \
+                                                                                 \
+private:                                                                         \
+  char __dummy;                                                                  \
+  __alt<_Index, _Tp> __head_;                                                    \
+  __union<destructible_trait, _Index + 1, _Types...> __tail_;                    \
+                                                                                 \
+  friend struct __access::__union;                                               \
+                                                                                 \
+public:                                                                          \
+  _CCCL_API explicit constexpr __union(__valueless_t) noexcept                   \
+      : __dummy{}                                                                \
+  {}                                                                             \
+                                                                                 \
+  template <class... _Args>                                                      \
+  _CCCL_API explicit constexpr __union(in_place_index_t<0>, _Args&&... __args)   \
+      : __head_(in_place_t{}, ::cuda::std::forward<_Args>(__args)...)            \
+  {}                                                                             \
+                                                                                 \
+  template <size_t _Ip, class... _Args>                                          \
+  _CCCL_API explicit constexpr __union(in_place_index_t<_Ip>, _Args&&... __args) \
+      : __tail_(in_place_index<_Ip - 1>, ::cuda::std::forward<_Args>(__args)...) \
+  {}                                                                             \
+                                                                                 \
+  _CCCL_HIDE_FROM_ABI __union(const __union&)            = default;              \
+  _CCCL_HIDE_FROM_ABI __union(__union&&)                 = default;              \
+  _CCCL_HIDE_FROM_ABI __union& operator=(const __union&) = default;              \
+  _CCCL_HIDE_FROM_ABI __union& operator=(__union&&)      = default;
+
+template <size_t _Index, class _Tp, class... _Types>
+union _CCCL_TYPE_VISIBILITY_DEFAULT __union<_Trait::_TriviallyAvailable, _Index, _Tp, _Types...>
+{
+  _LIBCUDACXX_VARIANT_UNION_BODY(_Trait::_TriviallyAvailable)
+  _CCCL_HIDE_FROM_ABI ~__union() = default;
+};
+
+template <size_t _Index, class _Tp, class... _Types>
+union _CCCL_TYPE_VISIBILITY_DEFAULT __union<_Trait::_Available, _Index, _Tp, _Types...>
+{
+  _LIBCUDACXX_VARIANT_UNION_BODY(_Trait::_Available)
+  _CCCL_API ~__union() {}
+};
+
+template <size_t _Index, class _Tp, class... _Types>
+union _CCCL_TYPE_VISIBILITY_DEFAULT __union<_Trait::_Unavailable, _Index, _Tp, _Types...>
+{
+  _LIBCUDACXX_VARIANT_UNION_BODY(_Trait::_Unavailable)
+  _CCCL_API ~__union() = delete;
+};
+
+#undef _LIBCUDACXX_VARIANT_UNION_BODY
+
+template <_Trait _DestructibleTrait, class... _Types>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __base
+{
+public:
+  using __index_t = __variant_index_t<sizeof...(_Types)>;
+
+  _CCCL_API explicit constexpr __base(__valueless_t __tag) noexcept
+      : __data_(__tag)
+      , __index_(__variant_npos<__index_t>)
+  {}
+
+  template <size_t _Ip, class... _Args>
+  _CCCL_API explicit constexpr __base(in_place_index_t<_Ip>, _Args&&... __args)
+      : __data_(in_place_index<_Ip>, ::cuda::std::forward<_Args>(__args)...)
+      , __index_(_Ip)
+  {}
+
+  [[nodiscard]] _CCCL_API constexpr bool valueless_by_exception() const noexcept
+  {
+    return index() == variant_npos;
+  }
+
+  [[nodiscard]] _CCCL_API constexpr size_t index() const noexcept
+  {
+    return __index_ == __variant_npos<__index_t> ? variant_npos : __index_;
+  }
+
+protected:
+  [[nodiscard]] _CCCL_API constexpr auto&& __as_base() &
+  {
+    return *this;
+  }
+
+  [[nodiscard]] _CCCL_API constexpr auto&& __as_base() &&
+  {
+    return ::cuda::std::move(*this);
+  }
+
+  [[nodiscard]] _CCCL_API constexpr auto&& __as_base() const&
+  {
+    return *this;
+  }
+
+  _CCCL_API constexpr auto&& __as_base() const&&
+  {
+    return ::cuda::std::move(*this);
+  }
+
+  [[nodiscard]] _CCCL_API static constexpr size_t __size() noexcept
+  {
+    return sizeof...(_Types);
+  }
+
+  __union<_DestructibleTrait, 0, _Types...> __data_;
+  __index_t __index_;
+
+  friend struct __access::__base;
+};
+
+template <class _Traits, _Trait = _Traits::__destructible_trait>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __dtor;
+
+#define _LIBCUDACXX_VARIANT_DESTRUCTOR_BODY(destructible_trait)   \
+  using __base_type = __base<destructible_trait, _Types...>;      \
+  using __index_t   = typename __base_type::__index_t;            \
+                                                                  \
+public:                                                           \
+  using __base_type::__base_type;                                 \
+  using __base_type::operator=;                                   \
+                                                                  \
+  _CCCL_HIDE_FROM_ABI __dtor(const __dtor&)            = default; \
+  _CCCL_HIDE_FROM_ABI __dtor(__dtor&&)                 = default; \
+  _CCCL_HIDE_FROM_ABI __dtor& operator=(const __dtor&) = default; \
+  _CCCL_HIDE_FROM_ABI __dtor& operator=(__dtor&&)      = default;
+
+template <class... _Types>
+class _CCCL_TYPE_VISIBILITY_DEFAULT
+__dtor<__traits<_Types...>, _Trait::_TriviallyAvailable> : public __base<_Trait::_TriviallyAvailable, _Types...>
+{
+  _LIBCUDACXX_VARIANT_DESTRUCTOR_BODY(_Trait::_TriviallyAvailable)
+  _CCCL_HIDE_FROM_ABI ~__dtor() = default;
+
+protected:
+  _CCCL_API void __destroy() noexcept
+  {
+    this->__index_ = __variant_npos<__index_t>;
+  }
+};
+
+template <class... _Types>
+class _CCCL_TYPE_VISIBILITY_DEFAULT
+__dtor<__traits<_Types...>, _Trait::_Available> : public __base<_Trait::_Available, _Types...>
+{
+  struct __visitor
+  {
+    _CCCL_EXEC_CHECK_DISABLE
+    template <class _Alt>
+    _CCCL_API void operator()(_Alt& __alt) const noexcept
+    {
+      using __alt_type = remove_cvref_t<decltype(__alt)>;
+      __alt.~__alt_type();
+    }
+  };
+
+  _LIBCUDACXX_VARIANT_DESTRUCTOR_BODY(_Trait::_Available)
+
+  _CCCL_API ~__dtor() noexcept
+  {
+    __destroy();
+  }
+
+protected:
+  _CCCL_API void __destroy() noexcept
+  {
+    if (!this->valueless_by_exception())
+    {
+      constexpr size_t __np = remove_cvref_t<__dtor>::__size();
+      __destroy(integral_constant<size_t, __np - 1>{}, this->__index_);
+    }
+    this->__index_ = __variant_npos<__index_t>;
+  }
+
+private:
+  template <size_t _CurrentIndex>
+  _CCCL_API void __destroy(integral_constant<size_t, _CurrentIndex>, const size_t __index_) noexcept
+  {
+    if (__index_ == _CurrentIndex)
+    {
+      using __alt_type = remove_cvref_t<decltype(__access::__base::__get_alt<_CurrentIndex>(this->__as_base()))>;
+      __access::__base::__get_alt<_CurrentIndex>(this->__as_base()).~__alt_type();
+      return;
+    }
+    __destroy(integral_constant<size_t, _CurrentIndex - 1>{}, __index_);
+  }
+  _CCCL_API void __destroy(integral_constant<size_t, 0>, const size_t __index_) noexcept
+  {
+    if (__index_ == 0)
+    {
+      using __alt_type = remove_cvref_t<decltype(__access::__base::__get_alt<0>(this->__as_base()))>;
+      __access::__base::__get_alt<0>(this->__as_base()).~__alt_type();
+      return;
+    }
+    // We already checked that every variant has a value, so we should never reach this line
+    _CCCL_UNREACHABLE();
+  }
+};
+
+template <class... _Types>
+class _CCCL_TYPE_VISIBILITY_DEFAULT
+__dtor<__traits<_Types...>, _Trait::_Unavailable> : public __base<_Trait::_Unavailable, _Types...>
+{
+  _LIBCUDACXX_VARIANT_DESTRUCTOR_BODY(_Trait::_Unavailable)
+  _CCCL_API ~__dtor() = delete;
+
+protected:
+  _CCCL_API void __destroy() noexcept = delete;
+};
+
+#undef _LIBCUDACXX_VARIANT_DESTRUCTOR_BODY
+
+template <class _Traits>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __ctor : public __dtor<_Traits>
+{
+  using __base_type = __dtor<_Traits>;
+
+  template <size_t _CurrentIndex, class _Rhs>
+  _CCCL_API static constexpr void
+  __generic_construct_impl(integral_constant<size_t, _CurrentIndex>, const size_t __index_, __ctor& __lhs, _Rhs&& __rhs)
+  {
+    if (__index_ == _CurrentIndex)
+    {
+      ::cuda::std::__construct_at(
+        ::cuda::std::addressof(__access::__base::__get_alt<_CurrentIndex>(__lhs.__as_base())),
+        in_place_t{},
+        __access::__base::__get_alt<_CurrentIndex>(::cuda::std::forward<_Rhs>(__rhs).__as_base()).__value);
+      return;
+    }
+    __generic_construct_impl(
+      integral_constant<size_t, _CurrentIndex - 1>{}, __index_, __lhs, ::cuda::std::forward<_Rhs>(__rhs));
+  }
+
+  template <class _Rhs>
+  _CCCL_API static constexpr void
+  __generic_construct_impl(integral_constant<size_t, 0>, const size_t __index_, __ctor& __lhs, _Rhs&& __rhs)
+  {
+    if (__index_ == 0)
+    {
+      ::cuda::std::__construct_at(
+        ::cuda::std::addressof(__access::__base::__get_alt<0>(__lhs.__as_base())),
+        in_place_t{},
+        __access::__base::__get_alt<0>(::cuda::std::forward<_Rhs>(__rhs).__as_base()).__value);
+      return;
+    }
+    // We already checked that every variant has a value, so we should never reach this line
+    _CCCL_UNREACHABLE();
+  }
+
+public:
+  using __base_type::__base_type;
+  using __base_type::operator=;
+
+protected:
+  template <size_t _Ip, class _Tp, class... _Args>
+  _CCCL_API static _Tp& __construct_alt(__alt<_Ip, _Tp>& __a, _Args&&... __args)
+  {
+    ::cuda::std::__construct_at(::cuda::std::addressof(__a), in_place_t{}, ::cuda::std::forward<_Args>(__args)...);
+    return __a.__value;
+  }
+
+  template <class _Rhs>
+  _CCCL_API static void __generic_construct(__ctor& __lhs, _Rhs&& __rhs)
+  {
+    __lhs.__destroy();
+    if (!__rhs.valueless_by_exception())
+    {
+      constexpr size_t __np = remove_cvref_t<__ctor>::__size();
+      __generic_construct_impl(
+        integral_constant<size_t, __np - 1>{}, __rhs.index(), __lhs, ::cuda::std::forward<_Rhs>(__rhs));
+      __lhs.__index_ = static_cast<decltype(__lhs.__index_)>(__rhs.index());
+    }
+  }
+};
+
+template <class _Traits, _Trait = _Traits::__move_constructible_trait>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __move_constructor;
+
+#define _LIBCUDACXX_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, move_constructor)                 \
+  template <class... _Types>                                                                             \
+  class _CCCL_TYPE_VISIBILITY_DEFAULT                                                                    \
+  __move_constructor<__traits<_Types...>, move_constructible_trait> : public __ctor<__traits<_Types...>> \
+  {                                                                                                      \
+    using __base_type = __ctor<__traits<_Types...>>;                                                     \
+                                                                                                         \
+  public:                                                                                                \
+    using __base_type::__base_type;                                                                      \
+    using __base_type::operator=;                                                                        \
+                                                                                                         \
+    _CCCL_HIDE_FROM_ABI __move_constructor(const __move_constructor&)            = default;              \
+    _CCCL_HIDE_FROM_ABI ~__move_constructor()                                    = default;              \
+    _CCCL_HIDE_FROM_ABI __move_constructor& operator=(const __move_constructor&) = default;              \
+    _CCCL_HIDE_FROM_ABI __move_constructor& operator=(__move_constructor&&)      = default;              \
+    move_constructor                                                                                     \
+  }
+
+_LIBCUDACXX_VARIANT_MOVE_CONSTRUCTOR(_Trait::_TriviallyAvailable,
+                                     _CCCL_HIDE_FROM_ABI __move_constructor(__move_constructor&& __that) = default;);
+
+_LIBCUDACXX_VARIANT_MOVE_CONSTRUCTOR(
+  _Trait::_Available,
+  _CCCL_API __move_constructor(__move_constructor&& __that) noexcept(
+    __fold_and_v<is_nothrow_move_constructible_v<_Types>...>) : __move_constructor(__valueless_t{}) {
+    this->__generic_construct(*this, ::cuda::std::move(__that));
+  });
+
+_LIBCUDACXX_VARIANT_MOVE_CONSTRUCTOR(_Trait::_Unavailable, __move_constructor(__move_constructor&&) = delete;);
+
+#undef _LIBCUDACXX_VARIANT_MOVE_CONSTRUCTOR
+
+template <class _Traits, _Trait = _Traits::__copy_constructible_trait>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __copy_constructor;
+
+#define _LIBCUDACXX_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, copy_constructor)                             \
+  template <class... _Types>                                                                                         \
+  class _CCCL_TYPE_VISIBILITY_DEFAULT                                                                                \
+  __copy_constructor<__traits<_Types...>, copy_constructible_trait> : public __move_constructor<__traits<_Types...>> \
+  {                                                                                                                  \
+    using __base_type = __move_constructor<__traits<_Types...>>;                                                     \
+                                                                                                                     \
+  public:                                                                                                            \
+    using __base_type::__base_type;                                                                                  \
+    using __base_type::operator=;                                                                                    \
+                                                                                                                     \
+    _CCCL_HIDE_FROM_ABI __copy_constructor(__copy_constructor&&)                 = default;                          \
+    _CCCL_HIDE_FROM_ABI ~__copy_constructor()                                    = default;                          \
+    _CCCL_HIDE_FROM_ABI __copy_constructor& operator=(const __copy_constructor&) = default;                          \
+    _CCCL_HIDE_FROM_ABI __copy_constructor& operator=(__copy_constructor&&)      = default;                          \
+    copy_constructor                                                                                                 \
+  }
+
+_LIBCUDACXX_VARIANT_COPY_CONSTRUCTOR(
+  _Trait::_TriviallyAvailable, _CCCL_HIDE_FROM_ABI __copy_constructor(const __copy_constructor& __that) = default;);
+
+_LIBCUDACXX_VARIANT_COPY_CONSTRUCTOR(
+  _Trait::_Available,
+  _CCCL_API __copy_constructor(const __copy_constructor& __that) : __copy_constructor(__valueless_t{}) {
+    this->__generic_construct(*this, __that);
+  });
+
+_LIBCUDACXX_VARIANT_COPY_CONSTRUCTOR(_Trait::_Unavailable, __copy_constructor(const __copy_constructor&) = delete;);
+
+#undef _LIBCUDACXX_VARIANT_COPY_CONSTRUCTOR
+
+template <class _Traits>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __assignment : public __copy_constructor<_Traits>
+{
+  using __base_type = __copy_constructor<_Traits>;
+
+  template <size_t _CurrentIndex, class _Other>
+  _CCCL_API constexpr void
+  __generic_assign(integral_constant<size_t, _CurrentIndex>, const size_t __index_, _Other&& __rhs)
+  {
+    if (__index_ == _CurrentIndex)
+    {
+      this->__assign_alt(
+        __access::__base::__get_alt<_CurrentIndex>(this->__as_base()),
+        __access::__base::__get_alt<_CurrentIndex>(::cuda::std::forward<_Other>(__rhs).__as_base()).__value);
+      return;
+    }
+    this->__generic_assign(
+      integral_constant<size_t, _CurrentIndex - 1>{}, __index_, ::cuda::std::forward<_Other>(__rhs));
+  }
+
+  template <class _Other>
+  _CCCL_API constexpr void __generic_assign(integral_constant<size_t, 0>, const size_t __index_, _Other&& __rhs)
+  {
+    if (__index_ == 0)
+    {
+      this->__assign_alt(__access::__base::__get_alt<0>(this->__as_base()),
+                         __access::__base::__get_alt<0>(::cuda::std::forward<_Other>(__rhs).__as_base()).__value);
+      return;
+    }
+    // We already checked that every variant has a value, so we should never reach this line
+    _CCCL_UNREACHABLE();
+  }
+
+public:
+  using __base_type::__base_type;
+  using __base_type::operator=;
+
+  template <size_t _Ip, class... _Args>
+  _CCCL_API auto& __emplace(_Args&&... __args)
+  {
+    this->__destroy();
+    auto& __res =
+      this->__construct_alt(__access::__base::__get_alt<_Ip>(*this), ::cuda::std::forward<_Args>(__args)...);
+    this->__index_ = _Ip;
+    return __res;
+  }
+
+protected:
+  _CCCL_EXEC_CHECK_DISABLE
+  template <size_t _Ip,
+            class _Tp,
+            class _Arg,
+            enable_if_t<is_nothrow_constructible_v<_Tp, _Arg> || !is_nothrow_move_constructible_v<_Tp>, int> = 0>
+  _CCCL_API void __assign_alt(__alt<_Ip, _Tp>& __a, _Arg&& __arg)
+  {
+    if (this->index() == _Ip)
+    {
+      __a.__value = ::cuda::std::forward<_Arg>(__arg);
+    }
+    else
+    {
+      this->__emplace<_Ip>(::cuda::std::forward<_Arg>(__arg));
+    }
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  template <size_t _Ip,
+            class _Tp,
+            class _Arg,
+            enable_if_t<!is_nothrow_constructible_v<_Tp, _Arg> && is_nothrow_move_constructible_v<_Tp>, int> = 0>
+  _CCCL_API void __assign_alt(__alt<_Ip, _Tp>& __a, _Arg&& __arg)
+  {
+    if (this->index() == _Ip)
+    {
+      __a.__value = ::cuda::std::forward<_Arg>(__arg);
+    }
+    else
+    {
+      this->__emplace<_Ip>(_Tp(::cuda::std::forward<_Arg>(__arg)));
+    }
+  }
+
+  template <class _That>
+  _CCCL_API void __generic_assign(_That&& __that)
+  {
+    if (this->valueless_by_exception() && __that.valueless_by_exception())
+    {
+      // do nothing.
+    }
+    else if (__that.valueless_by_exception())
+    {
+      this->__destroy();
+    }
+    else
+    {
+      constexpr size_t __np = remove_cvref_t<__assignment>::__size();
+      this->__generic_assign(integral_constant<size_t, __np - 1>{}, __that.index(), ::cuda::std::forward<_That>(__that));
+    }
+  }
+};
+
+template <class _Traits, _Trait = _Traits::__move_assignable_trait>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __move_assignment;
+
+#define _LIBCUDACXX_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, move_assignment)                        \
+  template <class... _Types>                                                                               \
+  class _CCCL_TYPE_VISIBILITY_DEFAULT                                                                      \
+  __move_assignment<__traits<_Types...>, move_assignable_trait> : public __assignment<__traits<_Types...>> \
+  {                                                                                                        \
+    using __base_type = __assignment<__traits<_Types...>>;                                                 \
+                                                                                                           \
+  public:                                                                                                  \
+    using __base_type::__base_type;                                                                        \
+    using __base_type::operator=;                                                                          \
+                                                                                                           \
+    _CCCL_HIDE_FROM_ABI __move_assignment(const __move_assignment&)            = default;                  \
+    _CCCL_HIDE_FROM_ABI __move_assignment(__move_assignment&&)                 = default;                  \
+    _CCCL_HIDE_FROM_ABI ~__move_assignment()                                   = default;                  \
+    _CCCL_HIDE_FROM_ABI __move_assignment& operator=(const __move_assignment&) = default;                  \
+    move_assignment                                                                                        \
+  }
+
+_LIBCUDACXX_VARIANT_MOVE_ASSIGNMENT(
+  _Trait::_TriviallyAvailable, _CCCL_HIDE_FROM_ABI __move_assignment& operator=(__move_assignment&& __that) = default;);
+
+_LIBCUDACXX_VARIANT_MOVE_ASSIGNMENT(
+  _Trait::_Available,
+  _CCCL_API __move_assignment&
+  operator=(__move_assignment&& __that) noexcept(
+    __fold_and_v<(is_nothrow_move_constructible_v<_Types> && is_nothrow_move_assignable_v<_Types>) ...>) {
+    this->__generic_assign(::cuda::std::move(__that));
+    return *this;
+  });
+
+_LIBCUDACXX_VARIANT_MOVE_ASSIGNMENT(_Trait::_Unavailable, __move_assignment& operator=(__move_assignment&&) = delete;);
+
+#undef _LIBCUDACXX_VARIANT_MOVE_ASSIGNMENT
+
+template <class _Traits, _Trait = _Traits::__copy_assignable_trait>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __copy_assignment;
+
+#define _LIBCUDACXX_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, copy_assignment)                             \
+  template <class... _Types>                                                                                    \
+  class _CCCL_TYPE_VISIBILITY_DEFAULT                                                                           \
+  __copy_assignment<__traits<_Types...>, copy_assignable_trait> : public __move_assignment<__traits<_Types...>> \
+  {                                                                                                             \
+    using __base_type = __move_assignment<__traits<_Types...>>;                                                 \
+                                                                                                                \
+  public:                                                                                                       \
+    using __base_type::__base_type;                                                                             \
+    using __base_type::operator=;                                                                               \
+                                                                                                                \
+    _CCCL_HIDE_FROM_ABI __copy_assignment(const __copy_assignment&)       = default;                            \
+    _CCCL_HIDE_FROM_ABI __copy_assignment(__copy_assignment&&)            = default;                            \
+    _CCCL_HIDE_FROM_ABI ~__copy_assignment()                              = default;                            \
+    _CCCL_HIDE_FROM_ABI __copy_assignment& operator=(__copy_assignment&&) = default;                            \
+    copy_assignment                                                                                             \
+  }
+
+_LIBCUDACXX_VARIANT_COPY_ASSIGNMENT(
+  _Trait::_TriviallyAvailable,
+  _CCCL_HIDE_FROM_ABI __copy_assignment& operator=(const __copy_assignment& __that) = default;);
+
+_LIBCUDACXX_VARIANT_COPY_ASSIGNMENT(
+  _Trait::_Available, _CCCL_API __copy_assignment& operator=(const __copy_assignment& __that) {
+    this->__generic_assign(__that);
+    return *this;
+  });
+
+_LIBCUDACXX_VARIANT_COPY_ASSIGNMENT(_Trait::_Unavailable,
+                                    __copy_assignment& operator=(const __copy_assignment&) = delete;);
+
+#undef _LIBCUDACXX_VARIANT_COPY_ASSIGNMENT
+
+template <class... _Types>
+class _CCCL_TYPE_VISIBILITY_DEFAULT __impl : public __copy_assignment<__traits<_Types...>>
+{
+  using __base_type = __copy_assignment<__traits<_Types...>>;
+
+  template <size_t _CurrentIndex>
+  _CCCL_API static constexpr void
+  __swap_value(integral_constant<size_t, _CurrentIndex>, const size_t __index_, __impl& __lhs, __impl& __rhs)
+  {
+    if (__index_ == _CurrentIndex)
+    {
+      using ::cuda::std::swap;
+      swap(__access::__base::__get_alt<_CurrentIndex>(__lhs.__as_base()).__value,
+           __access::__base::__get_alt<_CurrentIndex>(__rhs.__as_base()).__value);
+      return;
+    }
+    __swap_value(integral_constant<size_t, _CurrentIndex - 1>{}, __index_, __lhs, __rhs);
+  }
+
+  _CCCL_API static constexpr void
+  __swap_value(integral_constant<size_t, 0>, const size_t __index_, __impl& __lhs, __impl& __rhs)
+  {
+    if (__index_ == 0)
+    {
+      using ::cuda::std::swap;
+      swap(__access::__base::__get_alt<0>(__lhs.__as_base()).__value,
+           __access::__base::__get_alt<0>(__rhs.__as_base()).__value);
+      return;
+    }
+    // We already checked that every variant has a value, so we should never reach this line
+    _CCCL_UNREACHABLE();
+  }
+
+public:
+  using __base_type::__base_type; // get in_place_index_t constructor & friends
+  _CCCL_HIDE_FROM_ABI __impl(__impl const&)            = default;
+  _CCCL_HIDE_FROM_ABI __impl(__impl&&)                 = default;
+  _CCCL_HIDE_FROM_ABI __impl& operator=(__impl const&) = default;
+  _CCCL_HIDE_FROM_ABI __impl& operator=(__impl&&)      = default;
+
+  template <size_t _Ip, class _Arg>
+  _CCCL_API void __assign(_Arg&& __arg)
+  {
+    this->__assign_alt(__access::__base::__get_alt<_Ip>(*this), ::cuda::std::forward<_Arg>(__arg));
+  }
+
+  _CCCL_API void __swap(__impl& __that)
+  {
+    if (this->valueless_by_exception() && __that.valueless_by_exception())
+    {
+      // do nothing.
+    }
+    else if (this->index() == __that.index())
+    {
+      constexpr size_t __np = remove_cvref_t<__impl>::__size();
+      __swap_value(integral_constant<size_t, __np - 1>{}, this->index(), *this, __that);
+    }
+    else
+    {
+      __impl* __lhs = this;
+      __impl* __rhs = ::cuda::std::addressof(__that);
+      if (__lhs->__move_nothrow() && !__rhs->__move_nothrow())
+      {
+        ::cuda::std::swap(__lhs, __rhs);
+      }
+      else
+      {
+        __impl __tmp(::cuda::std::move(*__rhs));
+        this->__generic_construct(*__rhs, ::cuda::std::move(*__lhs));
+        this->__generic_construct(*__lhs, ::cuda::std::move(__tmp));
+      }
+    }
+  }
+
+private:
+  [[nodiscard]] _CCCL_API constexpr bool __move_nothrow() const
+  {
+    constexpr bool __results[] = {is_nothrow_move_constructible_v<_Types>...};
+    return this->valueless_by_exception() || __results[this->index()];
+  }
+};
+} // namespace __variant_detail
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_BASE_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant_constraints.h

@@ -0,0 +1,136 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_CONSTRAINTS_H
+#define _CUDA_STD___VARIANT_VARIANT_CONSTRAINTS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__tuple_dir/get.h>
+#include <cuda/std/__tuple_dir/sfinae_helpers.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_assignable.h>
+#include <cuda/std/__type_traits/is_constructible.h>
+#include <cuda/std/__type_traits/is_move_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_assignable.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
+#include <cuda/std/__type_traits/is_swappable.h>
+#include <cuda/std/__variant/variant_match.h>
+#include <cuda/std/initializer_list>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+namespace __find_detail
+{
+template <class _Tp, class... _Types>
+_CCCL_API constexpr size_t __find_index()
+{
+  constexpr bool __matches[] = {is_same_v<_Tp, _Types>...};
+  size_t __result            = __not_found;
+  for (size_t __i = 0; __i < sizeof...(_Types); ++__i)
+  {
+    if (__matches[__i])
+    {
+      if (__result != __not_found)
+      {
+        __result = __ambiguous;
+        break;
+      }
+      __result = __i;
+    }
+  }
+  return __result;
+}
+
+template <class _Tp, class... _Types>
+using __find_unambiguous_index_sfinae =
+  enable_if_t<::cuda::std::__find_detail::__find_index<_Tp, _Types...>()
+                != __not_found&& ::cuda::std::__find_detail::__find_index<_Tp, _Types...>() != __ambiguous,
+              integral_constant<size_t, ::cuda::std::__find_detail::__find_index<_Tp, _Types...>()>>;
+} // namespace __find_detail
+
+namespace __variant_detail
+{
+struct __invalid_variant_constraints
+{
+  static constexpr bool __constructible         = false;
+  static constexpr bool __nothrow_constructible = false;
+  static constexpr bool __assignable            = false;
+  static constexpr bool __nothrow_assignable    = false;
+};
+
+template <class... _Types>
+struct __variant_constraints
+{
+  template <class _Arg, class _Tp = __best_match_t<_Arg, _Types...>>
+  struct __match_construct
+  {
+    static constexpr size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value;
+
+    static constexpr bool __constructible         = is_constructible_v<_Tp, _Arg>;
+    static constexpr bool __nothrow_constructible = is_nothrow_constructible_v<_Tp, _Arg>;
+  };
+
+  template <size_t _Ip, class... _Args>
+  struct __variadic_construct
+  {
+    using _Tp = variant_alternative_t<_Ip, variant<_Types...>>;
+
+    static constexpr bool __constructible         = is_constructible_v<_Tp, _Args...>;
+    static constexpr bool __nothrow_constructible = is_nothrow_constructible_v<_Tp, _Args...>;
+  };
+
+  template <size_t _Ip, class _Up, class... _Args>
+  struct __variadic_ilist_construct
+  {
+    using _Tp = variant_alternative_t<_Ip, variant<_Types...>>;
+
+    static constexpr bool __constructible         = is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>;
+    static constexpr bool __nothrow_constructible = is_nothrow_constructible_v<_Tp, initializer_list<_Up>&, _Args...>;
+  };
+
+  template <class _Arg, class _Tp = __best_match_t<_Arg, _Types...>>
+  struct __match_assign
+  {
+    static constexpr size_t _Ip = __find_detail::__find_unambiguous_index_sfinae<_Tp, _Types...>::value;
+
+    static constexpr bool __assignable = is_assignable_v<_Tp&, _Arg> && is_constructible_v<_Tp, _Arg>;
+    static constexpr bool __nothrow_assignable =
+      is_nothrow_assignable_v<_Tp&, _Arg> && is_nothrow_constructible_v<_Tp, _Arg>;
+  };
+
+  template <bool>
+  struct __swappable
+  {
+    static constexpr bool __is_swappable_v = ((is_move_constructible_v<_Types> && is_swappable_v<_Types>) && ...);
+
+    static constexpr bool __is_nothrow_swappable_v =
+      ((is_nothrow_move_constructible_v<_Types> && is_nothrow_swappable_v<_Types>) && ...);
+  };
+};
+} // namespace __variant_detail
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_CONSTRAINTS_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant_match.h

@@ -0,0 +1,126 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_MATCH_H
+#define _CUDA_STD___VARIANT_VARIANT_MATCH_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/invoke.h>
+#include <cuda/std/__tuple_dir/tuple_indices.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_arithmetic.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__type_traits/type_identity.h>
+#include <cuda/std/__utility/integer_sequence.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+namespace __variant_detail
+{
+struct __no_narrowing_check
+{
+  template <class _Dest, class _Source>
+  using _Apply = type_identity<_Dest>;
+};
+
+struct __narrowing_check
+{
+  template <class _Dest, class _Source, bool = __cccl_internal::__is_non_narrowing_convertible<_Dest, _Source>::value>
+  struct __narrowing_check_impl
+  {};
+
+  template <class _Dest, class _Source>
+  struct __narrowing_check_impl<_Dest, _Source, true>
+  {
+    using type = type_identity<_Dest>;
+  };
+
+  template <class _Dest, class _Source>
+  using _Apply _CCCL_NODEBUG_ALIAS = typename __narrowing_check_impl<_Dest, _Source>::type;
+};
+
+template <class _Dest, class _Source>
+using __check_for_narrowing _CCCL_NODEBUG_ALIAS = typename conditional_t<
+#ifdef _LIBCUDACXX_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT
+  false &&
+#endif // _LIBCUDACXX_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT
+    is_arithmetic_v<_Dest>,
+  __narrowing_check,
+  __no_narrowing_check>::template _Apply<_Dest, _Source>;
+
+template <class _Tp, size_t _Idx>
+struct __overload
+{
+  template <class _Up>
+  _CCCL_API inline auto operator()(_Tp, _Up&&) const -> __check_for_narrowing<_Tp, _Up>;
+};
+
+template <class _Tp, size_t>
+struct __overload_bool
+{
+  template <class _Up, class _Ap = remove_cvref_t<_Up>>
+  _CCCL_API inline auto operator()(bool, _Up&&) const -> enable_if_t<is_same_v<_Ap, bool>, type_identity<_Tp>>;
+};
+
+template <size_t _Idx>
+struct __overload<bool, _Idx> : __overload_bool<bool, _Idx>
+{};
+template <size_t _Idx>
+struct __overload<bool const, _Idx> : __overload_bool<bool const, _Idx>
+{};
+template <size_t _Idx>
+struct __overload<bool volatile, _Idx> : __overload_bool<bool volatile, _Idx>
+{};
+template <size_t _Idx>
+struct __overload<bool const volatile, _Idx> : __overload_bool<bool const volatile, _Idx>
+{};
+
+template <class... _Bases>
+struct __all_overloads : _Bases...
+{
+  _CCCL_API inline void operator()() const;
+  using _Bases::operator()...;
+};
+
+template <class IdxSeq>
+struct __make_overloads_imp;
+
+template <size_t... _Idx>
+struct __make_overloads_imp<__tuple_indices<_Idx...>>
+{
+  template <class... _Types>
+  using _Apply _CCCL_NODEBUG_ALIAS = __all_overloads<__overload<_Types, _Idx>...>;
+};
+
+template <class... _Types>
+using _MakeOverloads _CCCL_NODEBUG_ALIAS =
+  typename __make_overloads_imp<__make_indices_imp<sizeof...(_Types), 0>>::template _Apply<_Types...>;
+
+template <class _Tp, class... _Types>
+using __best_match_t = typename invoke_result_t<_MakeOverloads<_Types...>, _Tp, _Tp>::type;
+} // namespace __variant_detail
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_MATCH_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant_traits.h

@@ -0,0 +1,184 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_TRAITS_H
+#define _CUDA_STD___VARIANT_VARIANT_TRAITS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__type_traits/add_const.h>
+#include <cuda/std/__type_traits/add_cv.h>
+#include <cuda/std/__type_traits/add_volatile.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_copy_assignable.h>
+#include <cuda/std/__type_traits/is_copy_constructible.h>
+#include <cuda/std/__type_traits/is_destructible.h>
+#include <cuda/std/__type_traits/is_move_assignable.h>
+#include <cuda/std/__type_traits/is_move_constructible.h>
+#include <cuda/std/__type_traits/is_trivially_copy_assignable.h>
+#include <cuda/std/__type_traits/is_trivially_copy_constructible.h>
+#include <cuda/std/__type_traits/is_trivially_destructible.h>
+#include <cuda/std/__type_traits/is_trivially_move_assignable.h>
+#include <cuda/std/__type_traits/is_trivially_move_constructible.h>
+#include <cuda/std/__type_traits/type_list.h>
+#include <cuda/std/initializer_list>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT variant_size<const _Tp> : variant_size<_Tp>
+{};
+
+template <class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT variant_size<volatile _Tp> : variant_size<_Tp>
+{};
+
+template <class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT variant_size<const volatile _Tp> : variant_size<_Tp>
+{};
+
+template <class... _Types>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT variant_size<variant<_Types...>> : integral_constant<size_t, sizeof...(_Types)>
+{};
+
+template <size_t _Ip, class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT variant_alternative<_Ip, const _Tp> : add_const<variant_alternative_t<_Ip, _Tp>>
+{};
+
+template <size_t _Ip, class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT
+variant_alternative<_Ip, volatile _Tp> : add_volatile<variant_alternative_t<_Ip, _Tp>>
+{};
+
+template <size_t _Ip, class _Tp>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT
+variant_alternative<_Ip, const volatile _Tp> : add_cv<variant_alternative_t<_Ip, _Tp>>
+{};
+
+template <size_t _Ip, class... _Types>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT variant_alternative<_Ip, variant<_Types...>>
+{
+  static_assert(_Ip < sizeof...(_Types), "Index out of bounds in cuda::std::variant_alternative<>");
+  using type = __type_index_c<_Ip, _Types...>;
+};
+
+[[nodiscard]] _CCCL_API _CCCL_CONSTEVAL int __choose_index_type(unsigned int __num_elem) noexcept
+{
+  constexpr unsigned char __small   = static_cast<unsigned char>(-1);
+  constexpr unsigned short __medium = static_cast<unsigned short>(-1);
+  if (__num_elem < static_cast<unsigned int>(__small))
+  {
+    return 0;
+  }
+  if (__num_elem < static_cast<unsigned int>(__medium))
+  {
+    return 1;
+  }
+  return 2;
+}
+
+template <size_t _NumAlts>
+using __variant_index_t =
+  conditional_t<::cuda::std::__choose_index_type(_NumAlts) == 0,
+                unsigned char,
+                conditional_t<::cuda::std::__choose_index_type(_NumAlts) == 1, unsigned short, unsigned int>>;
+namespace __variant_detail
+{
+struct __valueless_t
+{};
+
+enum class _Trait
+{
+  _TriviallyAvailable,
+  _Available,
+  _Unavailable
+};
+
+template <typename... _Types>
+struct __traits
+{
+  [[nodiscard]] _CCCL_API static _CCCL_CONSTEVAL _Trait __common_trait(initializer_list<_Trait> __traits) noexcept
+  {
+    _Trait __result = _Trait::_TriviallyAvailable;
+    for (_Trait __t : __traits)
+    {
+      if (static_cast<int>(__t) > static_cast<int>(__result))
+      {
+        __result = __t;
+      }
+    }
+    return __result;
+  }
+
+  template <class _Type>
+  static constexpr _Trait __copy_constructible_trait_ =
+    is_trivially_copy_constructible_v<_Type> ? _Trait::_TriviallyAvailable
+    : is_copy_constructible_v<_Type>
+      ? _Trait::_Available
+      : _Trait::_Unavailable;
+
+  static constexpr _Trait __copy_constructible_trait = __common_trait({__copy_constructible_trait_<_Types>...});
+
+  template <class _Type>
+  static constexpr _Trait __move_constructible_trait_ =
+    is_trivially_move_constructible_v<_Type> ? _Trait::_TriviallyAvailable
+    : is_move_constructible_v<_Type>
+      ? _Trait::_Available
+      : _Trait::_Unavailable;
+
+  static constexpr _Trait __move_constructible_trait = __common_trait({__move_constructible_trait_<_Types>...});
+
+  template <class _Type>
+  static constexpr _Trait __copy_assignable_trait_ =
+    is_trivially_copy_assignable_v<_Type> ? _Trait::_TriviallyAvailable
+    : is_copy_assignable_v<_Type>
+      ? _Trait::_Available
+      : _Trait::_Unavailable;
+
+  static constexpr _Trait __copy_assignable_trait =
+    __common_trait({__copy_constructible_trait, __copy_assignable_trait_<_Types>...});
+
+  template <class _Type>
+  static constexpr _Trait __move_assignable_trait_ =
+    is_trivially_move_assignable_v<_Type> ? _Trait::_TriviallyAvailable
+    : is_move_assignable_v<_Type>
+      ? _Trait::_Available
+      : _Trait::_Unavailable;
+
+  static constexpr _Trait __move_assignable_trait =
+    __common_trait({__move_constructible_trait, __move_assignable_trait_<_Types>...});
+
+  template <class _Type>
+  static constexpr _Trait __destructible_trait_ =
+    is_trivially_destructible_v<_Type> ? _Trait::_TriviallyAvailable
+    : is_destructible_v<_Type>
+      ? _Trait::_Available
+      : _Trait::_Unavailable;
+
+  static constexpr _Trait __destructible_trait = __common_trait({__destructible_trait_<_Types>...});
+};
+} // namespace __variant_detail
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_TRAITS_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/variant_visit.h

@@ -0,0 +1,225 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VARIANT_VISIT_H
+#define _CUDA_STD___VARIANT_VARIANT_VISIT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/invoke.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/integer_sequence.h>
+#include <cuda/std/__variant/variant_access.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+namespace __variant_detail::__visitation
+{
+struct __variant
+{
+  // We need to guard against the final invocation where we have processed all variants
+  template <size_t _Remaining, size_t _CurrentVariant, class... _Variants, enable_if_t<_Remaining == 0, int> = 0>
+  [[nodiscard]] _CCCL_API static constexpr size_t __get_runtime_index(const _Variants&...) noexcept
+  {
+    return 0;
+  }
+
+  template <size_t _Remaining,
+            size_t _CurrentVariant,
+            class _Variant,
+            class... _OtherVariants,
+            enable_if_t<(_Remaining != 0) && (_CurrentVariant == 0), int> = 0>
+  [[nodiscard]] _CCCL_API static constexpr size_t
+  __get_runtime_index(const _Variant& __v, const _OtherVariants&...) noexcept
+  {
+    return __v.__impl_.index();
+  }
+
+  template <size_t _Remaining,
+            size_t _CurrentVariant,
+            class _Variant,
+            class... _OtherVariants,
+            enable_if_t<(_Remaining != 0) && (_CurrentVariant != 0), int> = 0>
+  [[nodiscard]] _CCCL_API static constexpr size_t
+  __get_runtime_index(const _Variant&, const _OtherVariants&... __vs) noexcept
+  {
+    return __get_runtime_index<_Remaining, _CurrentVariant - 1>(__vs...);
+  }
+
+  // Terminal function call with all indexes determined
+  template <class _Visitor, class... _Vs, size_t... _ProcessedIndices>
+  [[nodiscard]] _CCCL_API static constexpr decltype(auto) __visit_impl(
+    index_sequence<_ProcessedIndices...>, integer_sequence<size_t>, const size_t, _Visitor&& __visitor, _Vs&&... __vs)
+  {
+    return ::cuda::std::__invoke(
+      ::cuda::std::forward<_Visitor>(__visitor),
+      __access::__base::__get_alt<_ProcessedIndices>(::cuda::std::forward<_Vs>(__vs).__impl_)...);
+  }
+
+  template <size_t _CurrentIndex,
+            class _Visitor,
+            class... _Vs,
+            size_t... _ProcessedIndices,
+            size_t... _UnprocessedIndices,
+            enable_if_t<_CurrentIndex != 0, int> = 0>
+  [[nodiscard]] _CCCL_API static constexpr decltype(auto) __visit_impl(
+    index_sequence<_ProcessedIndices...>,
+    index_sequence<_CurrentIndex, _UnprocessedIndices...>,
+    const size_t __current_index,
+    _Visitor&& __visitor,
+    _Vs&&... __vs)
+  {
+    // We found the right index, move to the next variant
+    if (__current_index == _CurrentIndex)
+    {
+      const size_t __next_index =
+        __get_runtime_index<sizeof...(_UnprocessedIndices), sizeof...(_ProcessedIndices) + 1>(__vs...);
+      return __visit_impl(
+        index_sequence<_ProcessedIndices..., _CurrentIndex>{},
+        index_sequence<_UnprocessedIndices...>{},
+        __next_index,
+        ::cuda::std::forward<_Visitor>(__visitor),
+        ::cuda::std::forward<_Vs>(__vs)...);
+    }
+
+    return __visit_impl(
+      index_sequence<_ProcessedIndices...>{},
+      index_sequence<_CurrentIndex - 1, _UnprocessedIndices...>{},
+      __current_index,
+      ::cuda::std::forward<_Visitor>(__visitor),
+      ::cuda::std::forward<_Vs>(__vs)...);
+  }
+
+  _CCCL_BEGIN_NV_DIAG_SUPPRESS(940) // Suppress no return at end of function
+  // This overload is needed to tell the compiler that the recursion is indeed limited
+  template <class _Visitor, class... _Vs, size_t... _ProcessedIndices, size_t... _UnprocessedIndices>
+  [[nodiscard]] _CCCL_API static constexpr decltype(auto) __visit_impl(
+    index_sequence<_ProcessedIndices...>,
+    index_sequence<0, _UnprocessedIndices...>,
+    const size_t __current_index,
+    _Visitor&& __visitor,
+    _Vs&&... __vs)
+  {
+    // We found the right index, move to the next variant
+    if (__current_index == 0)
+    {
+      const size_t __next_index =
+        __get_runtime_index<sizeof...(_UnprocessedIndices), sizeof...(_ProcessedIndices) + 1>(__vs...);
+      return __visit_impl(
+        index_sequence<_ProcessedIndices..., 0>{},
+        index_sequence<_UnprocessedIndices...>{},
+        __next_index,
+        ::cuda::std::forward<_Visitor>(__visitor),
+        ::cuda::std::forward<_Vs>(__vs)...);
+    }
+    _CCCL_UNREACHABLE();
+  }
+  _CCCL_END_NV_DIAG_SUPPRESS() // End suppression of no return at end of function
+
+  template <class _Visitor, class... _Vs>
+  [[nodiscard]] _CCCL_API static constexpr decltype(auto) __visit_value(_Visitor&& __visitor, _Vs&&... __vs)
+  {
+    // NOTE: We use a recursive implementation strategy here. That means we can omit the manual return type checks from
+    // the common function pointer implementation, as the compiler will abort if the return types do not match.
+    const size_t __first_index = __get_runtime_index<sizeof...(_Vs), 0>(__vs...);
+    return __visit_impl(
+      integer_sequence<size_t>{},
+      index_sequence<(remove_cvref_t<_Vs>::__size() - 1)...>{},
+      __first_index,
+      __make_value_visitor(::cuda::std::forward<_Visitor>(__visitor)),
+      ::cuda::std::forward<_Vs>(__vs)...);
+  }
+
+  template <class _Rp, class _Visitor, class... _Vs>
+  [[nodiscard]] _CCCL_API static constexpr _Rp __visit_value(_Visitor&& __visitor, _Vs&&... __vs)
+  {
+    const size_t __first_index = __get_runtime_index<sizeof...(_Vs), 0>(__vs...);
+    return __visit_impl(
+      integer_sequence<size_t>{},
+      index_sequence<(remove_cvref_t<_Vs>::__size() - 1)...>{},
+      __first_index,
+      __make_value_visitor<_Rp>(::cuda::std::forward<_Visitor>(__visitor)),
+      ::cuda::std::forward<_Vs>(__vs)...);
+  }
+
+  template <class _Visitor, class... _Values>
+  _CCCL_API static constexpr void __cccl_visit_exhaustive_visitor_check()
+  {
+    static_assert(is_invocable_v<_Visitor, _Values...>, "`std::visit` requires the visitor to be exhaustive.");
+  }
+
+  template <class _Visitor>
+  struct __value_visitor
+  {
+    template <class... _Alts>
+    [[nodiscard]] _CCCL_API constexpr decltype(auto) operator()(_Alts&&... __alts) const
+    {
+      __cccl_visit_exhaustive_visitor_check<_Visitor, decltype((::cuda::std::forward<_Alts>(__alts).__value))...>();
+      return ::cuda::std::__invoke(
+        ::cuda::std::forward<_Visitor>(__visitor), ::cuda::std::forward<_Alts>(__alts).__value...);
+    }
+    _Visitor&& __visitor;
+  };
+
+  template <class _Rp, class _Visitor>
+  struct __value_visitor_return_type
+  {
+    template <class... _Alts>
+    [[nodiscard]] _CCCL_API constexpr _Rp operator()(_Alts&&... __alts) const
+    {
+      __cccl_visit_exhaustive_visitor_check<_Visitor, decltype((::cuda::std::forward<_Alts>(__alts).__value))...>();
+      return ::cuda::std::__invoke(
+        ::cuda::std::forward<_Visitor>(__visitor), ::cuda::std::forward<_Alts>(__alts).__value...);
+    }
+    _Visitor&& __visitor;
+  };
+
+  template <class _Visitor>
+  struct __value_visitor_return_type<void, _Visitor>
+  {
+    template <class... _Alts>
+    _CCCL_API constexpr void operator()(_Alts&&... __alts) const
+    {
+      __cccl_visit_exhaustive_visitor_check<_Visitor, decltype((::cuda::std::forward<_Alts>(__alts).__value))...>();
+      ::cuda::std::__invoke(::cuda::std::forward<_Visitor>(__visitor), ::cuda::std::forward<_Alts>(__alts).__value...);
+    }
+    _Visitor&& __visitor;
+  };
+
+  template <class _Visitor>
+  [[nodiscard]] _CCCL_API static constexpr auto __make_value_visitor(_Visitor&& __visitor)
+  {
+    return __value_visitor<_Visitor>{::cuda::std::forward<_Visitor>(__visitor)};
+  }
+
+  template <class _Rp, class _Visitor>
+  [[nodiscard]] _CCCL_API static constexpr auto __make_value_visitor(_Visitor&& __visitor)
+  {
+    return __value_visitor_return_type<_Rp, _Visitor>{::cuda::std::forward<_Visitor>(__visitor)};
+  }
+};
+} // namespace __variant_detail::__visitation
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VARIANT_VISIT_H

cccl_upstream/libcudacxx/include/cuda/std/__variant/visit.h

@@ -0,0 +1,148 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___VARIANT_VISIT_H
+#define _CUDA_STD___VARIANT_VISIT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__fwd/variant.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/void_t.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/move.h>
+#include <cuda/std/__utility/unreachable.h>
+#include <cuda/std/__variant/bad_variant_access.h>
+#include <cuda/std/__variant/get.h>
+#include <cuda/std/__variant/variant.h>
+#include <cuda/std/__variant/variant_visit.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+struct __variant_binary_visitor
+{
+  template <class _BinaryOp, class _LeftVariant, class _RightVariant>
+  [[nodiscard]] _CCCL_API static constexpr auto
+  __visit(const size_t __index_, _BinaryOp&& __op, const _LeftVariant& __lhs, const _RightVariant& __rhs)
+  {
+    return __visit(integral_constant<size_t, variant_size_v<_LeftVariant> - 1>{},
+                   __index_,
+                   ::cuda::std::forward<_BinaryOp>(__op),
+                   __lhs,
+                   __rhs);
+  }
+
+private:
+  template <size_t _CurrentIndex, class _BinaryOp, class _LeftVariant, class _RightVariant>
+  [[nodiscard]] _CCCL_API static constexpr auto
+  __visit(integral_constant<size_t, _CurrentIndex>,
+          const size_t __index_,
+          _BinaryOp&& __op,
+          const _LeftVariant& __lhs,
+          const _RightVariant& __rhs)
+  {
+    if (__index_ == _CurrentIndex)
+    {
+      return __op(::cuda::std::get<_CurrentIndex>(__lhs), ::cuda::std::get<_CurrentIndex>(__rhs));
+    }
+    return __visit(
+      integral_constant<size_t, _CurrentIndex - 1>{}, __index_, ::cuda::std::forward<_BinaryOp>(__op), __lhs, __rhs);
+  }
+
+  template <class _BinaryOp, class _LeftVariant, class _RightVariant>
+  [[nodiscard]] _CCCL_API static constexpr auto
+  __visit(integral_constant<size_t, 0>,
+          const size_t __index_,
+          _BinaryOp&& __op,
+          const _LeftVariant& __lhs,
+          const _RightVariant& __rhs)
+  {
+    if (__index_ == 0)
+    {
+      return __op(::cuda::std::get<0>(__lhs), ::cuda::std::get<0>(__rhs));
+    }
+    // We already checked that every variant has a value, so we should never reach this line
+#if _CCCL_COMPILER(MSVC) // MSVC needs this to be wrapped in a function or it will error
+    ::cuda::std::unreachable();
+#else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+    _CCCL_UNREACHABLE();
+#endif // !_CCCL_COMPILER(MSVC)
+  }
+};
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr variant<_Types...>& __as_variant(variant<_Types...>& __vs) noexcept
+{
+  return __vs;
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr const variant<_Types...>& __as_variant(const variant<_Types...>& __vs) noexcept
+{
+  return __vs;
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr variant<_Types...>&& __as_variant(variant<_Types...>&& __vs) noexcept
+{
+  return ::cuda::std::move(__vs);
+}
+
+template <class... _Types>
+[[nodiscard]] _CCCL_API constexpr const variant<_Types...>&& __as_variant(const variant<_Types...>&& __vs) noexcept
+{
+  return ::cuda::std::move(__vs);
+}
+
+template <class... _Vs>
+_CCCL_API constexpr void __throw_if_valueless(_Vs&&... __vs)
+{
+  [[maybe_unused]] int __unused[] = {
+    (::cuda::std::__as_variant(__vs).valueless_by_exception() ? ::cuda::std::__throw_bad_variant_access() : void(),
+     0)...,
+    0};
+}
+
+template <class _Visitor,
+          class... _Vs,
+          typename = void_t<decltype(::cuda::std::__as_variant(::cuda::std::declval<_Vs>()))...>>
+_CCCL_API constexpr decltype(auto) visit(_Visitor&& __visitor, _Vs&&... __vs)
+{
+  using __variant_detail::__visitation::__variant;
+  ::cuda::std::__throw_if_valueless(::cuda::std::forward<_Vs>(__vs)...);
+  return __variant::__visit_value(::cuda::std::forward<_Visitor>(__visitor), ::cuda::std::forward<_Vs>(__vs)...);
+}
+
+template <class _Rp,
+          class _Visitor,
+          class... _Vs,
+          typename = void_t<decltype(::cuda::std::__as_variant(::cuda::std::declval<_Vs>()))...>>
+_CCCL_API constexpr _Rp visit(_Visitor&& __visitor, _Vs&&... __vs)
+{
+  using __variant_detail::__visitation::__variant;
+  ::cuda::std::__throw_if_valueless(::cuda::std::forward<_Vs>(__vs)...);
+  return __variant::__visit_value<_Rp>(::cuda::std::forward<_Visitor>(__visitor), ::cuda::std::forward<_Vs>(__vs)...);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___VARIANT_VISIT_H