[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/std/__simd/abi.h

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_ABI_H
+#define _CUDA_STD___SIMD_ABI_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+using __simd_size_type = ptrdiff_t;
+
+template <__simd_size_type _Np>
+using __simd_size_constant = integral_constant<__simd_size_type, _Np>;
+
+// [simd.expos.abi], simd ABI tags
+template <__simd_size_type _Np>
+struct __fixed_size; // internal ABI tag
+
+template <__simd_size_type _Np>
+using fixed_size = __fixed_size<_Np>; // implementation-defined ABI
+
+// TODO(fbusato): this could be optimized by using max access size / sizeof(T)
+template <typename>
+using native = fixed_size<1>; // implementation-defined ABI
+
+template <typename, __simd_size_type _Np>
+using __deduce_abi_t = fixed_size<_Np>; // exposition-only
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_ABI_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/algorithm.h

@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_ALGORITHM_H
+#define _CUDA_STD___SIMD_ALGORITHM_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/clamp.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/totally_ordered.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__simd/basic_mask.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/pair.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.alg], algorithms
+
+template <typename _Vec>
+struct __clamp_generator
+{
+  using __result_t = typename _Vec::value_type;
+
+  const _Vec& __v;
+  const _Vec& __lo;
+  const _Vec& __hi;
+
+  template <typename _Ip>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept
+  {
+    return ::cuda::std::clamp(__v[_Ip::value], __lo[_Ip::value], __hi[_Ip::value]);
+  }
+};
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+min(const basic_vec<_Tp, _Abi>& __a, const basic_vec<_Tp, _Abi>& __b) noexcept
+{
+  return __simd_min_impl(__a, __b); // ADL
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+max(const basic_vec<_Tp, _Abi>& __a, const basic_vec<_Tp, _Abi>& __b) noexcept
+{
+  return __simd_max_impl(__a, __b); // ADL
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr pair<basic_vec<_Tp, _Abi>, basic_vec<_Tp, _Abi>>
+minmax(const basic_vec<_Tp, _Abi>& __a, const basic_vec<_Tp, _Abi>& __b) noexcept
+{
+  using __result_t = pair<basic_vec<_Tp, _Abi>, basic_vec<_Tp, _Abi>>;
+  return __result_t{::cuda::std::simd::min(__a, __b), ::cuda::std::simd::max(__a, __b)};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+clamp(const basic_vec<_Tp, _Abi>& __v, const basic_vec<_Tp, _Abi>& __lo, const basic_vec<_Tp, _Abi>& __hi) noexcept
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__clamp_generator<__vec_t>{__v, __lo, __hi}};
+}
+
+// Scalar select
+template <typename _Tp, typename _Up>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto select(const bool __c, const _Tp& __a, const _Up& __b)
+  -> remove_cvref_t<decltype(__c ? __a : __b)>
+{
+  return __c ? __a : __b;
+}
+
+// Mask-based select: dispatches to the hidden-friend __simd_select_impl via ADL
+template <size_t _Bytes, typename _Abi, typename _Tp, typename _Up>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+select(const basic_mask<_Bytes, _Abi>& __c, const _Tp& __a, const _Up& __b) noexcept
+  -> decltype(__simd_select_impl(__c, __a, __b))
+{
+  return __simd_select_impl(__c, __a, __b);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_ALGORITHM_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/basic_mask.h

@@ -0,0 +1,396 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_BASIC_MASK_H
+#define _CUDA_STD___SIMD_BASIC_MASK_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/same_as.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__iterator/default_sentinel.h>
+#include <cuda/std/__simd/iterator.h>
+#include <cuda/std/__simd/specializations/fixed_size_mask.h>
+#include <cuda/std/__simd/utility.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/is_unsigned.h>
+#include <cuda/std/__type_traits/num_bits.h>
+#include <cuda/std/bitset>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <size_t _Bytes>
+inline constexpr bool __is_vectorizable_byte_size_v =
+  (_Bytes == 1 || _Bytes == 2 || _Bytes == 4 || _Bytes == 8
+#if _CCCL_HAS_INT128()
+   || _Bytes == 16
+#endif // _CCCL_HAS_INT128()
+  );
+
+// If basic_mask<Bytes, Abi> is disabled, the specialization has a deleted default constructor, deleted destructor,
+// deleted copy constructor, and deleted copy assignment. In addition only the value_type and abi_type members are
+// present.
+template <size_t _Bytes, typename _Abi, typename>
+class basic_mask
+{
+public:
+  using value_type = bool;
+  using abi_type   = _Abi;
+
+  _CCCL_HIDE_FROM_ABI basic_mask()                             = delete;
+  _CCCL_HIDE_FROM_ABI ~basic_mask()                            = delete;
+  _CCCL_HIDE_FROM_ABI basic_mask(const basic_mask&)            = delete;
+  _CCCL_HIDE_FROM_ABI basic_mask& operator=(const basic_mask&) = delete;
+};
+
+// basic_mask<Bytes, Abi> is enabled when there exists a vectorizable type T with sizeof(T) == Bytes and N in [1, 64]
+// derived from deduce-abi-t<T, N>
+template <size_t _Bytes, typename _Abi>
+class basic_mask<_Bytes, _Abi, enable_if_t<__is_vectorizable_byte_size_v<_Bytes> && __is_enabled_abi_v<_Abi>>>
+    : public __mask_operations<_Bytes, _Abi>
+{
+  template <typename, typename, typename>
+  friend class basic_vec;
+
+  using _Impl    = __mask_operations<_Bytes, _Abi>;
+  using _Storage = typename _Impl::_MaskStorage;
+
+  _Storage __s_;
+
+  struct __storage_tag_t
+  {};
+  static constexpr __storage_tag_t __storage_tag{};
+
+  _CCCL_HOST_DEVICE_API constexpr basic_mask(const _Storage __v, __storage_tag_t) noexcept
+      : __s_{__v}
+  {}
+
+public:
+  using value_type = bool;
+  using abi_type   = _Abi;
+
+  using iterator       = __simd_iterator<basic_mask>;
+  using const_iterator = __simd_iterator<const basic_mask>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr iterator begin() noexcept
+  {
+    return {*this, 0};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const_iterator begin() const noexcept
+  {
+    return {*this, 0};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const_iterator cbegin() const noexcept
+  {
+    return {*this, 0};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr default_sentinel_t end() const noexcept
+  {
+    return {};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr default_sentinel_t cend() const noexcept
+  {
+    return {};
+  }
+
+  static constexpr __simd_size_constant<__simd_size_v<__integer_from<_Bytes>, _Abi>> size{};
+
+  static constexpr auto __usize = size_t{size};
+  static constexpr auto __size  = __simd_size_type{size};
+
+  _CCCL_HIDE_FROM_ABI constexpr basic_mask() noexcept = default;
+
+  // [simd.mask.ctor], basic_mask constructors
+
+  _CCCL_TEMPLATE(typename _Up)
+  _CCCL_REQUIRES(same_as<_Up, value_type>)
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_mask(const _Up __v) noexcept
+      : __s_{_Impl::__broadcast(__v)}
+  {}
+
+  _CCCL_TEMPLATE(size_t _UBytes, typename _UAbi)
+  _CCCL_REQUIRES((__simd_size_v<__integer_from<_UBytes>, _UAbi> == __size))
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, __x[__i]);
+    }
+  }
+
+  _CCCL_TEMPLATE(typename _Generator)
+  _CCCL_REQUIRES(__can_generate_v<bool, _Generator, __size>)
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_mask(_Generator&& __g)
+      : __s_{_Impl::__generate(__g)}
+  {}
+
+  _CCCL_TEMPLATE(typename _Tp)
+  _CCCL_REQUIRES(same_as<_Tp, bitset<__usize>>)
+  _CCCL_HOST_DEVICE_API constexpr basic_mask(const _Tp& __b) noexcept
+      : __s_{_Impl::__broadcast(false)}
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, static_cast<bool>(__b[__i]));
+    }
+  }
+
+  _CCCL_TEMPLATE(typename _Tp)
+  _CCCL_REQUIRES(is_integral_v<_Tp> _CCCL_AND is_unsigned_v<_Tp> _CCCL_AND(!is_same_v<_Tp, value_type>))
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_mask(const _Tp __val) noexcept
+      : __s_{_Impl::__broadcast(false)}
+  {
+    constexpr auto __num_bits = __simd_size_type{__num_bits_v<_Tp>};
+    constexpr auto __m        = __size < __num_bits ? __size : __num_bits;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __m; ++__i)
+    {
+      __s_.__set(__i, static_cast<bool>((__val >> __i) & _Tp{1}));
+    }
+  }
+
+  // [simd.mask.subscr], basic_mask subscript operators
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr value_type operator[](const __simd_size_type __i) const noexcept
+  {
+    _CCCL_ASSERT(::cuda::in_range(__i, __simd_size_type{0}, __size), "Index is out of bounds");
+    return static_cast<bool>(__s_.__get(__i));
+  }
+
+  // TODO(fbusato): subscript with integral indices, requires permute()
+  // template<simd-integral I>
+  // constexpr resize_t<I::size(), basic_mask> operator[](const I& indices) const;
+
+  // [simd.mask.unary], basic_mask unary operators
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_mask operator!() const noexcept
+  {
+    return {_Impl::__bitwise_not(__s_), __storage_tag};
+  }
+
+  using __unary_return_t = basic_vec<__integer_from<_Bytes>, _Abi>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __unary_return_t operator+() const noexcept
+  {
+    return static_cast<__unary_return_t>(*this);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __unary_return_t operator-() const noexcept
+  {
+    return -static_cast<__unary_return_t>(*this);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __unary_return_t operator~() const noexcept
+  {
+    return ~static_cast<__unary_return_t>(*this);
+  }
+
+  // [simd.mask.conv], basic_mask conversions
+
+  _CCCL_TEMPLATE(typename _Up, typename _Ap)
+  _CCCL_REQUIRES((sizeof(_Up) != _Bytes && __simd_size_v<_Up, _Ap> == __size))
+  _CCCL_HOST_DEVICE_API constexpr explicit operator basic_vec<_Up, _Ap>() const noexcept
+  {
+    basic_vec<_Up, _Ap> __result{};
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __result.__s_.__set(__i, static_cast<_Up>((*this)[__i]));
+    }
+    return __result;
+  }
+
+  _CCCL_TEMPLATE(typename _Up, typename _Ap)
+  _CCCL_REQUIRES((sizeof(_Up) == _Bytes && __simd_size_v<_Up, _Ap> == __size))
+  _CCCL_HOST_DEVICE_API constexpr operator basic_vec<_Up, _Ap>() const noexcept
+  {
+    basic_vec<_Up, _Ap> __result{};
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __result.__s_.__set(__i, static_cast<_Up>((*this)[__i]));
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bitset<__usize> to_bitset() const noexcept
+  {
+    bitset<__usize> __result{};
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __result.set(__i, (*this)[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr unsigned long long to_ullong() const
+  {
+    constexpr __simd_size_type __nbits = __num_bits_v<unsigned long long>;
+    if constexpr (__size > __nbits)
+    {
+      for (auto __i = __nbits; __i < __size; ++__i)
+      {
+        _CCCL_ASSERT(!(*this)[__i], "Bit above unsigned long long width is set");
+      }
+    }
+    return to_bitset().to_ullong();
+  }
+
+  // [simd.mask.binary], basic_mask binary operators
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator&&(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return {_Impl::__logic_and(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator||(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return {_Impl::__logic_or(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator&(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return {_Impl::__bitwise_and(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator|(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return {_Impl::__bitwise_or(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator^(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return {_Impl::__bitwise_xor(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  // [simd.mask.cassign], basic_mask compound assignment
+
+  _CCCL_HOST_DEVICE_API friend constexpr basic_mask& operator&=(basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return __lhs = __lhs & __rhs;
+  }
+
+  _CCCL_HOST_DEVICE_API friend constexpr basic_mask& operator|=(basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return __lhs = __lhs | __rhs;
+  }
+
+  _CCCL_HOST_DEVICE_API friend constexpr basic_mask& operator^=(basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return __lhs = __lhs ^ __rhs;
+  }
+
+  // [simd.mask.comparison], basic_mask comparisons (element-wise)
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator==(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return !(__lhs ^ __rhs);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator!=(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return __lhs ^ __rhs;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator>=(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return __lhs || !__rhs;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator<=(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return !__lhs || __rhs;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator>(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return __lhs && !__rhs;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_mask
+  operator<(const basic_mask& __lhs, const basic_mask& __rhs) noexcept
+  {
+    return !__lhs && __rhs;
+  }
+
+  // [simd.mask.cond], basic_mask exposition-only conditional operators
+
+  [[nodiscard]] _CCCL_API friend constexpr basic_mask
+  __simd_select_impl(const basic_mask& __mask, const basic_mask& __a, const basic_mask& __b) noexcept
+  {
+    basic_mask __result{};
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __result.__s_.__set(__i, (__mask[__i] ? __a[__i] : __b[__i]));
+    }
+    return __result;
+  }
+
+  _CCCL_TEMPLATE(typename _TpA, typename _TpB)
+  _CCCL_REQUIRES(same_as<_TpA, bool> _CCCL_AND same_as<_TpB, bool>)
+  [[nodiscard]] _CCCL_API friend constexpr basic_mask
+  __simd_select_impl(const basic_mask& __mask, const _TpA __a, const _TpB __b) noexcept
+  {
+    basic_mask __result{};
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __result.__s_.__set(__i, (__mask[__i] ? __a : __b));
+    }
+    return __result;
+  }
+
+  _CCCL_TEMPLATE(typename _TpA, typename _TpB)
+  _CCCL_REQUIRES(same_as<_TpA, _TpB> _CCCL_AND __is_vectorizable_v<_TpA> _CCCL_AND(sizeof(_TpA) == _Bytes))
+  [[nodiscard]] _CCCL_API friend constexpr vec<_TpA, __size>
+  __simd_select_impl(const basic_mask& __mask, const _TpA& __a, const _TpB& __b) noexcept
+  {
+    using _Vec = vec<_TpA, __size>;
+    return __simd_select_impl(__mask, _Vec{__a}, _Vec{__b});
+  }
+};
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_BASIC_MASK_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/basic_vec.h

@@ -0,0 +1,788 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_BASIC_VEC_H
+#define _CUDA_STD___SIMD_BASIC_VEC_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__fwd/complex.h>
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__fwd/complex.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__iterator/default_sentinel.h>
+#include <cuda/std/__ranges/concepts.h>
+#include <cuda/std/__ranges/data.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/basic_mask.h>
+#include <cuda/std/__simd/concepts.h>
+#include <cuda/std/__simd/flag.h>
+#include <cuda/std/__simd/iterator.h>
+#include <cuda/std/__simd/specializations/fixed_size_float_vec.h>
+#include <cuda/std/__simd/specializations/fixed_size_integral_vec.h>
+#include <cuda/std/__simd/specializations/fixed_size_vec.h>
+#include <cuda/std/__simd/type_traits.h>
+#include <cuda/std/__simd/utility.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/operations.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.expos]
+
+// Disambiguates the converting constructor from the complex constructor when _ValueType is complex
+// and _Up is its underlying real type
+template <typename _ValueType, typename _Up, bool = __is_complex_vectorizable_v<_ValueType>>
+inline constexpr bool __is_complex_real_type_v = false;
+
+template <typename _ValueType, typename _Up>
+inline constexpr bool __is_complex_real_type_v<_ValueType, _Up, true> = is_same_v<_Up, typename _ValueType::value_type>;
+
+template <typename _Tp>
+inline constexpr bool __is_cccl_complex_v = __is_cuda_std_complex_v<_Tp> || ::cuda::__is_cuda_complex_v<_Tp>;
+
+// [simd.class], class template basic_vec
+
+// If basic_vec<T, Abi> is disabled, the specialization has a deleted default constructor, deleted destructor, deleted
+// copy constructor, and deleted copy assignment. In addition only the value_type, abi_type, and mask_type members are
+// present.
+template <typename _Tp, typename _Abi, typename>
+class basic_vec
+{
+public:
+  using value_type = _Tp;
+  using abi_type   = _Abi;
+  using mask_type  = basic_mask<sizeof(_Tp), _Abi>;
+
+  _CCCL_HIDE_FROM_ABI basic_vec()                            = delete;
+  _CCCL_HIDE_FROM_ABI ~basic_vec()                           = delete;
+  _CCCL_HIDE_FROM_ABI basic_vec(const basic_vec&)            = delete;
+  _CCCL_HIDE_FROM_ABI basic_vec& operator=(const basic_vec&) = delete;
+};
+
+// basic_vec<T, Abi> is enabled when:
+// - T is a vectorizable type
+// - there exists N in [1, 64] derived from deduce-abi-t<T, N>
+// - sizeof(T) is a vectorizable byte size, i.e. sizeof(T) fits in an integer, otherwise mask is not representable
+template <typename _Tp, typename _Abi>
+class basic_vec<
+  _Tp,
+  _Abi,
+  enable_if_t<__is_vectorizable_v<_Tp> && __is_vectorizable_byte_size_v<sizeof(_Tp)> && __is_enabled_abi_v<_Abi>>>
+    : public __simd_operations<_Tp, _Abi>
+{
+public:
+  using value_type = _Tp;
+  using mask_type  = basic_mask<sizeof(value_type), _Abi>;
+
+private:
+  template <typename, typename, typename>
+  friend class basic_vec;
+
+  template <size_t, typename, typename>
+  friend class basic_mask;
+
+  template <typename _Result, typename _Up, typename... _Flags>
+  _CCCL_HOST_DEVICE_API friend constexpr _Result
+  __partial_load_from_ptr(const _Up*, __simd_size_type, const typename _Result::mask_type&, flags<_Flags...>) noexcept;
+
+  template <typename _Result, typename _Up, typename... _Flags>
+  _CCCL_HOST_DEVICE_API friend constexpr _Result
+  __full_load_from_ptr(const _Up*, const typename _Result::mask_type&, flags<_Flags...>) noexcept;
+
+  using _Impl    = __simd_operations<_Tp, _Abi>;
+  using _Storage = typename _Impl::_SimdStorage;
+
+  _Storage __s_{};
+
+  struct __storage_tag_t
+  {};
+  static constexpr __storage_tag_t __storage_tag{};
+
+  _CCCL_HOST_DEVICE_API constexpr basic_vec(const _Storage& __s, __storage_tag_t) noexcept
+      : __s_{__s}
+  {}
+
+  // Friend comparison operators (e.g. operator==) cannot access basic_mask's private constructor directly (friendship
+  // is not transitive). This function is required to access the private constructor of basic_mask.
+  _CCCL_HOST_DEVICE_API static constexpr mask_type __make_mask(const typename mask_type::_Storage __s) noexcept
+  {
+    return mask_type{__s, mask_type::__storage_tag};
+  }
+
+  // operator[] is const only. We need this function to set values
+  _CCCL_HOST_DEVICE_API constexpr void __set(const __simd_size_type __i, const value_type __v) noexcept
+  {
+    __s_.__set(__i, __v);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  __simd_min_impl(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__min_simd(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  __simd_max_impl(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__max_simd(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+public:
+  using abi_type = _Abi;
+
+  using iterator       = __simd_iterator<basic_vec>;
+  using const_iterator = __simd_iterator<const basic_vec>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr iterator begin() noexcept
+  {
+    return {*this, 0};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const_iterator begin() const noexcept
+  {
+    return {*this, 0};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const_iterator cbegin() const noexcept
+  {
+    return {*this, 0};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr default_sentinel_t end() const noexcept
+  {
+    return {};
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr default_sentinel_t cend() const noexcept
+  {
+    return {};
+  }
+
+  static constexpr __simd_size_constant<__simd_size_v<value_type, abi_type>> size{};
+
+  static constexpr auto __usize = size_t{size};
+  static constexpr auto __size  = __simd_size_type{size};
+
+  _CCCL_HIDE_FROM_ABI basic_vec() noexcept = default;
+
+  // [simd.ctor], basic_vec constructors
+
+  // [simd.ctor] value broadcast constructor (explicit overload)
+  _CCCL_TEMPLATE(typename _Up)
+  _CCCL_REQUIRES((__explicitly_convertible_to<_Up, value_type>) _CCCL_AND(!__is_value_ctor_implicit<_Up, value_type>))
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_vec(_Up&& __v) noexcept
+      : __s_{_Impl::__broadcast(static_cast<value_type>(__v))}
+  {}
+
+  // [simd.ctor] value broadcast constructor (implicit overload)
+  _CCCL_TEMPLATE(typename _Up)
+  _CCCL_REQUIRES((__explicitly_convertible_to<_Up, value_type>) _CCCL_AND(__is_value_ctor_implicit<_Up, value_type>))
+  _CCCL_HOST_DEVICE_API constexpr basic_vec(_Up&& __v) noexcept
+      : __s_{_Impl::__broadcast(static_cast<value_type>(__v))}
+  {}
+
+  template <typename _Up>
+  static inline constexpr bool __can_implicitly_convert_v =
+    convertible_to<_Up, value_type> //
+    && !__is_vec_ctor_explicit<_Up, value_type> //
+    && !__is_complex_real_type_v<value_type, _Up>;
+
+  template <typename _Up>
+  static inline constexpr bool __can_explicitly_convert_v =
+    __explicitly_convertible_to<_Up, value_type> //
+    && !__can_implicitly_convert_v<_Up> //
+    && !__is_complex_real_type_v<value_type, _Up>;
+
+  // [simd.ctor] converting constructor from basic_vec<U, UAbi> (explicit overload)
+  _CCCL_TEMPLATE(typename _Up, typename _UAbi)
+  _CCCL_REQUIRES((__simd_size_v<_Up, _UAbi> == __size) _CCCL_AND(__can_explicitly_convert_v<_Up>))
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_vec(const basic_vec<_Up, _UAbi>& __v) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, static_cast<value_type>(__v[__i]));
+    }
+  }
+
+  // [simd.ctor] converting constructor from basic_vec<U, UAbi> (implicit overload)
+
+  _CCCL_EXEC_CHECK_DISABLE
+  _CCCL_TEMPLATE(typename _Up, typename _UAbi)
+  _CCCL_REQUIRES((__simd_size_v<_Up, _UAbi> == __size) _CCCL_AND(__can_implicitly_convert_v<_Up>))
+  _CCCL_HOST_DEVICE_API constexpr basic_vec(const basic_vec<_Up, _UAbi>& __v) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, static_cast<value_type>(__v[__i]));
+    }
+  }
+
+  // [simd.ctor] generator constructor
+  _CCCL_TEMPLATE(typename _Generator)
+  _CCCL_REQUIRES(__can_generate_v<value_type, _Generator, __size>)
+  _CCCL_HOST_DEVICE_API constexpr explicit basic_vec(_Generator&& __g)
+      : __s_{_Impl::__generate(__g)}
+  {}
+
+  // [simd.ctor] range constructor
+  template <typename _Range>
+  static constexpr bool __is_compatible_range = __is_compatible_range_v<value_type, __size, _Range>;
+
+  // [simd.ctor] range constructor
+  _CCCL_TEMPLATE(typename _Range, typename... _Flags)
+  _CCCL_REQUIRES(__is_compatible_range<_Range>)
+  _CCCL_HOST_DEVICE_API constexpr basic_vec(_Range&& __range, flags<_Flags...> = {})
+  {
+    static_assert(__has_convert_flag_v<_Flags...>
+                    || __is_value_preserving_v<::cuda::std::ranges::range_value_t<_Range>, value_type>,
+                  "Conversion from range_value_t<R> to value_type is not value-preserving; use flag_convert");
+    const auto __data = ::cuda::std::ranges::__data_cpo{}(__range);
+    ::cuda::std::simd::__assert_load_store_alignment<basic_vec, ::cuda::std::ranges::range_value_t<_Range>, _Flags...>(
+      __data);
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, static_cast<value_type>(__data[__i]));
+    }
+  }
+
+  // [simd.ctor] masked range constructor
+  _CCCL_TEMPLATE(typename _Range, typename... _Flags)
+  _CCCL_REQUIRES(__is_compatible_range<_Range>)
+  _CCCL_HOST_DEVICE_API constexpr basic_vec(_Range&& __range, const mask_type& __mask, flags<_Flags...> = {})
+  {
+    static_assert(__has_convert_flag_v<_Flags...>
+                    || __is_value_preserving_v<::cuda::std::ranges::range_value_t<_Range>, value_type>,
+                  "Conversion from range_value_t<R> to value_type is not value-preserving; use flag_convert");
+    const auto __data = ::cuda::std::ranges::__data_cpo{}(__range);
+    ::cuda::std::simd::__assert_load_store_alignment<basic_vec, ::cuda::std::ranges::range_value_t<_Range>, _Flags...>(
+      __data);
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, __mask[__i] ? static_cast<value_type>(__data[__i]) : value_type{});
+    }
+  }
+
+  // [simd.ctor] complex constructor
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_cccl_complex_v<_Tp2>)
+  _CCCL_HOST_DEVICE_API constexpr basic_vec(const rebind_t<typename _Tp2::value_type, basic_vec>& __reals,
+                                            const rebind_t<typename _Tp2::value_type, basic_vec>& __imags = {}) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, value_type{__reals[__i], __imags[__i]});
+    }
+  }
+
+#if _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_std_complex_v<_Tp2>)
+  _CCCL_HOST_API constexpr basic_vec(const rebind_t<typename _Tp2::value_type, basic_vec>& __reals,
+                                     const rebind_t<typename _Tp2::value_type, basic_vec>& __imags = {}) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, value_type{__reals[__i], __imags[__i]});
+    }
+  }
+
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+  // [simd.subscr], basic_vec subscript operators
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr value_type operator[](const __simd_size_type __i) const noexcept
+  {
+    _CCCL_ASSERT(::cuda::in_range(__i, __simd_size_type{0}, __size), "Index is out of bounds");
+    return __s_.__get(__i);
+  }
+
+  // TODO(fbusato): subscript with integral indices, requires permute()
+  // template<simd-integral _Idx>
+  //   constexpr resize_t<_Idx::size(), basic_vec> operator[](const _Idx& __indices) const;
+
+  // [simd.complex.access], basic_vec complex accessors
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_cccl_complex_v<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<typename _Tp2::value_type, basic_vec> real() const noexcept
+  {
+    rebind_t<typename _Tp2::value_type, basic_vec> __ret;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __ret.__s_.__set(__i, (*this)[__i].real());
+    }
+    return __ret;
+  }
+
+#if _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_std_complex_v<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_API constexpr rebind_t<typename _Tp2::value_type, basic_vec> real() const noexcept
+  {
+    rebind_t<typename _Tp2::value_type, basic_vec> __ret;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __ret.__s_.__set(__i, (*this)[__i].real());
+    }
+    return __ret;
+  }
+
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_cccl_complex_v<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<typename _Tp2::value_type, basic_vec> imag() const noexcept
+  {
+    rebind_t<typename _Tp2::value_type, basic_vec> __ret;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __ret.__s_.__set(__i, (*this)[__i].imag());
+    }
+    return __ret;
+  }
+
+#if _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_std_complex_v<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_API constexpr rebind_t<typename _Tp2::value_type, basic_vec> imag() const noexcept
+  {
+    rebind_t<typename _Tp2::value_type, basic_vec> __ret;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __ret.__s_.__set(__i, (*this)[__i].imag());
+    }
+    return __ret;
+  }
+
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_cccl_complex_v<_Tp2>)
+  _CCCL_HOST_DEVICE_API constexpr void real(const rebind_t<typename _Tp2::value_type, basic_vec>& __v) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, value_type{__v[__i], (*this)[__i].imag()});
+    }
+  }
+
+#if _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_std_complex_v<_Tp2>)
+  _CCCL_HOST_API constexpr void real(const rebind_t<typename _Tp2::value_type, basic_vec>& __v) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, value_type{__v[__i], (*this)[__i].imag()});
+    }
+  }
+
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_cccl_complex_v<_Tp2>)
+  _CCCL_HOST_DEVICE_API constexpr void imag(const rebind_t<typename _Tp2::value_type, basic_vec>& __v) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, value_type{(*this)[__i].real(), __v[__i]});
+    }
+  }
+
+#if _CCCL_HAS_HOST_STD_LIB()
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__is_std_complex_v<_Tp2>)
+  _CCCL_HOST_API constexpr void imag(const rebind_t<typename _Tp2::value_type, basic_vec>& __v) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __s_.__set(__i, value_type{(*this)[__i].real(), __v[__i]});
+    }
+  }
+
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+  // [simd.unary], basic_vec unary operators
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_pre_increment<_Tp2>)
+  _CCCL_HOST_DEVICE_API constexpr basic_vec& operator++() noexcept
+  {
+    _Impl::__increment(__s_);
+    return *this;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_post_increment<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec operator++(int) noexcept
+  {
+    const basic_vec __r = *this;
+    _Impl::__increment(__s_);
+    return __r;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_pre_decrement<_Tp2>)
+  _CCCL_HOST_DEVICE_API constexpr basic_vec& operator--() noexcept
+  {
+    _Impl::__decrement(__s_);
+    return *this;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_post_decrement<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec operator--(int) noexcept
+  {
+    const basic_vec __r = *this;
+    _Impl::__decrement(__s_);
+    return __r;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_negate<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr mask_type operator!() const noexcept
+  {
+    return mask_type{_Impl::__negate(__s_), mask_type::__storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_not<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec operator~() const noexcept
+  {
+    return basic_vec{_Impl::__bitwise_not(__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_unary_plus<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec operator+() const noexcept
+  {
+    return *this;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_unary_minus<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec operator-() const noexcept
+  {
+    return basic_vec{_Impl::__unary_minus(__s_), __storage_tag};
+  }
+
+  // [simd.binary], basic_vec binary operators
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_binary_plus<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator+(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__plus(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_binary_minus<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator-(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__minus(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_multiplies<_Tp2>)
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec operator*(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__multiplies(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_divides<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator/(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__divides(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_modulo<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator%(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__modulo(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_and<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator&(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__bitwise_and(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_or<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator|(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__bitwise_or(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_xor<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator^(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__bitwise_xor(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_left<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator<<(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__shift_left(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_right<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator>>(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return basic_vec{_Impl::__shift_right(__lhs.__s_, __rhs.__s_), __storage_tag};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_left_size<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator<<(const basic_vec& __lhs, const __simd_size_type __n) noexcept
+  {
+    return __lhs << basic_vec{__n};
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_right_size<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr basic_vec
+  operator>>(const basic_vec& __lhs, const __simd_size_type __n) noexcept
+  {
+    return __lhs >> basic_vec{__n};
+  }
+
+  // [simd.cassign], basic_vec compound assignment
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_binary_plus<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator+=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs + __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_binary_minus<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator-=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs - __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_multiplies<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator*=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs * __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_divides<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator/=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs / __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_modulo<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator%=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs % __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_and<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator&=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs & __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_or<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator|=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs | __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_bitwise_xor<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator^=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs ^ __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_left<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator<<=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs << __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_right<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator>>=(basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __lhs = __lhs >> __rhs;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_left_size<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator<<=(basic_vec& __lhs, const __simd_size_type __n) noexcept
+  {
+    return __lhs = __lhs << __n;
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_shift_right_size<_Tp2>)
+  _CCCL_HOST_DEVICE_API friend constexpr basic_vec& operator>>=(basic_vec& __lhs, const __simd_size_type __n) noexcept
+  {
+    return __lhs = __lhs >> __n;
+  }
+
+  // [simd.comparison], basic_vec compare operators
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_equal_to<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr mask_type
+  operator==(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __make_mask(_Impl::__equal_to(__lhs.__s_, __rhs.__s_));
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_not_equal_to<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr mask_type
+  operator!=(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __make_mask(_Impl::__not_equal_to(__lhs.__s_, __rhs.__s_));
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_greater_equal<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr mask_type
+  operator>=(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __make_mask(_Impl::__greater_equal(__lhs.__s_, __rhs.__s_));
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_less_equal<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr mask_type
+  operator<=(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __make_mask(_Impl::__less_equal(__lhs.__s_, __rhs.__s_));
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_greater<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr mask_type
+  operator>(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __make_mask(_Impl::__greater(__lhs.__s_, __rhs.__s_));
+  }
+
+  _CCCL_TEMPLATE(typename _Tp2 = _Tp)
+  _CCCL_REQUIRES(__has_less<_Tp2>)
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr mask_type
+  operator<(const basic_vec& __lhs, const basic_vec& __rhs) noexcept
+  {
+    return __make_mask(_Impl::__less(__lhs.__s_, __rhs.__s_));
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend basic_vec
+  __simd_fma_impl(const basic_vec& __x, const basic_vec& __y, const basic_vec& __z) noexcept
+  {
+    return basic_vec{_Impl::__fma(__x.__s_, __y.__s_, __z.__s_), __storage_tag};
+  }
+
+  // [simd.cond], basic_vec exposition-only conditional operators
+
+  [[nodiscard]] _CCCL_API friend constexpr basic_vec
+  __simd_select_impl(const mask_type& __mask, const basic_vec& __a, const basic_vec& __b) noexcept
+  {
+    basic_vec __result{};
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __size; ++__i)
+    {
+      __result.__set(__i, (__mask[__i] ? __a[__i] : __b[__i]));
+    }
+    return __result;
+  }
+};
+
+// [simd.ctor] deduction guide from contiguous sized range
+// Deduces vec<range_value_t<R>, static_cast<simd-size-type>(::cuda::std::ranges::size(r))>
+//    * it is not possible to use the alias "vec" for the deduction guide
+//    * "vec" is defined as basic_vec<_Tp, __deduce_abi_t<_Tp, _Np>>
+//    * where _Np is __simd_size_v<_Tp, __static_range_size_v<_Range>>
+_CCCL_TEMPLATE(typename _Range, typename... _Ts)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>
+                 _CCCL_AND __has_static_size<_Range>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES basic_vec(_Range&&, _Ts...)
+  -> basic_vec<::cuda::std::ranges::range_value_t<_Range>,
+               __deduce_abi_t<::cuda::std::ranges::range_value_t<_Range>, __static_range_size_v<_Range>>>;
+
+// [simd.ctor] deduction guide from basic_mask
+// basic_vec<__integer_from<Bytes>, Abi> is equivalent to decltype(+k):
+//   * k has type basic_mask<_Bytes, _Abi>
+//   * +k calls basic_mask::operator+()
+//   * the return type is basic_vec<__integer_from<_Bp>, _Abi>
+// The deduced type is equivalent to decltype(+k), i.e. basic_vec<__integer_from<Bytes>, Abi>
+_CCCL_TEMPLATE(size_t _Bytes, typename _Abi)
+_CCCL_REQUIRES(__has_unary_plus<basic_mask<_Bytes, _Abi>>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES basic_vec(basic_mask<_Bytes, _Abi>) -> basic_vec<__integer_from<_Bytes>, _Abi>;
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_BASIC_VEC_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/bit.h

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_BIT_H
+#define _CUDA_STD___SIMD_BIT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__simd/bit/scalar.h>
+
+#endif // _CUDA_STD___SIMD_BIT_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/bit/scalar.h

@@ -0,0 +1,498 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_BIT_SCALAR_H
+#define _CUDA_STD___SIMD_BIT_SCALAR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__bit/bit_reverse.h>
+#include <cuda/std/__bit/byteswap.h>
+#include <cuda/std/__bit/countl.h>
+#include <cuda/std/__bit/countr.h>
+#include <cuda/std/__bit/has_single_bit.h>
+#include <cuda/std/__bit/integral.h>
+#include <cuda/std/__bit/popcount.h>
+#include <cuda/std/__bit/rotate.h>
+#include <cuda/std/__bit/shl.h>
+#include <cuda/std/__bit/shr.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/exposition.h>
+#include <cuda/std/__simd/type_traits.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_unsigned_integer.h>
+#include <cuda/std/__type_traits/make_signed.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.bit] element-wise helpers
+
+template <typename _Vp>
+struct __simd_byteswap_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::byteswap(__v_[_Idx::value]);
+  }
+};
+
+template <typename _Vp>
+struct __simd_bit_reverse_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::bit_reverse(__v_[_Idx::value]);
+  }
+};
+
+template <typename _Vp>
+struct __simd_bit_ceil_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::bit_ceil(__v_[_Idx::value]);
+  }
+};
+
+template <typename _Vp>
+struct __simd_bit_floor_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::bit_floor(__v_[_Idx::value]);
+  }
+};
+
+template <typename _Vp>
+struct __simd_has_single_bit_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = bool;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::has_single_bit(__v_[_Idx::value]);
+  }
+};
+
+template <typename _Vp0, typename _Vp1>
+struct __simd_shl_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type;
+  const _Vp0& __v0_;
+  const _Vp1& __v1_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::shl(__v0_[_Idx::value], __v1_[_Idx::value]);
+  }
+};
+
+template <typename _Vp0, typename _Vp1>
+struct __simd_shr_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type;
+  const _Vp0& __v0_;
+  const _Vp1& __v1_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::shr(__v0_[_Idx::value], __v1_[_Idx::value]);
+  }
+};
+
+template <typename _Vp, typename _Sp>
+struct __simd_shl_scalar_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+  const _Sp __s_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::shl(__v_[_Idx::value], __s_);
+  }
+};
+
+template <typename _Vp, typename _Sp>
+struct __simd_shr_scalar_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+  const _Sp __s_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::shr(__v_[_Idx::value], __s_);
+  }
+};
+
+template <typename _Vp0, typename _Vp1>
+struct __simd_rotl_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type;
+  const _Vp0& __v0_;
+  const _Vp1& __v1_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    _CCCL_ASSERT(::cuda::std::in_range<int>(__v1_[_Idx::value]), "rotl: count is out of range");
+    return ::cuda::std::rotl(__v0_[_Idx::value], static_cast<int>(__v1_[_Idx::value]));
+  }
+};
+
+template <typename _Vp0, typename _Vp1>
+struct __simd_rotr_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type;
+  const _Vp0& __v0_;
+  const _Vp1& __v1_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    _CCCL_ASSERT(::cuda::std::in_range<int>(__v1_[_Idx::value]), "rotr: count is out of range");
+    return ::cuda::std::rotr(__v0_[_Idx::value], static_cast<int>(__v1_[_Idx::value]));
+  }
+};
+
+template <typename _Vp>
+struct __simd_rotl_scalar_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+  const int __s_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::rotl(__v_[_Idx::value], __s_);
+  }
+};
+
+template <typename _Vp>
+struct __simd_rotr_scalar_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type;
+  const _Vp& __v_;
+  const int __s_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return ::cuda::std::rotr(__v_[_Idx::value], __s_);
+  }
+};
+
+template <typename _Vp, typename _Result>
+struct __simd_bit_width_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return static_cast<__result_t>(::cuda::std::bit_width(__v_[_Idx::value]));
+  }
+};
+
+template <typename _Vp, typename _Result>
+struct __simd_countl_zero_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return static_cast<__result_t>(::cuda::std::countl_zero(__v_[_Idx::value]));
+  }
+};
+
+template <typename _Vp, typename _Result>
+struct __simd_countl_one_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return static_cast<__result_t>(::cuda::std::countl_one(__v_[_Idx::value]));
+  }
+};
+
+template <typename _Vp, typename _Result>
+struct __simd_countr_zero_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return static_cast<__result_t>(::cuda::std::countr_zero(__v_[_Idx::value]));
+  }
+};
+
+template <typename _Vp, typename _Result>
+struct __simd_countr_one_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return static_cast<__result_t>(::cuda::std::countr_one(__v_[_Idx::value]));
+  }
+};
+
+template <typename _Vp, typename _Result>
+struct __simd_popcount_generator
+{
+  using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type;
+  const _Vp& __v_;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept
+  {
+    return static_cast<__result_t>(::cuda::std::popcount(__v_[_Idx::value]));
+  }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.bit], bit manipulation
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(is_integral_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto byteswap(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_byteswap_generator<__vec_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_reverse(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_bit_reverse_generator<__vec_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_ceil(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_bit_ceil_generator<__vec_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_floor(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_bit_floor_generator<__vec_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto has_single_bit(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = typename __vec_t::mask_type;
+  return __result_t{__simd_has_single_bit_generator<__vec_t>{__v}};
+}
+
+template <typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1>
+inline constexpr bool __simd_is_valid_rotate_v =
+  (sizeof(_Tp0) == sizeof(_Tp1)) //
+  && __cccl_is_unsigned_integer_v<_Tp0> //
+  && is_integral_v<_Tp1> //
+  && (__simd_size_v<_Tp0, _Abi0> == __simd_size_v<_Tp1, _Abi1>);
+
+template <typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1>
+inline constexpr bool __simd_is_valid_shift_v =
+  (sizeof(_Tp0) == sizeof(_Tp1)) //
+  && __cccl_is_integer_v<_Tp0> //
+  && __cccl_is_integer_v<_Tp1> //
+  && (__simd_size_v<_Tp0, _Abi0> == __simd_size_v<_Tp1, _Abi1>);
+
+_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1)
+_CCCL_REQUIRES(__simd_is_valid_shift_v<_Tp0, _Abi0, _Tp1, _Abi1>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+shl(const basic_vec<_Tp0, _Abi0>& __v, const basic_vec<_Tp1, _Abi1>& __s) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS   = basic_vec<_Tp0, _Abi0>;
+  using __shift_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>;
+  return __vec_t{__simd_shl_generator<__vec_t, __shift_t>{__v, __s}};
+}
+
+_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1)
+_CCCL_REQUIRES(__simd_is_valid_shift_v<_Tp0, _Abi0, _Tp1, _Abi1>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+shr(const basic_vec<_Tp0, _Abi0>& __v, const basic_vec<_Tp1, _Abi1>& __s) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS   = basic_vec<_Tp0, _Abi0>;
+  using __shift_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>;
+  return __vec_t{__simd_shr_generator<__vec_t, __shift_t>{__v, __s}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Sp)
+_CCCL_REQUIRES(__cccl_is_integer_v<_Tp> _CCCL_AND __cccl_is_integer_v<_Sp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto shl(const basic_vec<_Tp, _Abi>& __v, const _Sp __s) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_shl_scalar_generator<__vec_t, _Sp>{__v, __s}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Sp)
+_CCCL_REQUIRES(__cccl_is_integer_v<_Tp> _CCCL_AND __cccl_is_integer_v<_Sp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto shr(const basic_vec<_Tp, _Abi>& __v, const _Sp __s) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_shr_scalar_generator<__vec_t, _Sp>{__v, __s}};
+}
+
+_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1)
+_CCCL_REQUIRES(__simd_is_valid_rotate_v<_Tp0, _Abi0, _Tp1, _Abi1>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+rotl(const basic_vec<_Tp0, _Abi0>& __v0, const basic_vec<_Tp1, _Abi1>& __v1) noexcept
+{
+  using __vec0_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp0, _Abi0>;
+  using __vec1_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>;
+  return __vec0_t{__simd_rotl_generator<__vec0_t, __vec1_t>{__v0, __v1}};
+}
+
+_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1)
+_CCCL_REQUIRES(__simd_is_valid_rotate_v<_Tp0, _Abi0, _Tp1, _Abi1>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+rotr(const basic_vec<_Tp0, _Abi0>& __v0, const basic_vec<_Tp1, _Abi1>& __v1) noexcept
+{
+  using __vec0_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp0, _Abi0>;
+  using __vec1_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>;
+  return __vec0_t{__simd_rotr_generator<__vec0_t, __vec1_t>{__v0, __v1}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rotl(const basic_vec<_Tp, _Abi>& __v, const int __s) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_rotl_scalar_generator<__vec_t>{__v, __s}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rotr(const basic_vec<_Tp, _Abi>& __v, const int __s) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_rotr_scalar_generator<__vec_t>{__v, __s}};
+}
+
+template <typename _Tp, typename _Vp>
+using __simd_bit_count_result_t _CCCL_NODEBUG_ALIAS = rebind_t<make_signed_t<_Tp>, _Vp>;
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_width(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>;
+  return __result_t{__simd_bit_width_generator<__vec_t, __result_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countl_zero(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>;
+  return __result_t{__simd_countl_zero_generator<__vec_t, __result_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countl_one(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>;
+  return __result_t{__simd_countl_one_generator<__vec_t, __result_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countr_zero(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>;
+  return __result_t{__simd_countr_zero_generator<__vec_t, __result_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countr_one(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>;
+  return __result_t{__simd_countr_one_generator<__vec_t, __result_t>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto popcount(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t _CCCL_NODEBUG_ALIAS    = basic_vec<_Tp, _Abi>;
+  using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>;
+  return __result_t{__simd_popcount_generator<__vec_t, __result_t>{__v}};
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_BIT_SCALAR_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/complex_math.h

@@ -0,0 +1,523 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_COMPLEX_MATH_H
+#define _CUDA_STD___SIMD_COMPLEX_MATH_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__complex/complex.h>
+#include <cuda/std/__complex/exponential_functions.h>
+#include <cuda/std/__complex/hyperbolic_functions.h>
+#include <cuda/std/__complex/inverse_hyperbolic_functions.h>
+#include <cuda/std/__complex/inverse_trigonometric_functions.h>
+#include <cuda/std/__complex/logarithms.h>
+#include <cuda/std/__complex/math.h>
+#include <cuda/std/__complex/roots.h>
+#include <cuda/std/__complex/trigonometric_functions.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/exposition.h>
+#include <cuda/std/__simd/type_traits.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.complex.math], helper functors for element-wise complex operations
+
+struct __fn_real
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return __z.real();
+  }
+};
+
+struct __fn_imag
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return __z.imag();
+  }
+};
+
+struct __fn_abs
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::abs(__z);
+  }
+};
+
+struct __fn_arg
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::arg(__z);
+  }
+};
+
+struct __fn_norm
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::norm(__z);
+  }
+};
+
+struct __fn_conj
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::conj(__z);
+  }
+};
+
+struct __fn_proj
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::proj(__z);
+  }
+};
+
+struct __fn_exp
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::exp(__z);
+  }
+};
+
+struct __fn_log
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::log(__z);
+  }
+};
+
+struct __fn_log10
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::log10(__z);
+  }
+};
+
+struct __fn_sqrt
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::sqrt(__z);
+  }
+};
+
+struct __fn_sin
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::sin(__z);
+  }
+};
+
+struct __fn_asin
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::asin(__z);
+  }
+};
+
+struct __fn_cos
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::cos(__z);
+  }
+};
+
+struct __fn_acos
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::acos(__z);
+  }
+};
+
+struct __fn_tan
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::tan(__z);
+  }
+};
+
+struct __fn_atan
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::atan(__z);
+  }
+};
+
+struct __fn_sinh
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::sinh(__z);
+  }
+};
+
+struct __fn_asinh
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::asinh(__z);
+  }
+};
+
+struct __fn_cosh
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::cosh(__z);
+  }
+};
+
+struct __fn_acosh
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::acosh(__z);
+  }
+};
+
+struct __fn_tanh
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::tanh(__z);
+  }
+};
+
+struct __fn_atanh
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __z) const noexcept
+  {
+    return ::cuda::std::atanh(__z);
+  }
+};
+
+// Generic generator: applies a scalar functor to each element of a vec
+template <typename _Vp, typename _Func>
+struct __gen_complex_apply_unary
+{
+  const _Vp& __v_;
+  _Func __func_ = {};
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Idx) const
+  {
+    return __func_(__v_[__simd_size_type{_Idx::value}]);
+  }
+};
+
+// Generic binary generator: applies a scalar functor to corresponding elements of two vecs
+template <typename _Vp, typename _Func>
+struct __gen_complex_apply_binary
+{
+  const _Vp& __x_;
+  const _Vp& __y_;
+  _Func __func_ = {};
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Idx) const
+  {
+    return __func_(__x_[__simd_size_type{_Idx::value}], __y_[__simd_size_type{_Idx::value}]);
+  }
+};
+
+// [simd.complex.math], unary complex functions returning real-valued result
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<__simd_complex_value_type_t<_Tp>, basic_vec<_Tp, _Abi>>
+real(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t    = basic_vec<_Tp, _Abi>;
+  using __result_t = rebind_t<__simd_complex_value_type_t<_Tp>, __vec_t>;
+  return __result_t{__gen_complex_apply_unary<__vec_t, __fn_real>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<__simd_complex_value_type_t<_Tp>, basic_vec<_Tp, _Abi>>
+imag(const basic_vec<_Tp, _Abi>& __v) noexcept
+{
+  using __vec_t    = basic_vec<_Tp, _Abi>;
+  using __result_t = rebind_t<__simd_complex_value_type_t<_Tp>, __vec_t>;
+  return __result_t{__gen_complex_apply_unary<__vec_t, __fn_imag>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<__simd_complex_value_type_t<_Tp>, basic_vec<_Tp, _Abi>>
+abs(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t    = basic_vec<_Tp, _Abi>;
+  using __result_t = rebind_t<__simd_complex_value_type_t<_Tp>, __vec_t>;
+  return __result_t{__gen_complex_apply_unary<__vec_t, __fn_abs>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<__simd_complex_value_type_t<_Tp>, basic_vec<_Tp, _Abi>>
+arg(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t    = basic_vec<_Tp, _Abi>;
+  using __result_t = rebind_t<__simd_complex_value_type_t<_Tp>, __vec_t>;
+  return __result_t{__gen_complex_apply_unary<__vec_t, __fn_arg>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr rebind_t<__simd_complex_value_type_t<_Tp>, basic_vec<_Tp, _Abi>>
+norm(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t    = basic_vec<_Tp, _Abi>;
+  using __result_t = rebind_t<__simd_complex_value_type_t<_Tp>, __vec_t>;
+  return __result_t{__gen_complex_apply_unary<__vec_t, __fn_norm>{__v}};
+}
+
+// [simd.complex.math], unary complex functions returning complex-valued result
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> conj(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_conj>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> proj(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_proj>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> exp(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_exp>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> log(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_log>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> log10(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_log10>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> sqrt(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_sqrt>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> sin(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_sin>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> asin(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_asin>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> cos(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_cos>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> acos(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_acos>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> tan(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_tan>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> atan(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_atan>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> sinh(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_sinh>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> asinh(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_asinh>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> cosh(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_cosh>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> acosh(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_acosh>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> tanh(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_tanh>{__v}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> atanh(const basic_vec<_Tp, _Abi>& __v)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_unary<__vec_t, __fn_atanh>{__v}};
+}
+
+// [simd.complex.math], binary complex function objects
+
+struct __fn_polar_binary
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __rho, const _Tp& __theta) const
+  {
+    return ::cuda::std::polar(__rho, __theta);
+  }
+};
+
+struct __fn_pow_binary
+{
+  template <typename _Tp>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(const _Tp& __x, const _Tp& __y) const
+  {
+    return ::cuda::std::pow(__x, __y);
+  }
+};
+
+// [simd.complex.math], binary complex functions
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(::cuda::is_floating_point_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API rebind_t<::cuda::std::complex<_Tp>, basic_vec<_Tp, _Abi>>
+polar(const basic_vec<_Tp, _Abi>& __x, const basic_vec<_Tp, _Abi>& __y = {})
+{
+  using __vec_t    = basic_vec<_Tp, _Abi>;
+  using __result_t = rebind_t<::cuda::std::complex<_Tp>, __vec_t>;
+  return __result_t{__gen_complex_apply_binary<__vec_t, __fn_polar_binary>{__x, __y}};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_complex_vectorizable_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi>
+pow(const basic_vec<_Tp, _Abi>& __x, const basic_vec<_Tp, _Abi>& __y)
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__gen_complex_apply_binary<__vec_t, __fn_pow_binary>{__x, __y}};
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_COMPLEX_MATH_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/concepts.h

@@ -0,0 +1,157 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_CONCEPTS_H
+#define _CUDA_STD___SIMD_CONCEPTS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__type_traits/is_floating_point.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/convertible_to.h>
+#include <cuda/std/__concepts/equality_comparable.h>
+#include <cuda/std/__concepts/same_as.h>
+#include <cuda/std/__floating_point/conversion_rank_order.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__limits/numeric_limits.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_arithmetic.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__type_traits/void_t.h>
+#include <cuda/std/__utility/declval.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.expos], explicitly-convertible-to concept
+
+template <typename _From, typename _To>
+_CCCL_CONCEPT __explicitly_convertible_to = _CCCL_REQUIRES_EXPR((_From, _To))((static_cast<_To>(declval<_From>())));
+
+// [simd.expos], simd-vec-type concept
+
+template <typename _Vp>
+_CCCL_CONCEPT __simd_vec_type = _CCCL_REQUIRES_EXPR(
+  (_Vp))(requires(same_as<_Vp, basic_vec<typename _Vp::value_type, typename _Vp::abi_type>>),
+         requires(is_default_constructible_v<_Vp>));
+
+// [simd.expos], constexpr-wrapper-like concept
+
+template <typename _Tp>
+_CCCL_CONCEPT __constexpr_wrapper_like = _CCCL_REQUIRES_EXPR((_Tp))(
+  requires(convertible_to<_Tp, decltype(_Tp::value)>),
+  requires(equality_comparable_with<_Tp, decltype(_Tp::value)>),
+  requires(bool_constant<(_Tp() == _Tp::value)>::value),
+  requires(bool_constant<(static_cast<decltype(_Tp::value)>(_Tp()) == _Tp::value)>::value));
+
+// Covers all integral types including character types (char16_t, char32_t, wchar_t, char8_t),
+// which are excluded by __cccl_is_integer_v
+template <typename _From, typename _To>
+inline constexpr bool __is_integral__value_preserving_v =
+  is_integral_v<_From> && is_integral_v<_To> && numeric_limits<_From>::digits <= numeric_limits<_To>::digits
+  && (!is_signed_v<_From> || is_signed_v<_To>);
+
+// [conv.rank], integer conversion rank for [simd.ctor] p7
+
+template <typename _Tp>
+inline constexpr int __integer_conversion_rank = 0;
+
+template <>
+inline constexpr int __integer_conversion_rank<signed char> = 1;
+template <>
+inline constexpr int __integer_conversion_rank<unsigned char> = 1;
+template <>
+inline constexpr int __integer_conversion_rank<char> = 1;
+template <>
+inline constexpr int __integer_conversion_rank<short> = 2;
+template <>
+inline constexpr int __integer_conversion_rank<unsigned short> = 2;
+template <>
+inline constexpr int __integer_conversion_rank<int> = 3;
+template <>
+inline constexpr int __integer_conversion_rank<unsigned int> = 3;
+template <>
+inline constexpr int __integer_conversion_rank<long> = 4;
+template <>
+inline constexpr int __integer_conversion_rank<unsigned long> = 4;
+template <>
+inline constexpr int __integer_conversion_rank<long long> = 5;
+template <>
+inline constexpr int __integer_conversion_rank<unsigned long long> = 5;
+#if _CCCL_HAS_INT128()
+template <>
+inline constexpr int __integer_conversion_rank<__int128_t> = 6;
+template <>
+inline constexpr int __integer_conversion_rank<__uint128_t> = 6;
+#endif // _CCCL_HAS_INT128()
+
+// The conversion from an arithmetic type U to a vectorizable type T is value-preserving if all possible
+// values of U can be represented with type T. For floating-point pairs we defer to
+// __fp_is_implicit_conversion_v, which correctly handles unordered pairs such as __half / __nv_bfloat16.
+template <typename _From, typename _To>
+inline constexpr bool __is_value_preserving_v =
+  __is_integral__value_preserving_v<_From, _To>
+  || (::cuda::is_floating_point_v<_From> && ::cuda::is_floating_point_v<_To>
+      && __fp_is_implicit_conversion_v<_From, _To>)
+  || (is_integral_v<_From> && ::cuda::is_floating_point_v<_To>
+      && numeric_limits<_From>::digits <= numeric_limits<_To>::digits);
+
+template <typename _From, typename _ValueType, typename = void>
+inline constexpr bool __is_constexpr_wrapper_value_preserving_v = false;
+
+// The standard requires checking whether the specific compile-time value From::value is representable by _ValueType,
+// not whether the entire source type is value-preserving.
+template <typename _From, typename _ValueType>
+inline constexpr bool __is_constexpr_wrapper_value_preserving_v<_From, _ValueType, void_t<decltype(_From::value)>> =
+  is_arithmetic_v<remove_cvref_t<decltype(_From::value)>>
+  && (static_cast<remove_cvref_t<decltype(_From::value)>>(static_cast<_ValueType>(_From::value)) == _From::value);
+
+// [simd.ctor] implicit value constructor
+// - From is not an arithmetic type and does not satisfy constexpr-wrapper-like,
+// - From is an arithmetic type and the conversion from From to value_type is value-preserving
+// - From satisfies constexpr-wrapper-like, remove_cvref_t<decltype(From::value)> is an arithmetic type, and
+//   From::value is representable by value_type.
+template <typename _Up, typename _ValueType, typename _From = remove_cvref_t<_Up>>
+_CCCL_CONCEPT __is_value_ctor_implicit =
+  convertible_to<_Up, _ValueType>
+  && ((!is_arithmetic_v<_From> && !__constexpr_wrapper_like<_From>)
+      || (is_arithmetic_v<_From> && __is_value_preserving_v<_From, _ValueType>)
+      || (__constexpr_wrapper_like<_From> && __is_constexpr_wrapper_value_preserving_v<_From, _ValueType>) );
+
+// [simd.ctor] p7: explicit(see below) for basic_vec(const basic_vec<U, UAbi>&)
+// explicit evaluates to true if either:
+//   - conversion from U to value_type is not value-preserving, or
+//   - both U and value_type are integral and integer_conversion_rank(U) > rank(value_type), or
+//   - both U and value_type are floating-point and fp_conversion_rank(U) > rank(value_type)
+template <typename _Up, typename _ValueType>
+inline constexpr bool __is_vec_ctor_explicit =
+  !__is_value_preserving_v<_Up, _ValueType>
+  || (is_integral_v<_Up> && is_integral_v<_ValueType>
+      && __integer_conversion_rank<_Up> > __integer_conversion_rank<_ValueType>)
+  || (::cuda::is_floating_point_v<_Up> && ::cuda::is_floating_point_v<_ValueType>
+      && __fp_conv_rank_order_v<_Up, _ValueType> == __fp_conv_rank_order::__greater);
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_CONCEPTS_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/creation.h

@@ -0,0 +1,305 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_CREATION_H
+#define _CUDA_STD___SIMD_CREATION_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/basic_mask.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/exposition.h>
+#include <cuda/std/__simd/type_traits.h>
+#include <cuda/std/__simd/utility.h>
+#include <cuda/std/__tuple_dir/get.h>
+#include <cuda/std/__tuple_dir/tuple.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__utility/integer_sequence.h>
+#include <cuda/std/array>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+// Creation traits
+
+// TODO(fbusato): remove duplications across other PRs, move to a common place for basic_vec.h and basic_mask.h
+template <typename _Tp>
+inline constexpr bool __is_enabled_basic_vec_v = false;
+
+template <typename _Tp, typename _Abi>
+inline constexpr bool __is_enabled_basic_vec_v<basic_vec<_Tp, _Abi>> =
+  __is_vectorizable_v<_Tp> && __is_enabled_abi_v<_Abi>;
+
+template <typename _Tp>
+inline constexpr bool __is_enabled_basic_mask_v = false;
+
+template <size_t _Bytes, typename _Abi>
+inline constexpr bool __is_enabled_basic_mask_v<basic_mask<_Bytes, _Abi>> =
+  __is_vectorizable_byte_size_v<_Bytes> && __is_enabled_abi_v<_Abi>;
+
+// get the element size of a basic_mask
+// TODO(fbusato): remove if duplicated in other PRs
+template <typename _Tp>
+inline constexpr size_t __mask_element_size_v = 0;
+
+template <size_t _Bytes, typename _Abi>
+inline constexpr size_t __mask_element_size_v<basic_mask<_Bytes, _Abi>> = _Bytes;
+
+// Shorthand for integer_sequence<__simd_size_type, N>
+template <__simd_size_type... _Ns>
+using __simd_size_seq = integer_sequence<__simd_size_type, _Ns...>;
+
+// Shorthand for make_integer_sequence<__simd_size_type, N>
+template <__simd_size_type _Np>
+using __make_simd_size_seq = make_integer_sequence<__simd_size_type, _Np>;
+
+// "If basic_vec<typename T::value_type, Abi>::size() % T::size() is not 0, then
+// resize_t<basic_vec<typename T::value_type, Abi>::size() % T::size(), T> is valid and denotes
+// a type."
+//
+// Vector: resize_t<V> is valid if __deduce_abi_t<V> is a specialized ABI type
+// Mask: resize_t<M> is valid if __deduce_abi_t<__integer_from<M>> is a specialized ABI type
+template <typename _Tp,
+          typename _Abi,
+          typename _ValueType = typename _Tp::value_type,
+          size_t _Rem         = (basic_vec<_ValueType, _Abi>::__size % _Tp::__size)>
+inline constexpr bool __chunk_vec_tail_valid_v = _Rem == 0 || __is_enabled_abi_v<__deduce_abi_t<_ValueType, _Rem>>;
+
+template <typename _Tp,
+          typename _Abi,
+          size_t _ElemSize = __mask_element_size_v<_Tp>,
+          size_t _Rem      = (basic_mask<_ElemSize, _Abi>::__size % _Tp::__size)>
+inline constexpr bool __chunk_mask_tail_ok_v =
+  _Rem == 0 || __is_enabled_abi_v<__deduce_abi_t<__integer_from<_ElemSize>, _Rem>>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.creation], chunk building blocks
+
+// extract _Src[Offset + {0, 1, ..., M}]
+template <typename _Src, __simd_size_type _Offset>
+struct __chunk_generator
+{
+  const _Src& __src;
+
+  template <typename _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Idx) const noexcept
+  {
+    return __src[_Offset + _Idx::value];
+  }
+};
+
+// wrapper for __chunk_generator
+template <typename _SubVec, __simd_size_type _Offset, typename _Src>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _SubVec __make_chunk(const _Src& __src) noexcept
+{
+  return _SubVec{__chunk_generator<_Src, _Offset>{__src}};
+}
+
+// Exact divisor case: return array<_SubVec, N>
+template <typename _SubVec, typename _Src, __simd_size_type... _Js>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+__make_chunk_array(const _Src& __src, __simd_size_seq<_Js...>) noexcept
+{
+  using __result_t = ::cuda::std::array<_SubVec, sizeof...(_Js)>;
+  return __result_t{::cuda::std::simd::__make_chunk<_SubVec, _Js * _SubVec::__size>(__src)...};
+}
+
+template <typename _SubVec, typename _Tail, __simd_size_type _NHead, __simd_size_type _Ip>
+using __select_head_or_tail_t = ::cuda::std::conditional_t<(_Ip < _NHead), _SubVec, _Tail>;
+
+template <typename _SubVec, typename _Tail, __simd_size_type _NHead, __simd_size_type _Ip, typename _Src>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __select_head_or_tail_t<_SubVec, _Tail, _NHead, _Ip>
+__make_chunk_tuple_element(const _Src& __src) noexcept
+{
+  if constexpr (_Ip < _NHead) // use _SubVec
+  {
+    return ::cuda::std::simd::__make_chunk<_SubVec, _Ip * _SubVec::__size>(__src);
+  }
+  else // use _Tail
+  {
+    return ::cuda::std::simd::__make_chunk<_Tail, _NHead * _SubVec::__size>(__src);
+  }
+}
+
+// Remainder case: return tuple<_SubVec, ..., _SubVec, _Tail>
+// where _Tail is resize_t<src.size() % _SubVec::size(), _SubVec>
+template <typename _SubVec, typename _Tail, typename _Src, __simd_size_type... _Js>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+__make_chunk_tuple(const _Src& __src, __simd_size_seq<_Js...>) noexcept
+{
+  constexpr __simd_size_type __nhead = sizeof...(_Js) - 1; // all elements except the last one (_Tail)
+  using __result_t                   = ::cuda::std::tuple<__select_head_or_tail_t<_SubVec, _Tail, __nhead, _Js>...>;
+  return __result_t{::cuda::std::simd::__make_chunk_tuple_element<_SubVec, _Tail, __nhead, _Js>(__src)...};
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.creation] chunk
+// split a SIMD vector of size N into a sequence of N/M sub-vectors of size M
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_enabled_basic_vec_v<_Tp> _CCCL_AND(__chunk_vec_tail_valid_v<_Tp, _Abi>))
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto chunk(const basic_vec<typename _Tp::value_type, _Abi>& __src) noexcept
+{
+  using __src_t                      = basic_vec<typename _Tp::value_type, _Abi>;
+  constexpr __simd_size_type __nhead = __src_t::__size / _Tp::__size;
+  constexpr __simd_size_type __rem   = __src_t::__size % _Tp::__size;
+  if constexpr (__rem == 0) // exact divisor case
+  {
+    return ::cuda::std::simd::__make_chunk_array<_Tp>(__src, __make_simd_size_seq<__nhead>{});
+  }
+  else // remainder case
+  {
+    using __tail_t = resize_t<__rem, _Tp>;
+    return ::cuda::std::simd::__make_chunk_tuple<_Tp, __tail_t>(__src, __make_simd_size_seq<__nhead + 1>{});
+  }
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(__is_enabled_basic_mask_v<_Tp> _CCCL_AND(__chunk_mask_tail_ok_v<_Tp, _Abi>))
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+chunk(const basic_mask<__mask_element_size_v<_Tp>, _Abi>& __src) noexcept
+{
+  using __src_t                      = basic_mask<__mask_element_size_v<_Tp>, _Abi>;
+  constexpr __simd_size_type __nhead = __src_t::__size / _Tp::__size;
+  constexpr __simd_size_type __rem   = __src_t::__size % _Tp::__size;
+  if constexpr (__rem == 0)
+  {
+    return ::cuda::std::simd::__make_chunk_array<_Tp>(__src, __make_simd_size_seq<__nhead>{});
+  }
+  else
+  {
+    using __tail_t = resize_t<__rem, _Tp>;
+    return ::cuda::std::simd::__make_chunk_tuple<_Tp, __tail_t>(__src, __make_simd_size_seq<__nhead + 1>{});
+  }
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.creation], chunk<M>, with the size of the sub-vector M is user-specified
+
+template <__simd_size_type _Mp, typename _Up, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto chunk(const basic_vec<_Up, _Abi>& __src) noexcept
+{
+  static_assert(_Mp > 0, "_Mp must be greater than 0"); // avoid division by zero
+  using __sub_vec_t = resize_t<_Mp, basic_vec<_Up, _Abi>>;
+  return ::cuda::std::simd::chunk<__sub_vec_t, _Abi>(__src);
+}
+
+template <__simd_size_type _Mp, size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto chunk(const basic_mask<_Bytes, _Abi>& __src) noexcept
+{
+  static_assert(_Mp > 0, "_Mp must be greater than 0"); // avoid division by zero
+  using __sub_vec_t = resize_t<_Mp, basic_mask<_Bytes, _Abi>>;
+  return ::cuda::std::simd::chunk<__sub_vec_t, _Abi>(__src);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.creation], cat
+// concatenate a sequence of SIMD vectors/masks
+
+// given the index _Ip, return the index of the corresponding SIMD vector/mask in the range [0, N)
+template <__simd_size_type _Ip, __simd_size_type... _Sizes>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL size_t __cat_arg_index(__simd_size_seq<_Sizes...>) noexcept
+{
+  const __simd_size_type __sizes[] = {_Sizes...};
+  __simd_size_type __prefix        = 0;
+  for (size_t __k = 0; __k < sizeof...(_Sizes); ++__k)
+  {
+    if (_Ip < __prefix + __sizes[__k])
+    {
+      return __k;
+    }
+    __prefix += __sizes[__k];
+  }
+  // this line is not reachable because _Ip is in the range [0, M), where M is the total size
+  // of the concatenation of the V1, V2, ..., VN
+  _CCCL_UNREACHABLE();
+  return 0; // MVSC workaround
+}
+
+// Compute the local prefix sum (number of elements before arg _Kp) for a given target arg index _Kp.
+template <size_t _Kp, __simd_size_type... _Sizes>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL __simd_size_type
+__cat_local_prefix(__simd_size_seq<_Sizes...>) noexcept
+{
+  __simd_size_type __prefix        = 0;
+  const __simd_size_type __sizes[] = {_Sizes...};
+  for (size_t __k = 0; __k != _Kp; ++__k)
+  {
+    __prefix += __sizes[__k];
+  }
+  return __prefix;
+}
+
+// Given a tuple of SIMD vectors/masks  {V1, V2, ..., VN}
+// - Let M be the total size of the concatenation of the V1, V2, ..., VN
+// - Create a generator where __i (_Ic) is in the range [0, M)
+// Algorithm:
+// 1. Given __i, compute the index __k of the corresponding SIMD vector/mask in the range [0, N)
+// 2. Compute the total number of elements before the __k-th SIMD vector/mask (prefix sum)
+// 3. Pick the __k-th SIMD vector/mask and return the element in the position (__i - __prefix)
+template <typename... _Vs>
+struct __cat_generator
+{
+  ::cuda::std::tuple<const _Vs&...> __args_;
+
+  template <typename _Ic>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(_Ic) const noexcept
+  {
+    constexpr __simd_size_seq<_Vs::__size...> __seq_sizes{};
+    constexpr __simd_size_type __i      = _Ic::value;
+    constexpr size_t __k                = ::cuda::std::simd::__cat_arg_index<__i>(__seq_sizes);
+    constexpr __simd_size_type __prefix = ::cuda::std::simd::__cat_local_prefix<__k>(__seq_sizes);
+    return ::cuda::std::get<__k>(__args_)[__i - __prefix];
+  }
+};
+
+template <typename... _Vs>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __cat_generator<_Vs...> __make_cat_generator(const _Vs&... __xs) noexcept
+{
+  return __cat_generator<_Vs...>{::cuda::std::tuple<const _Vs&...>{__xs...}};
+}
+
+// c++ specification has no explicit constraints for cat()
+template <typename _Tp, typename _Abi0, typename... _Abis>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+cat(const basic_vec<_Tp, _Abi0>& __x0, const basic_vec<_Tp, _Abis>&... __xs) noexcept
+{
+  constexpr __simd_size_type __total = (basic_vec<_Tp, _Abi0>::__size + ... + basic_vec<_Tp, _Abis>::__size);
+  using __result_t                   = resize_t<__total, basic_vec<_Tp, _Abi0>>;
+  return __result_t{::cuda::std::simd::__make_cat_generator(__x0, __xs...)};
+}
+
+template <size_t _Bytes, typename _Abi0, typename... _Abis>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto
+cat(const basic_mask<_Bytes, _Abi0>& __x0, const basic_mask<_Bytes, _Abis>&... __xs) noexcept
+{
+  constexpr __simd_size_type __total = (basic_mask<_Bytes, _Abi0>::__size + ... + basic_mask<_Bytes, _Abis>::__size);
+  using __result_t                   = resize_t<__total, basic_mask<_Bytes, _Abi0>>;
+  return __result_t{::cuda::std::simd::__make_cat_generator(__x0, __xs...)};
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_CREATION_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/exposition.h

@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_EXPOSITION_H
+#define _CUDA_STD___SIMD_EXPOSITION_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__fwd/complex.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/complex.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__type_traits/is_const.h>
+#include <cuda/std/__type_traits/is_extended_arithmetic.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/is_volatile.h>
+#include <cuda/std/__type_traits/make_nbit_int.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.expos], exposition-only helpers
+
+template <size_t _Bytes>
+using __integer_from = __make_nbit_int_t<_Bytes * 8, true>;
+
+// complex<T> where T is a vectorizable floating-point type
+template <typename _Tp>
+inline constexpr bool __is_complex_vectorizable_v = false;
+
+template <typename _Tp>
+inline constexpr bool __is_complex_vectorizable_v<::cuda::std::complex<_Tp>> = true;
+
+template <typename _Tp>
+inline constexpr bool __is_complex_vectorizable_v<::cuda::complex<_Tp>> = true;
+
+#if _CCCL_HAS_HOST_STD_LIB()
+
+template <typename _Tp>
+inline constexpr bool __is_complex_vectorizable_v<::std::complex<_Tp>> = true;
+
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+// [simd.expos], vectorizable types:
+// all standard integer types, character types, and the types float and double ([basic.fundamental]);
+// std::float16_t, std::float32_t, and std::float64_t if defined ([basic.extended.fp]); and
+// complex<T> where T is a vectorizable floating-point type.
+template <typename _Tp>
+inline constexpr bool __is_vectorizable_v =
+  (__is_extended_arithmetic_v<_Tp> || __is_complex_vectorizable_v<_Tp>)
+  && !is_same_v<_Tp, bool> && !is_const_v<_Tp> && !is_volatile_v<_Tp>;
+
+// [simd.expos], simd-complex-value-type
+template <typename _Tp>
+using __simd_complex_value_type_t = typename _Tp::value_type;
+
+template <typename _Tp, typename _Abi>
+inline constexpr __simd_size_type __simd_size_v = 0;
+
+template <typename _Tp, __simd_size_type _Np>
+inline constexpr __simd_size_type __simd_size_v<_Tp, fixed_size<_Np>> = _Np;
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_EXPOSITION_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/flag.h

@@ -0,0 +1,113 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_FLAG_H
+#define _CUDA_STD___SIMD_FLAG_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/pow2.h>
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__type_traits/is_same.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.expos], exposition-only flag types
+
+struct __convert_flag
+{};
+
+struct __aligned_flag
+{};
+
+template <size_t _Np>
+struct __overaligned_flag
+{
+  static_assert(::cuda::__is_valid_alignment(_Np), "Overaligned flag requires a power-of-2 alignment");
+};
+
+template <typename _Tp>
+inline constexpr bool __is_flag_type_v = false;
+
+template <>
+inline constexpr bool __is_flag_type_v<__convert_flag> = true;
+
+template <>
+inline constexpr bool __is_flag_type_v<__aligned_flag> = true;
+
+template <size_t _Np>
+inline constexpr bool __is_flag_type_v<__overaligned_flag<_Np>> = true;
+
+template <typename _Flag>
+inline constexpr size_t __overaligned_value_v = 0;
+
+template <size_t _Np>
+inline constexpr size_t __overaligned_value_v<__overaligned_flag<_Np>> = _Np;
+
+// [simd.flags.overview], class template flags
+
+template <typename... _Flags>
+struct flags
+{
+  static_assert((true && ... && __is_flag_type_v<_Flags>),
+                "Every flag type must be one of convert_flag, aligned_flag, or overaligned_flag<N>");
+  static_assert((0 + ... + static_cast<int>(__overaligned_value_v<_Flags> != 0)) <= 1,
+                "At most one overaligned_flag is allowed");
+  // we cannot use __is_valid_alignment because 0 has a different meaning
+  static_assert((true && ...
+                 && (__overaligned_value_v<_Flags> == 0 || ::cuda::is_power_of_two(__overaligned_value_v<_Flags>))),
+                "Overaligned flag requires a power-of-2 alignment");
+
+  // [simd.flags.oper], flags operators
+  template <typename... _Other>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend _CCCL_CONSTEVAL flags<_Flags..., _Other...>
+  operator|(flags, flags<_Other...>) noexcept
+  {
+    return {};
+  }
+};
+
+// [simd.flags], flag constants
+
+inline constexpr flags<> flag_default{};
+inline constexpr flags<__convert_flag> flag_convert{};
+inline constexpr flags<__aligned_flag> flag_aligned{};
+
+template <size_t _Np>
+inline constexpr flags<__overaligned_flag<_Np>> flag_overaligned{};
+
+template <typename... _Flags>
+inline constexpr bool __has_convert_flag_v = (false || ... || is_same_v<_Flags, __convert_flag>);
+
+template <typename... _Flags>
+inline constexpr bool __has_aligned_flag_v = (false || ... || is_same_v<_Flags, __aligned_flag>);
+
+template <typename... _Flags>
+inline constexpr bool __has_overaligned_flag_v = (false || ... || (__overaligned_value_v<_Flags> != 0));
+
+template <typename... _Flags>
+inline constexpr size_t __overaligned_alignment_v = (size_t{0} | ... | __overaligned_value_v<_Flags>);
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_FLAG_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/iterator.h

@@ -0,0 +1,323 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_ITERATOR_H
+#define _CUDA_STD___SIMD_ITERATOR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__iterator/advance.h>
+#include <cuda/std/__iterator/default_sentinel.h>
+#include <cuda/std/__iterator/distance.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__memory/addressof.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/remove_const.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.iterator], class template __simd_iterator
+template <typename _Vp>
+class __simd_iterator
+{
+  _Vp* __data_               = nullptr;
+  __simd_size_type __offset_ = 0;
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator(_Vp& __data, const __simd_size_type __offset) noexcept
+      : __data_{::cuda::std::addressof(__data)}
+      , __offset_{__offset}
+  {
+    _CCCL_ASSERT(__data_ != nullptr, "cuda::std::simd::__simd_iterator: data is nullptr");
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+  }
+
+  template <typename, typename, typename>
+  friend class basic_vec;
+
+  template <size_t, typename, typename>
+  friend class basic_mask;
+
+  template <typename>
+  friend class __simd_iterator;
+
+public:
+  using value_type        = typename _Vp::value_type;
+  using iterator_category = input_iterator_tag;
+  using iterator_concept  = random_access_iterator_tag;
+  using difference_type   = __simd_size_type;
+
+  _CCCL_HIDE_FROM_ABI constexpr __simd_iterator() noexcept                                  = default;
+  _CCCL_HIDE_FROM_ABI constexpr __simd_iterator(const __simd_iterator&) noexcept            = default;
+  _CCCL_HIDE_FROM_ABI constexpr __simd_iterator& operator=(const __simd_iterator&) noexcept = default;
+
+  // non-const to const converting constructor
+  // workaround for MSVC (cannot used is_const_v<_Vp>)
+  // _Vp = const T:  const T == const T
+  // _Vp = T:        const T != T
+  _CCCL_TEMPLATE(typename _Up = remove_const_t<_Vp>)
+  _CCCL_REQUIRES(is_same_v<const _Up, _Vp>)
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator(const __simd_iterator<_Up>& __i) noexcept
+      : __data_{__i.__data_}
+      , __offset_{__i.__offset_}
+  {}
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr value_type operator*() const noexcept
+  {
+    _CCCL_ASSERT(__data_ != nullptr, "cuda::std::simd::__simd_iterator: data is nullptr");
+    return (*__data_)[__offset_];
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator& operator++() noexcept
+  {
+    ++__offset_;
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return *this;
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator operator++(int) noexcept
+  {
+    const __simd_iterator __tmp = *this;
+    ++__offset_;
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return __tmp;
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator& operator--() noexcept
+  {
+    --__offset_;
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return *this;
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator operator--(int) noexcept
+  {
+    const __simd_iterator __tmp = *this;
+    --__offset_;
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return __tmp;
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator& operator+=(const difference_type __n) noexcept
+  {
+    __offset_ += __n;
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return *this;
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr __simd_iterator& operator-=(const difference_type __n) noexcept
+  {
+    __offset_ -= __n;
+    _CCCL_ASSERT(::cuda::in_range(__offset_, __simd_size_type{0}, _Vp::__size),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return *this;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr value_type operator[](const difference_type __n) const noexcept
+  {
+    _CCCL_ASSERT(__data_ != nullptr, "cuda::std::simd::__simd_iterator: data is nullptr");
+    _CCCL_ASSERT(::cuda::in_range(__offset_ + __n, __simd_size_type{0}, _Vp::__size - 1),
+                 "cuda::std::simd::__simd_iterator: offset is out of range");
+    return (*__data_)[__offset_ + __n];
+  }
+
+  // [simd.iterator] comparisons
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator==(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    return __a.__data_ == __b.__data_ && __a.__offset_ == __b.__offset_;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator==(const __simd_iterator __i, default_sentinel_t) noexcept
+  {
+    return __i.__offset_ == _Vp::__size;
+  }
+
+#if _CCCL_STD_VER <= 2017
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API friend constexpr bool operator!=(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    return !(__a == __b);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator!=(const __simd_iterator __i, const default_sentinel_t __s) noexcept
+  {
+    return !(__i == __s);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator!=(const default_sentinel_t __s, const __simd_iterator __i) noexcept
+  {
+    return !(__i == __s);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator==(const default_sentinel_t __s, const __simd_iterator __i) noexcept
+  {
+    return __i == __s;
+  }
+#endif // _CCCL_STD_VER <= 2017
+
+#if _LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API friend constexpr auto operator<=>(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    _CCCL_ASSERT(__a.__data_ == __b.__data_, "cuda::std::simd::__simd_iterator: iterators refer to different objects");
+    return __a.__offset_ <=> __b.__offset_;
+  }
+#else // ^^^ _LIBCUDACXX_HAS_SPACESHIP_OPERATOR() ^^^ / vvv !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR() vvv
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API friend constexpr bool operator<(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    _CCCL_ASSERT(__a.__data_ == __b.__data_, "cuda::std::simd::__simd_iterator: iterators refer to different objects");
+    return __a.__offset_ < __b.__offset_;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator>(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    _CCCL_ASSERT(__a.__data_ == __b.__data_, "cuda::std::simd::__simd_iterator: iterators refer to different objects");
+    return __b.__offset_ < __a.__offset_;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator<=(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    return !(__b < __a);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool
+  operator>=(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    return !(__a < __b);
+  }
+#endif // !_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()
+
+  // [simd.iterator] arithmetic
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __simd_iterator
+  operator+(__simd_iterator __i, const difference_type __n) noexcept
+  {
+    return __i += __n;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __simd_iterator
+  operator+(const difference_type __n, __simd_iterator __i) noexcept
+  {
+    return __i += __n;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __simd_iterator
+  operator-(__simd_iterator __i, const difference_type __n) noexcept
+  {
+    return __i -= __n;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr difference_type
+  operator-(const __simd_iterator __a, const __simd_iterator __b) noexcept
+  {
+    _CCCL_ASSERT(__a.__data_ == __b.__data_, "cuda::std::simd::__simd_iterator: iterators refer to different objects");
+    return __a.__offset_ - __b.__offset_;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr difference_type
+  operator-(const __simd_iterator __i, default_sentinel_t) noexcept
+  {
+    return __i.__offset_ - _Vp::__size;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr difference_type
+  operator-(default_sentinel_t, const __simd_iterator __i) noexcept
+  {
+    return _Vp::__size - __i.__offset_;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <typename _Vp>
+struct iterator_traits<simd::__simd_iterator<_Vp>>
+{
+  using _Iter             = simd::__simd_iterator<_Vp>;
+  using iterator_concept  = typename _Iter::iterator_concept;
+  using iterator_category = typename _Iter::iterator_category;
+  using value_type        = typename _Iter::value_type;
+  using difference_type   = typename _Iter::difference_type;
+  using pointer           = void;
+  using reference         = value_type;
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#if _CCCL_HAS_HOST_STD_LIB()
+_CCCL_BEGIN_NAMESPACE_STD
+
+template <typename _Diff, typename _Vp>
+_CCCL_HOST_API constexpr void advance(::cuda::std::simd::__simd_iterator<_Vp>& __iter, const _Diff __diff) noexcept
+{
+  ::cuda::std::advance(__iter, __diff);
+}
+
+template <typename _Vp>
+[[nodiscard]] _CCCL_HOST_API constexpr typename ::cuda::std::simd::__simd_iterator<_Vp>::difference_type distance(
+  const ::cuda::std::simd::__simd_iterator<_Vp> __first, const ::cuda::std::simd::__simd_iterator<_Vp> __last) noexcept
+{
+  return ::cuda::std::distance(__first, __last);
+}
+
+template <typename _Vp>
+[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::simd::__simd_iterator<_Vp>
+next(::cuda::std::simd::__simd_iterator<_Vp> __iter,
+     const typename ::cuda::std::simd::__simd_iterator<_Vp>::difference_type __n = 1) noexcept
+{
+  ::cuda::std::advance(__iter, __n);
+  return __iter;
+}
+
+template <typename _Vp>
+[[nodiscard]] _CCCL_HOST_API constexpr ::cuda::std::simd::__simd_iterator<_Vp>
+prev(::cuda::std::simd::__simd_iterator<_Vp> __iter,
+     const typename ::cuda::std::simd::__simd_iterator<_Vp>::difference_type __n = 1) noexcept
+{
+  ::cuda::std::advance(__iter, -__n);
+  return __iter;
+}
+
+_CCCL_END_NAMESPACE_STD
+#endif // _CCCL_HAS_HOST_STD_LIB()
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_ITERATOR_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/load.h

@@ -0,0 +1,343 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_LOAD_H
+#define _CUDA_STD___SIMD_LOAD_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/__memory/ptr_rebind.h>
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/same_as.h>
+#include <cuda/std/__cstring/memcpy.h>
+#include <cuda/std/__iterator/concepts.h>
+#include <cuda/std/__iterator/distance.h>
+#include <cuda/std/__iterator/incrementable_traits.h>
+#include <cuda/std/__iterator/readable_traits.h>
+#include <cuda/std/__memory/assume_aligned.h>
+#include <cuda/std/__memory/pointer_traits.h>
+#include <cuda/std/__ranges/concepts.h>
+#include <cuda/std/__ranges/data.h>
+#include <cuda/std/__ranges/size.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/concepts.h>
+#include <cuda/std/__simd/flag.h>
+#include <cuda/std/__simd/utility.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/cmp.h>
+#include <cuda/std/__utility/forward.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.loadstore] helper: resolves default V template parameter for load functions
+// When _Vp = void (default), resolves to basic_vec<_Up>; otherwise uses the explicit _Vp
+template <typename _Vp, typename _Up>
+using __load_vec_t = conditional_t<is_void_v<_Vp>, basic_vec<_Up>, _Vp>;
+
+template <typename _Result, typename _Up, typename... _Flags>
+_CCCL_HOST_DEVICE_API constexpr void
+__check_load_preconditions(const _Up* __ptr, flags<_Flags...>, const __simd_size_type __count = 1) noexcept
+{
+  using __value_t = typename _Result::value_type;
+  static_assert(same_as<remove_cvref_t<_Result>, _Result>, "V must not be a reference or cv-qualified type");
+
+  static_assert(__is_vectorizable_v<__value_t> && __is_enabled_abi_v<typename _Result::abi_type>,
+                "cuda::std::simd::load: V must be an enabled specialization of basic_vec");
+  static_assert(__is_vectorizable_v<_Up>, "range_value_t<R> must be a vectorizable type");
+
+  static_assert(__explicitly_convertible_to<_Up, __value_t>,
+                "cuda::std::simd::load: range_value_t<R> must satisfy explicitly-convertible-to<value_type>");
+
+  static_assert(__has_convert_flag_v<_Flags...> || __is_value_preserving_v<_Up, __value_t>,
+                "cuda::std::simd::load: Conversion from range_value_t<R> to value_type is not value-preserving; use "
+                "flag_convert");
+
+  _CCCL_ASSERT(__count == 0 || __ptr != nullptr, "cuda::std::simd::load: range data is nullptr");
+  ::cuda::std::simd::__assert_load_store_alignment<_Result, _Up, _Flags...>(__ptr);
+}
+
+// [simd.loadstore] helper: core partial load from pointer + count + mask
+template <typename _Result, typename _Up, typename... _Flags>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Result __partial_load_from_ptr(
+  const _Up* __ptr,
+  const __simd_size_type __count,
+  const typename _Result::mask_type& __mask,
+  flags<_Flags...> __flags = {}) noexcept
+{
+  using __value_t = typename _Result::value_type;
+  ::cuda::std::simd::__check_load_preconditions<_Result>(__ptr, __flags, __count);
+  constexpr auto __simd_size = _Result::__size;
+
+  _Result __result;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __simd_size; ++__i)
+  {
+    const auto __value = (__mask[__i] && __i < __count) ? static_cast<__value_t>(__ptr[__i]) : __value_t{};
+    __result.__set(__i, __value);
+  }
+  return __result;
+}
+
+template <typename _Result, typename _Up, typename... _Flags>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Result
+__full_load_from_ptr(const _Up* __ptr, const typename _Result::mask_type& __mask, flags<_Flags...> __flags) noexcept
+{
+  ::cuda::std::simd::__check_load_preconditions<_Result>(__ptr, __flags);
+  constexpr bool __has_aligned_flag = __has_aligned_flag_v<_Flags...>;
+
+  if constexpr (__has_aligned_flag || __has_overaligned_flag_v<_Flags...>)
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      // minimum condition for pointer alignment
+      constexpr auto __base_alignment = __has_aligned_flag ? alignment_v<_Result, _Up> : alignof(_Up);
+      constexpr auto __ptr_alignment  = ::cuda::std::max(__base_alignment, __overaligned_value_v<_Flags...>);
+      constexpr auto __simd_size      = _Result::__size;
+      constexpr auto __data_size      = __simd_size * sizeof(_Up);
+
+      _Up __tmp[__simd_size]{};
+      // vectorized load from pointer
+      if constexpr (__is_cuda_vectorizable_v<_Up> && __simd_size > 1 && __ptr_alignment >= __data_size
+                    && ::cuda::__is_valid_alignment(__data_size))
+      {
+        struct alignas(__data_size) __aligned_t
+        {
+          _Up __data[__simd_size];
+        };
+        // nvcc performance bug: memcpy from pointer could not be vectorized
+        const auto __aligned_ptr = ::cuda::ptr_rebind<__aligned_t>(__ptr);
+        const auto __data        = *::cuda::std::assume_aligned<__ptr_alignment>(__aligned_ptr);
+        ::cuda::std::memcpy(&__tmp, &__data, sizeof(__tmp));
+      }
+      // rely on compiler vectorization
+      else
+      {
+        const auto __aligned_ptr = ::cuda::std::assume_aligned<__ptr_alignment>(__ptr);
+        _CCCL_PRAGMA_UNROLL_FULL()
+        for (__simd_size_type __i = 0; __i < __simd_size; ++__i)
+        {
+          __tmp[__i] = __aligned_ptr[__i];
+        }
+      }
+      using __value_t = typename _Result::value_type;
+      _Result __result;
+      _CCCL_PRAGMA_UNROLL_FULL()
+      for (__simd_size_type __i = 0; __i < __simd_size; ++__i)
+      {
+        const auto __value = (!__mask[__i]) ? __value_t{} : static_cast<__value_t>(__tmp[__i]);
+        __result.__set(__i, __value);
+      }
+      return __result;
+    }
+  }
+  return ::cuda::std::simd::__partial_load_from_ptr<_Result>(__ptr, _Result::__size, __mask, __flags);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.loadstore] partial_load
+
+// partial_load: range, masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>
+partial_load(_Range&& __r,
+             const typename __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>::mask_type& __mask,
+             flags<_Flags...> __f = {})
+{
+  using __result_t        = __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>;
+  const auto __range_size = ::cuda::std::ranges::size(__r);
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__range_size),
+               "cuda::std::simd::partial_load: range size out of range");
+  const auto __size = static_cast<__simd_size_type>(__range_size);
+
+  return ::cuda::std::simd::__partial_load_from_ptr<__result_t>(::cuda::std::ranges::data(__r), __size, __mask, __f);
+}
+
+// partial_load: range, no mask
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>
+partial_load(_Range&& __r, flags<_Flags...> __f = {})
+{
+  using __result_t           = __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>;
+  constexpr auto __true_mask = typename __result_t::mask_type(true);
+
+  return ::cuda::std::simd::partial_load<_Vp>(::cuda::std::forward<_Range>(__r), __true_mask, __f);
+}
+
+// partial_load: iterator + count, masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>> partial_load(
+  const _Ip __first,
+  const iter_difference_t<_Ip> __n,
+  const typename __load_vec_t<_Vp, iter_value_t<_Ip>>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  using __result_t = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__n), "cuda::std::simd::partial_load: n out of range");
+  const auto __ptr  = ::cuda::std::to_address(__first);
+  const auto __size = static_cast<__simd_size_type>(__n);
+
+  return ::cuda::std::simd::__partial_load_from_ptr<__result_t>(__ptr, __size, __mask, __f);
+}
+
+// partial_load: iterator + count, no mask
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>>
+partial_load(const _Ip __first, const iter_difference_t<_Ip> __n, flags<_Flags...> __f = {})
+{
+  using __result_t           = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  constexpr auto __true_mask = typename __result_t::mask_type(true);
+
+  return ::cuda::std::simd::partial_load<_Vp>(__first, __n, __true_mask, __f);
+}
+
+// partial_load: iterator + sentinel, masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>> partial_load(
+  const _Ip __first,
+  const _Sp __last,
+  const typename __load_vec_t<_Vp, iter_value_t<_Ip>>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  using __result_t      = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  const auto __ptr      = ::cuda::std::to_address(__first);
+  const auto __distance = ::cuda::std::distance(__first, __last);
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__distance),
+               "cuda::std::simd::partial_load: distance(first, last) out of range");
+  const auto __size = static_cast<__simd_size_type>(__distance);
+
+  return ::cuda::std::simd::__partial_load_from_ptr<__result_t>(__ptr, __size, __mask, __f);
+}
+
+// partial_load: iterator + sentinel, no mask
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>>
+partial_load(const _Ip __first, const _Sp __last, flags<_Flags...> __f = {})
+{
+  using __result_t           = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  constexpr auto __true_mask = typename __result_t::mask_type(true);
+
+  return ::cuda::std::simd::partial_load<_Vp>(__first, __last, __true_mask, __f);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.loadstore] unchecked_load
+
+// unchecked_load: range, masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>
+unchecked_load(_Range&& __r,
+               const typename __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>::mask_type& __mask,
+               flags<_Flags...> __f = {})
+{
+  using __result_t = __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>;
+  if constexpr (__has_static_size<_Range>)
+  {
+    static_assert(__static_range_size_v<_Range> >= __result_t::__size,
+                  "cuda::std::simd::unchecked_load: requires ::cuda::std::ranges::size(r) >= V::size()");
+  }
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(::cuda::std::ranges::size(__r), __result_t::__size),
+               "cuda::std::simd::unchecked_load: requires ::cuda::std::ranges::size(r) >= V::size()");
+
+  return ::cuda::std::simd::__full_load_from_ptr<__result_t>(::cuda::std::ranges::data(__r), __mask, __f);
+}
+
+// unchecked_load: range, no mask
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>
+unchecked_load(_Range&& __r, flags<_Flags...> __f = {})
+{
+  using __result_t           = __load_vec_t<_Vp, ::cuda::std::ranges::range_value_t<_Range>>;
+  constexpr auto __true_mask = typename __result_t::mask_type(true);
+
+  return ::cuda::std::simd::unchecked_load<_Vp>(::cuda::std::forward<_Range>(__r), __true_mask, __f);
+}
+
+// unchecked_load: iterator + count, masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>> unchecked_load(
+  const _Ip __first,
+  const iter_difference_t<_Ip> __n,
+  const typename __load_vec_t<_Vp, iter_value_t<_Ip>>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  using __result_t = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(__n, __result_t::__size),
+               "cuda::std::simd::unchecked_load: requires n >= V::size()");
+  const auto __ptr = ::cuda::std::to_address(__first);
+
+  return ::cuda::std::simd::__full_load_from_ptr<__result_t>(__ptr, __mask, __f);
+}
+
+// unchecked_load: iterator + count, no mask
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>>
+unchecked_load(const _Ip __first, const iter_difference_t<_Ip> __n, flags<_Flags...> __f = {})
+{
+  using __result_t           = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  constexpr auto __true_mask = typename __result_t::mask_type(true);
+
+  return ::cuda::std::simd::unchecked_load<_Vp>(__first, __n, __true_mask, __f);
+}
+
+// unchecked_load: iterator + sentinel, masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>> unchecked_load(
+  const _Ip __first,
+  const _Sp __last,
+  const typename __load_vec_t<_Vp, iter_value_t<_Ip>>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  using __result_t = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(::cuda::std::distance(__first, __last), __result_t::__size),
+               "unchecked_load requires distance(first, last) >= V::size()");
+
+  return ::cuda::std::simd::__full_load_from_ptr<__result_t>(::cuda::std::to_address(__first), __mask, __f);
+}
+
+// unchecked_load: iterator + sentinel, no mask
+_CCCL_TEMPLATE(typename _Vp = void, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __load_vec_t<_Vp, iter_value_t<_Ip>>
+unchecked_load(const _Ip __first, const _Sp __last, flags<_Flags...> __f = {})
+{
+  using __result_t           = __load_vec_t<_Vp, iter_value_t<_Ip>>;
+  constexpr auto __true_mask = typename __result_t::mask_type(true);
+
+  return ::cuda::std::simd::unchecked_load<_Vp>(__first, __last, __true_mask, __f);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_LOAD_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math.h

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_H
+#define _CUDA_STD___SIMD_MATH_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__simd/math/abs.h>
+#include <cuda/std/__simd/math/classification.h>
+#include <cuda/std/__simd/math/common.h>
+#include <cuda/std/__simd/math/comparison.h>
+#include <cuda/std/__simd/math/exponential.h>
+#include <cuda/std/__simd/math/fma.h>
+#include <cuda/std/__simd/math/lerp.h>
+#include <cuda/std/__simd/math/manipulation.h>
+#include <cuda/std/__simd/math/min_max.h>
+#include <cuda/std/__simd/math/modulo.h>
+#include <cuda/std/__simd/math/rounding.h>
+#include <cuda/std/__simd/math/trigonometric.h>
+
+#endif // _CUDA_STD___SIMD_MATH_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/abs.h

@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_ABS_H
+#define _CUDA_STD___SIMD_MATH_ABS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/abs.h>
+#include <cuda/std/__limits/numeric_limits.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <typename _Result, typename _Vp>
+struct __simd_abs_generator
+{
+  using __result_t = typename _Result::value_type;
+
+  const _Vp& __x_;
+
+  template <typename _Ip>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept
+  {
+    const auto __x = __x_[_Ip::value];
+    if constexpr (is_unsigned_v<__result_t>)
+    {
+      return __x;
+    }
+    else if constexpr (is_integral_v<__result_t>)
+    {
+      _CCCL_ASSERT(__x >= -numeric_limits<__result_t>::max(),
+                   "cuda::std::simd::abs precondition: each element must be greater than the minimum value");
+      return (__x < __result_t{0}) ? static_cast<__result_t>(-__x) : __x;
+    }
+    else
+    {
+      return ::cuda::std::fabs(__x);
+    }
+  }
+};
+
+// signed integral
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(is_integral_v<_Tp> _CCCL_AND is_signed_v<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto abs(const basic_vec<_Tp, _Abi>& __x) noexcept
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  return __vec_t{__simd_abs_generator<__vec_t, __vec_t>{__x}};
+}
+
+// floating point
+_CCCL_TEMPLATE(typename _Vp)
+_CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto abs(const _Vp& __x) noexcept
+{
+  using __result_t = __deduced_vec_t<_Vp>;
+  return __result_t{__simd_abs_generator<__result_t, _Vp>{__x}};
+}
+
+// fabs
+_CCCL_TEMPLATE(typename _Vp)
+_CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto fabs(const _Vp& __x) noexcept
+{
+  using __result_t = __deduced_vec_t<_Vp>;
+  return __result_t{__simd_abs_generator<__result_t, _Vp>{__x}};
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::abs;
+using simd::fabs;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_ABS_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/classification.h

@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_CLASSIFICATION_H
+#define _CUDA_STD___SIMD_MATH_CLASSIFICATION_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/fpclassify.h>
+#include <cuda/std/__cmath/isfinite.h>
+#include <cuda/std/__cmath/isinf.h>
+#include <cuda/std/__cmath/isnan.h>
+#include <cuda/std/__cmath/isnormal.h>
+#include <cuda/std/__cmath/signbit.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_UNARY_GENERATOR(fpclassify);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(isfinite);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(isinf);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(isnan);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(isnormal);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(signbit);
+
+#define _CCCL_SIMD_MATH_MASK_FUNCTION(_NAME, _CONSTEXPR)                                    \
+  _CCCL_TEMPLATE(typename _Vp, typename _Result = typename __deduced_vec_t<_Vp>::mask_type) \
+  _CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)                                      \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR _Result _NAME(const _Vp& __x) noexcept     \
+  {                                                                                         \
+    return _Result{__simd_##_NAME##_generator<_Result, _Vp>{__x}};                          \
+  }
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(fpclassify, int, constexpr)
+
+_CCCL_SIMD_MATH_MASK_FUNCTION(isfinite, constexpr)
+_CCCL_SIMD_MATH_MASK_FUNCTION(isinf, constexpr)
+_CCCL_SIMD_MATH_MASK_FUNCTION(isnan, constexpr)
+_CCCL_SIMD_MATH_MASK_FUNCTION(isnormal, constexpr)
+_CCCL_SIMD_MATH_MASK_FUNCTION(signbit, constexpr)
+
+#undef _CCCL_SIMD_MATH_MASK_FUNCTION
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::fpclassify;
+using simd::isfinite;
+using simd::isinf;
+using simd::isnan;
+using simd::isnormal;
+using simd::signbit;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_CLASSIFICATION_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/common.h

@@ -0,0 +1,291 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_COMMON_H
+#define _CUDA_STD___SIMD_MATH_COMMON_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__type_traits/is_floating_point.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/type_traits.h> // rebind_t
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__type_traits/void_t.h>
+#include <cuda/std/__utility/declval.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// concept simd-vec-type, exposition only
+template <typename _Tp, typename _Up = remove_cvref_t<_Tp>>
+_CCCL_CONCEPT __is_simd_vec_type_v = _CCCL_REQUIRES_EXPR((_Up))(
+  typename(typename _Up::value_type),
+  typename(typename _Up::abi_type),
+  requires(is_same_v<_Up, basic_vec<typename _Up::value_type, typename _Up::abi_type>>),
+  requires(is_default_constructible_v<_Up>));
+
+template <typename _Tp, typename = void>
+struct __deduced_vec
+{};
+
+template <typename _Tp>
+struct __deduced_vec<_Tp, void_t<decltype(::cuda::std::declval<const _Tp&>() + ::cuda::std::declval<const _Tp&>())>>
+{
+  using type = decltype(::cuda::std::declval<const _Tp&>() + ::cuda::std::declval<const _Tp&>());
+};
+
+// using deduced-vec-t, exposition only
+template <typename _Tp>
+using __deduced_vec_t = typename __deduced_vec<_Tp>::type;
+
+// concept simd-floating-point, exposition only
+template <typename _Tp>
+_CCCL_CONCEPT __is_simd_floating_point_v = _CCCL_REQUIRES_EXPR(
+  (_Tp))(typename(typename _Tp::value_type),
+         requires(__is_simd_vec_type_v<_Tp>),
+         requires(::cuda::is_floating_point_v<typename _Tp::value_type>));
+
+// concept math-floating-point, exposition only
+template <typename _Tp>
+_CCCL_CONCEPT __is_simd_math_floating_point_v = _CCCL_REQUIRES_EXPR(
+  (_Tp))(typename(__deduced_vec_t<_Tp>), requires(__is_simd_floating_point_v<__deduced_vec_t<_Tp>>));
+
+//----------------------------------------------------------------------------------------------------------------------
+// unary macros
+
+// common macro to implement unary generator
+#define _CCCL_SIMD_MATH_UNARY_GENERATOR(_NAME)                                              \
+  template <typename _Result, typename _Vp>                                                 \
+  struct __simd_##_NAME##_generator                                                         \
+  {                                                                                         \
+    using __result_t = typename _Result::value_type;                                        \
+                                                                                            \
+    const _Vp& __x_;                                                                        \
+                                                                                            \
+    template <typename _Ip>                                                                 \
+    [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept \
+    {                                                                                       \
+      return static_cast<__result_t>(::cuda::std::_NAME(__x_[_Ip::value]));                 \
+    }                                                                                       \
+  }
+
+// common macro to implement unary function
+#define _CCCL_SIMD_MATH_UNARY_FUNCTION(_NAME, _CONSTEXPR)                               \
+  _CCCL_SIMD_MATH_UNARY_GENERATOR(_NAME);                                               \
+                                                                                        \
+  _CCCL_TEMPLATE(typename _Vp, typename _Result = __deduced_vec_t<_Vp>)                 \
+  _CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)                                  \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR _Result _NAME(const _Vp& __x) noexcept \
+  {                                                                                     \
+    return _Result{__simd_##_NAME##_generator<_Result, _Vp>{__x}};                      \
+  }
+
+// common macro to implement (unary) rebind function
+#define _CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(_NAME, _Tp, _CONSTEXPR)                \
+  _CCCL_TEMPLATE(typename _Vp)                                                       \
+  _CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)                               \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR auto _NAME(const _Vp& __x) noexcept \
+  {                                                                                  \
+    using __result_t = rebind_t<_Tp, __deduced_vec_t<_Vp>>;                          \
+    return __result_t{__simd_##_NAME##_generator<__result_t, _Vp>{__x}};             \
+  }
+
+//----------------------------------------------------------------------------------------------------------------------
+
+// The following overloads don't work with nvcc (while they work with clang and gcc) because they are recognized as
+// ambiguous with the above overload. The workaround consists in checking if the non-floating point type is convertible
+// to the floating point type.
+
+// For example:
+//  template<math-floating-point V>
+//  constexpr deduced-vec-t<V> pow(const V& x, const V& y);
+
+//  template<math-floating-point V>
+//  constexpr deduced-vec-t<V> pow(const deduced-vec-t<V>& x, const V& y);
+
+//  template<math-floating-point V>
+//  constexpr deduced-vec-t<V> pow(const V& x, const deduced-vec-t<V>& y);
+
+template <typename _Void, typename... _Args>
+struct __simd_math_result
+{};
+
+// the following code to deduce the result type of a SIMD math function doesn't work with NVRTC (13.3) with
+// expression like basic_vec<__half> + __half
+//   struct __simd_math_result<void_t<decltype((::cuda::std::declval<const _Args&>() + ...))>, _Args...>
+// This requires the following workaround:
+// 1. Find the first vector argument (__simd_math_first_vec)
+// 2. Uses that vector as the starting result type
+// 3. If there are multiple SIMD vector arguments, combines only vector-with-vector expressions to compute the final
+// result type. Treats scalar arguments as broadcast operands, not as participants in result-type arithmetic
+
+// Find the first vector argument
+template <typename... _Args>
+struct __simd_math_first_vec
+{};
+
+template <bool _IsVec, typename... _Args>
+struct __simd_math_first_vec_impl;
+
+template <typename _Arg, typename... _Args>
+struct __simd_math_first_vec_impl<true, _Arg, _Args...>
+{
+  using type = __deduced_vec_t<_Arg>;
+};
+
+template <typename _Arg, typename... _Args>
+struct __simd_math_first_vec_impl<false, _Arg, _Args...> : __simd_math_first_vec<_Args...>
+{};
+
+template <typename _Arg, typename... _Args>
+struct __simd_math_first_vec<_Arg, _Args...>
+    : __simd_math_first_vec_impl<__is_simd_math_floating_point_v<_Arg>, _Arg, _Args...>
+{};
+
+// Derive the result type (no arguments)
+template <typename _Result, typename... _Args>
+struct __simd_math_result_impl
+{
+  using type = _Result;
+};
+
+// Accumulate the result type
+// - no vector argument -> keep the result type
+// - vector argument -> combine with the result type
+template <bool _IsVec, typename _Result, typename _Arg, typename... _Args>
+struct __simd_math_accumulate_vec_result;
+
+template <typename _Result, typename _Arg, typename... _Args>
+struct __simd_math_accumulate_vec_result<false, _Result, _Arg, _Args...> : __simd_math_result_impl<_Result, _Args...>
+{};
+
+template <typename _Result, typename _Arg, typename... _Args>
+struct __simd_math_accumulate_vec_result<true, _Result, _Arg, _Args...>
+    : __simd_math_result_impl<
+        decltype(::cuda::std::declval<const _Result&>() + ::cuda::std::declval<const __deduced_vec_t<_Arg>&>()),
+        _Args...>
+{};
+
+// Derive the result type (with arguments)
+template <typename _Result, typename _Arg, typename... _Args>
+struct __simd_math_result_impl<_Result, _Arg, _Args...>
+    : __simd_math_accumulate_vec_result<__is_simd_math_floating_point_v<_Arg>, _Result, _Arg, _Args...>
+{};
+
+// there is at least one vector argument, invalid otherwise
+template <typename... _Args>
+struct __simd_math_result<void_t<typename __simd_math_first_vec<_Args...>::type>, _Args...>
+    : __simd_math_result_impl<typename __simd_math_first_vec<_Args...>::type, _Args...>
+{};
+
+template <typename... _Args>
+using __simd_math_result_t = typename __simd_math_result<void, _Args...>::type;
+
+// Check if the argument is a vector argument and the same as the result type
+template <typename _Arg, typename _Result, typename = void>
+inline constexpr bool __is_simd_math_same_vec_arg_v = false;
+
+template <typename _Arg, typename _Result>
+inline constexpr bool __is_simd_math_same_vec_arg_v<_Arg, _Result, void_t<__deduced_vec_t<_Arg>>> =
+  is_same_v<__deduced_vec_t<_Arg>, _Result>;
+
+// Check if the argument is a valid SIMD math argument: same vector argument or convertible to the result type
+template <typename _Arg, typename _Result>
+inline constexpr bool __is_simd_math_arg_v =
+  __is_simd_math_same_vec_arg_v<_Arg, _Result>
+  || (!__is_simd_math_floating_point_v<_Arg>
+      && (is_same_v<remove_cvref_t<_Arg>, typename _Result::value_type> || is_convertible_v<const _Arg&, _Result>) );
+
+template <typename _Result, typename... _Args>
+inline constexpr bool __is_simd_math_v =
+  __is_simd_floating_point_v<_Result> && (__is_simd_math_arg_v<_Args, _Result> && ...);
+
+//----------------------------------------------------------------------------------------------------------------------
+// binary macros
+
+// common macro to implement a binary generator
+#define _CCCL_SIMD_MATH_BINARY_GENERATOR(_NAME, _GENERATOR)                                   \
+  template <typename _Result, typename _Vp0, typename _Vp1>                                   \
+  struct __simd_##_GENERATOR##_generator                                                      \
+  {                                                                                           \
+    using __result_t = typename _Result::value_type;                                          \
+                                                                                              \
+    const _Vp0& __x_;                                                                         \
+    const _Vp1& __y_;                                                                         \
+                                                                                              \
+    template <typename _Ip>                                                                   \
+    [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept   \
+    {                                                                                         \
+      return static_cast<__result_t>(::cuda::std::_NAME(__x_[_Ip::value], __y_[_Ip::value])); \
+    }                                                                                         \
+  }
+
+#define _CCCL_SIMD_MATH_BINARY_FUNCTION(_NAME, _GENERATOR, _CONSTEXPR)                                        \
+  _CCCL_TEMPLATE(typename _Vp0, typename _Vp1)                                                                \
+  _CCCL_REQUIRES(__is_simd_math_v<__simd_math_result_t<_Vp0, _Vp1>, _Vp0, _Vp1>)                              \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR auto _NAME(const _Vp0& __x, const _Vp1& __y) noexcept        \
+  {                                                                                                           \
+    using __result_t = __simd_math_result_t<_Vp0, _Vp1>;                                                      \
+    const __result_t __x_vec{__x};                                                                            \
+    const __result_t __y_vec{__y};                                                                            \
+    return __result_t{__simd_##_GENERATOR##_generator<__result_t, __result_t, __result_t>{__x_vec, __y_vec}}; \
+  }
+
+//----------------------------------------------------------------------------------------------------------------------
+// ternary macros
+
+#define _CCCL_SIMD_MATH_TERNARY_GENERATOR(_NAME, _GENERATOR)                                                    \
+  template <typename _Result, typename _Vp0, typename _Vp1, typename _Vp2>                                      \
+  struct __simd_##_GENERATOR##_generator                                                                        \
+  {                                                                                                             \
+    using __result_t = typename _Result::value_type;                                                            \
+                                                                                                                \
+    const _Vp0& __x_;                                                                                           \
+    const _Vp1& __y_;                                                                                           \
+    const _Vp2& __z_;                                                                                           \
+                                                                                                                \
+    template <typename _Ip>                                                                                     \
+    [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept                     \
+    {                                                                                                           \
+      return static_cast<__result_t>(::cuda::std::_NAME(__x_[_Ip::value], __y_[_Ip::value], __z_[_Ip::value])); \
+    }                                                                                                           \
+  }
+
+#define _CCCL_SIMD_MATH_TERNARY_FUNCTION(_NAME, _GENERATOR, _CONSTEXPR)                                                 \
+  _CCCL_TEMPLATE(typename _Vp0, typename _Vp1, typename _Vp2)                                                           \
+  _CCCL_REQUIRES(__is_simd_math_v<__simd_math_result_t<_Vp0, _Vp1, _Vp2>, _Vp0, _Vp1, _Vp2>)                            \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR auto _NAME(const _Vp0& __x, const _Vp1& __y, const _Vp2& __z) noexcept \
+  {                                                                                                                     \
+    using __result_t = __simd_math_result_t<_Vp0, _Vp1, _Vp2>;                                                          \
+    const __result_t __x_vec{__x};                                                                                      \
+    const __result_t __y_vec{__y};                                                                                      \
+    const __result_t __z_vec{__z};                                                                                      \
+    return __result_t{                                                                                                  \
+      __simd_##_GENERATOR##_generator<__result_t, __result_t, __result_t, __result_t>{__x_vec, __y_vec, __z_vec}};      \
+  }
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_COMMON_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/comparison.h

@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_COMPARISON_H
+#define _CUDA_STD___SIMD_MATH_COMPARISON_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/traits.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_BINARY_GENERATOR(isgreater, isgreater);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(isgreaterequal, isgreaterequal);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(isless, isless);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(islessequal, islessequal);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(islessgreater, islessgreater);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(isunordered, isunordered);
+
+#define _CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(_NAME, _CONSTEXPR)                                           \
+  _CCCL_TEMPLATE(typename _Vp0,                                                                           \
+                 typename _Vp1,                                                                           \
+                 typename _Vec    = __simd_math_result_t<_Vp0, _Vp1>,                                     \
+                 typename _Result = typename _Vec::mask_type)                                             \
+  _CCCL_REQUIRES(__is_simd_math_v<_Vec, _Vp0, _Vp1>)                                                      \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR _Result _NAME(const _Vp0& __x, const _Vp1& __y) noexcept \
+  {                                                                                                       \
+    const _Vec __x_vec{__x};                                                                              \
+    const _Vec __y_vec{__y};                                                                              \
+    return _Result{__simd_##_NAME##_generator<_Result, _Vec, _Vec>{__x_vec, __y_vec}};                    \
+  }
+
+_CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(isgreater, )
+_CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(isgreaterequal, )
+_CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(isless, )
+_CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(islessequal, )
+_CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(islessgreater, )
+_CCCL_SIMD_MATH_BINARY_MASK_FUNCTION(isunordered, )
+
+#undef _CCCL_SIMD_MATH_BINARY_MASK_FUNCTION
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::isgreater;
+using simd::isgreaterequal;
+using simd::isless;
+using simd::islessequal;
+using simd::islessgreater;
+using simd::isunordered;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_COMPARISON_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/exponential.h

@@ -0,0 +1,84 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_EXPONENTIAL_H
+#define _CUDA_STD___SIMD_MATH_EXPONENTIAL_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/error_functions.h>
+#include <cuda/std/__cmath/exponential_functions.h>
+#include <cuda/std/__cmath/gamma.h>
+#include <cuda/std/__cmath/hypot.h>
+#include <cuda/std/__cmath/logarithms.h>
+#include <cuda/std/__cmath/roots.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_UNARY_FUNCTION(exp, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(exp2, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(expm1, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(log, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(log10, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(log1p, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(log2, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(cbrt, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(sqrt, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(erf, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(erfc, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(lgamma, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(tgamma, )
+
+_CCCL_SIMD_MATH_BINARY_GENERATOR(pow, pow);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(hypot, hypot_two_args);
+_CCCL_SIMD_MATH_TERNARY_GENERATOR(hypot, hypot_three_args);
+
+_CCCL_SIMD_MATH_BINARY_FUNCTION(pow, pow, )
+_CCCL_SIMD_MATH_BINARY_FUNCTION(hypot, hypot_two_args, )
+_CCCL_SIMD_MATH_TERNARY_FUNCTION(hypot, hypot_three_args, )
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::cbrt;
+using simd::erf;
+using simd::erfc;
+using simd::exp;
+using simd::exp2;
+using simd::expm1;
+using simd::hypot;
+using simd::lgamma;
+using simd::log;
+using simd::log10;
+using simd::log1p;
+using simd::log2;
+using simd::pow;
+using simd::sqrt;
+using simd::tgamma;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_EXPONENTIAL_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/fma.h

@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_FMA_H
+#define _CUDA_STD___SIMD_MATH_FMA_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_TEMPLATE(typename _Vp0, typename _Vp1, typename _Vp2)
+_CCCL_REQUIRES(__is_simd_math_v<__simd_math_result_t<_Vp0, _Vp1, _Vp2>, _Vp0, _Vp1, _Vp2>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API auto fma(const _Vp0& __x, const _Vp1& __y, const _Vp2& __z) noexcept
+{
+  using __result_t = __simd_math_result_t<_Vp0, _Vp1, _Vp2>;
+  const __result_t __x_vec{__x};
+  const __result_t __y_vec{__y};
+  const __result_t __z_vec{__z};
+  return __simd_fma_impl(__x_vec, __y_vec, __z_vec); // ADL
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::fma;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_FMA_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/lerp.h

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_LERP_H
+#define _CUDA_STD___SIMD_MATH_LERP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/lerp.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_TERNARY_GENERATOR(lerp, lerp);
+_CCCL_SIMD_MATH_TERNARY_FUNCTION(lerp, lerp, constexpr)
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::lerp;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_LERP_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/manipulation.h

@@ -0,0 +1,118 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_MANIPULATION_H
+#define _CUDA_STD___SIMD_MATH_MANIPULATION_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/copysign.h>
+#include <cuda/std/__cmath/exponential_functions.h>
+#include <cuda/std/__cmath/logarithms.h>
+#include <cuda/std/__cmath/rounding_functions.h>
+#include <cuda/std/__simd/math/common.h>
+#include <cuda/std/array>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_UNARY_GENERATOR(ilogb);
+_CCCL_SIMD_MATH_UNARY_FUNCTION(logb, constexpr)
+
+_CCCL_SIMD_MATH_BINARY_GENERATOR(ldexp, ldexp);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(scalbn, scalbn);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(scalbln, scalbln);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(nextafter, nextafter);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(copysign, copysign);
+
+_CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(ilogb, int, constexpr)
+
+_CCCL_SIMD_MATH_BINARY_FUNCTION(nextafter, nextafter, )
+_CCCL_SIMD_MATH_BINARY_FUNCTION(copysign, copysign, constexpr)
+
+//----------------------------------------------------------------------------------------------------------------------
+// ldexp, scalbn, scalbln
+
+#define _CCCL_SIMD_MATH_BINARY_REBIND_FUNCTION(_NAME, _Tp, _CONSTEXPR)                          \
+  _CCCL_TEMPLATE(typename _Vp, typename _Result = __deduced_vec_t<_Vp>)                         \
+  _CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)                                          \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API _CONSTEXPR _Result _NAME(                                 \
+    const _Vp& __x, const rebind_t<_Tp, _Result>& __y) noexcept                                 \
+  {                                                                                             \
+    return _Result{__simd_##_NAME##_generator<_Result, _Vp, rebind_t<_Tp, _Result>>{__x, __y}}; \
+  }
+
+_CCCL_SIMD_MATH_BINARY_REBIND_FUNCTION(ldexp, int, )
+_CCCL_SIMD_MATH_BINARY_REBIND_FUNCTION(scalbn, int, )
+_CCCL_SIMD_MATH_BINARY_REBIND_FUNCTION(scalbln, long, )
+
+#undef _CCCL_SIMD_MATH_BINARY_REBIND_FUNCTION
+
+// frexp
+
+template <typename _Result, typename _Vp>
+struct __simd_frexp_generator
+{
+  using __result_t = typename _Result::value_type;
+
+  const _Vp& __x_;
+  array<int, _Result::__usize>& __exponents_;
+
+  template <typename _Ip>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept
+  {
+    int __exponent           = 0;
+    const auto __result      = static_cast<__result_t>(::cuda::std::frexp(__x_[_Ip::value], &__exponent));
+    __exponents_[_Ip::value] = __exponent;
+    return __result;
+  }
+};
+
+_CCCL_TEMPLATE(typename _Vp, typename _Result = __deduced_vec_t<_Vp>)
+_CCCL_REQUIRES(__is_simd_math_floating_point_v<_Vp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API _Result frexp(const _Vp& __x, rebind_t<int, _Result>* __exp) noexcept
+{
+  using __exp_t = rebind_t<int, _Result>;
+  array<int, _Result::__usize> __exponents{};
+
+  const _Result __values{__simd_frexp_generator<_Result, _Vp>{__x, __exponents}};
+  *__exp = __exp_t{__exponents};
+  return __values;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::copysign;
+using simd::frexp;
+using simd::ilogb;
+using simd::ldexp;
+using simd::logb;
+using simd::nextafter;
+using simd::scalbln;
+using simd::scalbn;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_MANIPULATION_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/min_max.h

@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_MIN_MAX_H
+#define _CUDA_STD___SIMD_MATH_MIN_MAX_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/fdim.h>
+#include <cuda/std/__cmath/min_max.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_BINARY_GENERATOR(fdim, fdim);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(fmax, fmax);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(fmin, fmin);
+
+_CCCL_SIMD_MATH_BINARY_FUNCTION(fdim, fdim, )
+_CCCL_SIMD_MATH_BINARY_FUNCTION(fmax, fmax, constexpr)
+_CCCL_SIMD_MATH_BINARY_FUNCTION(fmin, fmin, constexpr)
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::fdim;
+using simd::fmax;
+using simd::fmin;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_MIN_MAX_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/modulo.h

@@ -0,0 +1,134 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_MODULO_H
+#define _CUDA_STD___SIMD_MATH_MODULO_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/modulo.h>
+#include <cuda/std/__cmath/remainder.h>
+#include <cuda/std/__simd/math/common.h>
+#include <cuda/std/__type_traits/type_identity.h>
+#include <cuda/std/array>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_BINARY_GENERATOR(fmod, fmod);
+_CCCL_SIMD_MATH_BINARY_GENERATOR(remainder, remainder);
+
+_CCCL_SIMD_MATH_BINARY_FUNCTION(fmod, fmod, )
+_CCCL_SIMD_MATH_BINARY_FUNCTION(remainder, remainder, )
+
+//----------------------------------------------------------------------------------------------------------------------
+// remquo
+
+template <typename _Result, typename _Vp0, typename _Vp1>
+struct __simd_remquo_generator
+{
+  using __result_t = typename _Result::value_type;
+
+  const _Vp0& __x_;
+  const _Vp1& __y_;
+  array<int, _Result::__usize>& __quotients_;
+
+  template <typename _Ip>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept
+  {
+    int __quotient           = 0;
+    const auto __remquo      = ::cuda::std::remquo(__x_[_Ip::value], __y_[_Ip::value], &__quotient);
+    const auto __result      = static_cast<__result_t>(__remquo);
+    __quotients_[_Ip::value] = __quotient;
+    return __result;
+  }
+};
+
+template <typename _Result, typename _Vp0, typename _Vp1>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _Result
+__simd_remquo_impl(const _Vp0& __x, const _Vp1& __y, rebind_t<int, _Result>* __quo) noexcept
+{
+  array<int, _Result::__usize> __quotients{};
+  const _Result __values{__simd_remquo_generator<_Result, _Vp0, _Vp1>{__x, __y, __quotients}};
+  *__quo = rebind_t<int, _Result>{__quotients};
+  return __values;
+}
+
+_CCCL_TEMPLATE(typename _Vp0, typename _Vp1, typename _Result = __simd_math_result_t<_Vp0, _Vp1>)
+_CCCL_REQUIRES(__is_simd_math_v<_Result, _Vp0, _Vp1>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API _Result
+remquo(const _Vp0& __x, const _Vp1& __y, rebind_t<int, _Result>* __quo) noexcept
+{
+  const _Result __x_vec{__x};
+  const _Result __y_vec{__y};
+  return ::cuda::std::simd::__simd_remquo_impl<_Result, _Result, _Result>(__x_vec, __y_vec, __quo);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// modf
+
+template <typename _Vp>
+struct __simd_modf_generator
+{
+  using __result_t = typename _Vp::value_type;
+
+  const _Vp& __x_;
+  array<__result_t, _Vp::__usize>& __integrals_;
+
+  template <typename _Ip>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Ip) const noexcept
+  {
+    __result_t __integral    = 0;
+    auto __modf              = ::cuda::std::modf(__x_[_Ip::value], &__integral);
+    const auto __result      = static_cast<__result_t>(__modf);
+    __integrals_[_Ip::value] = __integral;
+    return __result;
+  }
+};
+
+// modf is the only function that doesn't have constraints on the type, even if modf is only defined for floating point
+// types.
+
+template <typename _Tp, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API basic_vec<_Tp, _Abi>
+modf(const type_identity_t<basic_vec<_Tp, _Abi>>& __x, basic_vec<_Tp, _Abi>* __iptr) noexcept
+{
+  using _Vp = basic_vec<_Tp, _Abi>;
+  array<_Tp, _Vp::__usize> __integrals{};
+  const _Vp __values{__simd_modf_generator<_Vp>{__x, __integrals}};
+  *__iptr = _Vp{__integrals};
+  return __values;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::fmod;
+using simd::modf;
+using simd::remainder;
+using simd::remquo;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_MODULO_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/rounding.h

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_ROUNDING_H
+#define _CUDA_STD___SIMD_MATH_ROUNDING_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/rounding_functions.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_UNARY_FUNCTION(ceil, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(floor, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(nearbyint, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(rint, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(round, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(trunc, )
+
+_CCCL_SIMD_MATH_UNARY_GENERATOR(lrint);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(llrint);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(lround);
+_CCCL_SIMD_MATH_UNARY_GENERATOR(llround);
+
+_CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(lrint, long, )
+_CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(llrint, long long, )
+_CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(lround, long, )
+_CCCL_SIMD_MATH_UNARY_REBIND_FUNCTION(llround, long long, )
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::ceil;
+using simd::floor;
+using simd::llrint;
+using simd::llround;
+using simd::lrint;
+using simd::lround;
+using simd::nearbyint;
+using simd::rint;
+using simd::round;
+using simd::trunc;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_ROUNDING_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/math/trigonometric.h

@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_MATH_TRIGONOMETRIC_H
+#define _CUDA_STD___SIMD_MATH_TRIGONOMETRIC_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cmath/hyperbolic_functions.h>
+#include <cuda/std/__cmath/inverse_hyperbolic_functions.h>
+#include <cuda/std/__cmath/inverse_trigonometric_functions.h>
+#include <cuda/std/__cmath/trigonometric_functions.h>
+#include <cuda/std/__simd/math/common.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+_CCCL_SIMD_MATH_UNARY_FUNCTION(acos, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(asin, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(atan, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(cos, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(sin, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(tan, )
+
+_CCCL_SIMD_MATH_UNARY_FUNCTION(acosh, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(asinh, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(atanh, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(cosh, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(sinh, )
+_CCCL_SIMD_MATH_UNARY_FUNCTION(tanh, )
+
+_CCCL_SIMD_MATH_BINARY_GENERATOR(atan2, atan2);
+
+_CCCL_SIMD_MATH_BINARY_FUNCTION(atan2, atan2, )
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+using simd::acos;
+using simd::acosh;
+using simd::asin;
+using simd::asinh;
+using simd::atan;
+using simd::atan2;
+using simd::atanh;
+using simd::cos;
+using simd::cosh;
+using simd::sin;
+using simd::sinh;
+using simd::tan;
+using simd::tanh;
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_MATH_TRIGONOMETRIC_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/permute.h

@@ -0,0 +1,362 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_PERMUTE_H
+#define _CUDA_STD___SIMD_PERMUTE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__functional/invoke.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/basic_mask.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/exposition.h>
+#include <cuda/std/__simd/type_traits.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__type_traits/type_identity.h>
+#include <cuda/std/__type_traits/void_t.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.permute.static], sentinels for static permute
+
+inline constexpr __simd_size_type zero_element   = -1;
+inline constexpr __simd_size_type uninit_element = -2;
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.static], Constraints detection
+//
+// At least one of `invoke_result_t<IdxMap&, simd-size-type>` and `invoke_result_t<IdxMap&, simd-size-type,
+// simd-size-type>` must satisfy `integral`.
+
+template <typename _IdxMap, typename _Enable, typename... _Args>
+inline constexpr bool __idxmap_nargs_integral_v = false;
+
+template <typename _IdxMap, typename... _Args>
+inline constexpr bool __idxmap_nargs_integral_v<_IdxMap, void_t<invoke_result_t<_IdxMap&, _Args...>>, _Args...> =
+  is_integral_v<remove_cvref_t<invoke_result_t<_IdxMap&, _Args...>>>;
+
+template <typename _IdxMap>
+inline constexpr bool __idxmap_result_is_integral_v =
+  is_invocable_v<remove_cvref_t<_IdxMap>&, __simd_size_type, __simd_size_type>
+    ? __idxmap_nargs_integral_v<remove_cvref_t<_IdxMap>, void, __simd_size_type, __simd_size_type>
+    : __idxmap_nargs_integral_v<remove_cvref_t<_IdxMap>, void, __simd_size_type>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// gen-fn: idxmap(i, V::size()) if that expression is well-formed, and idxmap(i) otherwise.
+
+template <typename _IdxMap, __simd_size_type _Idx, __simd_size_type _Size>
+inline constexpr bool __idxmap_invocable_two_args_v =
+  is_invocable_v<_IdxMap&, __simd_size_constant<_Idx>, __simd_size_constant<_Size>>;
+
+template <typename _IdxMap, __simd_size_type _Idx, __simd_size_type _Size>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL __simd_size_type __permute_gen_fn() noexcept
+{
+  if constexpr (__idxmap_invocable_two_args_v<_IdxMap, _Idx, _Size>)
+  {
+    return static_cast<__simd_size_type>(_IdxMap{}(__simd_size_constant<_Idx>{}, __simd_size_constant<_Size>{}));
+  }
+  else
+  {
+    return static_cast<__simd_size_type>(_IdxMap{}(__simd_size_constant<_Idx>{}));
+  }
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// permute_generator
+
+template <typename _IdxMap, typename _Vp>
+struct __permute_generator
+{
+  using __value_type = typename _Vp::value_type;
+
+  const _Vp& __v_;
+
+  template <__simd_size_type _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __value_type operator()(__simd_size_constant<_Idx>) const noexcept
+  {
+    using __map_t                     = remove_cvref_t<_IdxMap>;
+    constexpr __simd_size_type __size = _Vp::__size;
+    constexpr __simd_size_type __src  = ::cuda::std::simd::__permute_gen_fn<__map_t, _Idx, __size>();
+    static_assert(__src == zero_element || __src == uninit_element || (__src >= 0 && __src < __size),
+                  "cuda::std::simd::permute: idxmap(i) must return zero_element, uninit_element, or a value in [0, "
+                  "V::size())");
+    if constexpr (__src == zero_element || __src == uninit_element)
+    {
+      return __value_type{}; // unspecified-value
+    }
+    else
+    {
+      return __v_[__src];
+    }
+  }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.static]
+
+// The default-N overloads below spell the return type as V because N is V::size(), so resize_t<N, V> is V.
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _IdxMap)
+_CCCL_REQUIRES(__idxmap_result_is_integral_v<_IdxMap>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> permute(const basic_vec<_Tp, _Abi>& __v, _IdxMap&&)
+{
+  return basic_vec<_Tp, _Abi>{__permute_generator<_IdxMap, basic_vec<_Tp, _Abi>>{__v}};
+}
+
+_CCCL_TEMPLATE(__simd_size_type _Np, typename _Tp, typename _Abi, typename _IdxMap)
+_CCCL_REQUIRES(__idxmap_result_is_integral_v<_IdxMap>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr resize_t<_Np, basic_vec<_Tp, _Abi>>
+permute(const basic_vec<_Tp, _Abi>& __v, _IdxMap&&)
+{
+  static_assert(_Np >= 0, "cuda::std::simd::permute: N must be non-negative");
+  using __result_t = resize_t<_Np, basic_vec<_Tp, _Abi>>;
+  return __result_t{__permute_generator<_IdxMap, basic_vec<_Tp, _Abi>>{__v}};
+}
+
+// The default-N overloads below spell the return type as V because N is V::size(), so resize_t<N, V> is V.
+_CCCL_TEMPLATE(typename _Abi, size_t _Bytes, typename _IdxMap)
+_CCCL_REQUIRES(__idxmap_result_is_integral_v<_IdxMap>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_mask<_Bytes, _Abi>
+permute(const basic_mask<_Bytes, _Abi>& __v, _IdxMap&&)
+{
+  return basic_mask<_Bytes, _Abi>{__permute_generator<_IdxMap, basic_mask<_Bytes, _Abi>>{__v}};
+}
+
+_CCCL_TEMPLATE(__simd_size_type _Np, size_t _Bytes, typename _Abi, typename _IdxMap)
+_CCCL_REQUIRES(__idxmap_result_is_integral_v<_IdxMap>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr resize_t<_Np, basic_mask<_Bytes, _Abi>>
+permute(const basic_mask<_Bytes, _Abi>& __v, _IdxMap&&)
+{
+  static_assert(_Np >= 0, "cuda::std::simd::permute: N must be non-negative");
+  using __result_t = resize_t<_Np, basic_mask<_Bytes, _Abi>>;
+  return __result_t{__permute_generator<_IdxMap, basic_mask<_Bytes, _Abi>>{__v}};
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.dynamic]
+
+template <typename _Vp, typename _Ip>
+struct __permute_dynamic_generator
+{
+  using __value_type = typename _Vp::value_type;
+
+  const _Vp& __v_;
+  const _Ip& __indices_;
+
+  template <__simd_size_type _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __value_type operator()(__simd_size_constant<_Idx>) const noexcept
+  {
+    const auto __src = static_cast<__simd_size_type>(__indices_[_Idx]);
+    _CCCL_ASSERT(::cuda::in_range(__src, __simd_size_type{0}, _Vp::size()),
+                 "cuda::std::simd::permute: indices[i] must be in [0, V::size())");
+    return __v_[__src];
+  }
+};
+
+template <typename _Vp, typename _Ip>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __permute_dynamic_generator<_Vp, _Ip>
+__make_permute_dynamic_generator(const _Vp& __v, const _Ip& __indices) noexcept
+{
+  return __permute_dynamic_generator<_Vp, _Ip>{__v, __indices};
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Up, typename _UAbi)
+_CCCL_REQUIRES(is_integral_v<_Up>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr resize_t<__simd_size_v<_Up, _UAbi>, basic_vec<_Tp, _Abi>>
+permute(const basic_vec<_Tp, _Abi>& __v, const basic_vec<_Up, _UAbi>& __indices)
+{
+  using __result_t = resize_t<__simd_size_v<_Up, _UAbi>, basic_vec<_Tp, _Abi>>;
+  return __result_t{::cuda::std::simd::__make_permute_dynamic_generator(__v, __indices)};
+}
+
+_CCCL_TEMPLATE(size_t _Bytes, typename _Abi, typename _Up, typename _UAbi)
+_CCCL_REQUIRES(is_integral_v<_Up>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr resize_t<__simd_size_v<_Up, _UAbi>, basic_mask<_Bytes, _Abi>>
+permute(const basic_mask<_Bytes, _Abi>& __v, const basic_vec<_Up, _UAbi>& __indices)
+{
+  using __result_t = resize_t<__simd_size_v<_Up, _UAbi>, basic_mask<_Bytes, _Abi>>;
+  return __result_t{::cuda::std::simd::__make_permute_dynamic_generator(__v, __indices)};
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.mask]
+
+// A data-parallel object where the i-th element is initialized to the result of select-value(i) for all i in the range
+// [0, V::size()).
+
+template <typename _Vp, typename _Mp>
+struct __compress_generator
+{
+  using __value_type = typename _Vp::value_type;
+
+  const _Vp& __v_;
+  const _Mp& __sel_;
+  const __value_type __fill_;
+
+  template <__simd_size_type _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __value_type operator()(__simd_size_constant<_Idx>) const noexcept
+  {
+    __simd_size_type __count = 0;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __j = 0; __j < _Vp::__size; ++__j)
+    {
+      if (__sel_[__j])
+      {
+        if (__count == _Idx)
+        {
+          return __v_[__j];
+        }
+        ++__count;
+      }
+    }
+    return __fill_;
+  }
+};
+
+template <typename _Vp, typename _Mp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __compress_generator<_Vp, _Mp>
+__make_compress_generator(const _Vp& __v, const _Mp& __sel, typename _Vp::value_type __fill) noexcept
+{
+  return __compress_generator<_Vp, _Mp>{__v, __sel, __fill};
+}
+
+// A data-parallel object where the i-th element is initialized to the result of select-value(i) for all i in the range
+// [0, V::size())
+
+template <typename _Vp, typename _Mp>
+struct __expand_generator
+{
+  using __value_type = typename _Vp::value_type;
+
+  const _Vp& __v_;
+  const _Mp& __sel_;
+  const _Vp& __orig_;
+
+  // example:
+  // v           = [10, 20, 30, 40, 50]
+  // selector    = [T, F, T, F, T]
+  // original    = [1, 2, 3, 4, 5]
+  //
+  // set-indices = [0, 2, 4] (where selector is true)
+  // bit-lookup  = [0, X, 1, X, 2] returns the index where b appears in set-indices
+  //
+  // i = 0 --> v[bit-lookup(0)] = v[0] = 10
+  // i = 1 --> original[1] = 2
+  // i = 2 --> v[bit-lookup(2)] = v[1] = 20
+  // i = 3 --> original[3] = 4
+  // i = 4 --> v[bit-lookup(4)] = v[2] = 30
+  //
+  // result = [10, 2, 20, 4, 30]
+  template <__simd_size_type _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __value_type operator()(__simd_size_constant<_Idx>) const noexcept
+  {
+    if (!__sel_[_Idx])
+    {
+      return __orig_[_Idx]; // otherwise returns original[i]
+    }
+    __simd_size_type __count = 0;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __j = 0; __j < _Idx; ++__j)
+    {
+      if (__sel_[__j]) // set-indices be a list of the index positions of true elements in selector
+      {
+        ++__count;
+      }
+    }
+    return __v_[__count]; // returns v[bit-lookup(i)]
+  }
+};
+
+template <typename _Vp, typename _Mp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __expand_generator<_Vp, _Mp>
+__make_expand_generator(const _Vp& __v, const _Mp& __sel, const _Vp& __orig) noexcept
+{
+  return __expand_generator<_Vp, _Mp>{__v, __sel, __orig};
+}
+
+// compress: basic_vec
+
+template <typename _Tp, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi>
+compress(const basic_vec<_Tp, _Abi>& __v, const typename basic_vec<_Tp, _Abi>::mask_type& __selector)
+{
+  return basic_vec<_Tp, _Abi>{::cuda::std::simd::__make_compress_generator(__v, __selector, _Tp{})};
+}
+
+// compress: basic_vec with fill_value
+
+template <typename _Tp, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi> compress(
+  const basic_vec<_Tp, _Abi>& __v, const typename basic_vec<_Tp, _Abi>::mask_type& __selector, const _Tp& __fill_value)
+{
+  return basic_vec<_Tp, _Abi>{::cuda::std::simd::__make_compress_generator(__v, __selector, __fill_value)};
+}
+
+// compress: basic_mask
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_mask<_Bytes, _Abi>
+compress(const basic_mask<_Bytes, _Abi>& __v, const type_identity_t<basic_mask<_Bytes, _Abi>>& __selector)
+{
+  return basic_mask<_Bytes, _Abi>{::cuda::std::simd::__make_compress_generator(__v, __selector, false)};
+}
+
+// compress: basic_mask with fill_value
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_mask<_Bytes, _Abi>
+compress(const basic_mask<_Bytes, _Abi>& __v,
+         const type_identity_t<basic_mask<_Bytes, _Abi>>& __selector,
+         const bool& __fill_value)
+{
+  return basic_mask<_Bytes, _Abi>{::cuda::std::simd::__make_compress_generator(__v, __selector, __fill_value)};
+}
+
+// expand: basic_vec
+
+template <typename _Tp, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_vec<_Tp, _Abi>
+expand(const basic_vec<_Tp, _Abi>& __v,
+       const typename basic_vec<_Tp, _Abi>::mask_type& __selector,
+       const basic_vec<_Tp, _Abi>& __original = {})
+{
+  return basic_vec<_Tp, _Abi>{::cuda::std::simd::__make_expand_generator(__v, __selector, __original)};
+}
+
+// expand: basic_mask
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr basic_mask<_Bytes, _Abi>
+expand(const basic_mask<_Bytes, _Abi>& __v,
+       const type_identity_t<basic_mask<_Bytes, _Abi>>& __selector,
+       const basic_mask<_Bytes, _Abi>& __original = {})
+{
+  return basic_mask<_Bytes, _Abi>{::cuda::std::simd::__make_expand_generator(__v, __selector, __original)};
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_PERMUTE_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/permute_memory.h

@@ -0,0 +1,339 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_PERMUTE_MEMORY_H
+#define _CUDA_STD___SIMD_PERMUTE_MEMORY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__iterator/concepts.h>
+#include <cuda/std/__ranges/access.h>
+#include <cuda/std/__ranges/concepts.h>
+#include <cuda/std/__ranges/data.h>
+#include <cuda/std/__ranges/size.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/basic_mask.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/concepts.h>
+#include <cuda/std/__simd/exposition.h>
+#include <cuda/std/__simd/flag.h>
+#include <cuda/std/__simd/utility.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__utility/cmp.h>
+#include <cuda/std/__utility/forward.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// [simd.permute.memory] gather
+
+//----------------------------------------------------------------------------------------------------------------------
+// gather helpers
+
+template <typename _Range, typename _Ip, typename _IAbi>
+using __default_gather_vec_t = vec<ranges::range_value_t<_Range>, __simd_size_v<_Ip, _IAbi>>;
+
+template <typename _Vp, typename _Range, typename _Ip, typename _IAbi>
+using __gather_result_t = conditional_t<is_same_v<_Vp, void>, __default_gather_vec_t<_Range, _Ip, _IAbi>, _Vp>;
+
+template <typename>
+inline constexpr bool __is_basic_vec_v = false;
+
+template <typename _Tp, typename _Abi>
+inline constexpr bool __is_basic_vec_v<basic_vec<_Tp, _Abi>> = __is_vectorizable_v<_Tp> && __is_enabled_abi_v<_Abi>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// gather constraints concept
+
+template <typename _Vp, typename _Range, typename _Ip, typename _IAbi>
+_CCCL_CONCEPT __gather_constraints =
+  ranges::contiguous_range<_Range> && ranges::sized_range<_Range> && is_integral_v<_Ip>
+  && __simd_vec_type<__gather_result_t<_Vp, _Range, _Ip, _IAbi>> && __is_vectorizable_v<ranges::range_value_t<_Range>>
+  && __explicitly_convertible_to<ranges::range_value_t<_Range>,
+                                 typename __gather_result_t<_Vp, _Range, _Ip, _IAbi>::value_type>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// gather generator
+
+template <typename _Vp, typename _Ptr, typename _Ip, typename _IAbi, typename _Mp>
+struct __gather_generator
+{
+  using __value_type = typename _Vp::value_type;
+
+  const _Ptr __data_;
+  const __simd_size_type __size_;
+  const basic_vec<_Ip, _IAbi>& __indices_;
+  const _Mp& __mask_;
+
+  template <__simd_size_type _Idx>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __value_type operator()(__simd_size_constant<_Idx>) const noexcept
+  {
+    if (!__mask_[_Idx])
+    {
+      return __value_type{};
+    }
+    const auto __raw_idx = __indices_[_Idx];
+    if (::cuda::std::cmp_greater_equal(__raw_idx, 0) && ::cuda::std::cmp_less(__raw_idx, __size_))
+    {
+      const auto __idx = static_cast<__simd_size_type>(__raw_idx);
+      return static_cast<__value_type>(__data_[__idx]);
+    }
+    return __value_type{};
+  }
+};
+
+template <typename _Result, typename _Range, typename _Ip, typename _IAbi, typename... _Flags>
+_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL void __check_gather_mandates() noexcept
+{
+  // same_as<remove_cvref_t<V>, V> is true (checked first so that later accesses to _Result's members are well-formed)
+  static_assert(is_same_v<remove_cvref_t<_Result>, _Result>,
+                "cuda::std::simd::partial_gather_from / unchecked_gather_from: V must not be cv- or ref-qualified");
+  // V is an enabled specialization of basic_vec
+  static_assert(__is_basic_vec_v<_Result>,
+                "cuda::std::simd::partial_gather_from / unchecked_gather_from: V must be a specialization of "
+                "basic_vec");
+  // ranges::range_value_t<R> is a vectorizable type
+  static_assert(__is_vectorizable_v<ranges::range_value_t<_Range>>,
+                "cuda::std::simd::partial_gather_from / unchecked_gather_from: range_value_t<R> must be vectorizable");
+  // V::size() == I::size() is true
+  static_assert(_Result::__size == __simd_size_v<_Ip, _IAbi>,
+                "cuda::std::simd::partial_gather_from / unchecked_gather_from: V::size() must equal indices.size()");
+  // if the template parameter pack Flags does not contain convert-flag, then the conversion from
+  // ranges::range_value_t<R> to T is value-preserving
+  static_assert(__has_convert_flag_v<_Flags...>
+                  || __is_value_preserving_v<ranges::range_value_t<_Range>, typename _Result::value_type>,
+                "cuda::std::simd::partial_gather_from / unchecked_gather_from: conversion from range_value_t<R> to "
+                "V::value_type is not value-preserving; use flag_convert");
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.memory] partial_gather_from
+
+// masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__gather_constraints<_Vp, _Range, _Ip, _IAbi>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __gather_result_t<_Vp, _Range, _Ip, _IAbi> partial_gather_from(
+  _Range&& __range,
+  const typename basic_vec<_Ip, _IAbi>::mask_type& __mask,
+  const basic_vec<_Ip, _IAbi>& __indices,
+  flags<_Flags...> = {})
+{
+  using _Result = __gather_result_t<_Vp, _Range, _Ip, _IAbi>;
+  ::cuda::std::simd::__check_gather_mandates<_Result, _Range, _Ip, _IAbi, _Flags...>();
+  const auto __range_size = ::cuda::std::ranges::size(__range);
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__range_size),
+               "cuda::std::simd::partial_gather_from: ranges::size(in) is not representable as __simd_size_type");
+
+  const auto __data = ::cuda::std::ranges::data(__range);
+  const auto __size = static_cast<__simd_size_type>(__range_size);
+  _CCCL_ASSERT(__size == 0 || __data != nullptr,
+               "cuda::std::simd::partial_gather_from: ranges::data(in) is null but ranges::size(in) > 0");
+  ::cuda::std::simd::__assert_load_store_alignment<_Result, ranges::range_value_t<_Range>, _Flags...>(__data);
+  using __mask_t      = typename basic_vec<_Ip, _IAbi>::mask_type;
+  using __generator_t = __gather_generator<_Result, decltype(__data), _Ip, _IAbi, __mask_t>;
+  return _Result{__generator_t{__data, __size, __indices, __mask}};
+}
+
+// unmasked: delegate to the masked overload with an all-true mask.
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__gather_constraints<_Vp, _Range, _Ip, _IAbi>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __gather_result_t<_Vp, _Range, _Ip, _IAbi>
+partial_gather_from(_Range&& __range, const basic_vec<_Ip, _IAbi>& __indices, flags<_Flags...> __f = {})
+{
+  constexpr auto __all_true = typename basic_vec<_Ip, _IAbi>::mask_type(true);
+  return ::cuda::std::simd::partial_gather_from<_Vp>(::cuda::std::forward<_Range>(__range), __all_true, __indices, __f);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.memory] unchecked_gather_from
+
+// masked
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__gather_constraints<_Vp, _Range, _Ip, _IAbi>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __gather_result_t<_Vp, _Range, _Ip, _IAbi> unchecked_gather_from(
+  _Range&& __range,
+  const typename basic_vec<_Ip, _IAbi>::mask_type& __mask,
+  const basic_vec<_Ip, _IAbi>& __indices,
+  flags<_Flags...> __f = {})
+{
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    const auto __range_size = ::cuda::std::ranges::size(__range);
+    _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__range_size),
+                 "cuda::std::simd::unchecked_gather_from: ranges::size(in) is not representable as __simd_size_type");
+    const auto __size = static_cast<__simd_size_type>(__range_size);
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < __simd_size_v<_Ip, _IAbi>; ++__i)
+    {
+      if (__mask[__i])
+      {
+        const auto __idx = static_cast<__simd_size_type>(__indices[__i]);
+        _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__indices[__i])
+                       && ::cuda::in_range(__idx, __simd_size_type{0}, __size),
+                     "cuda::std::simd::unchecked_gather_from: indices[i] must be in [0, ranges::size(in)) for every "
+                     "selected i");
+      }
+    }
+  }
+  return ::cuda::std::simd::partial_gather_from<_Vp>(::cuda::std::forward<_Range>(__range), __mask, __indices, __f);
+}
+
+// unmasked: delegate to the masked overload with an all-true mask to avoid duplicating the precondition check.
+_CCCL_TEMPLATE(typename _Vp = void, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__gather_constraints<_Vp, _Range, _Ip, _IAbi>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __gather_result_t<_Vp, _Range, _Ip, _IAbi>
+unchecked_gather_from(_Range&& __range, const basic_vec<_Ip, _IAbi>& __indices, flags<_Flags...> __f = {})
+{
+  constexpr auto __all_true = typename basic_vec<_Ip, _IAbi>::mask_type(true);
+  return ::cuda::std::simd::unchecked_gather_from<_Vp>(
+    ::cuda::std::forward<_Range>(__range), __all_true, __indices, __f);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.memory] scatter
+
+// scatter constraints concept
+
+template <typename _Tp, typename _Abi, typename _Range, typename _Ip, typename _IAbi>
+_CCCL_CONCEPT __scatter_constraints =
+  __simd_vec_type<basic_vec<_Tp, _Abi>> && ranges::contiguous_range<_Range> && ranges::sized_range<_Range>
+  && is_integral_v<_Ip> && (__simd_size_v<_Tp, _Abi> == __simd_size_v<_Ip, _IAbi>)
+  && __is_vectorizable_v<ranges::range_value_t<_Range>> && indirectly_writable<ranges::iterator_t<_Range>, _Tp>
+  && __explicitly_convertible_to<_Tp, ranges::range_value_t<_Range>>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// scatter mandates
+
+template <typename _Tp, typename _Range, typename... _Flags>
+_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL void __check_scatter_mandates() noexcept
+{
+  static_assert(__is_vectorizable_v<ranges::range_value_t<_Range>>,
+                "cuda::std::simd::partial_scatter_to / unchecked_scatter_to: range_value_t<R> must be vectorizable");
+  static_assert(__has_convert_flag_v<_Flags...> || __is_value_preserving_v<_Tp, ranges::range_value_t<_Range>>,
+                "cuda::std::simd::partial_scatter_to / unchecked_scatter_to: conversion from V::value_type to "
+                "range_value_t<R> is not value-preserving; use flag_convert");
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.memory] partial_scatter_to
+
+// masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__scatter_constraints<_Tp, _Abi, _Range, _Ip, _IAbi>)
+_CCCL_HOST_DEVICE_API constexpr void partial_scatter_to(
+  const basic_vec<_Tp, _Abi>& __v,
+  _Range&& __range,
+  const typename basic_vec<_Ip, _IAbi>::mask_type& __mask,
+  const basic_vec<_Ip, _IAbi>& __indices,
+  flags<_Flags...> = {})
+{
+  using __vec_t = basic_vec<_Tp, _Abi>;
+  ::cuda::std::simd::__check_scatter_mandates<_Tp, _Range, _Flags...>();
+  const auto __range_size = ::cuda::std::ranges::size(__range);
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__range_size),
+               "cuda::std::simd::partial_scatter_to: ranges::size(out) is not representable as __simd_size_type");
+
+  const auto __data     = ::cuda::std::ranges::data(__range);
+  const auto __out_size = static_cast<__simd_size_type>(__range_size);
+  _CCCL_ASSERT(__out_size == 0 || __data != nullptr,
+               "cuda::std::simd::partial_scatter_to: ranges::data(out) is null but ranges::size(out) > 0");
+  ::cuda::std::simd::__assert_load_store_alignment<__vec_t, ranges::range_value_t<_Range>, _Flags...>(__data);
+
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __vec_t::__size; ++__i)
+  {
+    if (!__mask[__i])
+    {
+      continue;
+    }
+    const auto __raw_idx = __indices[__i];
+    if (::cuda::std::cmp_greater_equal(__raw_idx, 0) && ::cuda::std::cmp_less(__raw_idx, __out_size))
+    {
+      const auto __idx = static_cast<__simd_size_type>(__raw_idx);
+      __data[__idx]    = static_cast<ranges::range_value_t<_Range>>(__v[__i]);
+    }
+  }
+}
+
+// unmasked: delegate to the masked overload with an all-true mask.
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__scatter_constraints<_Tp, _Abi, _Range, _Ip, _IAbi>)
+_CCCL_HOST_DEVICE_API constexpr void partial_scatter_to(
+  const basic_vec<_Tp, _Abi>& __v, _Range&& __range, const basic_vec<_Ip, _IAbi>& __indices, flags<_Flags...> __f = {})
+{
+  constexpr auto __all_true = typename basic_vec<_Ip, _IAbi>::mask_type(true);
+  ::cuda::std::simd::partial_scatter_to(__v, ::cuda::std::forward<_Range>(__range), __all_true, __indices, __f);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.permute.memory] unchecked_scatter_to
+
+// masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__scatter_constraints<_Tp, _Abi, _Range, _Ip, _IAbi>)
+_CCCL_HOST_DEVICE_API constexpr void unchecked_scatter_to(
+  const basic_vec<_Tp, _Abi>& __v,
+  _Range&& __range,
+  const typename basic_vec<_Ip, _IAbi>::mask_type& __mask,
+  const basic_vec<_Ip, _IAbi>& __indices,
+  flags<_Flags...> __f = {})
+{
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    const auto __range_size = ::cuda::std::ranges::size(__range);
+    _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__range_size),
+                 "cuda::std::simd::unchecked_scatter_to: ranges::size(out) is not representable as __simd_size_type");
+    const auto __size = static_cast<__simd_size_type>(__range_size);
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < basic_vec<_Tp, _Abi>::__size; ++__i)
+    {
+      if (__mask[__i])
+      {
+        const auto __idx = static_cast<__simd_size_type>(__indices[__i]);
+        _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__indices[__i])
+                       && ::cuda::in_range(__idx, __simd_size_type{0}, __size),
+                     "cuda::std::simd::unchecked_scatter_to: indices[i] must be in [0, ranges::size(out)) for every "
+                     "selected i");
+      }
+    }
+  }
+  ::cuda::std::simd::partial_scatter_to(__v, ::cuda::std::forward<_Range>(__range), __mask, __indices, __f);
+}
+
+// unmasked: delegate to the masked overload with an all-true mask to avoid duplicating the precondition check.
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename _Ip, typename _IAbi, typename... _Flags)
+_CCCL_REQUIRES(__scatter_constraints<_Tp, _Abi, _Range, _Ip, _IAbi>)
+_CCCL_HOST_DEVICE_API constexpr void unchecked_scatter_to(
+  const basic_vec<_Tp, _Abi>& __v, _Range&& __range, const basic_vec<_Ip, _IAbi>& __indices, flags<_Flags...> __f = {})
+{
+  constexpr auto __all_true = typename basic_vec<_Ip, _IAbi>::mask_type(true);
+  ::cuda::std::simd::unchecked_scatter_to(__v, ::cuda::std::forward<_Range>(__range), __all_true, __indices, __f);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_PERMUTE_MEMORY_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/reductions.h

@@ -0,0 +1,349 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_REDUCTIONS_H
+#define _CUDA_STD___SIMD_REDUCTIONS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__concepts/same_as.h>
+#include <cuda/std/__concepts/totally_ordered.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__functional/operations_traits.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__limits/numeric_limits.h>
+#include <cuda/std/__limits/numeric_limits_ext.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/basic_mask.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__type_traits/always_false.h>
+#include <cuda/std/__type_traits/type_identity.h>
+#include <cuda/std/__utility/declval.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+// [simd.expos], reduction-binary-operation concept
+
+template <typename _BinaryOp, typename _Tp>
+_CCCL_CONCEPT __reduction_binary_operation = _CCCL_REQUIRES_EXPR(
+  (_BinaryOp, _Tp), const _BinaryOp __binary_op, const vec<_Tp, 1> __v)(_Same_as(vec<_Tp, 1>) __binary_op(__v, __v));
+
+template <typename _BinaryOp, typename _Tp, bool = __reduction_binary_operation<_BinaryOp, _Tp>>
+inline constexpr bool __is_nothrow_reduction_binary_operation_v = false;
+
+template <typename _BinaryOp, typename _Tp>
+inline constexpr bool __is_nothrow_reduction_binary_operation_v<_BinaryOp, _Tp, true> = noexcept(
+  ::cuda::std::declval<const _BinaryOp&>()(::cuda::std::declval<vec<_Tp, 1>>(), ::cuda::std::declval<vec<_Tp, 1>>()));
+
+template <typename _BinaryOp>
+inline constexpr bool __is_reduce_default_supported_operation_v =
+  __is_plus_op_v<_BinaryOp> //
+  || __is_multiplies_op_v<_BinaryOp> //
+  || __is_bit_and_op_v<_BinaryOp> //
+  || __is_bit_or_op_v<_BinaryOp> //
+  || __is_bit_xor_op_v<_BinaryOp>;
+
+template <typename _Tp, typename _BinaryOp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp __default_identity_element() noexcept
+{
+  if constexpr (__is_plus_op_v<_BinaryOp> || __is_bit_or_op_v<_BinaryOp> || __is_bit_xor_op_v<_BinaryOp>)
+  {
+    return _Tp{};
+  }
+  else if constexpr (__is_multiplies_op_v<_BinaryOp>)
+  {
+    return _Tp(1);
+  }
+  else if constexpr (__is_bit_and_op_v<_BinaryOp>)
+  {
+    return static_cast<_Tp>(~_Tp{});
+  }
+  else
+  {
+    static_assert(__always_false_v<_Tp>,
+                  "No default identity element for this BinaryOperation; provide one explicitly");
+    return _Tp{};
+  }
+}
+
+// [simd.reductions], reduce
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _BinaryOperation = plus<>)
+_CCCL_REQUIRES(__reduction_binary_operation<_BinaryOperation, _Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp
+reduce(const basic_vec<_Tp, _Abi>& __x,
+       _BinaryOperation __binary_op = {}) noexcept(__is_nothrow_reduction_binary_operation_v<_BinaryOperation, _Tp>)
+{
+  vec<_Tp, 1> __result{__x[0]};
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 1; __i < __x.__size; ++__i)
+  {
+    __result = __binary_op(__result, vec<_Tp, 1>{__x[__i]});
+  }
+  return __result[0];
+}
+
+// We need two overloads:
+// 1) An argument for identity_element is provided for the invocation
+// 2) unless BinaryOperation is one of plus<>, multiplies<>, bit_and<>, bit_or<>, or bit_xor<>
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _BinaryOperation)
+_CCCL_REQUIRES(__reduction_binary_operation<_BinaryOperation, _Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp
+reduce(const basic_vec<_Tp, _Abi>& __x,
+       const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+       _BinaryOperation __binary_op,
+       const type_identity_t<_Tp> __identity_element) //
+  noexcept(__is_nothrow_reduction_binary_operation_v<_BinaryOperation, _Tp>)
+{
+  vec<_Tp, 1> __result{__identity_element};
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __x.__size; ++__i)
+  {
+    if (__mask[__i])
+    {
+      __result = __binary_op(__result, vec<_Tp, 1>{__x[__i]});
+    }
+  }
+  return __result[0];
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _BinaryOperation = plus<>)
+_CCCL_REQUIRES(__reduction_binary_operation<_BinaryOperation, _Tp> _CCCL_AND
+                 __is_reduce_default_supported_operation_v<_BinaryOperation>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp reduce(
+  const basic_vec<_Tp, _Abi>& __x,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  const _BinaryOperation __binary_op = {}) noexcept(__is_nothrow_reduction_binary_operation_v<_BinaryOperation, _Tp>)
+{
+  return ::cuda::std::simd::reduce(
+    __x, __mask, __binary_op, ::cuda::std::simd::__default_identity_element<_Tp, _BinaryOperation>());
+}
+
+// [simd.reductions], reduce_min
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp reduce_min(const basic_vec<_Tp, _Abi>& __x) noexcept
+{
+  auto __result = __x[0];
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 1; __i < __x.__size; ++__i)
+  {
+    const auto __val = __x[__i];
+    if (__val < __result)
+    {
+      __result = __val;
+    }
+  }
+  return __result;
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp
+reduce_min(const basic_vec<_Tp, _Abi>& __x, const typename basic_vec<_Tp, _Abi>::mask_type& __mask) noexcept
+{
+  auto __result = numeric_limits<_Tp>::max();
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __x.__size; ++__i)
+  {
+    if (__mask[__i])
+    {
+      const auto __val = __x[__i];
+      if (__val < __result)
+      {
+        __result = __val;
+      }
+    }
+  }
+  return __result;
+}
+
+// [simd.reductions], reduce_max
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp reduce_max(const basic_vec<_Tp, _Abi>& __x) noexcept
+{
+  auto __result = __x[0];
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 1; __i < __x.__size; ++__i)
+  {
+    const auto __val = __x[__i];
+    if (__result < __val)
+    {
+      __result = __val;
+    }
+  }
+  return __result;
+}
+
+_CCCL_TEMPLATE(typename _Tp, typename _Abi)
+_CCCL_REQUIRES(totally_ordered<_Tp>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp
+reduce_max(const basic_vec<_Tp, _Abi>& __x, const typename basic_vec<_Tp, _Abi>::mask_type& __mask) noexcept
+{
+  auto __result = numeric_limits<_Tp>::lowest();
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __x.__size; ++__i)
+  {
+    if (__mask[__i])
+    {
+      const auto __val = __x[__i];
+      if (__result < __val)
+      {
+        __result = __val;
+      }
+    }
+  }
+  return __result;
+}
+
+// [simd.mask.reductions], mask reductions
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool all_of(const basic_mask<_Bytes, _Abi>& __k) noexcept
+{
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __k.__size; ++__i)
+  {
+    if (!__k[__i])
+    {
+      return false;
+    }
+  }
+  return true;
+}
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool any_of(const basic_mask<_Bytes, _Abi>& __k) noexcept
+{
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __k.__size; ++__i)
+  {
+    if (__k[__i])
+    {
+      return true;
+    }
+  }
+  return false;
+}
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool none_of(const basic_mask<_Bytes, _Abi>& __k) noexcept
+{
+  return !::cuda::std::simd::any_of(__k);
+}
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __simd_size_type reduce_count(const basic_mask<_Bytes, _Abi>& __k) noexcept
+{
+  __simd_size_type __count = 0;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __k.__size; ++__i)
+  {
+    __count += static_cast<__simd_size_type>(__k[__i]);
+  }
+  return __count;
+}
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __simd_size_type
+reduce_min_index(const basic_mask<_Bytes, _Abi>& __k) noexcept
+{
+  _CCCL_ASSERT(::cuda::std::simd::any_of(__k), "No bits are set");
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __k.__size; ++__i)
+  {
+    if (__k[__i])
+    {
+      return __i;
+    }
+  }
+  return __simd_size_type{-1};
+}
+
+template <size_t _Bytes, typename _Abi>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __simd_size_type
+reduce_max_index(const basic_mask<_Bytes, _Abi>& __k) noexcept
+{
+  _CCCL_ASSERT(::cuda::std::simd::any_of(__k), "No bits are set");
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = __k.__size - 1; __i >= 0; --__i)
+  {
+    if (__k[__i])
+    {
+      return __i;
+    }
+  }
+  return __simd_size_type{-1};
+}
+
+// Scalar bool overloads
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(same_as<_Tp, bool>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool all_of(const _Tp __x) noexcept
+{
+  return __x;
+}
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(same_as<_Tp, bool>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool any_of(const _Tp __x) noexcept
+{
+  return __x;
+}
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(same_as<_Tp, bool>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool none_of(const _Tp __x) noexcept
+{
+  return !__x;
+}
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(same_as<_Tp, bool>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __simd_size_type reduce_count(const _Tp __x) noexcept
+{
+  return __x;
+}
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(same_as<_Tp, bool>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __simd_size_type reduce_min_index(const _Tp __x) noexcept
+{
+  _CCCL_ASSERT(__x, "No bits are set");
+  return 0;
+}
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(same_as<_Tp, bool>)
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __simd_size_type reduce_max_index(const _Tp __x) noexcept
+{
+  _CCCL_ASSERT(__x, "No bits are set");
+  return 0;
+}
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_REDUCTIONS_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fixed_size_float_vec.h

@@ -0,0 +1,133 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_FLOAT_VEC_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_FLOAT_VEC_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_HAS_SIMD_F32X2()
+
+#  include <cuda/std/__fwd/simd.h>
+#  include <cuda/std/__simd/abi.h>
+#  include <cuda/std/__simd/specializations/fixed_size_vec.h>
+#  include <cuda/std/__simd/specializations/fp32x2_intrinsics_array.h>
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <__simd_size_type _Np>
+inline constexpr __simd_operations_kind __simd_operations_kind_v<float, __fixed_size<_Np>> =
+  (_Np >= 2) ? __simd_operations_kind::__fixed_size_float : __simd_operations_kind::__default;
+
+// Simd operations for fixed_size ABI with float elements and F32x2 fast paths.
+template <__simd_size_type _Np>
+struct __simd_operations<float, __fixed_size<_Np>, __simd_operations_kind::__fixed_size_float>
+    : __fixed_size_operations<float, _Np>
+{
+  using __base       = __fixed_size_operations<float, _Np>;
+  using _SimdStorage = __simd_storage<float, __fixed_size<_Np>>;
+
+  _CCCL_HOST_DEVICE_API static constexpr void __increment(_SimdStorage& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, ({
+                     constexpr _SimdStorage __one = __base::__broadcast(1.0f);
+                     __s                          = ::cuda::std::simd::__plus_f32x2(__s, __one);
+                     return;
+                   }));
+    }
+    __base::__increment(__s);
+  }
+
+  _CCCL_HOST_DEVICE_API static constexpr void __decrement(_SimdStorage& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, ({
+                     constexpr _SimdStorage __one = __base::__broadcast(1.0f);
+                     __s                          = ::cuda::std::simd::__minus_f32x2(__s, __one);
+                     return;
+                   }));
+    }
+    __base::__decrement(__s);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage __unary_minus(const _SimdStorage& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, ({
+                     constexpr _SimdStorage __zero = __base::__broadcast(0.0f);
+                     return ::cuda::std::simd::__minus_f32x2(__zero, __s);
+                   }));
+    }
+    return __base::__unary_minus(__s);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __plus(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, (return ::cuda::std::simd::__plus_f32x2(__lhs, __rhs);))
+    }
+    return __base::__plus(__lhs, __rhs);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __minus(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, (return ::cuda::std::simd::__minus_f32x2(__lhs, __rhs);))
+    }
+    return __base::__minus(__lhs, __rhs);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __multiplies(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, (return ::cuda::std::simd::__multiplies_f32x2(__lhs, __rhs);))
+    }
+    return __base::__multiplies(__lhs, __rhs);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static _SimdStorage
+  __fma(const _SimdStorage& __lhs, const _SimdStorage& __rhs, const _SimdStorage& __add) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_PROVIDES_SM_100, (return ::cuda::std::simd::__fma_f32x2(__lhs, __rhs, __add);))
+    }
+    return __base::__fma(__lhs, __rhs, __add);
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_HAS_SIMD_F32X2()
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_FLOAT_VEC_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fixed_size_integral_vec.h

@@ -0,0 +1,282 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_INTEGRAL_VEC_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_INTEGRAL_VEC_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+// automatic vectorization for small integers is not supported (until CUDA 13.2)
+// TODO(fbusato): remove this path once the feature is supported
+// TODO(fbusato): extend to other GPU archs in the future
+
+#include <cuda/__cmath/ceil_div.h>
+#include <cuda/std/__simd/specializations/fixed_size_vec.h>
+#include <cuda/std/__simd/specializations/simd_intrinsics_array.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/array>
+#include <cuda/std/cstdint>
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <typename _Tp, __simd_size_type _Np>
+inline constexpr bool __is_fixed_size_small_integral_v =
+  is_integral_v<_Tp> && sizeof(_Tp) < sizeof(uint32_t) && _Np >= 2;
+
+inline constexpr auto __simd_operations_small_integral = __simd_operations_kind::__fixed_size_integral;
+
+template <typename _Tp, __simd_size_type _Np>
+inline constexpr __simd_operations_kind __simd_operations_kind_v<_Tp, __fixed_size<_Np>> =
+  __is_fixed_size_small_integral_v<_Tp, _Np> ? __simd_operations_small_integral : __simd_operations_kind::__default;
+
+#define _CCCL_SIMD_FIXED_SIZE_INTEGRAL_BINARY_BITWISE(_NAME, _OP)                           \
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t _NAME(              \
+    const __simd_storage_t& __lhs, const __simd_storage_t& __rhs) noexcept                  \
+  {                                                                                         \
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT                                                          \
+    {                                                                                       \
+      __unsigned_storage_t __result_u{};                                                    \
+      const auto __lhs_u = ::cuda::std::simd::__to_unsigned_storage(__lhs);                 \
+      const auto __rhs_u = ::cuda::std::simd::__to_unsigned_storage(__rhs);                 \
+      _CCCL_PRAGMA_UNROLL_FULL()                                                            \
+      for (__simd_size_type __i = 0; __i < __usize; ++__i)                                  \
+      {                                                                                     \
+        __result_u[__i] = __lhs_u[__i] _OP __rhs_u[__i];                                    \
+      }                                                                                     \
+      return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(__result_u); \
+    }                                                                                       \
+    return __base::_NAME(__lhs, __rhs);                                                     \
+  }
+
+// Simd operations for fixed_size ABI with small integral element types.
+template <typename _Tp, __simd_size_type _Np>
+struct __simd_operations<_Tp, __fixed_size<_Np>, __simd_operations_small_integral> : __fixed_size_operations<_Tp, _Np>
+{
+  using __base           = __fixed_size_operations<_Tp, _Np>;
+  using __simd_storage_t = __simd_storage<_Tp, __fixed_size<_Np>>;
+
+  // all computation is done on uint32_t, so the alignment must be at least the alignment of uint32_t
+  static_assert(alignof(__simd_storage_t) >= alignof(uint32_t));
+
+  static constexpr __simd_size_type __ratio = sizeof(uint32_t) / sizeof(_Tp);
+  static constexpr __simd_size_type __usize = ::cuda::ceil_div(_Np, __ratio);
+  using __unsigned_storage_t                = array<uint32_t, __usize>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t
+  __bitwise_not(const __simd_storage_t& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      auto __udata = ::cuda::std::simd::__to_unsigned_storage(__s);
+      _CCCL_PRAGMA_UNROLL_FULL()
+      for (__simd_size_type __i = 0; __i < __usize; ++__i)
+      {
+        __udata[__i] = ~__udata[__i];
+      }
+      return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(__udata);
+    }
+    return __fixed_size_operations<_Tp, _Np>::__bitwise_not(__s);
+  }
+
+  _CCCL_SIMD_FIXED_SIZE_INTEGRAL_BINARY_BITWISE(__bitwise_and, &)
+  _CCCL_SIMD_FIXED_SIZE_INTEGRAL_BINARY_BITWISE(__bitwise_or, |)
+  _CCCL_SIMD_FIXED_SIZE_INTEGRAL_BINARY_BITWISE(__bitwise_xor, ^)
+
+#if _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
+  // Unary arithmetic operations
+
+  // x++ = x + 1
+  _CCCL_HOST_DEVICE_API static constexpr void __increment(__simd_storage_t& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      [[maybe_unused]] constexpr __simd_storage_t __one = __base::__broadcast(1);
+      if constexpr (sizeof(_Tp) == 2)
+      {
+        NV_IF_TARGET(NV_PROVIDES_SM_90, (__s = __plus(__s, __one); return;))
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, (__s = __plus(__s, __one); return;))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    __base::__increment(__s);
+  }
+
+  // x-- = x - 1
+  _CCCL_HOST_DEVICE_API static constexpr void __decrement(__simd_storage_t& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      [[maybe_unused]] constexpr __simd_storage_t __minus_one = __base::__broadcast(static_cast<_Tp>(-1));
+      if constexpr (sizeof(_Tp) == 2)
+      {
+        NV_IF_TARGET(NV_PROVIDES_SM_90, (__s = __plus(__s, __minus_one); return;))
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, (__s = __plus(__s, __minus_one); return;))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    __base::__decrement(__s);
+  }
+
+  // -x = ~x + 1
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t __unary_minus(const __simd_storage_t& __s) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      [[maybe_unused]] constexpr __simd_storage_t __one = __base::__broadcast(1);
+      if constexpr (sizeof(_Tp) == 2)
+      {
+        NV_IF_TARGET(NV_PROVIDES_SM_90, (return __plus(__bitwise_not(__s), __one);))
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, (return __plus(__bitwise_not(__s), __one);))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    return __base::__unary_minus(__s);
+  }
+
+  // Binary arithmetic operations
+
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t
+  __plus(const __simd_storage_t& __lhs, const __simd_storage_t& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      [[maybe_unused]] const auto __lhs_u = ::cuda::std::simd::__to_unsigned_storage(__lhs);
+      [[maybe_unused]] const auto __rhs_u = ::cuda::std::simd::__to_unsigned_storage(__rhs);
+      if constexpr (sizeof(_Tp) == 2)
+      {
+        NV_IF_TARGET(NV_PROVIDES_SM_90,
+                     (return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(
+                               ::cuda::std::simd::__vadd_16bit_x2(__lhs_u, __rhs_u));))
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+                     (return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(
+                               ::cuda::std::simd::__vadd_8bit_x4(__lhs_u, __rhs_u));))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    return __fixed_size_operations<_Tp, _Np>::__plus(__lhs, __rhs);
+  }
+
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t
+  __minus(const __simd_storage_t& __lhs, const __simd_storage_t& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      if constexpr (sizeof(_Tp) == 2)
+      {
+        NV_IF_TARGET(NV_PROVIDES_SM_90, (return __plus(__lhs, __unary_minus(__rhs));))
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, (return __plus(__lhs, __unary_minus(__rhs));))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    return __base::__minus(__lhs, __rhs);
+  }
+
+  // Min/max operations
+
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t
+  __min_simd(const __simd_storage_t& __lhs, const __simd_storage_t& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      [[maybe_unused]] const auto __lhs_u = ::cuda::std::simd::__to_unsigned_storage(__lhs);
+      [[maybe_unused]] const auto __rhs_u = ::cuda::std::simd::__to_unsigned_storage(__rhs);
+      if constexpr (sizeof(_Tp) == 2)
+      {
+#  if !_CCCL_HAS_SIMD_16BIT_MIN_MAX_COMPILER_OPTIMIZATION()
+        NV_IF_TARGET(NV_PROVIDES_SM_90,
+                     (return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(
+                               ::cuda::std::simd::__vmin_16bit_x2<_Tp>(__lhs_u, __rhs_u));))
+#  endif // !_CCCL_HAS_SIMD_16BIT_MIN_MAX_COMPILER_OPTIMIZATION()
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+                     (return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(
+                               ::cuda::std::simd::__vmin_8bit_x4<_Tp>(__lhs_u, __rhs_u));))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    return __base::__min_simd(__lhs, __rhs);
+  }
+
+  [[nodiscard]]
+  _CCCL_HOST_DEVICE_API static constexpr __simd_storage_t
+  __max_simd(const __simd_storage_t& __lhs, const __simd_storage_t& __rhs) noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      [[maybe_unused]] const auto __lhs_u = ::cuda::std::simd::__to_unsigned_storage(__lhs);
+      [[maybe_unused]] const auto __rhs_u = ::cuda::std::simd::__to_unsigned_storage(__rhs);
+      if constexpr (sizeof(_Tp) == 2)
+      {
+#  if !_CCCL_HAS_SIMD_16BIT_MIN_MAX_COMPILER_OPTIMIZATION()
+        NV_IF_TARGET(NV_PROVIDES_SM_90,
+                     (return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(
+                               ::cuda::std::simd::__vmax_16bit_x2<_Tp>(__lhs_u, __rhs_u));))
+#  endif // !_CCCL_HAS_SIMD_16BIT_MIN_MAX_COMPILER_OPTIMIZATION()
+      }
+#  if _CCCL_HAS_SIMD_8BIT()
+      else if constexpr (sizeof(_Tp) == 1)
+      {
+        NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+                     (return ::cuda::std::simd::__copy_from_unsigned_storage<__simd_storage_t>(
+                               ::cuda::std::simd::__vmax_8bit_x4<_Tp>(__lhs_u, __rhs_u));))
+      }
+#  endif // _CCCL_HAS_SIMD_8BIT()
+    }
+    return __base::__max_simd(__lhs, __rhs);
+  }
+#endif // _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
+};
+
+#undef _CCCL_SIMD_FIXED_SIZE_INTEGRAL_BINARY_BITWISE
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_INTEGRAL_VEC_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fixed_size_mask.h

@@ -0,0 +1,173 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_MASK_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_MASK_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/specializations/fixed_size_storage.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__utility/integer_sequence.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// Bool-per-element mask storage for fixed_size ABI
+template <size_t _Bytes, __simd_size_type _Np>
+struct __mask_storage<_Bytes, __fixed_size<_Np>>
+{
+  static constexpr size_t __element_bytes = _Bytes;
+
+  bool __data[_Np]{};
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __get(const __simd_size_type __idx) const noexcept
+  {
+    _CCCL_ASSERT(::cuda::in_range(__idx, __simd_size_type{0}, _Np), "Index is out of bounds");
+    return __data[__idx];
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr void __set(const __simd_size_type __idx, const bool __v) noexcept
+  {
+    _CCCL_ASSERT(::cuda::in_range(__idx, __simd_size_type{0}, _Np), "Index is out of bounds");
+    __data[__idx] = __v;
+  }
+};
+
+// Mask operations for fixed_size ABI with bool-per-element storage
+template <size_t _Bytes, __simd_size_type _Np>
+struct __mask_operations<_Bytes, __fixed_size<_Np>>
+{
+  using _MaskStorage = __mask_storage<_Bytes, __fixed_size<_Np>>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage __broadcast(const bool __v) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __v;
+    }
+    return __result;
+  }
+
+  template <typename _Generator, __simd_size_type... _Is>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __generate_init(_Generator&& __g, integer_sequence<__simd_size_type, _Is...>)
+  {
+#if _CCCL_STD_VER >= 2020
+    _MaskStorage __result;
+    ((__result.__data[_Is] = static_cast<bool>(__g(integral_constant<__simd_size_type, _Is>()))), ...);
+    return __result;
+#else // ^^^ C++20 ^^^ / vvv C++17 vvv
+    return _MaskStorage{{ static_cast<bool>(__g(integral_constant<__simd_size_type, _Is>()))... }};
+#endif // _CCCL_STD_VER < 2020
+  }
+
+  template <typename _Generator>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage __generate(_Generator&& __g)
+  {
+    return __generate_init(__g, make_integer_sequence<__simd_size_type, _Np>());
+  }
+
+  // Logical operators (for operator&& and operator||)
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __logic_and(const _MaskStorage& __lhs, const _MaskStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __lhs.__data[__i] && __rhs.__data[__i];
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __logic_or(const _MaskStorage& __lhs, const _MaskStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __lhs.__data[__i] || __rhs.__data[__i];
+    }
+    return __result;
+  }
+
+  // Bitwise operators (for operator&, operator|, operator^)
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __bitwise_and(const _MaskStorage& __lhs, const _MaskStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __lhs.__data[__i] && __rhs.__data[__i];
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __bitwise_or(const _MaskStorage& __lhs, const _MaskStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __lhs.__data[__i] || __rhs.__data[__i];
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __bitwise_xor(const _MaskStorage& __lhs, const _MaskStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __lhs.__data[__i] != __rhs.__data[__i];
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage __bitwise_not(const _MaskStorage& __s) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = !__s.__data[__i];
+    }
+    return __result;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_MASK_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fixed_size_storage.h

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_STORAGE_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_STORAGE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__utility/in_range.h>
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__simd/abi.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <__simd_size_type _Np>
+struct __fixed_size
+{
+  static_assert(_Np > 0, "_Np must be greater than 0");
+
+  static constexpr __simd_size_type __simd_size = _Np;
+};
+
+// SIMD storage never directly interacts with memory. Users must use load/store API for that purpose.
+// However, SIMD storage could spill from register to cache/memory. This could break the alignment of the data for
+// vectorized instructions. For this reason, we align the SIMD storage to at least 8 bytes (max SIMD instruction size).
+// 8 bytes is a negligible constraint in case of spilling.
+template <typename _Tp>
+inline constexpr size_t __simd_storage_alignment_v = ::cuda::std::max(alignof(_Tp), size_t{8});
+
+// Element-per-slot simd storage for fixed_size ABI
+template <typename _Tp, __simd_size_type _Np>
+struct alignas(__simd_storage_alignment_v<_Tp>) __simd_storage<_Tp, __fixed_size<_Np>>
+{
+  using value_type = _Tp;
+
+  _Tp __data[_Np]{};
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Tp __get(const __simd_size_type __idx) const noexcept
+  {
+    _CCCL_ASSERT(::cuda::in_range(__idx, __simd_size_type{0}, _Np), "Index is out of bounds");
+    return __data[__idx];
+  }
+
+  _CCCL_HOST_DEVICE_API constexpr void __set(const __simd_size_type __idx, const _Tp __v) noexcept
+  {
+    _CCCL_ASSERT(::cuda::in_range(__idx, __simd_size_type{0}, _Np), "Index is out of bounds");
+    __data[__idx] = __v;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_STORAGE_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fixed_size_vec.h

@@ -0,0 +1,378 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_VEC_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_VEC_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__algorithm/min.h>
+#include <cuda/std/__cmath/fma.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/specializations/fixed_size_mask.h>
+#include <cuda/std/__simd/specializations/fixed_size_storage.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__utility/integer_sequence.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// Simd operations for fixed_size ABI
+
+template <typename _Tp, __simd_size_type _Np>
+struct __fixed_size_operations
+{
+  using _SimdStorage = __simd_storage<_Tp, __fixed_size<_Np>>;
+  using _MaskStorage = __mask_storage<sizeof(_Tp), __fixed_size<_Np>>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage __broadcast(const _Tp __v) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = __v;
+    }
+    return __result;
+  }
+
+  template <typename _Generator, __simd_size_type... _Is>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __generate_init(_Generator&& __g, integer_sequence<__simd_size_type, _Is...>)
+  {
+#if _CCCL_STD_VER >= 2020
+    _SimdStorage __result;
+    ((__result.__data[_Is] = __g(integral_constant<__simd_size_type, _Is>())), ...);
+    return __result;
+#else // ^^^ C++20 ^^^ / vvv C++17 vvv
+    return _SimdStorage{{__g(integral_constant<__simd_size_type, _Is>())...}};
+#endif // _CCCL_STD_VER < 2020
+  }
+
+  template <typename _Generator>
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage __generate(_Generator&& __g)
+  {
+    return __generate_init(__g, make_integer_sequence<__simd_size_type, _Np>());
+  }
+
+  // Unary operations
+
+  _CCCL_HOST_DEVICE_API static constexpr void __increment(_SimdStorage& __s) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      ++__s.__data[__i];
+    }
+  }
+
+  _CCCL_HOST_DEVICE_API static constexpr void __decrement(_SimdStorage& __s) noexcept
+  {
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      --__s.__data[__i];
+    }
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage __negate(const _SimdStorage& __s) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = !__s.__data[__i];
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage __bitwise_not(const _SimdStorage& __s) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = ~__s.__data[__i];
+    }
+    return __result;
+  }
+
+  _CCCL_DIAG_PUSH
+  _CCCL_DIAG_SUPPRESS_MSVC(4146) // unary minus applied to unsigned type
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage __unary_minus(const _SimdStorage& __s) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = -__s.__data[__i];
+    }
+    return __result;
+  }
+  _CCCL_DIAG_POP
+
+  // Binary arithmetic operations
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __plus(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] + __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __minus(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] - __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __multiplies(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] * __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __fma(const _SimdStorage& __lhs, const _SimdStorage& __rhs, const _SimdStorage& __add) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = ::cuda::std::fma(__lhs.__data[__i], __rhs.__data[__i], __add.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __divides(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] / __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __modulo(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] % __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  // Min/max operations
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __min_simd(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = ::cuda::std::min(__lhs.__data[__i], __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __max_simd(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = ::cuda::std::max(__lhs.__data[__i], __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  // Comparison operations
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __equal_to(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] == __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __not_equal_to(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] != __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __less(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] < __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __less_equal(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] <= __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __greater(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] > __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _MaskStorage
+  __greater_equal(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _MaskStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] >= __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  // Bitwise and shift operations
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __bitwise_and(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] & __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __bitwise_or(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] | __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __bitwise_xor(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] ^ __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __shift_left(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] << __rhs.__data[__i]);
+    }
+    return __result;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr _SimdStorage
+  __shift_right(const _SimdStorage& __lhs, const _SimdStorage& __rhs) noexcept
+  {
+    _SimdStorage __result;
+    _CCCL_PRAGMA_UNROLL_FULL()
+    for (__simd_size_type __i = 0; __i < _Np; ++__i)
+    {
+      __result.__data[__i] = (__lhs.__data[__i] >> __rhs.__data[__i]);
+    }
+    return __result;
+  }
+};
+
+// Default path (no optimizations)
+template <typename _Tp, __simd_size_type _Np>
+struct __simd_operations<_Tp, __fixed_size<_Np>, __simd_operations_kind::__default> : __fixed_size_operations<_Tp, _Np>
+{};
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FIXED_SIZE_VEC_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fp32x2_intrinsics.h

@@ -0,0 +1,161 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FP32X2_INTRINSICS_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FP32X2_INTRINSICS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+// automatic vectorization for float2 is not supported (until CUDA 13.2)
+// TODO(fbusato): extend for other GPU archs in the future
+// TODO(fbusato): check 5361571, remove this path once the feature is supported
+#if _CCCL_HAS_SIMD_F32X2()
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+[[nodiscard]] _CCCL_DEVICE_API inline ::float2
+__add_f32x2([[maybe_unused]] const ::float2 __lhs, [[maybe_unused]] const ::float2 __rhs) noexcept
+{
+  ::float2 __result{};
+#  if _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100,
+                    (__result = ::__fadd2_rn(__lhs, __rhs);),
+                    (_CCCL_VERIFY(false, "cuda::std::simd::__add_f32x2: Unsupported architecture");))
+#  elif _CCCL_HAS_SIMD_F32X2_PTX() // PTX ISA 8.6
+  NV_IF_ELSE_TARGET(
+    NV_PROVIDES_SM_100,
+    (asm("{"
+         ".reg .b64 __lhs, __rhs, __result;"
+         "mov.b64 __lhs, {%2, %3};"
+         "mov.b64 __rhs, {%4, %5};"
+         "add.f32x2 __result, __lhs, __rhs;"
+         "mov.b64 {%0, %1}, __result;"
+         "}" //
+         : "=f"(__result.x),
+         "=f"(__result.y) //
+         : "f"(__lhs.x),
+         "f"(__lhs.y),
+         "f"(__rhs.x),
+         "f"(__rhs.y));),
+    (_CCCL_VERIFY(false, "cuda::std::simd::__add_f32x2: Unsupported architecture");))
+#  endif // _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  return __result;
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline ::float2
+__mul_f32x2([[maybe_unused]] const ::float2 __lhs, [[maybe_unused]] const ::float2 __rhs) noexcept
+{
+  ::float2 __result{};
+#  if _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100,
+                    (__result = ::__fmul2_rn(__lhs, __rhs);),
+                    (_CCCL_VERIFY(false, "cuda::std::simd::__mul_f32x2: Unsupported architecture");))
+#  elif _CCCL_HAS_SIMD_F32X2_PTX() // PTX ISA 8.6
+  NV_IF_ELSE_TARGET(
+    NV_PROVIDES_SM_100,
+    (asm("{"
+         ".reg .b64 __lhs, __rhs, __result;"
+         "mov.b64 __lhs, {%2, %3};"
+         "mov.b64 __rhs, {%4, %5};"
+         "mul.f32x2 __result, __lhs, __rhs;"
+         "mov.b64 {%0, %1}, __result;"
+         "}" //
+         : "=f"(__result.x),
+         "=f"(__result.y) //
+         : "f"(__lhs.x),
+         "f"(__lhs.y),
+         "f"(__rhs.x),
+         "f"(__rhs.y));),
+    (_CCCL_VERIFY(false, "cuda::std::simd::__mul_f32x2: Unsupported architecture");))
+#  endif // _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  return __result;
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline ::float2
+__sub_f32x2([[maybe_unused]] const ::float2 __lhs, [[maybe_unused]] const ::float2 __rhs) noexcept
+{
+  ::float2 __result{};
+#  if _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100,
+                    (__result = ::__fadd2_rn(__lhs, ::float2{-__rhs.x, -__rhs.y});),
+                    (_CCCL_VERIFY(false, "cuda::std::simd::__sub_f32x2: Unsupported architecture");))
+#  elif _CCCL_HAS_SIMD_F32X2_PTX() // PTX ISA 8.6
+  NV_IF_ELSE_TARGET(
+    NV_PROVIDES_SM_100,
+    (asm("{"
+         ".reg .b64 __lhs, __rhs, __result;"
+         "mov.b64 __lhs, {%2, %3};"
+         "mov.b64 __rhs, {%4, %5};"
+         "sub.f32x2 __result, __lhs, __rhs;"
+         "mov.b64 {%0, %1}, __result;"
+         "}" //
+         : "=f"(__result.x),
+         "=f"(__result.y) //
+         : "f"(__lhs.x),
+         "f"(__lhs.y),
+         "f"(__rhs.x),
+         "f"(__rhs.y));),
+    (_CCCL_VERIFY(false, "cuda::std::simd::__sub_f32x2: Unsupported architecture");))
+#  endif // _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  return __result;
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline ::float2 __fma_f32x2(
+  [[maybe_unused]] const ::float2 __lhs,
+  [[maybe_unused]] const ::float2 __rhs,
+  [[maybe_unused]] const ::float2 __add) noexcept
+{
+  ::float2 __result{};
+#  if _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100,
+                    (__result = ::__ffma2_rn(__lhs, __rhs, __add);),
+                    (_CCCL_VERIFY(false, "cuda::std::simd::__fma_f32x2: Unsupported architecture");))
+#  elif _CCCL_HAS_SIMD_F32X2_PTX() // PTX ISA 8.6
+  NV_IF_ELSE_TARGET(
+    NV_PROVIDES_SM_100,
+    (asm("{"
+         ".reg .b64 __lhs, __rhs, __add, __result;"
+         "mov.b64 __lhs, {%2, %3};"
+         "mov.b64 __rhs, {%4, %5};"
+         "mov.b64 __add, {%6, %7};"
+         "fma.rn.f32x2 __result, __lhs, __rhs, __add;"
+         "mov.b64 {%0, %1}, __result;"
+         "}" //
+         : "=f"(__result.x),
+         "=f"(__result.y) //
+         : "f"(__lhs.x),
+         "f"(__lhs.y),
+         "f"(__rhs.x),
+         "f"(__rhs.y),
+         "f"(__add.x),
+         "f"(__add.y));),
+    (_CCCL_VERIFY(false, "cuda::std::simd::__fma_f32x2: Unsupported architecture");))
+#  endif // _CCCL_HAS_SIMD_F32X2_INTRINSICS()
+  return __result;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_HAS_SIMD_F32X2()
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FP32X2_INTRINSICS_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/fp32x2_intrinsics_array.h

@@ -0,0 +1,134 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_FP32X2_INTRINSICS_ARRAY_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_FP32X2_INTRINSICS_ARRAY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+// automatic vectorization for float2 is not supported (until CUDA 13.2)
+// TODO(fbusato): extend for other GPU archs in the future
+// TODO(fbusato): check 5361571, remove this path once the feature is supported
+#if _CCCL_HAS_SIMD_F32X2()
+
+#  include <cuda/std/__cmath/fma.h>
+#  include <cuda/std/__fwd/simd.h>
+#  include <cuda/std/__simd/abi.h>
+#  include <cuda/std/__simd/specializations/fixed_size_storage.h>
+#  include <cuda/std/__simd/specializations/fp32x2_intrinsics.h>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <__simd_size_type _Np>
+using __simd_storage_f32 = __simd_storage<float, __fixed_size<_Np>>;
+
+template <__simd_size_type _Np>
+[[nodiscard]] _CCCL_DEVICE_API constexpr __simd_storage_f32<_Np>
+__plus_f32x2(const __simd_storage_f32<_Np>& __lhs, const __simd_storage_f32<_Np>& __rhs) noexcept
+{
+  __simd_storage_f32<_Np> __result;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < (_Np / 2) * 2; __i += 2)
+  {
+    const auto __lhs_value   = ::float2{__lhs.__data[__i], __lhs.__data[__i + 1]};
+    const auto __rhs_value   = ::float2{__rhs.__data[__i], __rhs.__data[__i + 1]};
+    const auto __value       = ::cuda::std::simd::__add_f32x2(__lhs_value, __rhs_value);
+    __result.__data[__i]     = __value.x;
+    __result.__data[__i + 1] = __value.y;
+  }
+  if constexpr (_Np % 2 != 0)
+  {
+    __result.__data[_Np - 1] = __lhs.__data[_Np - 1] + __rhs.__data[_Np - 1];
+  }
+  return __result;
+}
+
+template <__simd_size_type _Np>
+[[nodiscard]] _CCCL_DEVICE_API constexpr __simd_storage_f32<_Np>
+__minus_f32x2(const __simd_storage_f32<_Np>& __lhs, const __simd_storage_f32<_Np>& __rhs) noexcept
+{
+  __simd_storage_f32<_Np> __result;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < (_Np / 2) * 2; __i += 2)
+  {
+    const auto __lhs_value   = ::float2{__lhs.__data[__i], __lhs.__data[__i + 1]};
+    const auto __rhs_value   = ::float2{__rhs.__data[__i], __rhs.__data[__i + 1]};
+    const auto __value       = ::cuda::std::simd::__sub_f32x2(__lhs_value, __rhs_value);
+    __result.__data[__i]     = __value.x;
+    __result.__data[__i + 1] = __value.y;
+  }
+  if constexpr (_Np % 2 != 0)
+  {
+    __result.__data[_Np - 1] = __lhs.__data[_Np - 1] - __rhs.__data[_Np - 1];
+  }
+  return __result;
+}
+
+template <__simd_size_type _Np>
+[[nodiscard]] _CCCL_DEVICE_API constexpr __simd_storage_f32<_Np>
+__multiplies_f32x2(const __simd_storage_f32<_Np>& __lhs, const __simd_storage_f32<_Np>& __rhs) noexcept
+{
+  __simd_storage_f32<_Np> __result;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < (_Np / 2) * 2; __i += 2)
+  {
+    const auto __lhs_value   = ::float2{__lhs.__data[__i], __lhs.__data[__i + 1]};
+    const auto __rhs_value   = ::float2{__rhs.__data[__i], __rhs.__data[__i + 1]};
+    const auto __value       = ::cuda::std::simd::__mul_f32x2(__lhs_value, __rhs_value);
+    __result.__data[__i]     = __value.x;
+    __result.__data[__i + 1] = __value.y;
+  }
+  if constexpr (_Np % 2 != 0)
+  {
+    __result.__data[_Np - 1] = __lhs.__data[_Np - 1] * __rhs.__data[_Np - 1];
+  }
+  return __result;
+}
+
+template <__simd_size_type _Np>
+[[nodiscard]] _CCCL_DEVICE_API constexpr __simd_storage_f32<_Np>
+__fma_f32x2(const __simd_storage_f32<_Np>& __lhs,
+            const __simd_storage_f32<_Np>& __rhs,
+            const __simd_storage_f32<_Np>& __add) noexcept
+{
+  __simd_storage_f32<_Np> __result;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < (_Np / 2) * 2; __i += 2)
+  {
+    const auto __lhs_value   = ::float2{__lhs.__data[__i], __lhs.__data[__i + 1]};
+    const auto __rhs_value   = ::float2{__rhs.__data[__i], __rhs.__data[__i + 1]};
+    const auto __add_value   = ::float2{__add.__data[__i], __add.__data[__i + 1]};
+    const auto __value       = ::cuda::std::simd::__fma_f32x2(__lhs_value, __rhs_value, __add_value);
+    __result.__data[__i]     = __value.x;
+    __result.__data[__i + 1] = __value.y;
+  }
+  if constexpr (_Np % 2 != 0)
+  {
+    __result.__data[_Np - 1] = ::cuda::std::fma(__lhs.__data[_Np - 1], __rhs.__data[_Np - 1], __add.__data[_Np - 1]);
+  }
+  return __result;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_HAS_SIMD_F32X2()
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_FP32X2_INTRINSICS_ARRAY_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/simd_intrinsics.h

@@ -0,0 +1,183 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_SIMD_INTRINSICS_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_SIMD_INTRINSICS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
+
+#  include <cuda/std/cstdint>
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+// Wrapping half-word/byte add produces bit-identical results for signed and unsigned operands (no saturation, no
+// cross-lane carry), so a single intrinsic is used for both signednesses.
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vadd_16x2([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90,
+               (return ::__vadd2(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vadd_16x2: Unsupported architecture"); return uint32_t{};));
+}
+
+#  if _CCCL_HAS_SIMD_8BIT()
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vadd_8x4([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+#    if _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               (return ::__vadd4(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vadd_8x4: Unsupported architecture"); return uint32_t{};));
+#    else // ^^^ _CCCL_HAS_SIMD_8BIT_INTRINSICS() ^^^ / vvv !_CCCL_HAS_SIMD_8BIT_INTRINSICS() vvv
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               ({
+                 uint32_t __result{};
+                 asm("add.u8x4 %0, %1, %2;" : "=r"(__result) : "r"(__lhs), "r"(__rhs));
+                 return __result;
+               }),
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vadd_8x4: Unsupported architecture"); return uint32_t{};));
+#    endif // _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+}
+
+#  endif // _CCCL_HAS_SIMD_8BIT()
+
+//----------------------------------------------------------------------------------------------------------------------
+// SIMD Packed Integer Min/Max
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmin_u16x2([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90,
+               (return ::__vminu2(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmin_u16x2: Unsupported architecture"); return uint32_t{};));
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmax_u16x2([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90,
+               (return ::__vmaxu2(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmax_u16x2: Unsupported architecture"); return uint32_t{};));
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmin_s16x2([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90,
+               (return ::__vmins2(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmin_s16x2: Unsupported architecture"); return uint32_t{};));
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmax_s16x2([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90,
+               (return ::__vmaxs2(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmax_s16x2: Unsupported architecture"); return uint32_t{};));
+}
+
+#  if _CCCL_HAS_SIMD_8BIT()
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmin_u8x4([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+#    if _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               (return ::__vminu4(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmin_u8x4: Unsupported architecture"); return uint32_t{};));
+#    else // ^^^ _CCCL_HAS_SIMD_8BIT_INTRINSICS() ^^^ / vvv !_CCCL_HAS_SIMD_8BIT_INTRINSICS() vvv
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               ({
+                 uint32_t __result{};
+                 asm("min.u8x4 %0, %1, %2;" : "=r"(__result) : "r"(__lhs), "r"(__rhs));
+                 return __result;
+               }),
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmin_u8x4: Unsupported architecture"); return uint32_t{};));
+#    endif // _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmin_s8x4([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+#    if _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               (return ::__vmins4(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmin_s8x4: Unsupported architecture"); return uint32_t{};));
+#    else // ^^^ _CCCL_HAS_SIMD_8BIT_INTRINSICS() ^^^ / vvv !_CCCL_HAS_SIMD_8BIT_INTRINSICS() vvv
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               ({
+                 uint32_t __result{};
+                 asm("min.s8x4 %0, %1, %2;" : "=r"(__result) : "r"(__lhs), "r"(__rhs));
+                 return __result;
+               }),
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmin_s8x4: Unsupported architecture"); return uint32_t{};));
+#    endif // _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmax_u8x4([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+#    if _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               (return ::__vmaxu4(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmax_u8x4: Unsupported architecture"); return uint32_t{};));
+#    else // ^^^ _CCCL_HAS_SIMD_8BIT_INTRINSICS() ^^^ / vvv !_CCCL_HAS_SIMD_8BIT_INTRINSICS() vvv
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               ({
+                 uint32_t __result{};
+                 asm("max.u8x4 %0, %1, %2;" : "=r"(__result) : "r"(__lhs), "r"(__rhs));
+                 return __result;
+               }),
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmax_u8x4: Unsupported architecture"); return uint32_t{};));
+#    endif // _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline uint32_t
+__vmax_s8x4([[maybe_unused]] const uint32_t __lhs, [[maybe_unused]] const uint32_t __rhs) noexcept
+{
+#    if _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               (return ::__vmaxs4(__lhs, __rhs);), //
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmax_s8x4: Unsupported architecture"); return uint32_t{};));
+#    else // ^^^ _CCCL_HAS_SIMD_8BIT_INTRINSICS() ^^^ / vvv !_CCCL_HAS_SIMD_8BIT_INTRINSICS() vvv
+  NV_IF_TARGET(NV_HAS_FEATURE_SM_120f,
+               ({
+                 uint32_t __result{};
+                 asm("max.s8x4 %0, %1, %2;" : "=r"(__result) : "r"(__lhs), "r"(__rhs));
+                 return __result;
+               }),
+               (_CCCL_VERIFY(false, "cuda::std::simd::__vmax_s8x4: Unsupported architecture"); return uint32_t{};));
+#    endif // _CCCL_HAS_SIMD_8BIT_INTRINSICS()
+}
+
+#  endif // _CCCL_HAS_SIMD_8BIT()
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_SIMD_INTRINSICS_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/specializations/simd_intrinsics_array.h

@@ -0,0 +1,217 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_SPECIALIZATIONS_SIMD_INTRINSICS_ARRAY_H
+#define _CUDA_STD___SIMD_SPECIALIZATIONS_SIMD_INTRINSICS_ARRAY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/ceil_div.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__cstring/memcpy.h>
+#include <cuda/std/__memory/assume_aligned.h>
+#include <cuda/std/__simd/specializations/fixed_size_storage.h>
+#include <cuda/std/__simd/specializations/simd_intrinsics.h>
+#include <cuda/std/__type_traits/is_unsigned.h>
+#include <cuda/std/array>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+//----------------------------------------------------------------------------------------------------------------------
+// conversion utilities
+
+template <typename _SimdStorage>
+inline constexpr size_t __simd_storage_size_u32 = 0;
+
+template <typename _Tp, __simd_size_type _Np>
+inline constexpr size_t __simd_storage_size_u32<__simd_storage<_Tp, __fixed_size<_Np>>> =
+  ::cuda::ceil_div(_Np, sizeof(uint32_t) / sizeof(_Tp));
+
+template <size_t _Np>
+using __array_u32_t = array<uint32_t, _Np>;
+
+template <typename _SimdStorage>
+using __simd_storage_u32_t = __array_u32_t<__simd_storage_size_u32<_SimdStorage>>;
+
+template <typename _SimdStorage>
+inline constexpr size_t __simd_storage_copy_size_u32 = 0;
+
+template <typename _Tp, __simd_size_type _Np>
+inline constexpr size_t __simd_storage_copy_size_u32<__simd_storage<_Tp, __fixed_size<_Np>>> = _Np * sizeof(_Tp);
+
+template <typename _SimdStorage, typename _SimdStorageU32 = __simd_storage_u32_t<_SimdStorage>>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _SimdStorageU32 __to_unsigned_storage(const _SimdStorage& __s) noexcept
+{
+  _SimdStorageU32 __tmp{};
+  const auto __input_data = ::cuda::std::assume_aligned<alignof(uint32_t)>(__s.__data);
+  ::cuda::std::memcpy(__tmp.data(), __input_data, __simd_storage_copy_size_u32<_SimdStorage>);
+  return __tmp;
+}
+
+template <typename _SimdStorage, typename _SimdStorageU32 = __simd_storage_u32_t<_SimdStorage>>
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _SimdStorage
+__copy_from_unsigned_storage(const _SimdStorageU32& __tmp) noexcept
+{
+  _SimdStorage __result{};
+  const auto __result_ptr = ::cuda::std::assume_aligned<alignof(uint32_t)>(__result.__data);
+  ::cuda::std::memcpy(__result_ptr, __tmp.data(), __simd_storage_copy_size_u32<_SimdStorage>);
+  return __result;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// device-only functions
+
+#if _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
+
+template <size_t _Np>
+[[nodiscard]] _CCCL_DEVICE_API constexpr __array_u32_t<_Np>
+__vadd_16bit_x2(const __array_u32_t<_Np>& __lhs_u, const __array_u32_t<_Np>& __rhs_u) noexcept
+{
+  __array_u32_t<_Np> __result_u;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (size_t __i = 0; __i < _Np; ++__i)
+  {
+    __result_u[__i] = ::cuda::std::simd::__vadd_16x2(__lhs_u[__i], __rhs_u[__i]);
+  }
+  return __result_u;
+}
+
+#  if _CCCL_HAS_SIMD_8BIT()
+
+template <size_t _Np>
+[[nodiscard]] _CCCL_DEVICE_API constexpr __array_u32_t<_Np>
+__vadd_8bit_x4(const __array_u32_t<_Np>& __lhs_u, const __array_u32_t<_Np>& __rhs_u) noexcept
+{
+  __array_u32_t<_Np> __result_u;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (size_t __i = 0; __i < _Np; ++__i)
+  {
+    __result_u[__i] = ::cuda::std::simd::__vadd_8x4(__lhs_u[__i], __rhs_u[__i]);
+  }
+  return __result_u;
+}
+
+#  endif // _CCCL_HAS_SIMD_8BIT()
+
+//----------------------------------------------------------------------------------------------------------------------
+// SIMD Packed Integer Min over Array<uint32_t, N>
+
+template <typename _Tp, size_t _Np>
+[[nodiscard]] _CCCL_DEVICE_API __array_u32_t<_Np>
+__vmin_16bit_x2(const __array_u32_t<_Np>& __lhs_u, const __array_u32_t<_Np>& __rhs_u) noexcept
+{
+  static_assert(sizeof(_Tp) == 2, "Unsupported element type");
+  __array_u32_t<_Np> __result_u;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (size_t __i = 0; __i < _Np; ++__i)
+  {
+    if constexpr (is_unsigned_v<_Tp>)
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmin_u16x2(__lhs_u[__i], __rhs_u[__i]);
+    }
+    else
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmin_s16x2(__lhs_u[__i], __rhs_u[__i]);
+    }
+  }
+  return __result_u;
+}
+
+#  if _CCCL_HAS_SIMD_8BIT()
+
+template <typename _Tp, size_t _Np>
+[[nodiscard]] _CCCL_DEVICE_API __array_u32_t<_Np>
+__vmin_8bit_x4(const __array_u32_t<_Np>& __lhs_u, const __array_u32_t<_Np>& __rhs_u) noexcept
+{
+  static_assert(sizeof(_Tp) == 1, "Unsupported element type");
+  __array_u32_t<_Np> __result_u;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (size_t __i = 0; __i < _Np; ++__i)
+  {
+    if constexpr (is_unsigned_v<_Tp>)
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmin_u8x4(__lhs_u[__i], __rhs_u[__i]);
+    }
+    else
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmin_s8x4(__lhs_u[__i], __rhs_u[__i]);
+    }
+  }
+  return __result_u;
+}
+
+#  endif // _CCCL_HAS_SIMD_8BIT()
+
+//----------------------------------------------------------------------------------------------------------------------
+// SIMD Packed Integer Max over Array<uint32_t, N>
+
+template <typename _Tp, size_t _Np>
+[[nodiscard]] _CCCL_DEVICE_API __array_u32_t<_Np>
+__vmax_16bit_x2(const __array_u32_t<_Np>& __lhs_u, const __array_u32_t<_Np>& __rhs_u) noexcept
+{
+  static_assert(sizeof(_Tp) == 2, "Unsupported element type");
+  __array_u32_t<_Np> __result_u;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (size_t __i = 0; __i < _Np; ++__i)
+  {
+    if constexpr (is_unsigned_v<_Tp>)
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmax_u16x2(__lhs_u[__i], __rhs_u[__i]);
+    }
+    else
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmax_s16x2(__lhs_u[__i], __rhs_u[__i]);
+    }
+  }
+  return __result_u;
+}
+
+#  if _CCCL_HAS_SIMD_8BIT()
+
+template <typename _Tp, size_t _Np>
+[[nodiscard]] _CCCL_DEVICE_API __array_u32_t<_Np>
+__vmax_8bit_x4(const __array_u32_t<_Np>& __lhs_u, const __array_u32_t<_Np>& __rhs_u) noexcept
+{
+  static_assert(sizeof(_Tp) == 1, "Unsupported element type");
+  __array_u32_t<_Np> __result_u;
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (size_t __i = 0; __i < _Np; ++__i)
+  {
+    if constexpr (is_unsigned_v<_Tp>)
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmax_u8x4(__lhs_u[__i], __rhs_u[__i]);
+    }
+    else
+    {
+      __result_u[__i] = ::cuda::std::simd::__vmax_s8x4(__lhs_u[__i], __rhs_u[__i]);
+    }
+  }
+  return __result_u;
+}
+
+#  endif // _CCCL_HAS_SIMD_8BIT()
+#endif // _CCCL_CUDA_COMPILATION() && !_CCCL_TILE_COMPILATION()
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_SPECIALIZATIONS_SIMD_INTRINSICS_ARRAY_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/store.h

@@ -0,0 +1,361 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_STORE_H
+#define _CUDA_STD___SIMD_STORE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/__memory/ptr_rebind.h>
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstring/memcpy.h>
+#include <cuda/std/__iterator/concepts.h>
+#include <cuda/std/__iterator/distance.h>
+#include <cuda/std/__iterator/incrementable_traits.h>
+#include <cuda/std/__iterator/readable_traits.h>
+#include <cuda/std/__memory/assume_aligned.h>
+#include <cuda/std/__memory/pointer_traits.h>
+#include <cuda/std/__ranges/access.h>
+#include <cuda/std/__ranges/concepts.h>
+#include <cuda/std/__ranges/data.h>
+#include <cuda/std/__ranges/size.h>
+#include <cuda/std/__simd/basic_vec.h>
+#include <cuda/std/__simd/concepts.h>
+#include <cuda/std/__simd/flag.h>
+#include <cuda/std/__simd/utility.h>
+#include <cuda/std/__utility/cmp.h>
+#include <cuda/std/__utility/forward.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <typename _Tp, typename _Abi, typename _Up, typename... _Flags>
+_CCCL_HOST_DEVICE_API constexpr void
+__check_store_preconditions(_Up* const __ptr, flags<_Flags...>, const __simd_size_type __count = 1) noexcept
+{
+  static_assert(__is_vectorizable_v<_Up>, "cuda::std::simd::store: range_value_t<R> must be a vectorizable type");
+
+  static_assert(__explicitly_convertible_to<_Tp, _Up>,
+                "cuda::std::simd::store: value_type must satisfy explicitly-convertible-to<range_value_t<R>>");
+
+  static_assert(__has_convert_flag_v<_Flags...> || __is_value_preserving_v<_Tp, _Up>,
+                "cuda::std::simd::store: Conversion from value_type to range_value_t<R> is not value-preserving; use "
+                "flag_convert");
+
+  _CCCL_ASSERT(__count == 0 || __ptr != nullptr, "cuda::std::simd::store: pointer is nullptr");
+  ::cuda::std::simd::__assert_load_store_alignment<basic_vec<_Tp, _Abi>, _Up, _Flags...>(__ptr);
+}
+
+// [simd.loadstore] helper: core partial store to pointer + count + mask
+template <typename _Tp, typename _Abi, typename _Up, typename... _Flags>
+_CCCL_HOST_DEVICE_API constexpr void __partial_store_to_ptr(
+  const basic_vec<_Tp, _Abi>& __v,
+  _Up* const __ptr,
+  const __simd_size_type __count,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __flags = {}) noexcept
+{
+  ::cuda::std::simd::__check_store_preconditions<_Tp, _Abi>(__ptr, __flags, __count);
+  constexpr auto __simd_size = basic_vec<_Tp, _Abi>::__size;
+
+  _CCCL_PRAGMA_UNROLL_FULL()
+  for (__simd_size_type __i = 0; __i < __simd_size; ++__i)
+  {
+    if (__mask[__i] && __i < __count)
+    {
+      __ptr[__i] = static_cast<_Up>(__v[__i]);
+    }
+  }
+}
+
+template <typename _Tp, typename _Abi, typename _Up, typename... _Flags>
+_CCCL_HOST_DEVICE_API constexpr void
+__full_store_to_ptr(const basic_vec<_Tp, _Abi>& __v, _Up* const __ptr, flags<_Flags...> __flags) noexcept
+{
+  ::cuda::std::simd::__check_store_preconditions<_Tp, _Abi>(__ptr, __flags);
+  using __vec_t                     = basic_vec<_Tp, _Abi>;
+  constexpr auto __simd_size        = __vec_t::__size;
+  constexpr bool __has_aligned_flag = __has_aligned_flag_v<_Flags...>;
+
+  if constexpr (__has_aligned_flag || __has_overaligned_flag_v<_Flags...>)
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      constexpr auto __base_alignment = __has_aligned_flag ? alignment_v<__vec_t, _Up> : alignof(_Up);
+      constexpr auto __ptr_alignment  = ::cuda::std::max(__base_alignment, __overaligned_value_v<_Flags...>);
+      constexpr auto __data_size      = __simd_size * sizeof(_Up);
+
+      _Up __tmp[__simd_size]{};
+      _CCCL_PRAGMA_UNROLL_FULL()
+      for (__simd_size_type __i = 0; __i < __simd_size; ++__i)
+      {
+        __tmp[__i] = static_cast<_Up>(__v[__i]);
+      }
+      // vectorized store to pointer
+      if constexpr (__is_cuda_vectorizable_v<_Up> && __simd_size > 1 && __ptr_alignment >= __data_size
+                    && ::cuda::__is_valid_alignment(__data_size))
+      {
+        struct alignas(__data_size) __aligned_t
+        {
+          char __data[__data_size];
+        };
+        // nvcc performance bug: memcpy to pointer could not be vectorized
+        const auto __aligned_ptr = ::cuda::ptr_rebind<__aligned_t>(__ptr);
+        __aligned_t __data{};
+        ::cuda::std::memcpy(&__data, &__tmp, sizeof(__tmp));
+        *::cuda::std::assume_aligned<__ptr_alignment>(__aligned_ptr) = __data;
+      }
+      // rely on compiler vectorization
+      else
+      {
+        const auto __aligned_ptr = ::cuda::std::assume_aligned<__ptr_alignment>(__ptr);
+        _CCCL_PRAGMA_UNROLL_FULL()
+        for (__simd_size_type __i = 0; __i < __simd_size; ++__i)
+        {
+          __aligned_ptr[__i] = __tmp[__i];
+        }
+      }
+      return;
+    }
+  }
+  constexpr auto __true_mask = typename basic_vec<_Tp, _Abi>::mask_type(true);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, __ptr, __simd_size, __true_mask, __flags);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.loadstore] partial_store
+
+// partial_store: range, masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> //
+                 _CCCL_AND ::cuda::std::ranges::sized_range<_Range> //
+                   _CCCL_AND indirectly_writable<::cuda::std::ranges::iterator_t<_Range>,
+                                                 ::cuda::std::ranges::range_value_t<_Range>> //
+                     _CCCL_AND __explicitly_convertible_to<_Tp, ::cuda::std::ranges::range_value_t<_Range>>)
+_CCCL_HOST_DEVICE_API constexpr void partial_store(
+  const basic_vec<_Tp, _Abi>& __v,
+  _Range&& __r,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  const auto __range_size = ::cuda::std::ranges::size(__r);
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__range_size),
+               "cuda::std::simd::partial_store: range size out of range");
+  const auto __size = static_cast<__simd_size_type>(__range_size);
+  const auto __ptr  = ::cuda::std::ranges::data(__r);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, __ptr, __size, __mask, __f);
+}
+
+// partial_store: range, no mask
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> //
+                 _CCCL_AND ::cuda::std::ranges::sized_range<_Range> //
+                   _CCCL_AND indirectly_writable<::cuda::std::ranges::iterator_t<_Range>,
+                                                 ::cuda::std::ranges::range_value_t<_Range>> //
+                     _CCCL_AND __explicitly_convertible_to<_Tp, ::cuda::std::ranges::range_value_t<_Range>>)
+_CCCL_HOST_DEVICE_API constexpr void
+partial_store(const basic_vec<_Tp, _Abi>& __v, _Range&& __r, flags<_Flags...> __f = {})
+{
+  constexpr auto __true_mask = typename basic_vec<_Tp, _Abi>::mask_type(true);
+
+  ::cuda::std::simd::partial_store(__v, ::cuda::std::forward<_Range>(__r), __true_mask, __f);
+}
+
+// partial_store: iterator + count, masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> //
+                 _CCCL_AND indirectly_writable<_Ip, iter_value_t<_Ip>> //
+                   _CCCL_AND __explicitly_convertible_to<_Tp, iter_value_t<_Ip>>)
+_CCCL_HOST_DEVICE_API constexpr void partial_store(
+  const basic_vec<_Tp, _Abi>& __v,
+  const _Ip __first,
+  const iter_difference_t<_Ip> __n,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__n), "cuda::std::simd::partial_store: n out of range");
+  const auto __size = static_cast<__simd_size_type>(__n);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, ::cuda::std::to_address(__first), __size, __mask, __f);
+}
+
+// partial_store: iterator + count, no mask
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> //
+                 _CCCL_AND indirectly_writable<_Ip, iter_value_t<_Ip>> //
+                   _CCCL_AND __explicitly_convertible_to<_Tp, iter_value_t<_Ip>>)
+_CCCL_HOST_DEVICE_API constexpr void partial_store(
+  const basic_vec<_Tp, _Abi>& __v, const _Ip __first, const iter_difference_t<_Ip> __n, flags<_Flags...> __f = {})
+{
+  constexpr auto __true_mask = typename basic_vec<_Tp, _Abi>::mask_type(true);
+
+  ::cuda::std::simd::partial_store(__v, __first, __n, __true_mask, __f);
+}
+
+// partial_store: iterator + sentinel, masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> //
+                 _CCCL_AND indirectly_writable<_Ip, iter_value_t<_Ip>> //
+                   _CCCL_AND __explicitly_convertible_to<_Tp, iter_value_t<_Ip>> //
+                     _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+_CCCL_HOST_DEVICE_API constexpr void partial_store(
+  const basic_vec<_Tp, _Abi>& __v,
+  const _Ip __first,
+  const _Sp __last,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  const auto __distance = ::cuda::std::distance(__first, __last);
+  _CCCL_ASSERT(::cuda::std::in_range<__simd_size_type>(__distance),
+               "cuda::std::simd::partial_store: distance(first, last) out of range");
+  const auto __size = static_cast<__simd_size_type>(__distance);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, ::cuda::std::to_address(__first), __size, __mask, __f);
+}
+
+// partial_store: iterator + sentinel, no mask
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> //
+                 _CCCL_AND indirectly_writable<_Ip, iter_value_t<_Ip>> //
+                   _CCCL_AND __explicitly_convertible_to<_Tp, iter_value_t<_Ip>> //
+                     _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+_CCCL_HOST_DEVICE_API constexpr void
+partial_store(const basic_vec<_Tp, _Abi>& __v, const _Ip __first, const _Sp __last, flags<_Flags...> __f = {})
+{
+  constexpr auto __true_mask = typename basic_vec<_Tp, _Abi>::mask_type(true);
+
+  ::cuda::std::simd::partial_store(__v, __first, __last, __true_mask, __f);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.loadstore] unchecked_store
+
+// unchecked_store: range, masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>)
+_CCCL_HOST_DEVICE_API constexpr void unchecked_store(
+  const basic_vec<_Tp, _Abi>& __v,
+  _Range&& __r,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  constexpr auto __simd_size = basic_vec<_Tp, _Abi>::__size;
+  if constexpr (__has_static_size<_Range>)
+  {
+    static_assert(__static_range_size_v<_Range> >= __simd_size,
+                  "cuda::std::simd::unchecked_store: requires ::cuda::std::ranges::size(r) >= V::size()");
+  }
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(::cuda::std::ranges::size(__r), __simd_size),
+               "cuda::std::simd::unchecked_store: requires ::cuda::std::ranges::size(r) >= V::size()");
+  const auto __ptr = ::cuda::std::ranges::data(__r);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, __ptr, __simd_size, __mask, __f);
+}
+
+// unchecked_store: range, no mask
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Range, typename... _Flags)
+_CCCL_REQUIRES(::cuda::std::ranges::contiguous_range<_Range> _CCCL_AND ::cuda::std::ranges::sized_range<_Range>)
+_CCCL_HOST_DEVICE_API constexpr void
+unchecked_store(const basic_vec<_Tp, _Abi>& __v, _Range&& __r, flags<_Flags...> __f = {})
+{
+  if constexpr (__has_static_size<_Range>)
+  {
+    static_assert(__static_range_size_v<_Range> >= basic_vec<_Tp, _Abi>::__size,
+                  "cuda::std::simd::unchecked_store: requires ::cuda::std::ranges::size(r) >= V::size()");
+  }
+  [[maybe_unused]] constexpr auto __simd_size = basic_vec<_Tp, _Abi>::__size;
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(::cuda::std::ranges::size(__r), __simd_size),
+               "cuda::std::simd::unchecked_store: requires ::cuda::std::ranges::size(r) >= V::size()");
+
+  ::cuda::std::simd::__full_store_to_ptr(__v, ::cuda::std::ranges::data(__r), __f);
+}
+
+// unchecked_store: iterator + count, masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip>)
+_CCCL_HOST_DEVICE_API constexpr void unchecked_store(
+  const basic_vec<_Tp, _Abi>& __v,
+  const _Ip __first,
+  const iter_difference_t<_Ip> __n,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  constexpr auto __simd_size = basic_vec<_Tp, _Abi>::size();
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(__n, __simd_size),
+               "cuda::std::simd::unchecked_store: requires n >= V::size()");
+  const auto __ptr = ::cuda::std::to_address(__first);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, __ptr, __simd_size, __mask, __f);
+}
+
+// unchecked_store: iterator + count, no mask
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip>)
+_CCCL_HOST_DEVICE_API constexpr void unchecked_store(
+  const basic_vec<_Tp, _Abi>& __v, const _Ip __first, const iter_difference_t<_Ip> __n, flags<_Flags...> __f = {})
+{
+  [[maybe_unused]] constexpr auto __simd_size = basic_vec<_Tp, _Abi>::__size;
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(__n, __simd_size),
+               "cuda::std::simd::unchecked_store: requires n >= V::size()");
+  const auto __ptr = ::cuda::std::to_address(__first);
+
+  ::cuda::std::simd::__full_store_to_ptr(__v, __ptr, __f);
+}
+
+// unchecked_store: iterator + sentinel, masked
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+_CCCL_HOST_DEVICE_API constexpr void unchecked_store(
+  const basic_vec<_Tp, _Abi>& __v,
+  const _Ip __first,
+  const _Sp __last,
+  const typename basic_vec<_Tp, _Abi>::mask_type& __mask,
+  flags<_Flags...> __f = {})
+{
+  constexpr auto __simd_size = basic_vec<_Tp, _Abi>::__size;
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(::cuda::std::distance(__first, __last), __simd_size),
+               "cuda::std::simd::unchecked_store: requires distance(first, last) >= V::size()");
+  const auto __ptr = ::cuda::std::to_address(__first);
+
+  ::cuda::std::simd::__partial_store_to_ptr(__v, __ptr, __simd_size, __mask, __f);
+}
+
+// unchecked_store: iterator + sentinel, no mask
+_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Ip, typename _Sp, typename... _Flags)
+_CCCL_REQUIRES(contiguous_iterator<_Ip> _CCCL_AND sized_sentinel_for<_Sp, _Ip>)
+_CCCL_HOST_DEVICE_API constexpr void
+unchecked_store(const basic_vec<_Tp, _Abi>& __v, const _Ip __first, const _Sp __last, flags<_Flags...> __f = {})
+{
+  [[maybe_unused]] constexpr auto __simd_size = basic_vec<_Tp, _Abi>::__size;
+  _CCCL_ASSERT(::cuda::std::cmp_greater_equal(::cuda::std::distance(__first, __last), __simd_size),
+               "cuda::std::simd::unchecked_store: requires distance(first, last) >= V::size()");
+  const auto __ptr = ::cuda::std::to_address(__first);
+
+  ::cuda::std::simd::__full_store_to_ptr(__v, __ptr, __f);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_STORE_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/type_traits.h

@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_TYPE_TRAITS_H
+#define _CUDA_STD___SIMD_TYPE_TRAITS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/simd.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/exposition.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+inline constexpr size_t __optimal_cuda_alignment = _CCCL_CTK_AT_LEAST(12, 9) ? 32 : 16;
+
+// The best alignment for a pointer to a SIMD type is the maximum of the type's alignment and the optimal CUDA
+// alignment.
+template <typename _Tp>
+inline constexpr size_t __simd_pointer_alignment_v = ::cuda::std::max(alignof(_Tp), __optimal_cuda_alignment);
+
+// [simd.traits], alignment
+template <typename _Tp, typename _Up = typename _Tp::value_type>
+struct alignment;
+
+template <typename _Tp, typename _Abi, typename _Up>
+struct alignment<basic_vec<_Tp, _Abi>, _Up> : integral_constant<size_t, __simd_pointer_alignment_v<_Up>>
+{
+  static_assert(__is_vectorizable_v<_Up>, "U must be a vectorizable type");
+};
+
+template <typename _Tp, typename _Up = typename _Tp::value_type>
+inline constexpr size_t alignment_v = alignment<_Tp, _Up>::value;
+
+// [simd.traits], rebind
+template <typename _Tp, typename _Vp>
+struct rebind;
+
+template <typename _Tp, typename _Up, typename _Abi>
+struct rebind<_Tp, basic_vec<_Up, _Abi>>
+{
+  static_assert(__is_vectorizable_v<_Tp>, "T must be a vectorizable type");
+  using type = basic_vec<_Tp, __deduce_abi_t<_Tp, __simd_size_v<_Up, _Abi>>>;
+};
+
+template <typename _Tp, size_t _Bytes, typename _Abi>
+struct rebind<_Tp, basic_mask<_Bytes, _Abi>>
+{
+  static_assert(__is_vectorizable_v<_Tp>, "T must be a vectorizable type");
+  using __integer_t       = __integer_from<sizeof(_Tp)>;
+  using __integer_bytes_t = __integer_from<_Bytes>;
+
+  using type = basic_mask<sizeof(_Tp), __deduce_abi_t<__integer_t, __simd_size_v<__integer_bytes_t, _Abi>>>;
+};
+
+template <typename _Tp, typename _Vp>
+using rebind_t = typename rebind<_Tp, _Vp>::type;
+
+// [simd.traits], resize
+template <__simd_size_type _Np, typename _Vp>
+struct resize;
+
+template <__simd_size_type _Np, typename _Tp, typename _Abi>
+struct resize<_Np, basic_vec<_Tp, _Abi>>
+{
+  using type = basic_vec<_Tp, __deduce_abi_t<_Tp, _Np>>;
+};
+
+template <__simd_size_type _Np, size_t _Bytes, typename _Abi>
+struct resize<_Np, basic_mask<_Bytes, _Abi>>
+{
+  using type = basic_mask<_Bytes, __deduce_abi_t<__integer_from<_Bytes>, _Np>>;
+};
+
+template <__simd_size_type _Np, typename _Vp>
+using resize_t = typename resize<_Np, _Vp>::type;
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_TYPE_TRAITS_H

cccl_upstream/libcudacxx/include/cuda/std/__simd/utility.h

@@ -0,0 +1,169 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++ in the CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___SIMD_UTILITY_H
+#define _CUDA_STD___SIMD_UTILITY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/pow2.h>
+#include <cuda/__memory/is_aligned.h>
+#include <cuda/__type_traits/is_trivially_copyable.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__fwd/span.h>
+#include <cuda/std/__ranges/concepts.h>
+#include <cuda/std/__simd/abi.h>
+#include <cuda/std/__simd/concepts.h>
+#include <cuda/std/__simd/flag.h>
+#include <cuda/std/__simd/specializations/fixed_size_vec.h>
+#include <cuda/std/__simd/type_traits.h>
+#include <cuda/std/__tuple_dir/tuple_size.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/remove_cvref.h>
+#include <cuda/std/__type_traits/void_t.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/integer_sequence.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD
+
+template <typename _Abi>
+inline constexpr bool __is_enabled_abi_v = false;
+
+// c++ specification sets 1 <= N <= 64
+template <__simd_size_type _Np>
+inline constexpr bool __is_enabled_abi_v<__fixed_size<_Np>> = (_Np >= 1 && _Np <= 64);
+
+//----------------------------------------------------------------------------------------------------------------------
+// __can_generate_v
+
+template <typename _Tp, typename _Generator, __simd_size_type _Idx, typename = void>
+inline constexpr bool __is_well_formed = false;
+
+template <typename _Tp, typename _Generator, __simd_size_type _Idx>
+inline constexpr bool
+  __is_well_formed<_Tp, _Generator, _Idx, void_t<decltype(declval<_Generator>()(__simd_size_constant<_Idx>()))>> =
+    is_convertible_v<decltype(declval<_Generator>()(__simd_size_constant<_Idx>())), _Tp>;
+
+template <typename _Tp, typename _Generator, __simd_size_type... _Indices>
+[[nodiscard]]
+_CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL bool __can_generate(integer_sequence<__simd_size_type, _Indices...>) noexcept
+{
+  return (true && ... && __is_well_formed<_Tp, _Generator, _Indices>);
+}
+
+template <typename _Tp, typename _Generator, __simd_size_type _Size>
+inline constexpr bool __can_generate_v =
+  __can_generate<_Tp, _Generator>(make_integer_sequence<__simd_size_type, _Size>());
+
+//----------------------------------------------------------------------------------------------------------------------
+// __is_compatible_range_v
+
+template <typename _Range, typename = void>
+inline constexpr bool __has_tuple_size_v = false;
+
+template <typename _Range>
+inline constexpr bool __has_tuple_size_v<_Range, void_t<decltype(tuple_size<remove_cvref_t<_Range>>::value)>> = true;
+
+template <typename _Range, typename = void>
+inline constexpr bool __has_static_extent_v = false;
+
+template <typename _Range>
+inline constexpr bool __has_static_extent_v<_Range, void_t<decltype(remove_cvref_t<_Range>::extent)>> =
+  remove_cvref_t<_Range>::extent != dynamic_extent;
+
+// Proxy for ranges::size(r) is a constant expression.
+template <typename _Range>
+_CCCL_CONCEPT __has_static_size = __has_tuple_size_v<_Range> || __has_static_extent_v<_Range>;
+
+template <typename _Range>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL __simd_size_type __get_static_range_size() noexcept
+{
+  using __range_t = remove_cvref_t<_Range>;
+  if constexpr (__has_tuple_size_v<_Range>)
+  {
+    return __simd_size_type{tuple_size_v<__range_t>};
+  }
+  else if constexpr (__has_static_extent_v<_Range>)
+  {
+    return __simd_size_type{__range_t::extent};
+  }
+  else
+  {
+    return 0;
+  }
+}
+
+template <typename _Range>
+inline constexpr __simd_size_type __static_range_size_v = __get_static_range_size<_Range>();
+
+// This trait is defined at namespace scope (not as a static member of basic_vec) because GCC 13 rejects partial
+// specialization of static member variable templates. The static-size detection intentionally avoids directly using
+// tuple_size_v<T> in the guard because that causes a hard error (instead of SFINAE) on NVCC with
+// clang-19/clang-14/nvc++ when T is an incomplete specialization of tuple_size.
+template <typename _Range>
+inline constexpr bool __is_compatible_range_guard_v =
+  __has_static_size<_Range> && ::cuda::std::ranges::contiguous_range<_Range>
+  && ::cuda::std::ranges::sized_range<_Range>;
+
+template <typename _Tp, __simd_size_type _Size, typename _Range, bool = __is_compatible_range_guard_v<_Range>>
+inline constexpr bool __is_compatible_range_v = false;
+
+template <typename _Tp, __simd_size_type _Size, typename _Range>
+inline constexpr bool __is_compatible_range_v<_Tp, _Size, _Range, true> =
+  (__static_range_size_v<_Range> == _Size) //
+  && __is_vectorizable_v<::cuda::std::ranges::range_value_t<_Range>> //
+  && __explicitly_convertible_to<::cuda::std::ranges::range_value_t<_Range>, _Tp>;
+
+//----------------------------------------------------------------------------------------------------------------------
+// [simd.flags] alignment assertion for load/store pointers
+
+template <typename _Vec, typename _Up, typename... _Flags>
+_CCCL_HOST_DEVICE_API constexpr void __assert_load_store_alignment([[maybe_unused]] const _Up* __data) noexcept
+{
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    if constexpr (__has_overaligned_flag_v<_Flags...>)
+    {
+      static_assert(__overaligned_alignment_v<_Flags...> >= alignof(_Up),
+                    "overaligned flag requires alignment >= alignof(_Up)");
+      _CCCL_ASSERT(::cuda::is_aligned(__data, __overaligned_alignment_v<_Flags...>),
+                   "flag_overaligned<N> requires data to be aligned to N");
+    }
+    if constexpr (__has_aligned_flag_v<_Flags...>)
+    {
+      _CCCL_ASSERT(::cuda::is_aligned(__data, alignment_v<_Vec, _Up>),
+                   "flag_aligned requires data to be aligned to alignment_v<V, range_value_t<R>>");
+    }
+    _CCCL_ASSERT(::cuda::is_aligned(__data, alignof(_Up)), "data is not aligned to alignof(_Up)");
+  }
+}
+
+// used in load/store preconditions
+// e.g. char3 doesn't work: alignof(char3) == 1, sizeof(char3) == 3
+template <typename _TpIn>
+inline constexpr bool __is_cuda_vectorizable_v =
+  ::cuda::is_trivially_copyable_v<_TpIn> && ::cuda::is_power_of_two(sizeof(_TpIn));
+
+_CCCL_END_NAMESPACE_CUDA_STD_SIMD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___SIMD_UTILITY_H