[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu.h

@@ -0,0 +1,868 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_H
+#define _CUDA___FP_FPEMU_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+//! @file fpemu.h
+//! @brief Main header file for the FPEMU floating point scalar emulation library
+//!
+//! This is the main header file that provides access to the complete FPEMU library.
+//! It includes all the necessary headers for:
+//!
+//! - Core definitions, macros and enumerations (fpemu_common.h)
+//! - Class templates (fpemu, fpemu_unpacked)
+//! - Public API functions (operators, builtins, conversions)
+//! - Implementation files for specific scalar operations:
+//!   - Comparison operations (fpemu_impl_cmp.h)
+//!   - Type conversions (fpemu_impl_cvt.h)
+//!   - Fused multiply-add (fpemu_impl_fma.h)
+//!   - Addition (fpemu_impl_add.h)
+//!   - Subtraction (fpemu_impl_sub.h)
+//!   - Multiplication (fpemu_impl_mul.h)
+//!   - Division (fpemu_impl_div.h)
+//!   - Square root (fpemu_impl_sqrt.h)
+//!   - Other operations (fpemu_impl_others.h)
+//!
+//! The library provides IEEE-754 compliant emulated scalar floating point operations
+//! with configurable rounding modes and computation methods.
+//!
+//! Accuracy levels (template parameter 'fpemu_accuracy'):
+//!   - fpemu_accuracy::high — correctly rounded, full IEEE-754 range including
+//!                        infinities, NaNs, and subnormals
+//!   - fpemu_accuracy::mid  — up to 1-2 least significant mantissa bits of error,
+//!                        limited INF, NaN and subnormal support
+//!   - fpemu_accuracy::low  — up to half of the mantissa bits may be lost,
+//!                        limited INF, NaN and subnormal support
+//!   - fpemu_accuracy::def  — default selector; equals high (IEEE-correct)
+//!
+//! The API supports both host and device code through appropriate decorators and
+//! can utilize different computational backends based on template parameters.
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__type_traits/is_arithmetic.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/make_nbit_int.h>
+#include <cuda/std/__type_traits/num_bits.h>
+#include <cuda/std/cstdint>
+
+// Public API surface (fpemu_accuracy selector + CCCL_FPEMU_LIB / CCCL_FPEMU_INLINE
+// compile-mode knobs) lives in fpemu_common.h; all library-internal machinery
+// (vocabulary types, decorator/ABI/declaration macros, helper functions) lives in
+// fpemu_impl.h. The class below stores raw __fpbits64 bits, so it needs both.
+#include <cuda/__fp/fpemu_common.h>
+#include <cuda/__fp/fpemu_impl.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// The public accuracy selector fpemu_accuracy is defined in
+// <cuda/__fp/fpemu_common.h> (the public API header); the internal vocabulary
+// types (__fpbits64 / __fpbits64_unpacked) and helpers come from
+// <cuda/__fp/fpemu_impl.h>. Both are included above so the class can store raw
+// bits while keeping every FP header self-contained.
+
+// Forward declaration of unpacked floating-point class
+template <typename _FpType, fpemu_accuracy _Met>
+class fpemu_unpacked;
+
+// Underlying element types accepted by the emulated classes. Only double is
+// implemented, but C++23's _Float64 (the type behind std::float64_t) is a
+// *distinct* type from double even though it is bit-identical, so accept it too
+// where the implementation provides it. The standard feature-test macro
+// __STDCPP_FLOAT64_T__ both guards the _Float64 token and guarantees the type is
+// available (so no compiler version table is needed); where _Float64 is merely an
+// alias for double (pre-C++23 GCC/clang) the double term below already covers it.
+template <typename _Tp>
+inline constexpr bool __fpemu_is_supported_fp_v =
+  ::cuda::std::is_same_v<_Tp, double>
+// nvcc currently doesn't support _Float64 in device code.
+#if __STDCPP_FLOAT64_T__ == 1 && !_CCCL_CUDA_COMPILER(NVCC)
+  || ::cuda::std::is_same_v<_Tp, _Float64>
+#endif // __STDCPP_FLOAT64_T__ == 1  && !_CCCL_CUDA_COMPILER(NVCC)
+  ;
+
+//! @brief Primary emulated double-precision floating-point class template
+//!
+//! The fpemu class template represents a double-precision (64-bit)
+//! floating-point number, emulated according to IEEE-754 semantics but with
+//! configurable accuracy level.
+//!
+//! @tparam met Accuracy level (fpemu_accuracy::high, mid, low; def == high)
+//!              - high: Correctly rounded with full IEEE-754 range
+//!              - mid: 1-2 LSB error with normal range
+//!              - low: Low accuracy with normal range
+//!
+//! This class provides:
+//!   - Storage of the value as __fpbits64 (raw IEEE-754 format)
+//!   - Construction from and conversion to standard C++ types (int, float, double)
+//!   - Arithmetic operators and mathematical functions
+//!   - Fine-grained control over rounding and accuracy level
+//!   - Portable host/device compatibility (CUDA/HIP/etc)
+//!
+//! Usage:
+//!   fpemu<double, fpemu_accuracy::high> x{1.5};
+//!   fpemu<double> y = x + 2.0;
+//!   double z = static_cast<double>(y);
+template <typename _FpType = double, fpemu_accuracy _Met = fpemu_accuracy::def>
+class fpemu
+{
+public:
+  // Only double emulation is implemented today; the _FpType axis exists for future
+  // extension. _Float64 is accepted as a bit-identical alias for double (see
+  // __fpemu_is_supported_fp_v).
+  static_assert(__fpemu_is_supported_fp_v<_FpType>,
+                "cuda::experimental::fpemu currently supports only _FpType == double (or the bit-identical _Float64), "
+                "possible future extension to other types emulation");
+
+private:
+  // Internal representation of the floating-point value (__fpbits64 is defined in
+  // fpemu_common.h). Private: fpemu<double> is trivially copyable and bit-identical
+  // to its 64-bit IEEE-754 representation, so use bit_cast to reinterpret it; no
+  // raw-bits accessor is provided.
+  __fpbits64 __bits_;
+
+public:
+  /*
+  // Constructors and assignment operators
+  */
+  // Basic constructors
+  _CCCL_API constexpr fpemu() noexcept
+      : __bits_{0u}
+  {}
+  /*
+  // Defaulted copy constructor (trivially copyable)
+  // Note: NVCC implicitly makes defaulted special members __host__ __device__
+  */
+  _CCCL_HIDE_FROM_ABI fpemu(const fpemu& __other) = default;
+
+  /*
+  // Copy constructor from volatile fpemu
+  // Template so it is NOT a copy constructor per the C++ standard.
+  // The volatile overloads are wrapped in dummy templates
+  // so that the C++ standard does not consider them copy constructors/assignment
+  // operators (a template is never a copy constructor or copy assignment operator),
+  // preserving trivial copyability while retaining volatile access support.
+  */
+  template <typename _Dummy = void>
+  _CCCL_API fpemu(const volatile fpemu& __other) noexcept
+      : __bits_{__other.__bits_}
+  {}
+
+  // Defaulted copy assignment operator (trivially copyable)
+  _CCCL_HIDE_FROM_ABI fpemu& operator=(const fpemu& __other) = default;
+
+  /*
+  // Assignment operator to volatile fpemu
+  // Template so it is NOT a copy assignment operator per the C++ standard
+  // Returns void to avoid C++20 -Wvolatile (deprecated volatile return)
+  */
+  template <typename _Dummy = void>
+  _CCCL_API void operator=(const fpemu& __other) volatile noexcept
+  {
+    __bits_ = __other.__bits_;
+  }
+
+  /*
+  // Assignment operator from volatile fpemu
+  // Template so it is NOT a copy assignment operator per the C++ standard
+  */
+  template <typename _Dummy = void>
+  _CCCL_API fpemu& operator=(const volatile fpemu& __other) noexcept
+  {
+    __bits_ = __other.__bits_;
+    return *this;
+  }
+
+  /*
+  // Conversion operators
+  */
+  // ==== Conversions from other types to fpemu:
+  // Implicit conversions from floating-point types
+  _CCCL_API fpemu(float __f) noexcept;
+  _CCCL_API fpemu(double __d) noexcept;
+  // Construction from any standard integer type (int / long / long long + unsigned).
+  // The value is canonicalized to the accuracy-correct 32- or 64-bit builtin: the
+  // target width comes from __num_bits_v and the signedness-correct fixed-width type
+  // from __make_nbit_int_t, so the static_cast selects the matching overloaded setter
+  // (signed vs unsigned) below. All widths are implicit, mirroring the implicit
+  // float/double ctors and the IEEE-754 `long -> double` conversion (64-bit values may
+  // lose precision). bool / character types are excluded by __cccl_is_integer_v.
+  _CCCL_TEMPLATE(class _Tp)
+  _CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
+  _CCCL_API fpemu(_Tp __i) noexcept
+  {
+    if constexpr (::cuda::std::__num_bits_v<_Tp> <= 32)
+    {
+      __set_from_int32(static_cast<::cuda::std::__make_nbit_int_t<32, ::cuda::std::is_signed_v<_Tp>>>(__i));
+    }
+    else
+    {
+      __set_from_int64(static_cast<::cuda::std::__make_nbit_int_t<64, ::cuda::std::is_signed_v<_Tp>>>(__i));
+    }
+  }
+  // bool and character types are excluded from __cccl_is_integer_v, but `1.0 + true`
+  // and `1.0 + 'a'` are valid for double, so mirror that behavior: widen the value to
+  // int32 and reuse the int32 constructor path (no dedicated char/bool handling).
+  _CCCL_TEMPLATE(class _Tp)
+  _CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND(!::cuda::std::__cccl_is_integer_v<_Tp>))
+  _CCCL_API fpemu(_Tp __i) noexcept
+      : fpemu(static_cast<int32_t>(__i))
+  {}
+#if _CCCL_HAS_INT128()
+  // 128-bit integers would silently truncate to 64 bits, so they are deleted until
+  // real 128-bit support is added (tracking issue: extended-precision fp <-> __int128).
+  _CCCL_API fpemu(__int128_t)  = delete;
+  _CCCL_API fpemu(__uint128_t) = delete;
+#endif // _CCCL_HAS_INT128()
+#if _CCCL_HAS_FLOAT128()
+  // __float128 -> double would silently lose precision (and today makes construction
+  // ambiguous with the float/double ctors), so it is deleted for parity with the
+  // 128-bit integer ctors until real extended-precision support exists.
+  _CCCL_API fpemu(__float128) = delete;
+#endif // _CCCL_HAS_FLOAT128()
+  // Converting constructor from another accuracy (same packed representation, so a
+  // pure reinterpretation). Explicit: an accuracy change must be opted into via
+  // direct-init / static_cast, mirroring fpmp2 and the IEEE-754 narrowing ctors.
+  template <fpemu_accuracy _Acc2>
+  _CCCL_API explicit fpemu(const fpemu<double, _Acc2>& __src) noexcept;
+  // Converting constructor from the unpacked representation (packs to the 64-bit form).
+  template <fpemu_accuracy _Acc2>
+  _CCCL_API explicit fpemu(const fpemu_unpacked<double, _Acc2>& __src) noexcept;
+
+  // ==== Conversion from fpemu to other types:
+  // Implicit conversion to double
+  _CCCL_API operator double() const noexcept;
+  // Explicit conversions to other types
+  _CCCL_API explicit operator float() const noexcept;
+  // Explicit conversion to any standard integer type (int / long / long long + unsigned).
+  // The target width comes from __num_bits_v and the signedness-correct fixed-width type
+  // from __make_nbit_int_t, selecting the matching overloaded __to_integer helper below;
+  // excludes bool / character types.
+  _CCCL_TEMPLATE(class _Tp)
+  _CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
+  _CCCL_API explicit operator _Tp() const noexcept
+  {
+    using _Up =
+      ::cuda::std::__make_nbit_int_t<(::cuda::std::__num_bits_v<_Tp> <= 32) ? 32 : 64, ::cuda::std::is_signed_v<_Tp>>;
+    return static_cast<_Tp>(__to_integer(_Up{}));
+  }
+#if _CCCL_HAS_INT128()
+  // See the deleted 128-bit constructors above: avoid silent 64-bit truncation.
+  _CCCL_API explicit operator __int128_t() const  = delete;
+  _CCCL_API explicit operator __uint128_t() const = delete;
+#endif // _CCCL_HAS_INT128()
+
+private:
+  // Accuracy-correct integer <-> value helpers (defined out-of-line where the fpemu
+  // builtins are visible). Kept non-template so the definitions stay out-of-line.
+  _CCCL_API void __set_from_int32(int32_t) noexcept;
+  _CCCL_API void __set_from_int32(uint32_t) noexcept;
+  _CCCL_API void __set_from_int64(int64_t) noexcept;
+  _CCCL_API void __set_from_int64(uint64_t) noexcept;
+  _CCCL_API int32_t __to_integer(int32_t) const noexcept;
+  _CCCL_API uint32_t __to_integer(uint32_t) const noexcept;
+  _CCCL_API int64_t __to_integer(int64_t) const noexcept;
+  _CCCL_API uint64_t __to_integer(uint64_t) const noexcept;
+
+public:
+  /*
+  // Arithmetic operations:
+  */
+  // === mul ===
+  // (*)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu<double, _Acc>
+  operator*(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept;
+  // (*) mixed-type
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu operator*(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) * fpemu(__y);
+  }
+
+  // === div ===
+  // (/)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu<double, _Acc>
+  operator/(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept;
+  // (/) mixed-type
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu operator/(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) / fpemu(__y);
+  }
+
+  // === add ===
+  // (+)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu<double, _Acc>
+  operator+(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept;
+  // (+) mixed-type
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu operator+(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) + fpemu(__y);
+  }
+
+  // === sub ===
+  // (-)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu<double, _Acc>
+  operator-(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept;
+  // (-) mixed-type
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu operator-(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) - fpemu(__y);
+  }
+
+  // Prefix increment/decrement
+  _CCCL_API fpemu& operator++() noexcept
+  {
+    this = this + fpemu(1.0);
+    return *this;
+  }
+  _CCCL_API fpemu& operator--() noexcept
+  {
+    this = this - fpemu(1.0);
+    return *this;
+  }
+  // Postfix increment/decrement
+  _CCCL_API fpemu operator++(int) noexcept
+  {
+    fpemu __temp(*this);
+    this = this + fpemu(1.0);
+    return __temp;
+  }
+  _CCCL_API fpemu operator--(int) noexcept
+  {
+    fpemu __temp(*this);
+    this = this - fpemu(1.0);
+    return __temp;
+  }
+  // Compound assignment operators
+  _CCCL_API fpemu& operator+=(const fpemu& __other) noexcept
+  {
+    *this = *this + __other;
+    return *this;
+  }
+  _CCCL_API fpemu& operator-=(const fpemu& __other) noexcept
+  {
+    *this = *this - __other;
+    return *this;
+  }
+  _CCCL_API fpemu& operator*=(const fpemu& __other) noexcept
+  {
+    *this = *this * __other;
+    return *this;
+  }
+  _CCCL_API fpemu& operator/=(const fpemu& __other) noexcept
+  {
+    *this = *this / __other;
+    return *this;
+  }
+  // Unary negation operator (implementation in fpemu_impl_others.h)
+  _CCCL_API fpemu operator-() const noexcept;
+
+  /*
+  // Comparison operators:
+  */
+  // equality (==)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator==(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) == fpemu(__y);
+  }
+#if _CCCL_STD_VER <= 2017
+  // inequality (!=) — in C++20 this is synthesized from operator==
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator!=(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) != fpemu(__y);
+  }
+#endif // _CCCL_STD_VER <= 2017
+  // less than (<)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator<(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) < fpemu(__y);
+  }
+  // greater than (>)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator>(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) > fpemu(__y);
+  }
+  // less than or equal to (<=)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator<=(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) <= fpemu(__y);
+  }
+  // greater than or equal to (>=)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu> || ::cuda::std::is_same_v<_T2, fpemu>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator>=(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu(__x) >= fpemu(__y);
+  }
+}; // class fpemu
+
+//! @brief Unpacked emulated double-precision floating-point class template
+//!
+//! The fpemu_unpacked class template represents a double-precision (64-bit)
+//! floating-point number in a decomposed (sign / exponent / mantissa) form,
+//! emulated according to IEEE-754 semantics but with configurable accuracy level.
+//! It trades the compact packed layout of fpemu for direct field access, which the
+//! emulation builtins use to avoid repeated pack/unpack work in chained operations.
+//!
+//! @tparam met Accuracy level (fpemu_accuracy::high, mid, low; def == high)
+//!              - high: Correctly rounded with full IEEE-754 range
+//!              - mid: 1-2 LSB error with normal range
+//!              - low: Low accuracy with normal range
+//!
+//! This class provides:
+//!   - Storage of the value as __fpbits64_unpacked (sign, exponent, mantissa)
+//!   - Construction from and conversion to standard C++ types (int, float, double)
+//!   - Arithmetic operators and mathematical functions
+//!   - Fine-grained control over rounding and accuracy level
+//!   - Portable host/device compatibility (CUDA/HIP/etc)
+//!
+//! Usage:
+//!   fpemu_unpacked<double, fpemu_accuracy::high> x{1.5};
+//!   fpemu_unpacked<double> y = x + 2.0;
+//!   double z = static_cast<double>(y);
+template <typename _FpType = double, fpemu_accuracy _Met = fpemu_accuracy::def>
+class fpemu_unpacked
+{
+public:
+  // Only double emulation is implemented today; the _FpType axis exists for future
+  // extension. _Float64 is accepted as a bit-identical alias for double (see
+  // __fpemu_is_supported_fp_v).
+  static_assert(__fpemu_is_supported_fp_v<_FpType>,
+                "cuda::experimental::fpemu_unpacked currently supports only _FpType == double (or the bit-identical "
+                "_Float64)");
+
+private:
+  // Internal representation of the unpacked floating-point value (__fpbits64_unpacked
+  // is defined in fpemu_common.h). Private: fpemu_unpacked<double> is trivially
+  // copyable and bit-identical to its __fpbits64_unpacked representation, so use
+  // bit_cast to reinterpret it; no raw-bits accessor is provided.
+  __fpbits64_unpacked __bits_;
+
+public:
+  /*
+  // Constructors and assignment operators
+  */
+  // Basic constructors
+  _CCCL_API constexpr fpemu_unpacked() noexcept
+      : __bits_{0u, 0, 0}
+  {}
+  /*
+  // Defaulted copy constructor (trivially copyable)
+  // Note: NVCC implicitly makes defaulted special members __host__ __device__
+  */
+  _CCCL_HIDE_FROM_ABI fpemu_unpacked(const fpemu_unpacked& __other) = default;
+
+  /*
+  // Copy constructor from volatile fpemu_unpacked
+  // Template so it is NOT a copy constructor per the C++ standard.
+  // The volatile overloads are wrapped in dummy templates
+  // so that the C++ standard does not consider them copy constructors/assignment
+  // operators (a template is never a copy constructor or copy assignment operator),
+  // preserving trivial copyability while retaining volatile access support.
+  */
+  template <typename _Dummy = void>
+  _CCCL_API fpemu_unpacked(const volatile fpemu_unpacked& __other) noexcept
+  {
+    __bits_.sign     = __other.__bits_.sign;
+    __bits_.exponent = __other.__bits_.exponent;
+    __bits_.mantissa = __other.__bits_.mantissa;
+  }
+
+  // Defaulted copy assignment operator (trivially copyable)
+  _CCCL_HIDE_FROM_ABI fpemu_unpacked& operator=(const fpemu_unpacked& __other) = default;
+
+  /*
+  // Assignment operator to volatile fpemu_unpacked
+  // Template so it is NOT a copy assignment operator per the C++ standard
+  // Returns void to avoid C++20 -Wvolatile (deprecated volatile return)
+  */
+  template <typename _Dummy = void>
+  _CCCL_API void operator=(const fpemu_unpacked& __other) volatile noexcept
+  {
+    __bits_.sign     = __other.__bits_.sign;
+    __bits_.exponent = __other.__bits_.exponent;
+    __bits_.mantissa = __other.__bits_.mantissa;
+  }
+
+  /*
+  // Assignment operator from volatile fpemu_unpacked
+  // Template so it is NOT a copy assignment operator per the C++ standard
+  */
+  template <typename _Dummy = void>
+  _CCCL_API fpemu_unpacked& operator=(const volatile fpemu_unpacked& __other) noexcept
+  {
+    __bits_.sign     = __other.__bits_.sign;
+    __bits_.exponent = __other.__bits_.exponent;
+    __bits_.mantissa = __other.__bits_.mantissa;
+    return *this;
+  }
+  /*
+  // Conversion operators
+  */
+  // ==== Conversions from other types to fpemu_unpacked:
+  // Explicit conversions from floating-point types. Unlike the packed fpemu (which is
+  // implicitly constructible from float/double, like a built-in number), the unpacked
+  // representation always requires an explicit conversion. This keeps float/double
+  // construction unambiguous between the packed and unpacked classes and, crucially,
+  // gives fpemu_unpacked the same public API regardless of whether the translation unit
+  // is compiled by nvcc or a host-only compiler.
+  _CCCL_API explicit fpemu_unpacked(float __f) noexcept;
+  _CCCL_API explicit fpemu_unpacked(double __d) noexcept;
+  // Construction from any standard integer type (int / long / long long + unsigned).
+  // The value is canonicalized to the accuracy-correct 32- or 64-bit builtin: the target
+  // width comes from __num_bits_v and the signedness-correct fixed-width type from
+  // __make_nbit_int_t, so the static_cast selects the matching overloaded setter (signed
+  // vs unsigned) below. Explicit, matching the float/double ctors above; 64-bit values
+  // may lose precision. bool / character types are excluded by __cccl_is_integer_v.
+  _CCCL_TEMPLATE(class _Tp)
+  _CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
+  _CCCL_API explicit fpemu_unpacked(_Tp __i) noexcept
+  {
+    if constexpr (::cuda::std::__num_bits_v<_Tp> <= 32)
+    {
+      __set_from_int32(static_cast<::cuda::std::__make_nbit_int_t<32, ::cuda::std::is_signed_v<_Tp>>>(__i));
+    }
+    else
+    {
+      __set_from_int64(static_cast<::cuda::std::__make_nbit_int_t<64, ::cuda::std::is_signed_v<_Tp>>>(__i));
+    }
+  }
+  // bool and character types are excluded from __cccl_is_integer_v, but `1.0 + true`
+  // and `1.0 + 'a'` are valid for double, so mirror that behavior: widen the value to
+  // int32 and reuse the int32 constructor path (no dedicated char/bool handling).
+  _CCCL_TEMPLATE(class _Tp)
+  _CCCL_REQUIRES(::cuda::std::is_integral_v<_Tp> _CCCL_AND(!::cuda::std::__cccl_is_integer_v<_Tp>))
+  _CCCL_API explicit fpemu_unpacked(_Tp __i) noexcept
+      : fpemu_unpacked(static_cast<int32_t>(__i))
+  {}
+#if _CCCL_HAS_INT128()
+  // 128-bit integers would silently truncate to 64 bits, so they are deleted until
+  // real 128-bit support is added (tracking issue: extended-precision fp <-> __int128).
+  _CCCL_API explicit fpemu_unpacked(__int128_t)  = delete;
+  _CCCL_API explicit fpemu_unpacked(__uint128_t) = delete;
+#endif // _CCCL_HAS_INT128()
+#if _CCCL_HAS_FLOAT128()
+  // __float128 -> double would silently lose precision (and today makes construction
+  // ambiguous with the float/double ctors), so it is deleted for parity with the
+  // 128-bit integer ctors until real extended-precision support exists.
+  _CCCL_API explicit fpemu_unpacked(__float128) = delete;
+#endif // _CCCL_HAS_FLOAT128()
+  // Converting constructor from another accuracy (same unpacked representation, so a
+  // pure reinterpretation). Explicit for the same reason as the packed class.
+  template <fpemu_accuracy _Acc2>
+  _CCCL_API explicit fpemu_unpacked(const fpemu_unpacked<double, _Acc2>& __src) noexcept;
+  // Converting constructor from the packed representation (unpacks the 64-bit form).
+  template <fpemu_accuracy _Acc2>
+  _CCCL_API explicit fpemu_unpacked(const fpemu<double, _Acc2>& __src) noexcept;
+
+  // ==== Conversion from fpemu_unpacked to other types:
+  // Implicit conversion to double
+  _CCCL_API operator double() const noexcept;
+  // Explicit conversions to other types
+  _CCCL_API explicit operator float() const noexcept;
+  // Explicit conversion to any standard integer type (int / long / long long + unsigned).
+  // The target width comes from __num_bits_v and the signedness-correct fixed-width type
+  // from __make_nbit_int_t, selecting the matching overloaded __to_integer helper below;
+  // excludes bool / character types.
+  _CCCL_TEMPLATE(class _Tp)
+  _CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp>)
+  _CCCL_API explicit operator _Tp() const noexcept
+  {
+    using _Up =
+      ::cuda::std::__make_nbit_int_t<(::cuda::std::__num_bits_v<_Tp> <= 32) ? 32 : 64, ::cuda::std::is_signed_v<_Tp>>;
+    return static_cast<_Tp>(__to_integer(_Up{}));
+  }
+#if _CCCL_HAS_INT128()
+  // See the deleted 128-bit constructors above: avoid silent 64-bit truncation.
+  _CCCL_API explicit operator __int128_t() const  = delete;
+  _CCCL_API explicit operator __uint128_t() const = delete;
+#endif // _CCCL_HAS_INT128()
+
+private:
+  // Accuracy-correct integer <-> value helpers (defined out-of-line where the fpemu
+  // builtins are visible). Kept non-template so the definitions stay out-of-line.
+  _CCCL_API void __set_from_int32(int32_t) noexcept;
+  _CCCL_API void __set_from_int32(uint32_t) noexcept;
+  _CCCL_API void __set_from_int64(int64_t) noexcept;
+  _CCCL_API void __set_from_int64(uint64_t) noexcept;
+  _CCCL_API int32_t __to_integer(int32_t) const noexcept;
+  _CCCL_API uint32_t __to_integer(uint32_t) const noexcept;
+  _CCCL_API int64_t __to_integer(int64_t) const noexcept;
+  _CCCL_API uint64_t __to_integer(uint64_t) const noexcept;
+
+public:
+  /*
+  // Arithmetic operations:
+  */
+  // === mul ===
+  // (*)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu_unpacked<double, _Acc>
+  operator*(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept;
+  // (/)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu_unpacked<double, _Acc>
+  operator/(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept;
+  // (+)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu_unpacked<double, _Acc>
+  operator+(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept;
+  // (-)
+  template <fpemu_accuracy _Acc>
+  _CCCL_API friend fpemu_unpacked<double, _Acc>
+  operator-(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept;
+
+  // == mul ==
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu_unpacked operator*(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) * fpemu_unpacked(__y);
+  }
+
+  // === div ===
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu_unpacked operator/(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) / fpemu_unpacked(__y);
+  }
+
+  // === add ===
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu_unpacked operator+(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) + fpemu_unpacked(__y);
+  }
+
+  // === sub ===
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend fpemu_unpacked operator-(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) - fpemu_unpacked(__y);
+  }
+
+  // Prefix increment/decrement
+  _CCCL_API fpemu_unpacked& operator++() noexcept
+  {
+    this = this + fpemu_unpacked(1.0);
+    return *this;
+  }
+  _CCCL_API fpemu_unpacked& operator--() noexcept
+  {
+    this = this - fpemu_unpacked(1.0);
+    return *this;
+  }
+  // Postfix increment/decrement
+  _CCCL_API fpemu_unpacked operator++(int) noexcept
+  {
+    fpemu_unpacked __temp(*this);
+    this = this + fpemu_unpacked(1.0);
+    return __temp;
+  }
+  _CCCL_API fpemu_unpacked operator--(int) noexcept
+  {
+    fpemu_unpacked __temp(*this);
+    this = this - fpemu_unpacked(1.0);
+    return __temp;
+  }
+  // Compound assignment operators
+  _CCCL_API fpemu_unpacked& operator+=(const fpemu_unpacked& __other) noexcept
+  {
+    *this = *this + __other;
+    return *this;
+  }
+  _CCCL_API fpemu_unpacked& operator-=(const fpemu_unpacked& __other) noexcept
+  {
+    *this = *this - __other;
+    return *this;
+  }
+  _CCCL_API fpemu_unpacked& operator*=(const fpemu_unpacked& __other) noexcept
+  {
+    *this = *this * __other;
+    return *this;
+  }
+  _CCCL_API fpemu_unpacked& operator/=(const fpemu_unpacked& __other) noexcept
+  {
+    *this = *this / __other;
+    return *this;
+  }
+  // Unary negation operator (implementation in fpemu_impl_others.h)
+  _CCCL_API fpemu_unpacked operator-() const noexcept;
+
+  /*
+  // Comparison operators:
+  */
+  // equality (==)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator==(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) == fpemu_unpacked(__y);
+  }
+  // inequality (!=)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator!=(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) != fpemu_unpacked(__y);
+  }
+  // less than (<)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator<(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) < fpemu_unpacked(__y);
+  }
+  // greater than (>)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator>(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) > fpemu_unpacked(__y);
+  }
+  // less than or equal to (<=)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator<=(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) <= fpemu_unpacked(__y);
+  }
+  // greater than or equal to (>=)
+  _CCCL_TEMPLATE(typename _T1, typename _T2)
+  _CCCL_REQUIRES(((::cuda::std::is_same_v<_T1, fpemu_unpacked> || ::cuda::std::is_same_v<_T2, fpemu_unpacked>)
+                  && (::cuda::std::is_arithmetic_v<_T1> || ::cuda::std::is_arithmetic_v<_T2>) ))
+  _CCCL_API friend bool operator>=(const _T1& __x, const _T2& __y) noexcept
+  {
+    return fpemu_unpacked(__x) >= fpemu_unpacked(__y);
+  }
+
+}; // class fpemu_unpacked
+
+/*
+// Aliases for the emulated floating-point types
+*/
+using fp64emu      = fpemu<double, fpemu_accuracy::def>;
+using fp64emu_low  = fpemu<double, fpemu_accuracy::low>;
+using fp64emu_mid  = fpemu<double, fpemu_accuracy::mid>;
+using fp64emu_high = fpemu<double, fpemu_accuracy::high>;
+
+using fp64emu_unpacked      = fpemu_unpacked<double, fpemu_accuracy::def>;
+using fp64emu_unpacked_low  = fpemu_unpacked<double, fpemu_accuracy::low>;
+using fp64emu_unpacked_mid  = fpemu_unpacked<double, fpemu_accuracy::mid>;
+using fp64emu_unpacked_high = fpemu_unpacked<double, fpemu_accuracy::high>;
+
+// Trait machinery for the mixed-operand free-function builtins (fma, __dadd_rn, dot,
+// cmul, ...) shared by the packed fpemu and unpacked fpemu_unpacked classes.
+// __is_fpemu_v detects an fpemu / fpemu_unpacked specialization; __fpemu_pick_t selects
+// the fpemu-family type among a set of operands; __fpemu_mixed_v is the constraint "at
+// least one fpemu-family operand AND at least one arithmetic operand" (so pure
+// fpemu-only calls bind to the exact-match cores, and pure-arithmetic calls are left to
+// the built-in types).
+template <class _Tp>
+inline constexpr bool __is_fpemu_v = false;
+template <class _FpType, fpemu_accuracy _Acc>
+inline constexpr bool __is_fpemu_v<fpemu<_FpType, _Acc>> = true;
+template <class _FpType, fpemu_accuracy _Acc>
+inline constexpr bool __is_fpemu_v<fpemu_unpacked<_FpType, _Acc>> = true;
+
+template <class... _Ts>
+inline constexpr bool __fpemu_mixed_v = (__is_fpemu_v<_Ts> || ...) && (::cuda::std::is_arithmetic_v<_Ts> || ...);
+
+template <class... _Ts>
+struct __fpemu_pick
+{
+  using type = void;
+};
+template <class _T0, class... _Ts>
+struct __fpemu_pick<_T0, _Ts...>
+{
+  using type = ::cuda::std::conditional_t<__is_fpemu_v<_T0>, _T0, typename __fpemu_pick<_Ts...>::type>;
+};
+template <class... _Ts>
+using __fpemu_pick_t = typename __fpemu_pick<_Ts...>::type;
+
+// Define this macro so that the API sections in _impl.hpp files are activated.
+// The _impl.hpp files are structured with implementation code under their own
+// include guard, and API code (operators, class methods) under this guard.
+// This ensures API code is only compiled after class definitions are complete.
+#define _CCCL_FPEMU_API_CLASSES_DEFINED
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#include <cuda/__fp/fpemu_impl_add.h>
+#include <cuda/__fp/fpemu_impl_cmp.h>
+#include <cuda/__fp/fpemu_impl_cvt.h>
+#include <cuda/__fp/fpemu_impl_div.h>
+#include <cuda/__fp/fpemu_impl_fma.h>
+#include <cuda/__fp/fpemu_impl_mul.h>
+#include <cuda/__fp/fpemu_impl_others.h>
+#include <cuda/__fp/fpemu_impl_sqrt.h>
+#include <cuda/__fp/fpemu_impl_sub.h>
+
+#endif // _CUDA___FP_FPEMU_H

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_common.h

@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_COMMON_H
+#define _CUDA___FP_FPEMU_COMMON_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+//! @file fpemu_common.h
+//! @brief Public API surface shared by the FPEMU headers
+//!
+//! This header carries only the user-facing pieces that both the fpemu class
+//! (<cuda/__fp/fpemu.h>) and the emulation cores (<cuda/__fp/fpemu_impl.h> and the
+//! per-operation fpemu_impl_<op>.h headers) need to agree on:
+//!
+//! - The public accuracy selector fpemu_accuracy
+//! - The public compile-mode knobs CCCL_FPEMU_LIB / CCCL_FPEMU_INLINE
+//!
+//! All library-internal machinery (decorator/ABI/declaration macros, the raw-bits
+//! vocabulary types __fpbits64/__fpbits64_unpacked, the internal __fpemu_rounding
+//! enum, and the helper functions) lives in <cuda/__fp/fpemu_impl.h>. Keeping the
+//! public and internal pieces apart lets every FP header compile standalone.
+
+// ---------------------------------------------------------------------------
+// User-facing configuration (public compile-mode knobs)
+// ---------------------------------------------------------------------------
+// CCCL_FPEMU_LIB: Compilation mode control.
+//   1 = link against precompiled library (maps to _CCCL_FPEMU_USE_LIB)
+//   0 = header-only inline mode (default)
+// CCCL_FPEMU_INLINE is the inverse alias: CCCL_FPEMU_INLINE=1 is equivalent to CCCL_FPEMU_LIB=0.
+#ifndef CCCL_FPEMU_LIB
+#  ifdef CCCL_FPEMU_INLINE
+#    if CCCL_FPEMU_INLINE == 1
+#      define CCCL_FPEMU_LIB 0
+#    else
+#      define CCCL_FPEMU_LIB 1
+#    endif
+#  else
+#    define CCCL_FPEMU_LIB 0
+#  endif
+#endif
+#ifndef CCCL_FPEMU_INLINE
+#  if CCCL_FPEMU_LIB == 1
+#    define CCCL_FPEMU_INLINE 0
+#  else
+#    define CCCL_FPEMU_INLINE 1
+#  endif
+#endif
+#if CCCL_FPEMU_LIB == 1 && !defined(_CCCL_FPEMU_USE_LIB)
+#  define _CCCL_FPEMU_USE_LIB
+#endif
+
+// The prologue/epilogue pair and the standard-library include are skipped in
+// __CUDA_LIBDEVICE__ builds. The namespace and the
+// enum below are plain C++ and are always emitted so the braces stay balanced and
+// the emulation cores (which take fpemu_accuracy as a template parameter) can see
+// it in every build.
+#if !defined(__CUDA_LIBDEVICE__)
+#  include <cuda/std/cstdint>
+
+#  include <cuda/std/__cccl/prologue.h>
+#endif
+
+namespace cuda::experimental
+{
+//! @brief Accuracy level for floating-point emulation (public).
+//!
+//! Named fpemu_accuracy, so callers write e.g. fpemu<double, fpemu_accuracy::high>.
+//! - high: Correctly rounded with full IEEE-754 range (infinities, NaNs, subnormals)
+//! - mid:  High accuracy (1-2 ULP) with normal range
+//! - low:  Low accuracy (up to half mantissa) with normal range
+//! - def:  Default selector; equals high so the default is IEEE-correct.
+enum struct fpemu_accuracy
+{
+  unset = -1,
+  low   = 1,
+  mid   = 2,
+  high  = 3,
+  def   = 3,
+};
+} // namespace cuda::experimental
+
+#if !defined(__CUDA_LIBDEVICE__)
+#  include <cuda/std/__cccl/epilogue.h>
+#endif
+
+#endif // _CUDA___FP_FPEMU_COMMON_H

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_cmp.h

@@ -0,0 +1,404 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_CMP_H
+#define _CUDA___FP_FPEMU_IMPL_CMP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_impl_cmp.h
+//! @brief Implementation of comparison operations for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of comparison operations for the FPEMU library.
+//! It includes:
+//!
+//! - Comparison functions for equality, less than, greater than, etc
+//! - Special case handling for NaN, inf, zero, etc
+//!
+//! The implementation is designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ------------------------------------------------------------------------
+// Bit-level helpers (IEEE-754 binary64 layout). No SoftFloat dependency;
+// the comparison logic mirrors SoftFloat's f64_eq / f64_lt / f64_le.
+// ------------------------------------------------------------------------
+
+// Magnitude mask: all bits except the sign bit.
+static constexpr __fpbits64 __fp64emu_cmp_abs_mask = _CCCL_FPEMU_ABS_64;
+
+//! @brief True if the bit pattern encodes a NaN (max exponent, nonzero mantissa).
+_CCCL_TRIVIAL_API bool __internal_fp64emu_is_nan_bits(__fpbits64 __ui) noexcept
+{
+  return ((~__ui & _CCCL_FPEMU_EXP_64) == 0) && ((__ui & _CCCL_FPEMU_MANT_64) != 0);
+} // __internal_fp64emu_is_nan_bits
+
+//! @brief IEEE-754 equality. Unordered (NaN) compares false; +0 equals -0.
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_eq(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  if (__internal_fp64emu_is_nan_bits(__x) || __internal_fp64emu_is_nan_bits(__y))
+  {
+    return false;
+  }
+  // Equal bit patterns, or both are zero (+0 / -0 ignore the sign bit).
+  return (__x == __y) || (((__x | __y) & __fp64emu_cmp_abs_mask) == 0);
+} // __internal_fp64emu_cmp_eq
+
+//! @brief IEEE-754 less-than. Unordered (NaN) compares false.
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_lt(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  if (__internal_fp64emu_is_nan_bits(__x) || __internal_fp64emu_is_nan_bits(__y))
+  {
+    return false;
+  }
+  const bool __sign_x = (__x >> 63) != 0;
+  const bool __sign_y = (__y >> 63) != 0;
+  // Different signs: x < y only if x is negative and not both zero.
+  // Same sign: ordering of magnitudes, inverted when both are negative.
+  return (__sign_x != __sign_y)
+         ? (__sign_x && (((__x | __y) & __fp64emu_cmp_abs_mask) != 0))
+         : ((__x != __y) && (__sign_x ^ (__x < __y)));
+} // __internal_fp64emu_cmp_lt
+
+//! @brief IEEE-754 less-or-equal. Unordered (NaN) compares false.
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_le(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  if (__internal_fp64emu_is_nan_bits(__x) || __internal_fp64emu_is_nan_bits(__y))
+  {
+    return false;
+  }
+  const bool __sign_x = (__x >> 63) != 0;
+  const bool __sign_y = (__y >> 63) != 0;
+  return (__sign_x != __sign_y)
+         ? (__sign_x || (((__x | __y) & __fp64emu_cmp_abs_mask) == 0))
+         : ((__x == __y) || (__sign_x ^ (__x < __y)));
+} // __internal_fp64emu_cmp_le
+
+// ne is the logical negation of eq, so unordered (NaN) compares true.
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_ne(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return !__internal_fp64emu_cmp_eq(__x, __y);
+}
+// gt / ge are lt / le with swapped operands, preserving IEEE unordered=false.
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_gt(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_lt(__y, __x);
+}
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_ge(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_le(__y, __x);
+}
+
+// ---- True unpacked comparisons -------------------------------------
+// Operate directly on the fully-accurate unpacked fields (no pack round
+// trip). The full unpack yields an order-preserving form:
+//   * NaN  -> exponent == 0x0007ff00 (magic, distinct from any finite/inf);
+//   * Inf  -> exponent == 0x00007ff0;
+//   * the SIGNED exponent is monotonic with magnitude (zero/denormal <= 0,
+//     normals 1..2046, inf, nan), so (exponent, mantissa) is a magnitude key;
+//   * a zero value has mantissa == 0 (its exponent is unconstrained, e.g. an
+//     additive cancellation), so zero is detected by the mantissa, and
+//     +0 / -0 differ only in the sign field.
+// eq and lt are the primitives; le/ne/gt/ge derive from them and inherit the
+// IEEE unordered (NaN) semantics for free.
+static constexpr int32_t __fp64emu_unp_nan_exp = 0x0007ff00;
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_unp_is_nan(__fpbits64_unpacked __u) noexcept
+{
+  return static_cast<int32_t>(__u.exponent) == __fp64emu_unp_nan_exp;
+}
+_CCCL_TRIVIAL_API bool __internal_fp64emu_unp_is_zero(__fpbits64_unpacked __u) noexcept
+{
+  return __u.mantissa == 0;
+}
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_eq_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  if (__internal_fp64emu_unp_is_nan(__x) || __internal_fp64emu_unp_is_nan(__y))
+  {
+    return false;
+  }
+  const bool __zx = __internal_fp64emu_unp_is_zero(__x);
+  const bool __zy = __internal_fp64emu_unp_is_zero(__y);
+  if (__zx || __zy)
+  {
+    return __zx && __zy; // +0 == -0; zero != nonzero
+  }
+  // Non-zero, non-NaN: equal iff same sign and identical magnitude key.
+  return (__x.sign == __y.sign) && (__x.exponent == __y.exponent) && (__x.mantissa == __y.mantissa);
+}
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_lt_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  if (__internal_fp64emu_unp_is_nan(__x) || __internal_fp64emu_unp_is_nan(__y))
+  {
+    return false;
+  }
+  const bool __zx = __internal_fp64emu_unp_is_zero(__x);
+  const bool __zy = __internal_fp64emu_unp_is_zero(__y);
+  if (__zx && __zy)
+  {
+    return false; // +/-0 are equal
+  }
+  const bool __sx = (__x.sign != 0);
+  const bool __sy = (__y.sign != 0);
+  // Different signs (and not both zero): x < y exactly when x is negative.
+  // (A signed zero against an opposite-sign nonzero also resolves here.)
+  if (__sx != __sy)
+  {
+    return __sx;
+  }
+  // Same sign. Magnitude order with zero as the smallest magnitude.
+  bool __mag_x_lt_y;
+  if (__zx)
+  {
+    __mag_x_lt_y = true; // 0 < |y|  (y nonzero)
+  }
+  else if (__zy)
+  {
+    __mag_x_lt_y = false; // |x| > 0
+  }
+  else
+  {
+    __mag_x_lt_y = (static_cast<int32_t>(__x.exponent) < static_cast<int32_t>(__y.exponent))
+                || ((__x.exponent == __y.exponent) && (__x.mantissa < __y.mantissa));
+  }
+  // Both negative reverses the magnitude order; both positive keeps it.
+  return __sx ? (!__mag_x_lt_y && !__internal_fp64emu_cmp_eq_unpacked(__x, __y)) : __mag_x_lt_y;
+}
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_le_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_lt_unpacked(__x, __y) || __internal_fp64emu_cmp_eq_unpacked(__x, __y);
+}
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_gt_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_lt_unpacked(__y, __x);
+}
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_ge_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_le_unpacked(__y, __x);
+}
+
+_CCCL_TRIVIAL_API bool __internal_fp64emu_cmp_ne_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return !__internal_fp64emu_cmp_eq_unpacked(__x, __y);
+}
+
+// ============================================================================
+// Builtin declarations/implementations for comparison operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+#  if (_CCCL_FPEMU_PACKED_VIA_UNPACKED == 1)
+// Packed-via-unpacked (testing): route the packed comparison builtins through the
+// unpacked cores. unpack(x) yields the fully-accurate, order-preserving form the
+// unpacked comparators expect; comparison is rounding-independent, so no pack step.
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_eq(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_eq_unpacked(__internal_fp64emu_unpack(__x), __internal_fp64emu_unpack(__y));
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_ne(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_ne_unpacked(__internal_fp64emu_unpack(__x), __internal_fp64emu_unpack(__y));
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_le(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_le_unpacked(__internal_fp64emu_unpack(__x), __internal_fp64emu_unpack(__y));
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_lt(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_lt_unpacked(__internal_fp64emu_unpack(__x), __internal_fp64emu_unpack(__y));
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_ge(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_ge_unpacked(__internal_fp64emu_unpack(__x), __internal_fp64emu_unpack(__y));
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_gt(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_gt_unpacked(__internal_fp64emu_unpack(__x), __internal_fp64emu_unpack(__y));
+}
+#  else
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_eq(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_eq(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_ne(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_ne(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_le(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_le(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_lt(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_lt(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_ge(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_ge(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_gt(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_cmp_gt(__x, __y);
+}
+#  endif // _CCCL_FPEMU_PACKED_VIA_UNPACKED
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_eq(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_eq_unpacked(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_ne(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_ne_unpacked(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_le(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_le_unpacked(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_lt(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_lt_unpacked(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_ge(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_ge_unpacked(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_gt(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_cmp_gt_unpacked(__x, __y);
+}
+#else
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_eq(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_ne(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_le(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_lt(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_ge(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_cmp_gt(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_eq(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_ne(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_le(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_lt(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_ge(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL bool __fp64emu_unpacked_cmp_gt(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FP_FPEMU_IMPL_CMP_H
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_CMP_API_MERGED)
+#define _CCCL_FPEMU_CMP_API_MERGED
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_cmp_api.hpp)
+// ============================================================================
+
+// Comparison operators
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool operator==(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_cmp_eq(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool operator!=(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_cmp_ne(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool operator<(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_cmp_lt(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool operator>(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_cmp_gt(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool operator<=(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_cmp_le(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool operator>=(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_cmp_ge(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y));
+}
+
+// Unpacked comparison operators
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool
+operator==(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_unpacked_cmp_eq(
+    ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool
+operator!=(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_unpacked_cmp_ne(
+    ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool
+operator<(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_unpacked_cmp_lt(
+    ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool
+operator>(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_unpacked_cmp_gt(
+    ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool
+operator<=(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_unpacked_cmp_le(
+    ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API inline bool
+operator>=(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  return __fp64emu_unpacked_cmp_ge(
+    ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y));
+}
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_FPEMU_CMP_API_MERGED

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_div.h

@@ -0,0 +1,674 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_DIV_H
+#define _CUDA___FP_FPEMU_IMPL_DIV_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_ddiv_impl.hpp
+//! @brief Implementation of double-precision division operations for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of double-precision division operations for the FPEMU library.
+//! It includes:
+//!
+//! - Division functions for fpemu
+//! - Division operators for fpemu
+//! - Division functions to other types
+//!
+//! The division functions are designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+
+#include <cuda/__cmath/mul_hi.h>
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+#include <cuda/std/__bit/countl.h>
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ========================================================================
+// Native fp64 division.
+//
+// Split sign/exp/mantissa, normalize subnormals, form a 32-bit reciprocal of the divisor
+// significand, then refine the quotient with fixed-point integer remainder
+// arithmetic and round/pack. We use the fp32 reciprocal builtin (1/x in float) and a single
+// Newton-Raphson step, which meets/exceeds the accuracy the remainder
+// refinement needs for correctly-rounded results.
+// ========================================================================
+
+//! @brief Approximation of floor(2^63 / b32) for b32 in [2^31, 2^32).
+//!        Seeded by the fp32 reciprocal builtin, refined by one Newton step.
+_CCCL_TRIVIAL_API uint32_t __internal_fp64emu_div_recip32(uint32_t __b32) noexcept
+{
+  // fp32 seed: interpret b32 as bf = b32 / 2^31 in [1, 2); 1/bf in (0.5, 1].
+  // r ~ (1/bf) * 2^32 = 2^63 / b32.
+  float __bf = (float) __b32 * (1.0f / 2147483648.0f); // 1/2^31
+  // Fast fp32 reciprocal seed. The Newton step below refines it and the
+  // final trim guarantees a strict underestimate, so the low accuracy of
+  // the SFU approximation (rcp.approx) is acceptable here.
+  float __rf{};
+  NV_IF_ELSE_TARGET(
+    NV_IS_DEVICE, ({ asm("rcp.approx.ftz.f32 %0, %1;"
+                         : "=f"(__rf)
+                         : "f"(__bf)); }), ({ __rf = 1.0f / __bf; }))
+  uint64_t __r = (uint64_t) (__rf * 4294967296.0f); // rf * 2^32
+
+  if (__r < 0x80000000ULL)
+  {
+    __r = 0x80000000ULL;
+  }
+  if (__r > 0xFFFFFFFFULL)
+  {
+    __r = 0xFFFFFFFFULL;
+  }
+
+  // One Newton-Raphson step: r <- r + r*(2^63 - b32*r)/2^63.
+  uint64_t __prod = (uint64_t) __b32 * __r; // ~2^63
+  int64_t __e     = (int64_t) (_CCCL_FPEMU_SIGN_64 - __prod); // 2^63 - prod
+  uint64_t __ae   = (uint64_t) (__e < 0 ? -__e : __e);
+  uint64_t __lo   = __r * __ae; // low 64 bits of r*ae
+  uint64_t __hi   = ::cuda::mul_hi(__r, __ae);
+  uint64_t __corr = (__hi << 1) | (__lo >> 63); // (r*ae) >> 63
+  __r             = (__e < 0) ? (__r - __corr) : (__r + __corr);
+  if (__r > 0xFFFFFFFFULL)
+  {
+    __r = 0xFFFFFFFFULL;
+  }
+
+  // The division algorithm requires a strict underestimate of 2^63/b32:
+  // if the estimate overshoots floor(2^63/b32), sig32Z exceeds the true quotient
+  // and the unsigned remainder underflows. The fp32 seed + Newton step is
+  // accurate to +/-1 ULP, so trim any positive overshoot.
+  while ((uint64_t) __b32 * (uint32_t) __r > _CCCL_FPEMU_SIGN_64)
+  {
+    --__r; // > 2^63
+  }
+  return (uint32_t) __r;
+} // __internal_fp64emu_div_recip32
+
+//! @brief True if the bit pattern encodes a NaN.
+_CCCL_TRIVIAL_API bool __internal_fp64emu_div_is_nan(uint64_t __ui) noexcept
+{
+  return ((~__ui & _CCCL_FPEMU_EXP_64) == 0) && (__ui & _CCCL_FPEMU_MANT_64);
+} // __internal_fp64emu_div_is_nan
+
+//! Propagate NaN. Precise (8086) propagation is only done for correctly-rounded
+//! accuracy; other modes return a cheap NaN (ui64_a | ui64_b is always a NaN when at
+//! least one operand is a NaN) to keep the implementation light.
+template <fpemu_accuracy _Acc = fpemu_accuracy::high>
+_CCCL_TRIVIAL_API uint64_t __internal_fp64emu_div_propagate_nan(uint64_t __ui64_a, uint64_t __ui64_b) noexcept
+{
+  if constexpr (_Acc != fpemu_accuracy::high)
+  {
+    return __ui64_a | __ui64_b;
+  }
+  else
+  {
+    // NB: this block lives in the `else` (not after an early `return`) so that
+    // non-high instantiations discard it entirely; otherwise MSVC flags every
+    // line below as unreachable code (C4702, promoted to an error under /WX).
+    bool __is_sig_nan_a =
+      ((__ui64_a & _CCCL_FPEMU_QNAN_64) == _CCCL_FPEMU_EXP_64) && (__ui64_a & _CCCL_FPEMU_SNAN_PAYLOAD_64);
+    bool __is_sig_nan_b =
+      ((__ui64_b & _CCCL_FPEMU_QNAN_64) == _CCCL_FPEMU_EXP_64) && (__ui64_b & _CCCL_FPEMU_SNAN_PAYLOAD_64);
+    uint64_t __nonsig64_a = __ui64_a | _CCCL_FPEMU_QNAN_BIT_64;
+    uint64_t __nonsig64_b = __ui64_b | _CCCL_FPEMU_QNAN_BIT_64;
+
+    if (__is_sig_nan_a && !__is_sig_nan_b)
+    {
+      return __internal_fp64emu_div_is_nan(__ui64_b) ? __nonsig64_b : __nonsig64_a;
+    }
+    if (__is_sig_nan_b && !__is_sig_nan_a)
+    {
+      return __internal_fp64emu_div_is_nan(__ui64_a) ? __nonsig64_a : __nonsig64_b;
+    }
+    // Both signaling or neither signaling: return the larger-magnitude NaN.
+    uint64_t __mag64_a = __ui64_a & _CCCL_FPEMU_ABS_64;
+    uint64_t __mag64_b = __ui64_b & _CCCL_FPEMU_ABS_64;
+    if (__mag64_a < __mag64_b)
+    {
+      return __nonsig64_b;
+    }
+    if (__mag64_b < __mag64_a)
+    {
+      return __nonsig64_a;
+    }
+    return (__nonsig64_a < __nonsig64_b) ? __nonsig64_a : __nonsig64_b;
+  }
+} // __internal_fp64emu_div_propagate_nan
+
+// Forward declaration: the unpacked divide core is defined below, but the
+// packed wrapper references it for the packed-via-unpacked (testing) path.
+template <fpemu_accuracy _Acc>
+_CCCL_TRIVIAL_API __fpbits64_unpacked
+__internal_fp64emu_ddiv_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept;
+
+//! @brief Divide two double-precision floating point numbers
+//!
+//! This function divides two double-precision floating point numbers.
+//! It works by splitting the numbers into sign, exponent, and mantissa, normalizing the mantissa,
+//! and then computing the division of the mantissa.
+//!
+//! @param __x The first double-precision floating point number
+//! @param __y The second double-precision floating point number
+//! @return The result of the division
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_ddiv(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+#if (_CCCL_FPEMU_PACKED_VIA_UNPACKED == 1)
+  // Packed-via-unpacked (testing): pack(ddiv_unpacked(unpack(x), unpack(y))).
+  // The ddiv_unpacked core handles special operands and method selection; the
+  // universal unpack/pack are the shared prologue/epilogue. Rounding is applied
+  // only at pack, so the packed builtins keep their per-mode behavior.
+  {
+    __fpbits64_unpacked __a = __internal_fp64emu_unpack(__x);
+    __fpbits64_unpacked __b = __internal_fp64emu_unpack(__y);
+    __fpbits64_unpacked __r = __internal_fp64emu_ddiv_unpacked<_Acc>(__a, __b);
+    return __internal_fp64emu_pack<_Rm>(__r);
+  }
+#else
+  const uint64_t __ui64_a = (uint64_t) __x;
+  const uint64_t __ui64_b = (uint64_t) __y;
+
+  bool __sign_a     = (__ui64_a >> 63) != 0;
+  int32_t __exp_a   = (int32_t) ((__ui64_a >> 52) & 0x7FF);
+  uint64_t __mant_a = __ui64_a & _CCCL_FPEMU_MANT_64;
+
+  bool __sign_b     = (__ui64_b >> 63) != 0;
+  int32_t __exp_b   = (int32_t) ((__ui64_b >> 52) & 0x7FF);
+  uint64_t __mant_b = __ui64_b & _CCCL_FPEMU_MANT_64;
+
+  bool __sign_z = __sign_a ^ __sign_b;
+
+  // -------- special operands (NaN / Inf / zero) --------
+  if (__exp_a == 0x7FF)
+  {
+    if (__mant_a)
+    {
+      return (__fpbits64) __internal_fp64emu_div_propagate_nan<_Acc>(__ui64_a, __ui64_b);
+    }
+    if (__exp_b == 0x7FF)
+    {
+      if (__mant_b)
+      {
+        return (__fpbits64) __internal_fp64emu_div_propagate_nan<_Acc>(__ui64_a, __ui64_b);
+      }
+      else
+      {
+        return (__fpbits64) _CCCL_FPEMU_DEFNAN_64; // inf / inf -> NaN
+      }
+    }
+    return (__fpbits64) (((uint64_t) __sign_z << 63) | _CCCL_FPEMU_INF_64); // inf / finite
+  }
+  if (__exp_b == 0x7FF)
+  {
+    if (__mant_b)
+    {
+      return (__fpbits64) __internal_fp64emu_div_propagate_nan<_Acc>(__ui64_a, __ui64_b);
+    }
+    else
+    {
+      return (__fpbits64) ((uint64_t) __sign_z << 63); // finite / inf -> 0
+    }
+  }
+
+  // -------- subnormals & division by zero --------
+  if (!__exp_b)
+  {
+    if (!__mant_b)
+    {
+      if (!__exp_a && !__mant_a)
+      {
+        return (__fpbits64) _CCCL_FPEMU_DEFNAN_64; // 0 / 0 -> NaN
+      }
+      else
+      {
+        return (__fpbits64) (((uint64_t) __sign_z << 63) | _CCCL_FPEMU_INF_64); // x / 0 -> inf
+      }
+    }
+    int __mant_b_shft = ::cuda::std::countl_zero((uint64_t) __mant_b) - 11;
+    // normalize subnormal b
+    __exp_b  = 1 - __mant_b_shft;
+    __mant_b = __mant_b << __mant_b_shft;
+  }
+  if (!__exp_a)
+  {
+    if (!__mant_a)
+    {
+      return (__fpbits64) ((uint64_t) __sign_z << 63); // 0 / x -> 0
+    }
+    int __mant_a_shft = ::cuda::std::countl_zero((uint64_t) __mant_a) - 11;
+    // normalize subnormal a
+    __exp_a  = 1 - __mant_a_shft;
+    __mant_a = __mant_a << __mant_a_shft;
+  }
+
+  // -------- fixed-point reciprocal division --------
+  int32_t __exp_z = __exp_a - __exp_b + 0x3FE;
+
+  __mant_a |= _CCCL_FPEMU_HIDDEN_64;
+  __mant_b |= _CCCL_FPEMU_HIDDEN_64;
+
+  if (__mant_a < __mant_b)
+  {
+    --__exp_z;
+    __mant_a <<= 11;
+  }
+  else
+  {
+    __mant_a <<= 10;
+  }
+
+  __mant_b <<= 11;
+
+  uint32_t __recip32   = __internal_fp64emu_div_recip32((uint32_t) (__mant_b >> 32)) - 2;
+  uint32_t __mant32_z  = (uint32_t) (((uint64_t) (uint32_t) (__mant_a >> 32) * (uint64_t) __recip32) >> 32);
+  uint32_t __mant32_z2 = __mant32_z << 1;
+  uint64_t __rem64     = ((__mant_a - (uint64_t) __mant32_z2 * (uint32_t) (__mant_b >> 32)) << 28)
+                   - (uint64_t) __mant32_z2 * ((uint32_t) __mant_b >> 4);
+  uint32_t __q32      = (uint32_t) (((uint64_t) (uint32_t) (__rem64 >> 32) * (uint64_t) __recip32) >> 32) + 4;
+  uint64_t __mant64_z = ((uint64_t) __mant32_z << 32) + ((uint64_t) __q32 << 4);
+
+  // Refine if the quotient is close to a rounding boundary (exact remainder).
+  if ((__mant64_z & 0x1FF) < (4u << 4))
+  {
+    __q32 &= ~7u;
+    __mant64_z &= ~(uint64_t) 0x7F;
+    __mant32_z2 = __q32 << 1;
+
+    __rem64 = ((__rem64 - (uint64_t) __mant32_z2 * (uint32_t) (__mant_b >> 32)) << 28)
+            - (uint64_t) __mant32_z2 * ((uint32_t) __mant_b >> 4);
+
+    if (__rem64 & _CCCL_FPEMU_SIGN_64)
+    {
+      __mant64_z -= 1 << 7;
+    }
+    else
+    {
+      if (__rem64)
+      {
+        __mant64_z |= 1;
+      }
+    }
+  }
+
+  return __internal_fp64emu_round_pack<_Rm>(__sign_z, __exp_z, __mant64_z);
+#endif // _CCCL_FPEMU_PACKED_VIA_UNPACKED
+} // __internal_fp64emu_ddiv
+
+//! @brief Divide two double-precision floating point numbers
+//!
+//! This function divides two double-precision floating point numbers.
+//! It works by splitting the numbers into sign, exponent, and mantissa, normalizing the mantissa,
+//! and then computing the division of the mantissa.
+//!
+//! @param __x The first double-precision floating point number
+//! @param __y The second double-precision floating point number
+//! @return The result of the division
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked
+__internal_fp64emu_ddiv_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  // ---- True unpacked divide -------------------------------------------
+  // Operates directly on the fully-accurate unpacked operands (no operand
+  // pack, no legacy packed kernel). The full unpack has already normalized
+  // denormals and encoded inf/nan in the exponent band, so the significand
+  // is mantissa>>EXTRA_BITS (implicit bit at 52) and the exponent is the
+  // IEEE-biased value -- no subnormal renormalization needed. The proven
+  // fixed-point reciprocal quotient is computed exactly as the packed core,
+  // then expressed on the universal unpacked scale (implicit bit at 61, a
+  // sticky LSB) so the full pack does the single correctly-rounded
+  // finalization (subnormal / overflow / rounding). Division has no def/fast
+  // arithmetic variant -- the quotient is correctly rounded for every method.
+  constexpr int32_t __nan_exp = 0x0007ff00;
+  constexpr int32_t __inf_exp = 0x00007ff0;
+
+  const int32_t __exp_x     = (int32_t) __x.exponent;
+  const int32_t __exp_y     = (int32_t) __y.exponent;
+  const bool __sign_z       = ((__x.sign != 0) ^ (__y.sign != 0));
+  const uint64_t __sign_bit = (uint64_t) __sign_z << 63;
+
+  const bool __nan_x  = (__exp_x == __nan_exp);
+  const bool __nan_y  = (__exp_y == __nan_exp);
+  const bool __inf_x  = (__exp_x == __inf_exp);
+  const bool __inf_y  = (__exp_y == __inf_exp);
+  const bool __zero_x = (__x.mantissa == 0);
+  const bool __zero_y = (__y.mantissa == 0);
+
+  // Special operands: build the canonical packed result and unpack it (rare,
+  // off the hot path -- no arithmetic round trip).
+  if (__nan_x || __nan_y)
+  {
+    return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_DEFNAN_64);
+  }
+  if (__inf_x)
+  {
+    if (__inf_y)
+    {
+      return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_DEFNAN_64); // inf/inf
+    }
+    return __internal_fp64emu_unpack((__fpbits64) (__sign_bit | _CCCL_FPEMU_INF_64)); // inf/finite
+  }
+  if (__inf_y)
+  {
+    return __internal_fp64emu_unpack((__fpbits64) __sign_bit); // finite/inf -> 0
+  }
+  if (__zero_y)
+  {
+    if (__zero_x)
+    {
+      return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_DEFNAN_64); // 0/0
+    }
+    return __internal_fp64emu_unpack((__fpbits64) (__sign_bit | _CCCL_FPEMU_INF_64)); // x/0
+  }
+  if (__zero_x)
+  {
+    return __internal_fp64emu_unpack((__fpbits64) __sign_bit); // 0/finite -> 0
+  }
+
+  // ---- finite / finite : fixed-point reciprocal division --------------
+  uint64_t __mant_a = __x.mantissa >> EXTRA_BITS; // 53-bit significand, implicit bit at 52
+  uint64_t __mant_b = __y.mantissa >> EXTRA_BITS;
+  int32_t __exp_z   = __exp_x - __exp_y + 0x3FE;
+
+  if (__mant_a < __mant_b)
+  {
+    --__exp_z;
+    __mant_a <<= 11;
+  }
+  else
+  {
+    __mant_a <<= 10;
+  }
+  __mant_b <<= 11;
+
+  uint32_t __recip32   = __internal_fp64emu_div_recip32((uint32_t) (__mant_b >> 32)) - 2;
+  uint32_t __mant32_z  = (uint32_t) (((uint64_t) (uint32_t) (__mant_a >> 32) * (uint64_t) __recip32) >> 32);
+  uint32_t __mant32_z2 = __mant32_z << 1;
+  uint64_t __rem64     = ((__mant_a - (uint64_t) __mant32_z2 * (uint32_t) (__mant_b >> 32)) << 28)
+                   - (uint64_t) __mant32_z2 * ((uint32_t) __mant_b >> 4);
+  uint32_t __q32      = (uint32_t) (((uint64_t) (uint32_t) (__rem64 >> 32) * (uint64_t) __recip32) >> 32) + 4;
+  uint64_t __mant64_z = ((uint64_t) __mant32_z << 32) + ((uint64_t) __q32 << 4);
+
+  // Refine if the quotient is close to a rounding boundary (exact remainder).
+  if ((__mant64_z & 0x1FF) < (4u << 4))
+  {
+    __q32 &= ~7u;
+    __mant64_z &= ~(uint64_t) 0x7F;
+    __mant32_z2 = __q32 << 1;
+    __rem64     = ((__rem64 - (uint64_t) __mant32_z2 * (uint32_t) (__mant_b >> 32)) << 28)
+            - (uint64_t) __mant32_z2 * ((uint32_t) __mant_b >> 4);
+    if (__rem64 & _CCCL_FPEMU_SIGN_64)
+    {
+      __mant64_z -= 1 << 7;
+    }
+    else
+    {
+      if (__rem64)
+      {
+        __mant64_z |= 1;
+      }
+    }
+  }
+
+  // round_pack expects the leading significand bit at 62 and exp == biased-1.
+  // The universal unpacked scale puts the implicit bit at 61 with EXTRA_BITS
+  // round bits and exponent == IEEE-biased (== exp_z + 1); shift the leading
+  // bit down one place (preserving the dropped bit as sticky) and let the
+  // full pack round + emit subnormal / saturate to inf.
+  __fpbits64_unpacked __r;
+  __r.sign     = __sign_z ? (1u << 31) : 0u;
+  __r.exponent = (uint32_t) (__exp_z + 1);
+  __r.mantissa = (__mant64_z >> 1) | (__mant64_z & 1);
+  return __r;
+} // __internal_fp64emu_ddiv_unpacked
+
+// ============================================================================
+// Builtin declarations/implementations for division operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::rz, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::ru, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::rd, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_ddiv_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_ddiv_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_ddiv_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_ddiv<__fpemu_rounding::rn, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_ddiv(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_ddiv_unpacked<fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_ddiv(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_ddiv_unpacked<fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_ddiv(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_ddiv_unpacked<fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_ddiv(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_ddiv_unpacked<fpemu_accuracy::low>(__x, __y);
+}
+#else
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_ddiv_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_ddiv_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_ddiv_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_ddiv_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_ddiv(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_ddiv(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_ddiv(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_ddiv(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_DIV_H (builtins)
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_DDIV_API_MERGED)
+#define _CCCL_FPEMU_DDIV_API_MERGED
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_ddiv_api.hpp)
+// ============================================================================
+
+// Default API implementation
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> operator/(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_ddiv_rn(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_ddiv_rn(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_ddiv_rn(__x.__bits_, __y.__bits_));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_ddiv_rn(__x.__bits_, __y.__bits_));
+  }
+} // operator /
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __ddiv_rn(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_high_ddiv_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_ddiv_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_ddiv_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __ddiv_rz(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+    __fp64emu_ddiv_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __ddiv_ru(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+    __fp64emu_ddiv_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __ddiv_rd(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+    __fp64emu_ddiv_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+}
+
+// Operator/ for unpacked division
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+operator/(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_ddiv(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_ddiv(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_ddiv(__x.__bits_, __y.__bits_));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_ddiv(__x.__bits_, __y.__bits_));
+  }
+} // operator/
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+__ddiv_rn(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_ddiv(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_ddiv(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_ddiv(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+}
+
+// Mixed-operand promoters (relocated from the class body; formerly hidden
+// friends). Enabled only when at least one operand is an fpemu and at least
+// one is a built-in arithmetic type: both operands are promoted to the fpemu
+// type and the exact-match core above is called. Pure fpemu/fpemu calls bind
+// to the cores directly; pure arithmetic calls are left to the language.
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __ddiv_rn(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __ddiv_rn(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __ddiv_rz(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __ddiv_rz(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __ddiv_ru(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __ddiv_ru(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __ddiv_rd(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __ddiv_rd(_Fp(__x), _Fp(__y));
+}
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_DIV_H

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_fma.h

@@ -0,0 +1,656 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_FMA_H
+#define _CUDA___FP_FPEMU_IMPL_FMA_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_impl_fma.h
+//! @brief Implementation of fused multiply-add operations (FMA & MAD) for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of fused multiply-add operations for the FPEMU library.
+//! It includes:
+//!   - Fused multiply-add functions for different accuracy and range configurations
+//!   - Special case handling for NaN, inf, zero, etc
+//!
+//! The implementation is designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+#include <cuda/std/__bit/countl.h>
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+#if _CCCL_HOST_COMPILATION()
+// Host seed: the libm symbol. The exception spec must match the platform's
+// <math.h> prototype exactly, otherwise this extern-"C" redeclaration conflicts
+// with ::fma when <cmath> is also in the TU (in C++17+ the exception spec is
+// part of the type, so a mismatch is an error, not just a warning):
+//   - glibc marks fma __THROW (noexcept), so the redeclaration must be noexcept.
+//   - MSVC's CRT/CUDA prototype carries no exception specification, so a
+//     noexcept redeclaration is a mismatched extern-"C" overload (C2382/C2733
+//     under C++20); declare it without noexcept to match.
+#  if _CCCL_COMPILER(MSVC)
+extern "C" double fma(double __x, double __y, double __z);
+#  else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+extern "C" double fma(double __x, double __y, double __z) noexcept;
+#  endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
+#endif // _CCCL_HOST_COMPILATION()
+
+//! @brief Pure FMA core operating on the unpacked representation.
+//!
+//! Consumes/produces __fpbits64_unpacked exactly as produced by the universal
+//! __internal_fp64emu_unpack and consumed by __internal_fp64emu_pack.
+//! Inputs carry a normalized mantissa (implicit bit set, denormals normalized)
+//! with inf/nan encoded in the exponent band (around the 0x00007ff0 / 0x0007ff00
+//! magics), so the floating-point class is read from the exponent rather than a
+//! separate field. The returned value is the pre-rounding intermediate: a 64-bit
+//! mantissa with a sticky LSB and an exponent that may be <= 0 (subnormal) or in
+//! the inf/nan band. Final rounding, subnormal shifting and inf/saturate are the
+//! job of pack. Templated on the rounding mode (sign-of-zero, rd handling) and
+//! the method (product accuracy via __mul_128 and range-based special handling).
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked
+__internal_fp64emu_fma_unpacked(__fpbits64_unpacked __a, __fpbits64_unpacked __b, __fpbits64_unpacked __c) noexcept
+{
+  constexpr fpemu_accuracy __acc_forced = fpemu_accuracy::_CCCL_FPEMU_FMA_METHOD;
+  constexpr fpemu_accuracy __acc_used   = (__acc_forced != fpemu_accuracy::unset) ? __acc_forced : _Acc;
+  // Unpacked cores always run on the fully-accurate full-range unpack/pack
+  // boundary (method-independent): the inf/nan folds stay live and underflow
+  // flows to the full pack (no FTZ, correct subnormal + min_normal round-up).
+
+  // Inf/Nan exponent magics produced by the universal unpack.
+  constexpr uint32_t __inf_exp = 0x00007ff0u;
+  constexpr int32_t __nan_exp  = 0x0007ff00;
+
+  __fpbits64_unpacked __r;
+  __fpemu_uint128 __mantissa_ab;
+  __uint64x2 __ab_res;
+
+  // MUL START:
+  // Compute mantissa_ab - the product of a and b in 128-bit
+  __uint32x2 __a_32x2 = ::cuda::std::bit_cast<__uint32x2>(__a.mantissa);
+  __uint32x2 __b_32x2 = ::cuda::std::bit_cast<__uint32x2>(__b.mantissa);
+  __ab_res            = ::cuda::std::bit_cast<__uint64x2>(__mul_128<__acc_used>(__a_32x2, __b_32x2));
+
+  __mantissa_ab              = ::cuda::std::bit_cast<__fpemu_uint128>(__ab_res);
+  __uint32x4 __mantissa_ab32 = ::cuda::std::bit_cast<__uint32x4>(__mantissa_ab);
+
+  // Exponents/signs (read with explicit signedness: the public field is
+  // uint32 but the core needs signed arithmetic for subnormal exponents).
+  int32_t __exponent_ab = (int32_t) __a.exponent + (int32_t) __b.exponent - (int32_t) __fpemu_bias;
+  int32_t __exponent_c  = (int32_t) __c.exponent;
+  int32_t __sign_ab     = (int32_t) (__a.sign ^ __b.sign);
+  int32_t __sign_c      = (int32_t) __c.sign;
+
+  // Compute exponent_ab_new - the exponent of the product of a and b
+  int __mul_nzeros          = __mantissa_ab32.hi.x[1] < 0x08000000;
+  int32_t __exponent_ab_new = __exponent_ab - __mul_nzeros + 1;
+  // Shift mantissa_ab
+  __mantissa_ab = __mantissa_ab << (11 - EXTRA_BITS + __mul_nzeros);
+  // Compute mantissa_c - the mantissa of c
+  __fpemu_uint128 __mantissa_c = __c.mantissa;
+  // Compute mantissa_r - the result of the product of a and b and c
+  __fpemu_uint128 __mantissa_r;
+
+  {
+    // Check if a or b is inf and c is inf and sign_ab != sign_c then return NaN
+    if ((__a.exponent == __inf_exp || __b.exponent == __inf_exp) && __c.exponent == __inf_exp && __sign_ab != __sign_c)
+    {
+      __exponent_ab_new = __nan_exp;
+    }
+  }
+
+  // Check if exponent_ab_new is INF_ZERO then return NaN
+  if (__exponent_ab_new == (int32_t) __fpemu_inf_zero)
+  {
+    __exponent_ab_new = __nan_exp;
+  }
+
+  // ADD START:
+  //  Compute exponent_r - the larger of exponent_ab_new and exponent_c
+  int32_t __exponent_r = _CCCL_FPEMU_MAX(__exponent_ab_new, __exponent_c);
+
+  // Compute delta_a and delta_b for mantissas shift
+  int32_t __delta_a = __exponent_r - __exponent_ab_new;
+  int32_t __delta_b = __exponent_r - __exponent_c;
+
+  NV_IF_TARGET(NV_IS_HOST, ({
+                 __delta_a = (__delta_a > 127) ? 127 : __delta_a;
+                 __delta_b = (__delta_b > 127) ? 127 : __delta_b;
+               }))
+
+  // Shift mantissas with jam only (SoftFloat shiftRightJam*); round at pack
+  __mantissa_ab = __shr_128_jam(__mantissa_ab, __delta_a);
+  __mantissa_c  = __shr_128_jam(__mantissa_c << 64, __delta_b);
+
+  // Add or subtract mantissas
+  uint32_t __sign_r = __sign_ab;
+  if (__sign_ab == __sign_c)
+  {
+    __mantissa_r = __mantissa_ab + __mantissa_c;
+  }
+  else if (__mantissa_ab == __mantissa_c)
+  {
+    __mantissa_r = 0;
+  }
+  else if ((__mantissa_ab > __mantissa_c))
+  {
+    __mantissa_r = __mantissa_ab - __mantissa_c;
+  }
+  else // mantissa_ab < mantissa_c
+  {
+    __sign_r     = __sign_c;
+    __mantissa_r = __mantissa_c - __mantissa_ab;
+  }
+
+  if (__mantissa_r == 0)
+  {
+    // Exact cancellation -> zero. IEEE-754 6.3 would make this -0 under
+    // round-toward-negative (rd); that rounding-dependent zero sign is
+    // intentionally NOT honored here (the core is rounding-independent), so
+    // the zero is -0 only when both effective signs are negative.
+    __sign_r = __sign_ab & __sign_c;
+  }
+
+  // Normalize mantissa_r
+  // use reinterpret_cast to avoid slowdown from bit_cast
+  uint64_t* __m = reinterpret_cast<uint64_t*>(&__mantissa_r);
+  int __nzeros  = (__m[1] == 0) ? (::cuda::std::countl_zero((uint64_t) (__m[0] + 64)))
+                                : (::cuda::std::countl_zero((uint64_t) (__m[1] << 1)));
+
+  // Shift mantissa_r
+  __mantissa_r = (__nzeros == 0) ? (__mantissa_r >> 1) : (__mantissa_r << (__nzeros - 1));
+
+  __uint64x2 __mantissa_r64 = ::cuda::std::bit_cast<__uint64x2>(__mantissa_r);
+
+  // The result class (inf vs finite-overflow) is recoverable from the
+  // exponent band by pack: genuine infinities inherit the huge exponent of
+  // their inf operand, finite results never reach it. So no class field is
+  // written here; the inf-inf -> NaN and inf*0 -> NaN cases were already
+  // folded into exponent_ab_new (NAN_EXP) above.
+  __r.sign = __sign_r;
+  // +1 matches the unified pack's "mask the implicit bit" convention
+  // (pack reconstitutes via exp-1); the +1/-1 cancel so packed FMA is
+  // bit-exact with the legacy add-convention packer.
+  __r.exponent = static_cast<uint32_t>(__exponent_r - __nzeros + 1);
+  __r.mantissa = __mantissa_r64.x[1] | (__mantissa_r64.x[0] != 0);
+
+  // The unpacked core runs on the full-range boundary: underflow (and the rare
+  // top-subnormal -> min_normal round-up) flows to the full pack, which has the
+  // complete mantissa and resolves the correct subnormal / min_normal for every
+  // rounding mode. No FTZ here, so no rounding-dependent fix-up is needed.
+
+  return __r;
+} // __internal_fp64emu_fma_unpacked
+
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_fma(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  // Forced parameters for the fused multiply-add operation
+  constexpr fpemu_accuracy __acc_forced = fpemu_accuracy::_CCCL_FPEMU_FMA_METHOD;
+  constexpr fpemu_accuracy __acc_used   = (__acc_forced != fpemu_accuracy::unset) ? __acc_forced : _Acc;
+
+  {
+    {
+      // FMA = pack(fma_unpacked(unpack(x), unpack(y), unpack(z))). The fma_unpacked
+      // core selects accurate/def/fast internally; the universal full-range
+      // unpack/pack are the shared prologue/epilogue (def/fast are full-range here).
+      __fpbits64_unpacked __a = __internal_fp64emu_unpack(__x);
+      __fpbits64_unpacked __b = __internal_fp64emu_unpack(__y);
+      __fpbits64_unpacked __c = __internal_fp64emu_unpack(__z);
+      __fpbits64_unpacked __r = __internal_fp64emu_fma_unpacked<__acc_used>(__a, __b, __c);
+      __fpbits64 __result     = __internal_fp64emu_pack<_Rm>(__r);
+
+      if constexpr (_Rm == __fpemu_rounding::rd)
+      {
+        // Exact cancellation (a*b + c == 0 with opposite effective signs)
+        // must yield -0 under round-toward-negative. The rounding-independent
+        // core packs an exact zero to +0 (r.mantissa == 0); a misaligned
+        // remainder can also surface as a tiny artifact. Both map to -0 here.
+        // A genuine underflow keeps a nonzero core mantissa and stays +0.
+        const bool __opposite_signs = ((__a.sign ^ __b.sign) != __c.sign);
+        const bool __exact_zero     = (__r.mantissa == 0) && ((__result << 1) == 0);
+        const bool __tiny_artifact  = (__result == UINT64_C(0x0000000100000000));
+        if (__opposite_signs && (__exact_zero || __tiny_artifact))
+        {
+          __fpbits64_unpacked __zneg;
+          __zneg.sign     = 1U << 31;
+          __zneg.exponent = 0;
+          __zneg.mantissa = 0;
+          __result        = __internal_fp64emu_pack<_Rm>(__zneg);
+        }
+      }
+      return __result;
+    }
+  }
+} // __internal_fp64emu_fma
+
+// ============================================================================
+// Builtin declarations/implementations for FMA operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_rz(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rz, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_ru(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::ru, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_rd(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rd, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_fma_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_rz(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rz, fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_ru(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::ru, fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_rd(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rd, fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rn, fpemu_accuracy::low>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_rz(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rz, fpemu_accuracy::low>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_ru(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::ru, fpemu_accuracy::low>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_rd(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_fma<__fpemu_rounding::rd, fpemu_accuracy::low>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_fma(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_fma_unpacked<fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_fma(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_fma_unpacked<fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_fma(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_fma_unpacked<fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_fma(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_fma_unpacked<fpemu_accuracy::low>(__x, __y, __z);
+}
+#else
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_rz(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_ru(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_fma_rd(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_fma_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_rz(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_ru(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_fma_rd(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_rz(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_ru(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_fma_rd(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_fma(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_fma(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_fma(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_fma(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_FMA_H
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_FMA_API_MERGED)
+#define _CCCL_FPEMU_FMA_API_MERGED
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_fma_api.hpp)
+// ============================================================================
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+fma(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_fma_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_fma_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_fma_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+__fma_rn(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_fma_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_fma_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_fma_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+__fma_rz(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_fma_rz(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_fma_rz(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_fma_rz(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_fma_rz(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+__fma_ru(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_fma_ru(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_fma_ru(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_fma_ru(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_fma_ru(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+__fma_rd(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_fma_rd(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_fma_rd(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_fma_rd(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_fma_rd(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+fma(const fpemu_unpacked<double, _Acc>& __x,
+    const fpemu_unpacked<double, _Acc>& __y,
+    const fpemu_unpacked<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_fma(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_fma(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_fma(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+__fma_rn(const fpemu_unpacked<double, _Acc>& __x,
+         const fpemu_unpacked<double, _Acc>& __y,
+         const fpemu_unpacked<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_fma(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_fma(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_fma(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+}
+
+// Mixed-operand promoters (relocated from the class body; formerly hidden
+// friends). Enabled only when at least one operand is an fpemu and at least
+// one is a built-in arithmetic type: both operands are promoted to the fpemu
+// type and the exact-match core above is called. Pure fpemu/fpemu calls bind
+// to the cores directly; pure arithmetic calls are left to the language.
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> fma(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return fma(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> __fma_rn(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return __fma_rn(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> __fma_rz(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return __fma_rz(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> __fma_ru(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return __fma_ru(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> __fma_rd(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return __fma_rd(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+} // namespace cuda::experimental
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+// Overloads of fma for the emulated double types so the standard spelling
+// cuda::std::fma selects the emulated implementation. A qualified cuda::std::fma
+// call suppresses ADL, so without these it would silently narrow fpemu -> double
+// (via the implicit conversion) and compute a native-double fma. These forward to
+// cuda::experimental::fma, which unqualified/ADL calls already resolve to. The
+// exact-type overloads cover pure fpemu/fpemu/fpemu calls (which __fpemu_mixed_v
+// excludes), while the constrained overload handles mixed fpemu + arithmetic.
+template <::cuda::experimental::fpemu_accuracy _Acc>
+[[nodiscard]] _CCCL_API ::cuda::experimental::fpemu<double, _Acc>
+fma(const ::cuda::experimental::fpemu<double, _Acc>& __x,
+    const ::cuda::experimental::fpemu<double, _Acc>& __y,
+    const ::cuda::experimental::fpemu<double, _Acc>& __z) noexcept
+{
+  return ::cuda::experimental::fma(__x, __y, __z);
+}
+template <::cuda::experimental::fpemu_accuracy _Acc>
+[[nodiscard]] _CCCL_API ::cuda::experimental::fpemu_unpacked<double, _Acc>
+fma(const ::cuda::experimental::fpemu_unpacked<double, _Acc>& __x,
+    const ::cuda::experimental::fpemu_unpacked<double, _Acc>& __y,
+    const ::cuda::experimental::fpemu_unpacked<double, _Acc>& __z) noexcept
+{
+  return ::cuda::experimental::fma(__x, __y, __z);
+}
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(::cuda::experimental::__fpemu_mixed_v<_T1, _T2, _T3>)
+[[nodiscard]] _CCCL_API ::cuda::experimental::__fpemu_pick_t<_T1, _T2, _T3>
+fma(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  return ::cuda::experimental::fma(__x, __y, __z);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CCCL_FPEMU_FMA_API_MERGED

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_mul.h

@@ -0,0 +1,966 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_MUL_H
+#define _CUDA___FP_FPEMU_IMPL_MUL_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_dmul_impl.hpp
+//! @brief Implementation of double-precision multiplication operations for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of double-precision multiplication operations for the FPEMU library.
+//! It includes:
+//!
+//! - Multiplication functions for fpemu
+//! - Multiplication operators for fpemu
+//! - Multiplication functions to other types
+//!
+//! The multiplication functions are designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+#define _CCCL_FP64EMU_USE_MUL_UNPACKED      0
+#define _CCCL_FP64EMU_DMUL_FP32_FAST_ENABLE 1
+
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+//! @brief v1.0 Emulation of double-precision multiplication for FPEMU
+//!
+//! This function implements the reference (v1.0) emulation of double-precision
+//! floating-point multiplication for the FPEMU format. It operates on the
+//! internal bitwise representation (__fpbits64) of the operands and produces
+//! a bit-exact result matching the IEEE-754 standard for double-precision
+//! multiplication.
+//!
+//! The algorithm performs the following steps:
+//!   - Extracts the sign, exponent, and mantissa from both operands.
+//!   - Handles special cases such as zero, denormalized numbers, infinities, and NaNs.
+//!   - Multiplies the mantissas with full precision, including the implicit leading bit.
+//!   - Computes the resulting exponent, taking into account normalization and bias.
+//!   - Normalizes the result and applies rounding according to the specified mode.
+//!   - Packs the sign, exponent, and mantissa back into the __fpbits64 result.
+//!
+//! This implementation is designed for correctness and bitwise reproducibility,
+//! serving as a reference for optimized or hardware-accelerated versions.
+//!
+//! @tparam rm   Rounding mode (default: nearest-even)
+//! @tparam _Acc Accuracy level (fpemu_accuracy; default: high)
+//! @param x     First operand (__fpbits64)
+//! @param y     Second operand (__fpbits64)
+//! @return      Product as __fpbits64
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_high_dmul(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  uint64_t __a = __x;
+  uint64_t __b = __y;
+
+  __uint32x2 __a_32x2 = ::cuda::std::bit_cast<__uint32x2>(__a);
+  __uint32x2 __b_32x2 = ::cuda::std::bit_cast<__uint32x2>(__b);
+
+  __uint32x2 __man_a_32x2;
+  __uint32x2 __man_b_32x2;
+  __uint32x2 __man_c_32x2;
+  int32_t __exp_a;
+  int32_t __exp_b;
+  int32_t __exp_c;
+  int32_t __shift;
+  bool __is_sign_a;
+  bool __is_sign_b;
+  bool __is_sign_c;
+  bool __is_a_exp_zero;
+  bool __is_b_exp_zero;
+  bool __is_impl_bit;
+  __fpbits64 __result;
+
+  // Extract sign and exponent for input A
+  __exp_a = __unpack_exp<_Acc>(__a_32x2);
+  // Extract sign and exponent for input B
+  __exp_b = __unpack_exp<_Acc>(__b_32x2);
+
+  // Check if input A is denormal or zero
+  __is_a_exp_zero = (__exp_a == 0);
+  __is_b_exp_zero = (__exp_b == 0);
+
+  // Extract mantissa for input A
+  __man_a_32x2 = __unpack_mant<_Acc>(&__is_sign_a, __a_32x2, __is_a_exp_zero);
+  // Extract mantissa for input B
+  __man_b_32x2 = __unpack_mant<_Acc>(&__is_sign_b, __b_32x2, __is_b_exp_zero);
+
+  if constexpr (_Acc != fpemu_accuracy::high)
+  {
+    if (__is_a_exp_zero)
+    {
+      __man_a_32x2 = {0, 0};
+    }
+    if (__is_b_exp_zero)
+    {
+      __man_b_32x2 = {0, 0};
+    }
+  }
+
+  // Correct exp and mantissa for denormal
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    __exp_a      = (__is_a_exp_zero) ? 12 - __flo_u64(__a_32x2) : __exp_a;
+    __exp_b      = (__is_b_exp_zero) ? 12 - __flo_u64(__b_32x2) : __exp_b;
+    __man_a_32x2 = (__is_a_exp_zero) ? __shl_64(__man_a_32x2, 1 - __exp_a) : __man_a_32x2;
+    __man_b_32x2 = (__is_b_exp_zero) ? __shl_64(__man_b_32x2, 1 - __exp_b) : __man_b_32x2;
+  }
+  else
+  {
+    __exp_a = (!__is_a_exp_zero) ? __exp_a : -52;
+    __exp_b = (!__is_b_exp_zero) ? __exp_b : -52;
+  }
+
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    __uint32x4 __man_c_32x4 = __mul_128<_Acc>(__man_a_32x2, __man_b_32x2);
+    // Rounding: Set least significant bit to "1"
+    // if low 64 bits are non-zero
+    if ((__man_c_32x4.lo.x[0] | __man_c_32x4.lo.x[1]) != 0)
+    {
+      __man_c_32x4.hi.x[0] |= 1;
+    }
+    __man_c_32x2 = __man_c_32x4.hi;
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    __man_c_32x2 = __mul_64<_Acc>(__man_a_32x2, __man_b_32x2);
+  }
+  else
+  {
+    __man_c_32x2.x[1] = __mul_32<_Acc>(__man_a_32x2, __man_b_32x2);
+    __man_c_32x2.x[0] = 0;
+  }
+
+  // Check implicit-bit position
+  __is_impl_bit = (__man_c_32x2.x[1] >= 0x08000000);
+
+  // Calculate exponent
+  __exp_c = static_cast<int32_t>(__exp_a + __exp_b - (__fpemu_bias + 1) + __is_impl_bit);
+
+  // Calculate SIGN
+  __is_sign_c = __is_sign_a ^ __is_sign_b;
+
+  // Check for negative exponent
+  bool __is_exp_c_neg = (__exp_c < 0);
+
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    if (__is_exp_c_neg)
+    {
+      // shift is negative, so we need to add it to exp_c
+      __shift = -__exp_c;
+      __exp_c = __exp_c + __shift;
+      // Shift mantissa to the right with rounding
+      // in case of negative exponent (DENORM)
+      __man_c_32x2 = __sar_64_rnd<_Acc, _Rm>(__man_c_32x2, __shift, __is_sign_c);
+    }
+  }
+  else
+  {
+    // Flush denormals to zero
+    __exp_c = (__is_exp_c_neg) ? 0 : __exp_c;
+    if (__is_exp_c_neg)
+    {
+      __man_c_32x2 = {0, 0};
+    }
+  }
+
+  if (__is_impl_bit)
+  {
+    __man_c_32x2 = __round<_Rm>(__man_c_32x2, -2, __is_sign_c);
+  }
+  else
+  {
+    __man_c_32x2 = __round<_Rm>(__man_c_32x2, -3, __is_sign_c);
+  }
+
+  // Pack sign, exponent and mantissa back to FP64
+  // (+checks for NAN,INF,0)
+  __result = __pack<_Acc, _Rm>(__is_sign_c, __exp_c, __man_c_32x2);
+  return __result;
+} // __internal_fp64emu_high_dmul
+
+//! @brief Version 2.0 emulation of the double-precision multiplication function.
+//!
+//! This implementation provides an emulation of IEEE-754 double-precision (fp64) multiplication,
+//! supporting configurable rounding modes, accuracy, and range options.
+//!
+//! The function operates by:
+//!   - Extracting the exponent and mantissa fields from the input operands.
+//!   - Computing the result sign as the XOR of the input signs.
+//!   - Adding the exponents and subtracting the fp64 bias to obtain the result exponent.
+//!   - Multiplying the mantissas by a dedicated helper (__mul_mant), which handles
+//!     normalization, carry, and accuracy-specific bit manipulations.
+//!   - Adjusting the result for exponent overflow and underflow, and setting the sign and exponent bits.
+//!   - Returning the correctly packed fp64 result.
+//!
+//! This version is designed for improved accuracy and performance, and is suitable for both
+//! host and device execution.
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_mid_dmul(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  __uint32x2 __a_32x2 = ::cuda::std::bit_cast<__uint32x2>(__x);
+  __uint32x2 __b_32x2 = ::cuda::std::bit_cast<__uint32x2>(__y);
+  __fpbits64 __result;
+
+  // Extract exponents
+  uint32_t __exp_a    = (__a_32x2.x[1] >> _CCCL_FP64_HI_MANT_SHIFT) & _CCCL_FP64_LO_EXP_MASK;
+  uint32_t __exp_b    = (__b_32x2.x[1] >> _CCCL_FP64_HI_MANT_SHIFT) & _CCCL_FP64_LO_EXP_MASK;
+  bool __is_exps_zero = ((__exp_a == 0) || (__exp_b == 0));
+
+  // Compute result sign (XOR of input signs)
+  uint32_t __result_sign = (__a_32x2.x[1] ^ __b_32x2.x[1]) & _CCCL_FP64_HI_SIGN_MASK;
+
+  // Add exponents and subtract fp64 bias (1023)
+  int32_t __result_exp = (int32_t) __exp_a + (int32_t) __exp_b - _CCCL_FP64_BIAS;
+
+  // Multiply the mantissas
+  __uint32x2 __result_32x2;
+
+  // Integer based mantissa processing
+  {
+    uint32_t __carry_bit;
+
+    // Clear the exponent/sign bits
+    __a_32x2.x[1] &= _CCCL_FP64_HI_MANT_MASK;
+    __b_32x2.x[1] &= _CCCL_FP64_HI_MANT_MASK;
+
+    // Set the implicit 1 bit
+    __a_32x2.x[1] |= (1 << _CCCL_FP64_HI_MANT_SHIFT);
+    __b_32x2.x[1] |= (1 << _CCCL_FP64_HI_MANT_SHIFT);
+
+    if constexpr (_Acc == fpemu_accuracy::mid)
+    {
+      // Shift the mantissa to the left by _CCCL_FP64_EXTRA_BITS (9) bits to preserve low bits
+      __a_32x2 = __shl_64(__a_32x2, _CCCL_FP64_EXTRA_BITS);
+      __b_32x2 = __shl_64(__b_32x2, _CCCL_FP64_EXTRA_BITS);
+
+      // Multiply the mantissas
+      __result_32x2 = __mul_64<_Acc>(__a_32x2, __b_32x2);
+
+      // Check if the carry bit is set
+      __carry_bit = (__result_32x2.x[1] >= (1 << (_CCCL_FP64_MANT_MUL_CARRY_BIT + _CCCL_FP64_EXTRA_BITS * 2 + 1)));
+
+      // Shift the mantissa back with directed rounding for ru/rd
+      const int __mant_shift = (__carry_bit) ? (((_CCCL_FP64_EXTRA_BITS * 2) - _CCCL_FP64_MANT_MUL_SHIFT) + 1)
+                                             : ((_CCCL_FP64_EXTRA_BITS * 2) - _CCCL_FP64_MANT_MUL_SHIFT);
+      __result_32x2          = __shr_64_rnd<_Rm>(__result_32x2, __mant_shift, __result_sign != 0);
+    }
+    else
+    {
+      // Multiply the mantissas
+      __result_32x2 = __mul_64<_Acc>(__a_32x2, __b_32x2);
+
+      // Check if the carry bit is set
+      __carry_bit = (__result_32x2.x[1] >= (1 << (_CCCL_FP64_MANT_MUL_CARRY_BIT + 1)));
+
+      // Shift the mantissa to the left by the correct amount taking into account the carry bit
+      __result_32x2 =
+        __shl_64(__result_32x2, (__carry_bit) ? (_CCCL_FP64_MANT_MUL_SHIFT - 1) : _CCCL_FP64_MANT_MUL_SHIFT);
+    }
+
+    // Add the carry bit to the exponent
+    __result_exp = __result_exp + static_cast<int32_t>(__carry_bit);
+
+    // Clear the unused high bits
+    __result_32x2.x[1] &= _CCCL_FP64_HI_MANT_MASK;
+  }
+
+  // Check for exponent overflow
+  bool __is_exp_ovfl = (__result_exp > (_CCCL_FP64_BIAS * 2));
+
+  /*
+  // Correct exponent:
+  - Check for exponent zero
+  - Adjust exponent if overflow occurs
+  - Check for negative exponent
+  - Adjust exponent if negative
+  - Check for exponent overflow or zero
+  - Adjust exponent if overflow or zero
+  - Check for exponent zero
+  - Adjust exponent if zero
+  */
+  const bool __is_sign_c = (__result_sign != 0);
+
+  if (__is_exps_zero)
+  {
+    __result_exp  = 0;
+    __result_32x2 = {0, 0};
+  }
+  else if (__is_exp_ovfl)
+  {
+    __fp64_ovfl_sat<_Rm>(__is_sign_c, __result_exp, __result_32x2);
+  }
+  else
+  {
+    // Check for negative exponent
+    if (__result_exp < 0)
+    {
+      __result_exp = 0;
+    }
+  }
+
+  // Set the sign bit
+  __result_32x2.x[1] |= (uint64_t) __result_sign;
+
+  // Set the exponent
+  __result_32x2.x[1] |= (uint64_t) __result_exp << _CCCL_FP64_HI_MANT_SHIFT;
+
+  __result = ::cuda::std::bit_cast<uint64_t>(__result_32x2);
+  return __result;
+} // __internal_fp64emu_mid_dmul
+
+//! @brief Fast double-precision multiplication for FPEMU
+//!
+//! This function performs double-precision multiplication on two FPEMU floating-point numbers,
+//! by single precision multiplication for the mantissa (fast mode).
+//! It takes two __fpbits64 structures as input, representing the packed sign, exponent,
+//! and mantissa fields of the operands. The multiplication is performed according to the specified rounding mode,
+//! accuracy, range, and engine template parameters.
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_low_dmul(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  __uint32x2 __a_32x2 = ::cuda::std::bit_cast<__uint32x2>(__x);
+  __uint32x2 __b_32x2 = ::cuda::std::bit_cast<__uint32x2>(__y);
+  __fpbits64 __result;
+
+  // Extract exponents
+  uint32_t __exp_a    = (__a_32x2.x[1] >> _CCCL_FP64_HI_MANT_SHIFT) & _CCCL_FP64_LO_EXP_MASK;
+  uint32_t __exp_b    = (__b_32x2.x[1] >> _CCCL_FP64_HI_MANT_SHIFT) & _CCCL_FP64_LO_EXP_MASK;
+  bool __is_exps_zero = ((__exp_a == 0) || (__exp_b == 0));
+
+  // Compute result sign (XOR of input signs)
+  uint32_t __result_sign = (__a_32x2.x[1] ^ __b_32x2.x[1]) & _CCCL_FP64_HI_SIGN_MASK;
+
+  // Add exponents and subtract fp64 bias (1023)
+  int32_t __result_exp = (int32_t) __exp_a + (int32_t) __exp_b - _CCCL_FP64_BIAS;
+
+  // Multiply the mantissas
+  __uint32x2 __result_32x2;
+
+  // Convert mantissas to single precision for multiplication
+  // Extract the upper 24 bits of the 53-bit mantissa
+  // by shifting the bits left by 3
+  uint32_t __mant_a_sp =
+    ((__a_32x2.x[0] >> _CCCL_FP64_MANT_TO_FP32_LO_SHIFT) | (__a_32x2.x[1] << _CCCL_FP64_MANT_TO_FP32_HI_SHIFT))
+    & _CCCL_FP32_MANT_MASK;
+  uint32_t __mant_b_sp =
+    ((__b_32x2.x[0] >> _CCCL_FP64_MANT_TO_FP32_LO_SHIFT) | (__b_32x2.x[1] << _CCCL_FP64_MANT_TO_FP32_HI_SHIFT))
+    & _CCCL_FP32_MANT_MASK;
+
+  // Normalize to [1.0, 2.0) by adding the 1.0 exponent
+  __mant_a_sp = _CCCL_FP32_ONE | __mant_a_sp;
+  __mant_b_sp = _CCCL_FP32_ONE | __mant_b_sp;
+
+  // Convert to single precision for multiplication
+  float __mant_a_float = ::cuda::std::bit_cast<float>(__mant_a_sp);
+  float __mant_b_float = ::cuda::std::bit_cast<float>(__mant_b_sp);
+
+  // Perform single precision multiplication with directed rounding on device
+  float __mant_product_float = __fmul_dir<_Rm>(__mant_a_float, __mant_b_float);
+
+  // Extract mantissa directly from the float result
+  uint32_t __mant_product_bits = ::cuda::std::bit_cast<uint32_t>(__mant_product_float);
+
+  // Extract 23-bit mantissa
+  uint32_t __result_mant = __mant_product_bits & _CCCL_FP32_MANT_MASK;
+
+  // Convert back to fp64 by shifting the mantissa to the right by 3 bits
+  __result_32x2.x[0] = __result_mant << _CCCL_FP64_MANT_TO_FP32_LO_SHIFT;
+  __result_32x2.x[1] = __result_mant >> _CCCL_FP64_MANT_TO_FP32_HI_SHIFT;
+
+  // Correct exponent field by 1 if the product is >= 2.0
+  __result_exp = (__mant_product_bits >= _CCCL_FP32_TWO) ? __result_exp + 1 : __result_exp;
+
+  // Check for exponent overflow
+  bool __is_exp_ovfl = (__result_exp > (_CCCL_FP64_BIAS * 2));
+
+  const bool __is_sign_c = (__result_sign != 0);
+
+  if (__is_exps_zero)
+  {
+    __result_exp  = 0;
+    __result_32x2 = {0, 0};
+  }
+  else if (__is_exp_ovfl)
+  {
+    __fp64_ovfl_sat<_Rm>(__is_sign_c, __result_exp, __result_32x2);
+  }
+  else
+  {
+    if (__result_exp < 0)
+    {
+      __result_exp = 0;
+    }
+  }
+
+  // Set the sign bit
+  __result_32x2.x[1] |= (uint64_t) __result_sign;
+
+  // Set the exponent
+  __result_32x2.x[1] |= (uint64_t) __result_exp << _CCCL_FP64_HI_MANT_SHIFT;
+
+  __result = ::cuda::std::bit_cast<uint64_t>(__result_32x2);
+  return __result;
+} // __internal_fp64emu_low_dmul
+
+//! @brief Pure MUL core on the unpacked representation (unified path).
+//!
+//! Consumes/produces __fpbits64_unpacked exactly as the universal
+//! __internal_fp64emu_unpack / __internal_fp64emu_pack. This
+//! is the multiply section of the unified FMA core, made standalone: the
+//! significand product normalized so the high 64 bits carry the universal
+//! mantissa (implicit bit at position 61) with a sticky LSB, and the product
+//! exponent in the pack's "+1" convention.
+//!
+//!   - accurate (cr/full): returns the pre-rounding intermediate; the universal
+//!     full pack does the single correct rounding plus inf/denormal/overflow.
+//!     inf*0 -> NaN is folded into the exponent band; inf*finite -> inf rides
+//!     the huge unpack exponent that pack classifies as inf.
+//!   - def (ha/normal): same 128-bit product, but the lean pack does no special
+//!     handling, so the core flushes denormal-inputs / underflow to signed zero
+//!     and saturates overflow, exactly as the legacy fused def kernel did.
+//!   - fast (la/normal): fp32 product of the top significand bits (the legacy
+//!     fast kernel), re-expressed on the universal scale, same flush/saturate.
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked
+__internal_fp64emu_dmul_unpacked(__fpbits64_unpacked __a, __fpbits64_unpacked __b) noexcept
+{
+  constexpr fpemu_accuracy __acc_forced = fpemu_accuracy::_CCCL_FPEMU_MUL_METHOD;
+  constexpr fpemu_accuracy __acc_used   = (__acc_forced != fpemu_accuracy::unset) ? __acc_forced : _Acc;
+  // Unpacked cores always run on the fully-accurate full-range unpack/pack
+  // boundary (accuracy-independent), so the inf*0 -> NaN exponent fold is
+  // always live regardless of the accuracy level's arithmetic precision.
+  // The accuracy level selects only the *arithmetic*: high uses the 128-bit
+  // product, mid the high-64 only, low the fp32 product. The
+  // boundary contract is the SAME for every accuracy level here -- this is the
+  // full (high) bit-61 boundary: encode inf*0 -> NaN in the exponent band
+  // and defer subnormal emission / inf / overflow / rounding to the (full)
+  // pack. So mid/low cores compose correctly with the high pack/unpack
+  // (Model 1 subsumption) with no range parameter -- range stays internal to
+  // the pack/unpack, never threaded through the core. For mid/low the lean
+  // (FTZ/DAZ) boundary handling lives in the mid/low pack/unpack, not here.
+  constexpr int32_t __nan_exp = 0x0007ff00;
+
+  const uint32_t __sign_ab = __a.sign ^ __b.sign;
+  __fpbits64_unpacked __r;
+
+  if constexpr (__acc_used == fpemu_accuracy::high)
+  {
+    // ---- accurate: full product -> correctly-rounded by pack ----
+    // The low product bits feed the sticky LSB so the universal full pack
+    // does a single correct rounding.
+    __uint32x2 __a_32x2 = ::cuda::std::bit_cast<__uint32x2>(__a.mantissa);
+    __uint32x2 __b_32x2 = ::cuda::std::bit_cast<__uint32x2>(__b.mantissa);
+    __uint32x4 __prod   = __mul_128<__acc_used>(__a_32x2, __b_32x2);
+
+    int32_t __exponent_ab = (int32_t) __a.exponent + (int32_t) __b.exponent - (int32_t) __fpemu_bias;
+    // mul_nzeros == 1 when the leading product bit landed one place low
+    // (significand in [1,2) rather than [2,4)); normalize the implicit bit
+    // to position 61 in the high word.
+    int __mul_nzeros = __prod.hi.x[1] < 0x08000000;
+    int32_t __e      = __exponent_ab - __mul_nzeros + 1;
+    // Full range: inf*0 -> NaN (inf*finite -> inf rides the exponent band).
+    if (__e == (int32_t) __fpemu_inf_zero)
+    {
+      __e = __nan_exp;
+    }
+
+    const int __sh = (11 - EXTRA_BITS) + __mul_nzeros; // 2 or 3
+    // 64-bit normalization (no __uint128_t). Shift the high 64 bits up by
+    // sh, pulling in the top sh bits of the low word (the guard bits), and
+    // fold the remaining low bits into the sticky LSB -- this is how the
+    // legacy accurate kernel stays in 64-bit.
+    uint64_t __hi     = ::cuda::std::bit_cast<uint64_t>(__prod.hi);
+    uint64_t __lo     = ::cuda::std::bit_cast<uint64_t>(__prod.lo);
+    uint64_t __hi_n   = (__hi << __sh) | (__lo >> (64 - __sh));
+    uint64_t __sticky = ((__lo << __sh) != 0) ? 1u : 0u;
+    __r.sign          = __sign_ab;
+    __r.exponent      = static_cast<uint32_t>(__e);
+    __r.mantissa      = __hi_n | __sticky;
+    return __r;
+  }
+  else if constexpr (__acc_used == fpemu_accuracy::mid)
+  {
+    // ---- def: high-64 product only (no low-64 / sticky) -----------------
+    // High-accuracy tolerates dropping the low product, so def uses the
+    // cheaper __mul_64 (top 64 bits) exactly like the legacy fused def
+    // kernel -- this keeps the GPU multiply lean. The high word carries the
+    // implicit bit at position 58/59 (significand in [2,4)/[1,2)); shift it
+    // up to position 61 (the universal scale). The exponent arithmetic and
+    // special handling are identical to the cr branch (only the multiply is
+    // cheaper): inf*0 -> NaN folds into the exponent band, and inf/overflow/
+    // subnormal/rounding are all emitted by the full pack. This is what lets
+    // a def core run on the accurate pack/unpack (~2 ulp on normals).
+    __uint32x2 __a_32x2 = ::cuda::std::bit_cast<__uint32x2>(__a.mantissa);
+    __uint32x2 __b_32x2 = ::cuda::std::bit_cast<__uint32x2>(__b.mantissa);
+    __uint32x2 __hi     = __mul_64<__acc_used>(__a_32x2, __b_32x2);
+
+    int32_t __exponent_ab = (int32_t) __a.exponent + (int32_t) __b.exponent - (int32_t) __fpemu_bias;
+    int __mul_nzeros      = __hi.x[1] < 0x08000000;
+    int32_t __e           = __exponent_ab - __mul_nzeros + 1;
+    // inf*0 -> NaN: dead on the lean def/fast boundary (no INF magic), so
+    // gate it out of the hot exponent chain. Kept only for full (cr).
+    if (__e == (int32_t) __fpemu_inf_zero)
+    {
+      __e = __nan_exp;
+    }
+
+    uint64_t __m = ::cuda::std::bit_cast<uint64_t>(__hi) << (11 - EXTRA_BITS + __mul_nzeros);
+    __r.sign     = __sign_ab;
+    __r.exponent = static_cast<uint32_t>(__e);
+    __r.mantissa = __m;
+    return __r;
+  }
+  else
+  {
+    // ---- fp32 significand product (fast / la) ----
+    // The legacy fast kernel multiplies the top 24 significand bits in fp32.
+    // At ~half mantissa precision directed rounding of the fp32 product is
+    // meaningless, so this uses plain round-to-nearest fp32 (the final
+    // directed rounding is the pack's job). The universal mantissa carries
+    // the top 23 fraction bits at positions 60..38 (implicit bit at 61), so
+    // we rebuild the fp32 [1,2) operands directly and re-expand the product.
+    int32_t __exp_a = (int32_t) __a.exponent;
+    int32_t __exp_b = (int32_t) __b.exponent;
+
+    uint32_t __mant_a_sp = ((uint32_t) (__a.mantissa >> 38) & _CCCL_FP32_MANT_MASK) | _CCCL_FP32_ONE;
+    uint32_t __mant_b_sp = ((uint32_t) (__b.mantissa >> 38) & _CCCL_FP32_MANT_MASK) | _CCCL_FP32_ONE;
+
+    float __fa       = ::cuda::std::bit_cast<float>(__mant_a_sp);
+    float __fb       = ::cuda::std::bit_cast<float>(__mant_b_sp);
+    float __fp       = __fmul_dir<__fpemu_rounding::rn>(__fa, __fb);
+    uint32_t __pbits = ::cuda::std::bit_cast<uint32_t>(__fp);
+
+    // Product of two [1,2) significands lands in [1,4): a binade bump when
+    // it reaches [2,4) carries into the result exponent. Express the exponent
+    // in the cr convention (E = exponent_ab + binade) so the inf*0 -> NaN
+    // fold is identical to cr; inf/overflow/subnormal are deferred to pack.
+    int32_t __binade       = (__pbits >= _CCCL_FP32_TWO) ? 1 : 0;
+    int32_t __exponent_ab  = __exp_a + __exp_b - (int32_t) __fpemu_bias;
+    int32_t __e            = __exponent_ab + __binade;
+    uint32_t __result_mant = __pbits & _CCCL_FP32_MANT_MASK;
+    // inf*0 -> NaN: dead on the lean def/fast boundary; gate it off the hot
+    // path (kept only for full / cr).
+    if (__e == (int32_t) __fpemu_inf_zero)
+    {
+      __e = __nan_exp;
+    }
+
+    // Re-expand the 24-bit fp32 significand back to the universal scale.
+    uint64_t __sig = ((uint64_t) ((1u << _CCCL_FP32_MANT_BITS) | __result_mant)) << 38;
+    // The fp32 rebuild fabricates a 1.0 significand even for a zero operand
+    // (on the full boundary zero has a normalized exponent, not exp==0), so a
+    // true zero would leak a tiny subnormal. Force the significand to zero; a
+    // finite zero then packs to signed zero, while 0*inf already folded E to
+    // NAN_EXP above and the pack overrides the mantissa to NaN.
+    if (__a.mantissa == 0 || __b.mantissa == 0)
+    {
+      __sig = 0;
+    }
+    __r.sign     = __sign_ab;
+    __r.exponent = static_cast<uint32_t>(__e);
+    __r.mantissa = __sig;
+    return __r;
+  }
+} // __internal_fp64emu_dmul_unpacked
+//! @brief Emulation of double-precision floating-point multiplication.
+//!
+//! This function provides an emulated implementation of the IEEE-754
+//! double-precision (fp64) multiplication operation. It operates on the
+//! internal bitwise representation of fp64 numbers (__fpbits64) and
+//! supports configurable rounding modes, accuracy, and range
+//! selection via template parameters.
+//!
+//! The emulation performs the following steps:
+//!   - Extracts the exponent and mantissa fields from the input operands.
+//!   - Computes the result sign as the XOR of the input signs.
+//!   - Adds the exponents and subtracts the fp64 bias to obtain the result exponent.
+//!   - Multiplies the mantissas by a dedicated helper function, handling
+//!     normalization and carry propagation.
+//!   - Checks for exponent overflow and underflow, saturating or zeroing the
+//!     result as appropriate.
+//!   - Assembles the final result by setting the sign, exponent, and mantissa
+//!     fields in the output bit pattern.
+//!
+//! This emulation is designed to be bit-accurate and to match the behavior
+//! of hardware or reference software implementations, including correct
+//! handling of special cases such as zero, infinity, and NaN.
+//!
+//! The function is intended for use in both host and device code, and is
+//! selected as the optimized multiplication path when the appropriate
+//! macros are enabled.
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_dmul(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  // Forced method override for the multiplication operation
+  constexpr fpemu_accuracy __acc_forced = fpemu_accuracy::_CCCL_FPEMU_MUL_METHOD;
+  constexpr fpemu_accuracy __acc_used   = (__acc_forced != fpemu_accuracy::unset) ? __acc_forced : _Acc;
+  {
+#if (_CCCL_FPEMU_PACKED_VIA_UNPACKED == 1)
+    {
+      // Packed-via-unpacked (testing): pack(dmul_unpacked(unpack(x), unpack(y))). One
+      // bit-61 core for every accuracy level (high/mid/low); the core
+      // selects only the multiply precision and defers subnormal/inf/
+      // overflow to the pack. The boundary is the accuracy level's own:
+      // high -> full (IEEE), mid/low -> normal (FTZ/DAZ).
+      __fpbits64_unpacked __a = __internal_fp64emu_unpack(__x);
+      __fpbits64_unpacked __b = __internal_fp64emu_unpack(__y);
+      __fpbits64_unpacked __r = __internal_fp64emu_dmul_unpacked<__acc_used>(__a, __b);
+      return __internal_fp64emu_pack<_Rm>(__r);
+    }
+#else
+    if constexpr (__acc_used == fpemu_accuracy::high)
+    {
+      return __internal_fp64emu_high_dmul<_Rm, __acc_used>(__x, __y);
+    }
+#  if _CCCL_FP64EMU_DMUL_FP32_FAST_ENABLE == 1
+    else if constexpr (__acc_used == fpemu_accuracy::low)
+    {
+      return __internal_fp64emu_low_dmul<_Rm, __acc_used>(__x, __y);
+    }
+#  endif
+    else
+    {
+      // mid and def -- and low when the fp32 fast path is disabled -- all use mid.
+      return __internal_fp64emu_mid_dmul<_Rm, __acc_used>(__x, __y);
+    }
+#endif
+  }
+} // __internal_fp64emu_dmul
+
+// ============================================================================
+// Builtin declarations/implementations for multiplication operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rz, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::ru, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rd, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_dmul_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rz, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::ru, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rd, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rn, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rz, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::ru, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dmul<__fpemu_rounding::rd, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_dmul(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dmul_unpacked<fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_dmul(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dmul_unpacked<fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_dmul(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dmul_unpacked<fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_dmul(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dmul_unpacked<fpemu_accuracy::low>(__x, __y);
+}
+#else
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dmul_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_dmul_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dmul_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dmul_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_dmul(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_dmul(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_dmul(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_dmul(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_MUL_H (builtins)
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_DMUL_API_MERGED)
+#define _CCCL_FPEMU_DMUL_API_MERGED
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_dmul_api.hpp)
+// ============================================================================
+
+// Default API implementation
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> operator*(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_dmul_rn(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_dmul_rn(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_dmul_rn(__x.__bits_, __y.__bits_));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_dmul_rn(__x.__bits_, __y.__bits_));
+  }
+} // operator*
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dmul_rn(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_high_dmul_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dmul_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dmul_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dmul_rz(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dmul_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dmul_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dmul_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dmul_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dmul_ru(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dmul_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dmul_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dmul_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dmul_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dmul_rd(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dmul_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dmul_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dmul_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dmul_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+
+// Operator* for unpacked multiplication
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+operator*(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_dmul(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_dmul(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_dmul(__x.__bits_, __y.__bits_));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_dmul(__x.__bits_, __y.__bits_));
+  }
+} // operator*
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+__dmul_rn(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_dmul(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_dmul(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_dmul(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+}
+
+// Mixed-operand promoters (relocated from the class body; formerly hidden
+// friends). Enabled only when at least one operand is an fpemu and at least
+// one is a built-in arithmetic type: both operands are promoted to the fpemu
+// type and the exact-match core above is called. Pure fpemu/fpemu calls bind
+// to the cores directly; pure arithmetic calls are left to the language.
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dmul_rn(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dmul_rn(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dmul_rz(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dmul_rz(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dmul_ru(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dmul_ru(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dmul_rd(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dmul_rd(_Fp(__x), _Fp(__y));
+}
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_MUL_H

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_others.h

@@ -0,0 +1,775 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_OTHERS_H
+#define _CUDA___FP_FPEMU_IMPL_OTHERS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_impl_others.h
+//! @brief Implementation of MAD, DOT, and CMUL operations for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of other operations for the FPEMU library.
+//! It includes:
+//!   - MAD (Multiply-Add with intermediate rounding) functions for different accuracy and range configurations
+//!   - DOT (dot product) functions
+//!   - CMUL (complex multiply) functions
+//!   - Special case handling for NaN, inf, zero, etc
+//!
+//! The implementation is designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+
+#define _CCCL_FP64EMU_USE_OPT_MAD_UNPACKED  1
+#define _CCCL_FP64EMU_USE_OPT_DOT_UNPACKED  1
+#define _CCCL_FP64EMU_USE_OPT_CMUL_UNPACKED 1
+
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_add.h>
+#include <cuda/__fp/fpemu_impl_mul.h>
+#include <cuda/__fp/fpemu_impl_sub.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// MAD unpacked implementation
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked
+__internal_fp64emu_mad_unpacked(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_dadd_unpacked<_Acc>(__internal_fp64emu_dmul_unpacked<_Acc>(__x, __y), __z);
+}
+
+// DOT unpacked implementation
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked __internal_fp64emu_dot_unpacked(
+  __fpbits64_unpacked __x1, __fpbits64_unpacked __y1, __fpbits64_unpacked __x2, __fpbits64_unpacked __y2) noexcept
+{
+  return __internal_fp64emu_dadd_unpacked<_Acc>(
+    __internal_fp64emu_dmul_unpacked<_Acc>(__x1, __x2), __internal_fp64emu_dmul_unpacked<_Acc>(__y1, __y2));
+}
+
+// CMPLX MUL unpacked implementation
+// (a+bi) * (c+di) = (ac-bd) + (ad+bc)i
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API void __internal_fp64emu_cmul_unpacked(
+  __fpbits64_unpacked __x_re,
+  __fpbits64_unpacked __x_im,
+  __fpbits64_unpacked __y_re,
+  __fpbits64_unpacked __y_im,
+  __fpbits64_unpacked& __r_re,
+  __fpbits64_unpacked& __r_im) noexcept
+{
+  __r_re = __internal_fp64emu_dsub_unpacked<_Acc>(
+    __internal_fp64emu_dmul_unpacked<_Acc>(__x_re, __y_re), __internal_fp64emu_dmul_unpacked<_Acc>(__x_im, __y_im));
+  __r_im = __internal_fp64emu_dadd_unpacked<_Acc>(
+    __internal_fp64emu_dmul_unpacked<_Acc>(__x_re, __y_im), __internal_fp64emu_dmul_unpacked<_Acc>(__x_im, __y_re));
+  return;
+}
+
+// MAD implementation
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_mad(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    __fpbits64_unpacked __x_unpacked = __internal_fp64emu_unpack(__x);
+    __fpbits64_unpacked __y_unpacked = __internal_fp64emu_unpack(__y);
+    __fpbits64_unpacked __z_unpacked = __internal_fp64emu_unpack(__z);
+
+    __fpbits64_unpacked __r_unpacked = __internal_fp64emu_mad_unpacked<_Acc>(__x_unpacked, __y_unpacked, __z_unpacked);
+    return __internal_fp64emu_pack<_Rm>(__r_unpacked);
+  }
+  else
+  {
+    return __internal_fp64emu_dadd<_Rm, _Acc>(__internal_fp64emu_dmul<_Rm, _Acc>(__x, __y), __z);
+  }
+}
+
+// DOT implementation
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64
+__internal_fp64emu_dot(__fpbits64 __x1, __fpbits64 __y1, __fpbits64 __x2, __fpbits64 __y2) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    __fpbits64_unpacked __x1_unpacked = __internal_fp64emu_unpack(__x1);
+    __fpbits64_unpacked __y1_unpacked = __internal_fp64emu_unpack(__y1);
+    __fpbits64_unpacked __x2_unpacked = __internal_fp64emu_unpack(__x2);
+    __fpbits64_unpacked __y2_unpacked = __internal_fp64emu_unpack(__y2);
+
+    __fpbits64_unpacked __r_unpacked =
+      __internal_fp64emu_dot_unpacked<_Acc>(__x1_unpacked, __y1_unpacked, __x2_unpacked, __y2_unpacked);
+    __fpbits64 __r = __internal_fp64emu_pack<_Rm>(__r_unpacked);
+
+    return __r;
+  }
+  else
+  {
+    __fpbits64 __r = __internal_fp64emu_dadd<_Rm, _Acc>(
+      __internal_fp64emu_dmul<_Rm, _Acc>(__x1, __x2), __internal_fp64emu_dmul<_Rm, _Acc>(__y1, __y2));
+    return __r;
+  }
+}
+
+// CMUL implementation
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API void __internal_fp64emu_cmul(
+  __fpbits64 __x_re,
+  __fpbits64 __x_im,
+  __fpbits64 __y_re,
+  __fpbits64 __y_im,
+  __fpbits64& __r_re,
+  __fpbits64& __r_im) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    __fpbits64_unpacked __x_re_unpacked = __internal_fp64emu_unpack(__x_re);
+    __fpbits64_unpacked __y_re_unpacked = __internal_fp64emu_unpack(__y_re);
+    __fpbits64_unpacked __x_im_unpacked = __internal_fp64emu_unpack(__x_im);
+    __fpbits64_unpacked __y_im_unpacked = __internal_fp64emu_unpack(__y_im);
+    __fpbits64_unpacked __r_re_unpacked;
+    __fpbits64_unpacked __r_im_unpacked;
+
+    __internal_fp64emu_cmul_unpacked<_Acc>(
+      __x_re_unpacked, __x_im_unpacked, __y_re_unpacked, __y_im_unpacked, __r_re_unpacked, __r_im_unpacked);
+
+    __r_re = __internal_fp64emu_pack<_Rm>(__r_re_unpacked);
+    __r_im = __internal_fp64emu_pack<_Rm>(__r_im_unpacked);
+
+    return;
+  }
+  else
+  {
+    __fpbits64 __r_re_y_re = __internal_fp64emu_dmul<_Rm, _Acc>(__x_re, __y_re);
+    __fpbits64 __r_im_y_im = __internal_fp64emu_dmul<_Rm, _Acc>(__x_im, __y_im);
+    __fpbits64 __r_re_y_im = __internal_fp64emu_dmul<_Rm, _Acc>(__x_re, __y_im);
+    __fpbits64 __r_im_y_re = __internal_fp64emu_dmul<_Rm, _Acc>(__x_im, __y_re);
+
+    __r_re = __internal_fp64emu_dsub<_Rm, _Acc>(__r_re_y_re, __r_im_y_im);
+    __r_im = __internal_fp64emu_dadd<_Rm, _Acc>(__r_re_y_im, __r_im_y_re);
+
+    return;
+  }
+}
+
+_CCCL_TRIVIAL_API __fpbits64_unpacked __internal_fp64emu_neg_unpacked(__fpbits64_unpacked __x) noexcept
+{
+  __x.sign = __invert_msb(__x.sign);
+  return __x;
+}
+
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_neg(__fpbits64 __x) noexcept
+{
+  __uint32x2 __t = ::cuda::std::bit_cast<__uint32x2>(__x);
+  __t.x[1]       = __invert_msb(__t.x[1]);
+  __x            = ::cuda::std::bit_cast<uint64_t>(__t);
+  return __x;
+}
+
+// ============================================================================
+// Builtin declarations/implementations for MAD, DOT, CMUL, NEG operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+
+// mad (packed)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mad_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_mad<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_mad_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_mad<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_mad_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_mad<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_mad_rn(__fpbits64 __x, __fpbits64 __y, __fpbits64 __z) noexcept
+{
+  return __internal_fp64emu_mad<__fpemu_rounding::rn, fpemu_accuracy::low>(__x, __y, __z);
+}
+
+// dot (packed)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_dot_rn(__fpbits64 __x1, __fpbits64 __y1, __fpbits64 __x2, __fpbits64 __y2) noexcept
+{
+  return __internal_fp64emu_dot<__fpemu_rounding::rn, fpemu_accuracy::high>(__x1, __y1, __x2, __y2);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_high_dot_rn(__fpbits64 __x1, __fpbits64 __y1, __fpbits64 __x2, __fpbits64 __y2) noexcept
+{
+  return __internal_fp64emu_dot<__fpemu_rounding::rn, fpemu_accuracy::high>(__x1, __y1, __x2, __y2);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_mid_dot_rn(__fpbits64 __x1, __fpbits64 __y1, __fpbits64 __x2, __fpbits64 __y2) noexcept
+{
+  return __internal_fp64emu_dot<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x1, __y1, __x2, __y2);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_low_dot_rn(__fpbits64 __x1, __fpbits64 __y1, __fpbits64 __x2, __fpbits64 __y2) noexcept
+{
+  return __internal_fp64emu_dot<__fpemu_rounding::rn, fpemu_accuracy::low>(__x1, __y1, __x2, __y2);
+}
+
+// cmul (packed)
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_cmul_rn(
+  __fpbits64 __x_re,
+  __fpbits64 __x_im,
+  __fpbits64 __y_re,
+  __fpbits64 __y_im,
+  __fpbits64& __r_re,
+  __fpbits64& __r_im) noexcept
+{
+  __internal_fp64emu_cmul<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_high_cmul_rn(
+  __fpbits64 __x_re,
+  __fpbits64 __x_im,
+  __fpbits64 __y_re,
+  __fpbits64 __y_im,
+  __fpbits64& __r_re,
+  __fpbits64& __r_im) noexcept
+{
+  __internal_fp64emu_cmul<__fpemu_rounding::rn, fpemu_accuracy::high>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_mid_cmul_rn(
+  __fpbits64 __x_re,
+  __fpbits64 __x_im,
+  __fpbits64 __y_re,
+  __fpbits64 __y_im,
+  __fpbits64& __r_re,
+  __fpbits64& __r_im) noexcept
+{
+  __internal_fp64emu_cmul<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_low_cmul_rn(
+  __fpbits64 __x_re,
+  __fpbits64 __x_im,
+  __fpbits64 __y_re,
+  __fpbits64 __y_im,
+  __fpbits64& __r_re,
+  __fpbits64& __r_im) noexcept
+{
+  __internal_fp64emu_cmul<__fpemu_rounding::rn, fpemu_accuracy::low>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+
+// neg (packed)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_neg(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_neg(__x);
+}
+
+// mad (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mad(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_mad_unpacked<fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_mad(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_mad_unpacked<fpemu_accuracy::high>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_mad(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_mad_unpacked<fpemu_accuracy::mid>(__x, __y, __z);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_mad(__fpbits64_unpacked __x, __fpbits64_unpacked __y, __fpbits64_unpacked __z) noexcept
+{
+  return __internal_fp64emu_mad_unpacked<fpemu_accuracy::low>(__x, __y, __z);
+}
+
+// dot (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_dot(
+  __fpbits64_unpacked __x1, __fpbits64_unpacked __y1, __fpbits64_unpacked __x2, __fpbits64_unpacked __y2) noexcept
+{
+  return __internal_fp64emu_dot_unpacked<fpemu_accuracy::mid>(__x1, __y1, __x2, __y2);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_high_dot(
+  __fpbits64_unpacked __x1, __fpbits64_unpacked __y1, __fpbits64_unpacked __x2, __fpbits64_unpacked __y2) noexcept
+{
+  return __internal_fp64emu_dot_unpacked<fpemu_accuracy::high>(__x1, __y1, __x2, __y2);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_mid_dot(
+  __fpbits64_unpacked __x1, __fpbits64_unpacked __y1, __fpbits64_unpacked __x2, __fpbits64_unpacked __y2) noexcept
+{
+  return __internal_fp64emu_dot_unpacked<fpemu_accuracy::mid>(__x1, __y1, __x2, __y2);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_low_dot(
+  __fpbits64_unpacked __x1, __fpbits64_unpacked __y1, __fpbits64_unpacked __x2, __fpbits64_unpacked __y2) noexcept
+{
+  return __internal_fp64emu_dot_unpacked<fpemu_accuracy::low>(__x1, __y1, __x2, __y2);
+}
+
+// cmul (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_cmul(
+  __fpbits64_unpacked __x_re,
+  __fpbits64_unpacked __x_im,
+  __fpbits64_unpacked __y_re,
+  __fpbits64_unpacked __y_im,
+  __fpbits64_unpacked& __r_re,
+  __fpbits64_unpacked& __r_im) noexcept
+{
+  __internal_fp64emu_cmul_unpacked<fpemu_accuracy::mid>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_high_cmul(
+  __fpbits64_unpacked __x_re,
+  __fpbits64_unpacked __x_im,
+  __fpbits64_unpacked __y_re,
+  __fpbits64_unpacked __y_im,
+  __fpbits64_unpacked& __r_re,
+  __fpbits64_unpacked& __r_im) noexcept
+{
+  __internal_fp64emu_cmul_unpacked<fpemu_accuracy::high>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_mid_cmul(
+  __fpbits64_unpacked __x_re,
+  __fpbits64_unpacked __x_im,
+  __fpbits64_unpacked __y_re,
+  __fpbits64_unpacked __y_im,
+  __fpbits64_unpacked& __r_re,
+  __fpbits64_unpacked& __r_im) noexcept
+{
+  __internal_fp64emu_cmul_unpacked<fpemu_accuracy::mid>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_low_cmul(
+  __fpbits64_unpacked __x_re,
+  __fpbits64_unpacked __x_im,
+  __fpbits64_unpacked __y_re,
+  __fpbits64_unpacked __y_im,
+  __fpbits64_unpacked& __r_re,
+  __fpbits64_unpacked& __r_im) noexcept
+{
+  __internal_fp64emu_cmul_unpacked<fpemu_accuracy::low>(__x_re, __x_im, __y_re, __y_im, __r_re, __r_im);
+}
+
+// neg (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_neg(__fpbits64_unpacked __x) noexcept
+{
+  return __internal_fp64emu_neg_unpacked(__x);
+}
+
+#else // LTO mode - declarations only
+
+// mad (packed)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mad_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_mad_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_mad_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_mad_rn(__fpbits64 x, __fpbits64 y, __fpbits64 z) noexcept;
+
+// dot (packed)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_dot_rn(__fpbits64 x1, __fpbits64 y1, __fpbits64 x2, __fpbits64 y2) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_high_dot_rn(__fpbits64 x1, __fpbits64 y1, __fpbits64 x2, __fpbits64 y2) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_mid_dot_rn(__fpbits64 x1, __fpbits64 y1, __fpbits64 x2, __fpbits64 y2) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64
+__fp64emu_low_dot_rn(__fpbits64 x1, __fpbits64 y1, __fpbits64 x2, __fpbits64 y2) noexcept;
+
+// cmul (packed)
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_cmul_rn(
+  __fpbits64 x_re, __fpbits64 x_im, __fpbits64 y_re, __fpbits64 y_im, __fpbits64& r_re, __fpbits64& r_im) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_high_cmul_rn(
+  __fpbits64 x_re, __fpbits64 x_im, __fpbits64 y_re, __fpbits64 y_im, __fpbits64& r_re, __fpbits64& r_im) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_mid_cmul_rn(
+  __fpbits64 x_re, __fpbits64 x_im, __fpbits64 y_re, __fpbits64 y_im, __fpbits64& r_re, __fpbits64& r_im) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_low_cmul_rn(
+  __fpbits64 x_re, __fpbits64 x_im, __fpbits64 y_re, __fpbits64 y_im, __fpbits64& r_re, __fpbits64& r_im) noexcept;
+
+// neg (packed)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_neg(__fpbits64 x) noexcept;
+
+// mad (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mad(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_mad(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_mad(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_mad(__fpbits64_unpacked x, __fpbits64_unpacked y, __fpbits64_unpacked z) noexcept;
+
+// dot (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_dot(
+  __fpbits64_unpacked x1, __fpbits64_unpacked y1, __fpbits64_unpacked x2, __fpbits64_unpacked y2) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_high_dot(
+  __fpbits64_unpacked x1, __fpbits64_unpacked y1, __fpbits64_unpacked x2, __fpbits64_unpacked y2) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_mid_dot(
+  __fpbits64_unpacked x1, __fpbits64_unpacked y1, __fpbits64_unpacked x2, __fpbits64_unpacked y2) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_low_dot(
+  __fpbits64_unpacked x1, __fpbits64_unpacked y1, __fpbits64_unpacked x2, __fpbits64_unpacked y2) noexcept;
+
+// cmul (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_cmul(
+  __fpbits64_unpacked x_re,
+  __fpbits64_unpacked x_im,
+  __fpbits64_unpacked y_re,
+  __fpbits64_unpacked y_im,
+  __fpbits64_unpacked& r_re,
+  __fpbits64_unpacked& r_im) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_high_cmul(
+  __fpbits64_unpacked x_re,
+  __fpbits64_unpacked x_im,
+  __fpbits64_unpacked y_re,
+  __fpbits64_unpacked y_im,
+  __fpbits64_unpacked& r_re,
+  __fpbits64_unpacked& r_im) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_mid_cmul(
+  __fpbits64_unpacked x_re,
+  __fpbits64_unpacked x_im,
+  __fpbits64_unpacked y_re,
+  __fpbits64_unpacked y_im,
+  __fpbits64_unpacked& r_re,
+  __fpbits64_unpacked& r_im) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL void __fp64emu_unpacked_low_cmul(
+  __fpbits64_unpacked x_re,
+  __fpbits64_unpacked x_im,
+  __fpbits64_unpacked y_re,
+  __fpbits64_unpacked y_im,
+  __fpbits64_unpacked& r_re,
+  __fpbits64_unpacked& r_im) noexcept;
+
+// neg (unpacked)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_neg(__fpbits64_unpacked x) noexcept;
+
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_OTHERS_H
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_OTHERS_API_MERGED)
+#define _CCCL_FPEMU_OTHERS_API_MERGED
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_others_api.hpp)
+// ============================================================================
+
+// Unary negation operator - member function implementation
+template <typename _FpType, fpemu_accuracy _Acc>
+_CCCL_API fpemu<_FpType, _Acc> fpemu<_FpType, _Acc>::operator-() const noexcept
+{
+  fpemu __temp(*this);
+  __temp.__bits_ = __fp64emu_neg(__temp.__bits_);
+  return __temp;
+}
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+mad(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_mad_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_mad_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_mad_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+__mad_rn(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y, const fpemu<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_mad_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_mad_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_mad_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x),
+      ::cuda::std::bit_cast<__fpbits64>(__y),
+      ::cuda::std::bit_cast<__fpbits64>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc>
+dot(const fpemu<double, _Acc>& __x1,
+    const fpemu<double, _Acc>& __y1,
+    const fpemu<double, _Acc>& __x2,
+    const fpemu<double, _Acc>& __y2) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_dot_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x1),
+      ::cuda::std::bit_cast<__fpbits64>(__y1),
+      ::cuda::std::bit_cast<__fpbits64>(__x2),
+      ::cuda::std::bit_cast<__fpbits64>(__y2)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_dot_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x1),
+      ::cuda::std::bit_cast<__fpbits64>(__y1),
+      ::cuda::std::bit_cast<__fpbits64>(__x2),
+      ::cuda::std::bit_cast<__fpbits64>(__y2)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_dot_rn(
+      ::cuda::std::bit_cast<__fpbits64>(__x1),
+      ::cuda::std::bit_cast<__fpbits64>(__y1),
+      ::cuda::std::bit_cast<__fpbits64>(__x2),
+      ::cuda::std::bit_cast<__fpbits64>(__y2)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API void
+cmul(const fpemu<double, _Acc>& __x_re,
+     const fpemu<double, _Acc>& __x_im,
+     const fpemu<double, _Acc>& __y_re,
+     const fpemu<double, _Acc>& __y_im,
+     fpemu<double, _Acc>& __r_re,
+     fpemu<double, _Acc>& __r_im) noexcept
+{
+  const __fpbits64 __xr = ::cuda::std::bit_cast<__fpbits64>(__x_re);
+  const __fpbits64 __xi = ::cuda::std::bit_cast<__fpbits64>(__x_im);
+  const __fpbits64 __yr = ::cuda::std::bit_cast<__fpbits64>(__y_re);
+  const __fpbits64 __yi = ::cuda::std::bit_cast<__fpbits64>(__y_im);
+  // The builtins write their results through non-const references, so compute into
+  // local raw-bits temporaries and construct the outputs from them (bits is private).
+  __fpbits64 __rr{};
+  __fpbits64 __ri{};
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    __fp64emu_high_cmul_rn(__xr, __xi, __yr, __yi, __rr, __ri);
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    __fp64emu_low_cmul_rn(__xr, __xi, __yr, __yi, __rr, __ri);
+  }
+  else
+  {
+    __fp64emu_mid_cmul_rn(__xr, __xi, __yr, __yi, __rr, __ri);
+  }
+  __r_re = ::cuda::std::bit_cast<fpemu<double, _Acc>>(__rr);
+  __r_im = ::cuda::std::bit_cast<fpemu<double, _Acc>>(__ri);
+}
+
+// Unary negation operator for unpacked - member function implementation
+template <typename _FpType, fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<_FpType, _Acc> fpemu_unpacked<_FpType, _Acc>::operator-() const noexcept
+{
+  fpemu_unpacked __temp(*this);
+  __temp.__bits_ = __fp64emu_unpacked_neg(__temp.__bits_);
+  return __temp;
+}
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+mad(const fpemu_unpacked<double, _Acc>& __x,
+    const fpemu_unpacked<double, _Acc>& __y,
+    const fpemu_unpacked<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_mad(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_mad(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_mad(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+__mad_rn(const fpemu_unpacked<double, _Acc>& __x,
+         const fpemu_unpacked<double, _Acc>& __y,
+         const fpemu_unpacked<double, _Acc>& __z) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_mad(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_mad(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_mad(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__z)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+dot(const fpemu_unpacked<double, _Acc>& __x1,
+    const fpemu_unpacked<double, _Acc>& __y1,
+    const fpemu_unpacked<double, _Acc>& __x2,
+    const fpemu_unpacked<double, _Acc>& __y2) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_dot(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x1),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y1),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x2),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y2)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_dot(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x1),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y1),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x2),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y2)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_dot(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x1),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y1),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x2),
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__y2)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API void
+cmul(const fpemu_unpacked<double, _Acc>& __x_re,
+     const fpemu_unpacked<double, _Acc>& __x_im,
+     const fpemu_unpacked<double, _Acc>& __y_re,
+     const fpemu_unpacked<double, _Acc>& __y_im,
+     fpemu_unpacked<double, _Acc>& __r_re,
+     fpemu_unpacked<double, _Acc>& __r_im) noexcept
+{
+  const __fpbits64_unpacked __xr = ::cuda::std::bit_cast<__fpbits64_unpacked>(__x_re);
+  const __fpbits64_unpacked __xi = ::cuda::std::bit_cast<__fpbits64_unpacked>(__x_im);
+  const __fpbits64_unpacked __yr = ::cuda::std::bit_cast<__fpbits64_unpacked>(__y_re);
+  const __fpbits64_unpacked __yi = ::cuda::std::bit_cast<__fpbits64_unpacked>(__y_im);
+  // The builtins write their results through non-const references, so compute into
+  // local raw-bits temporaries and construct the outputs from them (bits is private).
+  __fpbits64_unpacked __rr{};
+  __fpbits64_unpacked __ri{};
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    __fp64emu_unpacked_high_cmul(__xr, __xi, __yr, __yi, __rr, __ri);
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    __fp64emu_unpacked_low_cmul(__xr, __xi, __yr, __yi, __rr, __ri);
+  }
+  else
+  {
+    __fp64emu_unpacked_mid_cmul(__xr, __xi, __yr, __yi, __rr, __ri);
+  }
+  __r_re = ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__rr);
+  __r_im = ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__ri);
+}
+
+// Mixed-operand promoters (relocated from the class body; formerly hidden
+// friends). Enabled only when at least one operand is an fpemu and at least
+// one is a built-in arithmetic type: both operands are promoted to the fpemu
+// type and the exact-match core above is called. Pure fpemu/fpemu calls bind
+// to the cores directly; pure arithmetic calls are left to the language.
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> mad(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return mad(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3> __mad_rn(const _T1& __x, const _T2& __y, const _T3& __z) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3>;
+  return __mad_rn(_Fp(__x), _Fp(__y), _Fp(__z));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3, class _T4)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3, _T4>)
+_CCCL_API __fpemu_pick_t<_T1, _T2, _T3, _T4>
+dot(const _T1& __x1, const _T2& __y1, const _T3& __x2, const _T4& __y2) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3, _T4>;
+  return dot(_Fp(__x1), _Fp(__y1), _Fp(__x2), _Fp(__y2));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2, class _T3, class _T4)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2, _T3, _T4>)
+_CCCL_API void
+cmul(const _T1& __x_re,
+     const _T2& __x_im,
+     const _T3& __y_re,
+     const _T4& __y_im,
+     __fpemu_pick_t<_T1, _T2, _T3, _T4>& __r_re,
+     __fpemu_pick_t<_T1, _T2, _T3, _T4>& __r_im) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2, _T3, _T4>;
+  cmul(_Fp(__x_re), _Fp(__x_im), _Fp(__y_re), _Fp(__y_im), __r_re, __r_im);
+}
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CCCL_FPEMU_OTHERS_API_MERGED

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_sqrt.h

@@ -0,0 +1,541 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_SQRT_H
+#define _CUDA___FP_FPEMU_IMPL_SQRT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_dsqrt_impl.hpp
+//! @brief Implementation of double-precision square root operations for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of double-precision square root operations for the FPEMU library.
+//! It includes:
+//!
+//! - Square root functions for fpemu
+//! - Square root operators for fpemu
+//! - Square root functions to other types
+//!
+//! The square root functions are designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+
+#include <cuda/__cmath/mul_hi.h>
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+#include <cuda/std/__bit/countl.h>
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+#if _CCCL_HOST_COMPILATION()
+// Host seeds: the libm symbols. On glibc these are declared noexcept to match
+// the standard <math.h> prototypes (marked __THROW); otherwise the extern-"C"
+// redeclarations conflict with ::sqrt/::sqrtf when <cmath> is also in the TU
+// (-Werror). On MSVC the CRT/CUDA prototypes carry no exception specification,
+// so a noexcept redeclaration is a mismatched extern-"C" overload (C2382/C2733
+// under C++20); declare them without noexcept to match.
+#  if _CCCL_COMPILER(MSVC)
+extern "C" double sqrt(double __x);
+extern "C" float sqrtf(float __x); // host seed for the reciprocal-sqrt builtin
+#  else // ^^^ _CCCL_COMPILER(MSVC) ^^^ / vvv !_CCCL_COMPILER(MSVC) vvv
+extern "C" double sqrt(double __x) noexcept;
+extern "C" float sqrtf(float __x) noexcept; // host seed for the reciprocal-sqrt builtin
+#  endif // ^^^ !_CCCL_COMPILER(MSVC) ^^^
+#endif // _CCCL_HOST_COMPILATION()
+
+// ========================================================================
+// Native fp64 square root.
+//
+// Split sign/exp/mantissa, normalize subnormals, form a 32-bit reciprocal
+// square root of the significand, then refine with fixed-point integer
+// remainder arithmetic and round/pack. The reciprocal-sqrt seed comes from
+// the fp32 rsqrt builtin (rsqrt.approx on device, 1/sqrtf on host) and is
+// refined by a single Newton step plus a trim that guarantees a strict
+// underestimate, which the remainder refinement needs for correctly-rounded
+// results.
+// ========================================================================
+
+//! @brief Approximation of 2^47 / sqrt(a / 2^odd_exp), a in [2^31, 2^32),
+//!        in [2^31, 2^32). Seeded by the fp32 rsqrt builtin, refined by one
+//!        Newton step, then trimmed to a strict underestimate so that the
+//!        derived root stays a lower bound (keeps the remainder unsigned).
+_CCCL_TRIVIAL_API uint32_t __internal_fp64emu_sqrt_recip_sqrt32(uint32_t __odd_exp, uint32_t __a) noexcept
+{
+  // Seed: r ~ 2^47 * rsqrt(a / 2^odd_exp). Halving the radicand for the
+  // odd-exponent case folds the sqrt(2) factor into the same 2^47 scale.
+  float __af = __odd_exp ? (float) __a * 0.5f : (float) __a;
+  float __rf{};
+  NV_IF_ELSE_TARGET(
+    NV_IS_DEVICE, ({ asm("rsqrt.approx.ftz.f32 %0, %1;"
+                         : "=f"(__rf)
+                         : "f"(__af)); }), ({ __rf = 1.0f / sqrtf(__af); }))
+  uint64_t __r = (uint64_t) (__rf * 140737488355328.0f); // * 2^47
+
+  if (__r < 0x80000000ULL)
+  {
+    __r = 0x80000000ULL;
+  }
+  if (__r > 0xFFFFFFFFULL)
+  {
+    __r = 0xFFFFFFFFULL;
+  }
+
+  // One Newton step (reciprocal-sqrt): r <- r*(3K - a*r^2)/(2K), K = 2^(94+odd).
+  uint64_t __r2      = __r * __r; // < 2^64
+  uint64_t __a_r2_hi = ::cuda::mul_hi((uint64_t) __a, __r2); // a*r^2 >> 64
+  uint64_t __a_r2_lo = (uint64_t) __a * __r2; // low 64 bits (wraps)
+  uint64_t __k3_hi   = 3ULL << (30 + __odd_exp); // (3 * 2^(94+odd)) >> 64
+  uint64_t __t_lo    = 0ULL - __a_r2_lo;
+  uint64_t __borrow  = (__a_r2_lo != 0) ? 1u : 0u;
+  uint64_t __t_hi    = __k3_hi - __a_r2_hi - __borrow; // t = 3K - a*r^2 (128-bit)
+  // r' = (r * t) >> (95 + odd); pre-shift t by 33 so the product fits 128 bits.
+  uint64_t __t_s   = (__t_hi << 31) | (__t_lo >> 33);
+  uint64_t __pr_hi = ::cuda::mul_hi(__r, __t_s);
+  uint64_t __pr_lo = __r * __t_s;
+  __r              = (__pr_hi << (2 - __odd_exp)) | (__pr_lo >> (62 + __odd_exp));
+  if (__r < 0x80000000ULL)
+  {
+    __r = 0x80000000ULL;
+  }
+  if (__r > 0xFFFFFFFFULL)
+  {
+    __r = 0xFFFFFFFFULL;
+  }
+
+  // Trim to a strict underestimate of 2^47*2^(odd/2)/sqrt(a): a*r^2 <= 2^(94+odd).
+  while (__r > 0x80000000ULL)
+  {
+    uint64_t __rr2 = __r * __r;
+    uint64_t __hi  = ::cuda::mul_hi((uint64_t) __a, __rr2);
+    uint64_t __lo  = (uint64_t) __a * __rr2;
+    uint64_t __thr = 1ULL << (30 + __odd_exp);
+    if ((__hi > __thr) || (__hi == __thr && __lo != 0))
+    {
+      --__r;
+    }
+    else
+    {
+      break;
+    }
+  }
+
+  return (uint32_t) __r;
+} // __internal_fp64emu_sqrt_recip_sqrt32
+
+// Forward declaration: the unpacked sqrt core is defined below, but the packed
+// wrapper references it for the packed-via-unpacked (testing) path.
+template <fpemu_accuracy _Acc>
+_CCCL_TRIVIAL_API __fpbits64_unpacked __internal_fp64emu_dsqrt_unpacked(__fpbits64_unpacked __x) noexcept;
+
+//! @brief Square root of a double-precision floating point number
+//!
+//! This function computes the square root of a double-precision floating point number.
+//! It works by splitting the number into sign, exponent, and mantissa, normalizing the mantissa,
+//! and then computing the square root of the mantissa.
+//!
+//! @param __x The double-precision floating point number to compute the square root of
+//! @return The square root of the double-precision floating point number
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_dsqrt(__fpbits64 __x) noexcept
+{
+#if (_CCCL_FPEMU_PACKED_VIA_UNPACKED == 1)
+  // Packed-via-unpacked (testing): pack(dsqrt_unpacked(unpack(x))). The
+  // dsqrt_unpacked core handles special operands and method selection; the
+  // universal unpack/pack are the shared prologue/epilogue. Rounding is
+  // applied only at pack, preserving the packed builtins' per-mode behavior.
+  {
+    __fpbits64_unpacked __a = __internal_fp64emu_unpack(__x);
+    __fpbits64_unpacked __r = __internal_fp64emu_dsqrt_unpacked<_Acc>(__a);
+    return __internal_fp64emu_pack<_Rm>(__r);
+  }
+#else
+
+  const uint64_t __ui64_x = (uint64_t) __x;
+
+  bool __sign_x     = (__ui64_x >> 63) != 0;
+  int32_t __exp_x   = (int32_t) ((__ui64_x >> 52) & 0x7FF);
+  uint64_t __mant_x = __ui64_x & _CCCL_FPEMU_MANT_64;
+
+  // -------- special operands (NaN / Inf / negative / zero) --------
+  if (__exp_x == 0x7FF)
+  {
+    if (__mant_x)
+    {
+      return (__fpbits64) (__ui64_x | _CCCL_FPEMU_QNAN_BIT_64); // NaN -> quiet NaN
+    }
+    if (!__sign_x)
+    {
+      return (__fpbits64) __ui64_x; // +inf -> +inf
+    }
+    return (__fpbits64) _CCCL_FPEMU_DEFNAN_64; // sqrt(-inf) -> NaN
+  }
+  if (__sign_x)
+  {
+    if (!(__exp_x | (int32_t) (__mant_x != 0)))
+    {
+      return (__fpbits64) __ui64_x; // -0 -> -0
+    }
+    return (__fpbits64) _CCCL_FPEMU_DEFNAN_64; // sqrt(negative) -> NaN
+  }
+  if (!__exp_x)
+  {
+    if (!__mant_x)
+    {
+      return (__fpbits64) __ui64_x; // +0 -> +0
+    }
+
+    int __mant_shft = ::cuda::std::countl_zero((uint64_t) __mant_x) - 11; // normalize subnormal
+
+    __exp_x  = 1 - __mant_shft;
+    __mant_x = __mant_x << __mant_shft;
+  }
+
+  // -------- fixed-point reciprocal-sqrt root --------
+  int32_t __exp_z = ((__exp_x - 0x3FF) >> 1) + 0x3FE;
+  __exp_x &= 1;
+  __mant_x |= _CCCL_FPEMU_HIDDEN_64;
+
+  uint32_t __mant32_x = (uint32_t) (__mant_x >> 21);
+  uint32_t __rcp32    = __internal_fp64emu_sqrt_recip_sqrt32((uint32_t) __exp_x, __mant32_x);
+  uint32_t __mant32_z = (uint32_t) (((uint64_t) __mant32_x * __rcp32) >> 32);
+
+  if (__exp_x)
+  {
+    __mant_x <<= 8;
+    __mant32_z >>= 1;
+  }
+  else
+  {
+    __mant_x <<= 9;
+  }
+
+  uint64_t __rem64 = __mant_x - (uint64_t) __mant32_z * __mant32_z;
+  uint32_t __q32   = (uint32_t) (((uint32_t) (__rem64 >> 2) * (uint64_t) __rcp32) >> 32);
+  // form mantissa: (1 << 52) + (mant32_z << 21) + (q32 << 3)
+  uint64_t __mant64_z = ((uint64_t) __mant32_z << 32 | (1u << 5)) + ((uint64_t) __q32 << 3);
+
+  // Refine if the root is close to a rounding boundary (exact remainder).
+  if ((__mant64_z & 0x1FF) < 0x22)
+  {
+    __mant64_z &= ~(uint64_t) 0x3F;
+    uint64_t __mant64_z_shftd = __mant64_z >> 6;
+    __rem64                   = (__mant_x << 52) - __mant64_z_shftd * __mant64_z_shftd;
+
+    if (__rem64 & _CCCL_FPEMU_SIGN_64)
+    {
+      --__mant64_z;
+    }
+    else
+    {
+      if (__rem64)
+      {
+        __mant64_z |= 1;
+      }
+    }
+  }
+
+  return __internal_fp64emu_round_pack<_Rm>(false, __exp_z, __mant64_z);
+#endif // _CCCL_FPEMU_PACKED_VIA_UNPACKED
+} // __internal_fp64emu_dsqrt
+
+//! @brief Square root of a double-precision floating point number
+//!
+//! This function computes the square root of a double-precision floating point number.
+//! It works by splitting the number into sign, exponent, and mantissa, normalizing the mantissa,
+//! and then computing the square root of the mantissa.
+//!
+//! @param __x The double-precision floating point number to compute the square root of
+//! @return The square root of the double-precision floating point number
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked __internal_fp64emu_dsqrt_unpacked(__fpbits64_unpacked __x) noexcept
+{
+  constexpr int32_t __nan_exp = 0x0007ff00;
+  constexpr int32_t __inf_exp = 0x00007ff0;
+
+  const int32_t __exp_x = (int32_t) __x.exponent;
+  const bool __sign_x   = (__x.sign != 0);
+  const bool __zero_x   = (__x.mantissa == 0);
+
+  // Special operands (canonical packed result, then unpack -- rare path).
+  if (__exp_x == __nan_exp)
+  {
+    return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_DEFNAN_64);
+  }
+  if (__exp_x == __inf_exp)
+  {
+    if (!__sign_x)
+    {
+      return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_INF_64); // +inf -> +inf
+    }
+    return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_DEFNAN_64); // sqrt(-inf) -> NaN
+  }
+  if (__sign_x)
+  {
+    if (__zero_x)
+    {
+      return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_SIGN_64); // -0 -> -0
+    }
+    return __internal_fp64emu_unpack((__fpbits64) _CCCL_FPEMU_DEFNAN_64); // sqrt(negative) -> NaN
+  }
+  if (__zero_x)
+  {
+    return __internal_fp64emu_unpack((__fpbits64) 0); // +0 -> +0
+  }
+
+  // ---- finite positive : fixed-point reciprocal-sqrt root -------------
+  int32_t __exp_z   = ((__exp_x - 0x3FF) >> 1) + 0x3FE;
+  int32_t __odd     = __exp_x & 1; // exponent parity
+  uint64_t __mant_x = __x.mantissa >> EXTRA_BITS; // 53-bit significand, implicit bit at 52
+
+  uint32_t __mant32_x = (uint32_t) (__mant_x >> 21);
+  uint32_t __rcp32    = __internal_fp64emu_sqrt_recip_sqrt32((uint32_t) __odd, __mant32_x);
+  uint32_t __mant32_z = (uint32_t) (((uint64_t) __mant32_x * __rcp32) >> 32);
+
+  if (__odd)
+  {
+    __mant_x <<= 8;
+    __mant32_z >>= 1;
+  }
+  else
+  {
+    __mant_x <<= 9;
+  }
+
+  uint64_t __rem64    = __mant_x - (uint64_t) __mant32_z * __mant32_z;
+  uint32_t __q32      = (uint32_t) (((uint32_t) (__rem64 >> 2) * (uint64_t) __rcp32) >> 32);
+  uint64_t __mant64_z = ((uint64_t) __mant32_z << 32 | (1u << 5)) + ((uint64_t) __q32 << 3);
+
+  // Refine if the root is close to a rounding boundary (exact remainder).
+  if ((__mant64_z & 0x1FF) < 0x22)
+  {
+    __mant64_z &= ~(uint64_t) 0x3F;
+    uint64_t __mant64_z_shftd = __mant64_z >> 6;
+    __rem64                   = (__mant_x << 52) - __mant64_z_shftd * __mant64_z_shftd;
+    if (__rem64 & _CCCL_FPEMU_SIGN_64)
+    {
+      --__mant64_z;
+    }
+    else
+    {
+      if (__rem64)
+      {
+        __mant64_z |= 1;
+      }
+    }
+  }
+
+  // Same conversion as the unpacked divide: leading bit 62 -> 61 (sticky
+  // preserved), exponent biased-1 -> IEEE-biased; full pack rounds.
+  __fpbits64_unpacked __r;
+  __r.sign     = 0u; // sqrt result is non-negative
+  __r.exponent = (uint32_t) (__exp_z + 1);
+  __r.mantissa = (__mant64_z >> 1) | (__mant64_z & 1);
+  return __r;
+} // __internal_fp64emu_dsqrt_unpacked
+
+// ============================================================================
+// Builtin declarations/implementations for sqrt operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_rn(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::rn, fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_rz(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::rz, fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_ru(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::ru, fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_rd(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::rd, fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_dsqrt_rn(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::rn, fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsqrt_rn(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsqrt_rn(__fpbits64 __x) noexcept
+{
+  return __internal_fp64emu_dsqrt<__fpemu_rounding::rn, fpemu_accuracy::low>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_dsqrt(__fpbits64_unpacked __x) noexcept
+{
+  return __internal_fp64emu_dsqrt_unpacked<fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_high_dsqrt(__fpbits64_unpacked __x) noexcept
+{
+  return __internal_fp64emu_dsqrt_unpacked<fpemu_accuracy::high>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_mid_dsqrt(__fpbits64_unpacked __x) noexcept
+{
+  return __internal_fp64emu_dsqrt_unpacked<fpemu_accuracy::mid>(__x);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_low_dsqrt(__fpbits64_unpacked __x) noexcept
+{
+  return __internal_fp64emu_dsqrt_unpacked<fpemu_accuracy::low>(__x);
+}
+#else
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_rn(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_rz(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_ru(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsqrt_rd(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_dsqrt_rn(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsqrt_rn(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsqrt_rn(__fpbits64 x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_dsqrt(__fpbits64_unpacked x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_high_dsqrt(__fpbits64_unpacked x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_mid_dsqrt(__fpbits64_unpacked x) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked __fp64emu_unpacked_low_dsqrt(__fpbits64_unpacked x) noexcept;
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_SQRT_H (builtins)
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_DSQRT_API_MERGED)
+#define _CCCL_FPEMU_DSQRT_API_MERGED
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_dsqrt_api.hpp)
+// ============================================================================
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> sqrt(const fpemu<double, _Acc>& __x) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_dsqrt_rn(::cuda::std::bit_cast<__fpbits64>(__x)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_dsqrt_rn(::cuda::std::bit_cast<__fpbits64>(__x)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_dsqrt_rn(::cuda::std::bit_cast<__fpbits64>(__x)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsqrt_rn(const fpemu<double, _Acc>& __x) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_dsqrt_rn(::cuda::std::bit_cast<__fpbits64>(__x)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_dsqrt_rn(::cuda::std::bit_cast<__fpbits64>(__x)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_dsqrt_rn(::cuda::std::bit_cast<__fpbits64>(__x)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsqrt_rz(const fpemu<double, _Acc>& __x) noexcept
+{
+  return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_dsqrt_rz(::cuda::std::bit_cast<__fpbits64>(__x)));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsqrt_ru(const fpemu<double, _Acc>& __x) noexcept
+{
+  return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_dsqrt_ru(::cuda::std::bit_cast<__fpbits64>(__x)));
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsqrt_rd(const fpemu<double, _Acc>& __x) noexcept
+{
+  return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_dsqrt_rd(::cuda::std::bit_cast<__fpbits64>(__x)));
+}
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc> sqrt(const fpemu_unpacked<double, _Acc>& __x) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(
+      __fp64emu_unpacked_high_dsqrt(::cuda::std::bit_cast<__fpbits64_unpacked>(__x)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(
+      __fp64emu_unpacked_low_dsqrt(::cuda::std::bit_cast<__fpbits64_unpacked>(__x)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(
+      __fp64emu_unpacked_mid_dsqrt(::cuda::std::bit_cast<__fpbits64_unpacked>(__x)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc> __dsqrt_rn(const fpemu_unpacked<double, _Acc>& __x) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(
+      __fp64emu_unpacked_high_dsqrt(::cuda::std::bit_cast<__fpbits64_unpacked>(__x)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(
+      __fp64emu_unpacked_low_dsqrt(::cuda::std::bit_cast<__fpbits64_unpacked>(__x)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(
+      __fp64emu_unpacked_mid_dsqrt(::cuda::std::bit_cast<__fpbits64_unpacked>(__x)));
+  }
+}
+} // namespace cuda::experimental
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+// Overloads of sqrt for the emulated double types so the standard spelling
+// cuda::std::sqrt selects the emulated implementation instead of silently
+// narrowing fpemu -> double (a qualified call suppresses ADL). These forward to
+// cuda::experimental::sqrt, which unqualified/ADL calls already resolve to.
+template <::cuda::experimental::fpemu_accuracy _Acc>
+[[nodiscard]] _CCCL_API ::cuda::experimental::fpemu<double, _Acc>
+sqrt(const ::cuda::experimental::fpemu<double, _Acc>& __x) noexcept
+{
+  return ::cuda::experimental::sqrt(__x);
+}
+template <::cuda::experimental::fpemu_accuracy _Acc>
+[[nodiscard]] _CCCL_API ::cuda::experimental::fpemu_unpacked<double, _Acc>
+sqrt(const ::cuda::experimental::fpemu_unpacked<double, _Acc>& __x) noexcept
+{
+  return ::cuda::experimental::sqrt(__x);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_SQRT_H

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_sub.h

@@ -0,0 +1,392 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_SUB_H
+#define _CUDA___FP_FPEMU_IMPL_SUB_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_dsub_impl.hpp
+//! @brief Implementation of double-precision subtraction operations for FPEMU floating point emulation library
+//!
+//! This header provides the implementation of double-precision subtraction operations for the FPEMU library.
+//! It includes:
+//!
+//! - Subtraction functions for fpemu
+//! - Subtraction operators for fpemu
+//! - Subtraction functions to other types
+//!
+//! The subtraction functions are designed to work across both host and device code
+//! through appropriate decorators and provide bit-exact results matching hardware
+//! floating point units.
+
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/__fp/fpemu_impl_add.h>
+#include <cuda/__fp/fpemu_impl_unpack.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+//! @brief Subtract two __fpbits64_unpacked
+//!
+//! This function subtracts two __fpbits64_unpacked.
+//!
+//! @param a The first __fpbits64_unpacked
+//! @param b The second __fpbits64_unpacked
+//! @return The result of the subtraction
+template <fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64_unpacked
+__internal_fp64emu_dsub_unpacked(__fpbits64_unpacked __a, __fpbits64_unpacked __b) noexcept
+{
+  return __internal_fp64emu_dadd_unpacked<_Acc, true>(__a, __b);
+}
+
+//! @brief Subtract two __fpbits64
+//!
+//! This function subtracts two __fpbits64.
+//!
+//! @param x The first __fpbits64
+//! @param y The second __fpbits64
+//! @return The result of the subtraction
+template <__fpemu_rounding _Rm = __fpemu_rounding::def, fpemu_accuracy _Acc = fpemu_accuracy::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_dsub(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  // Forced parameters for the subtraction operation
+  constexpr fpemu_accuracy __acc_forced = fpemu_accuracy::_CCCL_FPEMU_ADD_METHOD;
+  constexpr fpemu_accuracy __acc_used   = (__acc_forced != fpemu_accuracy::unset) ? __acc_forced : _Acc;
+
+  {
+    // Pass true to the dadd function to indicate that we are subtracting
+    return __internal_fp64emu_dadd<_Rm, __acc_used, true>(__x, __y);
+  }
+} // __internal_fp64emu_dsub
+
+// ============================================================================
+// Builtin declarations/implementations for subtraction operations
+// ============================================================================
+#if defined(_CCCL_FPEMU_INLINE)
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rz, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::ru, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rd, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_dsub_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rn, fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rn, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rz, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::ru, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rd, fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_rn(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rn, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_rz(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rz, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_ru(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::ru, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_rd(__fpbits64 __x, __fpbits64 __y) noexcept
+{
+  return __internal_fp64emu_dsub<__fpemu_rounding::rd, fpemu_accuracy::low>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_dsub(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dsub_unpacked<fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_dsub(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dsub_unpacked<fpemu_accuracy::high>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_dsub(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dsub_unpacked<fpemu_accuracy::mid>(__x, __y);
+}
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_dsub(__fpbits64_unpacked __x, __fpbits64_unpacked __y) noexcept
+{
+  return __internal_fp64emu_dsub_unpacked<fpemu_accuracy::low>(__x, __y);
+}
+#else
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_dsub_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_high_dsub_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_mid_dsub_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_rn(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_rz(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_ru(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64 __fp64emu_low_dsub_rd(__fpbits64 x, __fpbits64 y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_dsub(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_high_dsub(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_mid_dsub(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+_CCCL_FPEMU_BUILTIN_DECL __fpbits64_unpacked
+__fp64emu_unpacked_low_dsub(__fpbits64_unpacked x, __fpbits64_unpacked y) noexcept;
+#endif // _CCCL_FPEMU_INLINE
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_SUB_H (builtins)
+
+#if defined(_CCCL_FPEMU_API_CLASSES_DEFINED) && !defined(_CCCL_FPEMU_DSUB_API_MERGED)
+#define _CCCL_FPEMU_DSUB_API_MERGED
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+// ============================================================================
+// API (merged from fp64emu_dsub_api.hpp)
+// ============================================================================
+
+// Default API implementation - binary subtraction operator
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> operator-(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_high_dsub_rn(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_mid_dsub_rn(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_low_dsub_rn(__x.__bits_, __y.__bits_));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(__fp64emu_dsub_rn(__x.__bits_, __y.__bits_));
+  }
+} // operator-
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsub_rn(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_high_dsub_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dsub_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dsub_rn(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsub_rz(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dsub_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dsub_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dsub_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dsub_rz(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsub_ru(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dsub_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dsub_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dsub_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dsub_ru(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu<double, _Acc> __dsub_rd(const fpemu<double, _Acc>& __x, const fpemu<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dsub_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_mid_dsub_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_low_dsub_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu<double, _Acc>>(
+      __fp64emu_dsub_rd(::cuda::std::bit_cast<__fpbits64>(__x), ::cuda::std::bit_cast<__fpbits64>(__y)));
+  }
+}
+
+// Operator- for unpacked subtraction
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+operator-(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_dsub(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::mid)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_dsub(__x.__bits_, __y.__bits_));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_dsub(__x.__bits_, __y.__bits_));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_dsub(__x.__bits_, __y.__bits_));
+  }
+} // operator-
+
+template <fpemu_accuracy _Acc>
+_CCCL_API fpemu_unpacked<double, _Acc>
+__dsub_rn(const fpemu_unpacked<double, _Acc>& __x, const fpemu_unpacked<double, _Acc>& __y) noexcept
+{
+  if constexpr (_Acc == fpemu_accuracy::high)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_high_dsub(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+  else if constexpr (_Acc == fpemu_accuracy::low)
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_low_dsub(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+  else
+  {
+    return ::cuda::std::bit_cast<fpemu_unpacked<double, _Acc>>(__fp64emu_unpacked_mid_dsub(
+      ::cuda::std::bit_cast<__fpbits64_unpacked>(__x), ::cuda::std::bit_cast<__fpbits64_unpacked>(__y)));
+  }
+}
+
+// Mixed-operand promoters (relocated from the class body; formerly hidden
+// friends). Enabled only when at least one operand is an fpemu and at least
+// one is a built-in arithmetic type: both operands are promoted to the fpemu
+// type and the exact-match core above is called. Pure fpemu/fpemu calls bind
+// to the cores directly; pure arithmetic calls are left to the language.
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dsub_rn(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dsub_rn(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dsub_rz(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dsub_rz(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dsub_ru(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dsub_ru(_Fp(__x), _Fp(__y));
+}
+
+_CCCL_TEMPLATE(class _T1, class _T2)
+_CCCL_REQUIRES(__fpemu_mixed_v<_T1, _T2>)
+_CCCL_API __fpemu_pick_t<_T1, _T2> __dsub_rd(const _T1& __x, const _T2& __y) noexcept
+{
+  using _Fp = __fpemu_pick_t<_T1, _T2>;
+  return __dsub_rd(_Fp(__x), _Fp(__y));
+}
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+#endif // _CUDA___FP_FPEMU_IMPL_SUB_H

cccl_upstream/libcudacxx/include/cuda/__fp/fpemu_impl_unpack.h

@@ -0,0 +1,215 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FP_FPEMU_IMPL_UNPACK_H
+#define _CUDA___FP_FPEMU_IMPL_UNPACK_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+//! @file fpemu_impl_unpack.h
+//! @brief Common pack/unpack routines for the FPEMU library
+//!
+//! This header holds the two routines that convert between the packed IEEE-754
+//! binary64 representation (__fpbits64) and the public unpacked ABI
+//! (__fpbits64_unpacked) used by the arithmetic cores:
+//!
+//!   - __internal_fp64emu_unpack  (packed   -> unpacked)
+//!   - __internal_fp64emu_pack    (unpacked -> packed)
+//!
+//! They are the single, shared prologue/epilogue for every unpacked operation
+//! (add / sub / mul / mad / dot / cmul / div / sqrt / cvt / cmp / fma) and every
+//! accuracy level (high / mid / low). Because the unpacked approach only crosses the
+//! packed<->unpacked boundary outside hot loops, both routines are always the
+//! richest, fully-accurate full-range form and are accuracy-INDEPENDENT: denormals
+//! are normalized (clz) and inf/nan are encoded in the exponent band on unpack,
+//! and the matching full-range epilogue (correctly-rounded by rm, subnormal
+//! emission, inf saturation, nan) finalizes every accuracy level on pack. A mid/low core
+//! (lower mantissa precision) simply rides on the richer form (subsumption); the
+//! accuracy level only affects the precision the core produced, not the range handling
+//! here. The packed (legacy non-unified) add/mul/fma kernels do their own inlined
+//! lean unpack/pack and do not use these routines.
+//!
+//! They depend only on the primitives/constants in fpemu_impl.h
+//! (bit_cast, __round, __fp64_ovfl_sat, ::cuda::std::countl_zero, the FP64_* masks,
+//! EXTRA_BITS, BIAS, ...).
+
+#include <cuda/__fp/fpemu_impl.h>
+#include <cuda/std/__bit/countl.h>
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+namespace cuda::experimental
+{
+//! @brief Unpack a packed binary64 value into the public unpacked ABI.
+//!
+//! Fully-accurate, method-independent, full-range prologue: the sign/exponent/
+//! mantissa are extracted, denormals are normalized via clz, and inf/nan are
+//! encoded in the exponent band so the matching pack can recover them. The
+//! exponent is stored as (biased exponent + 1) with the implicit bit kept in the
+//! mantissa (bit 61); pack consumes exactly this convention.
+//!
+//! @param  x The packed 64-bit value to unpack
+//! @return The unpacked representation
+_CCCL_TRIVIAL_API __fpbits64_unpacked __internal_fp64emu_unpack(__fpbits64 __x) noexcept
+{
+  __fpbits64_unpacked __a_unpacked;
+  __uint32x2 __a32  = ::cuda::std::bit_cast<__uint32x2>(__x);
+  __a_unpacked.sign = __a32.x[1] & (1U << 31);
+  __a32.x[1] &= 0x7fffffff;
+  int32_t __exponent = static_cast<int32_t>(__a32.x[1] >> 20);
+
+  // Normalize denormals: leading-zero count of the magnitude (clamped so a
+  // normal stays at shift == EXTRA_BITS, and a true zero maps to the zero band).
+  uint64_t __abs_a = ::cuda::std::bit_cast<uint64_t>(__a32);
+  int __nzeros     = ::cuda::std::countl_zero(__abs_a);
+  if (__nzeros < 11)
+  {
+    __nzeros = 11;
+  }
+  if (__nzeros == 64)
+  {
+    __nzeros = 2049;
+  }
+  __a32.x[1] = __a32.x[1] & 0x000fffff;
+
+  if (__exponent == 0x7ff)
+  {
+    // inf -> 0x00007ff0 band, nan -> 0x0007ff00 band (recovered on pack).
+    __exponent = (__a32.x[1] == 0 && __a32.x[0] == 0) ? 0x00007ff0 : 0x0007ff00;
+  }
+  if (__exponent != 0)
+  {
+    __a32.x[1] = __a32.x[1] | (1 << 20); // set the implicit bit
+  }
+  if (__exponent == 0)
+  {
+    __exponent = 12 - __nzeros; // denormal / zero
+  }
+
+  int __shift    = EXTRA_BITS + __nzeros - 11;
+  uint64_t __a64 = ::cuda::std::bit_cast<uint64_t>(__a32);
+
+  __a_unpacked.exponent = static_cast<uint32_t>(__exponent);
+  __a_unpacked.mantissa = __a64 << __shift;
+  return __a_unpacked;
+}
+
+//! @brief Pack a public unpacked value back into packed binary64.
+//!
+//! Fully-accurate, full-range epilogue: subnormal emission, inf/nan
+//! classification and correctly-rounded overflow saturation (per rm).
+//!
+//! It is the exact inverse of unpack (pack(unpack(x)) == x), which the converters
+//! (cvt/div/sqrt/cmp) and the unpacked class rely on. unpack stores the *biased*
+//! exponent + 1 with the implicit bit kept in the mantissa (bit 61); the proven
+//! epilogue expects an exponent one smaller, so we round/place with (exp - 1).
+//! The cores emit a matching (exp + 1) so the +1/-1 cancel and op results stay
+//! bit-exact while the round-trip identity holds. inf is recovered from the
+//! exponent band unpack/cores encode (finite results never reach it); nan is
+//! detected from the exponent and wins the overflow branch.
+//!
+//! @tparam rm Rounding mode
+//! @param  x  The unpacked value to pack
+//! @return The packed 64-bit value
+template <__fpemu_rounding _Rm = __fpemu_rounding::def>
+_CCCL_TRIVIAL_API __fpbits64 __internal_fp64emu_pack(__fpbits64_unpacked __x) noexcept
+{
+  const bool __sign   = __x.sign != 0;
+  const bool __is_inf = (static_cast<int32_t>(__x.exponent) >= 0x2000);
+  const int32_t __e   = static_cast<int32_t>(__x.exponent) - 1;
+  int32_t __exponent  = __e > 0 ? __e : 0;
+
+  int __shift = __e > 0 ? 0 : -__e;
+  NV_IF_TARGET(NV_IS_HOST, ({ __shift = (__shift > 0) ? (__shift > 63) ? 63 : __shift : 0; }))
+
+  if (__shift > 0)
+  {
+    const uint64_t __mask                 = (__shift >= 64) ? ~0ULL : ((1ULL << __shift) - 1);
+    [[maybe_unused]] const bool __inexact = (__x.mantissa & __mask) != 0;
+    __x.mantissa >>= __shift;
+    if constexpr (_Rm == __fpemu_rounding::rn)
+    {
+      if (__inexact)
+      {
+        __x.mantissa |= 1;
+      }
+    }
+    else if constexpr (_Rm == __fpemu_rounding::ru)
+    {
+      if (!__sign && __inexact)
+      {
+        __x.mantissa |= 1;
+      }
+    }
+    else if constexpr (_Rm == __fpemu_rounding::rd)
+    {
+      if (__sign && __inexact)
+      {
+        __x.mantissa |= 1;
+      }
+    }
+  }
+
+  __uint32x2 __mantissa32 = ::cuda::std::bit_cast<__uint32x2>(__x.mantissa);
+  __mantissa32            = __round<_Rm>(__mantissa32, 0, __sign);
+
+  const bool __is_nan = (__exponent >= (int) (0x0007ff00 - __fpemu_bias - 2048 - 1 - 128 + 0xC));
+
+  if (__mantissa32.x[0] == 0 && __mantissa32.x[1] == 0 && __exponent < 0x000007ff)
+  {
+    __exponent = 0;
+  }
+
+  if (__exponent >= 0x000007ff)
+  {
+    __exponent = 0x000007ff;
+  }
+
+  __exponent <<= 20;
+  __mantissa32.x[1] += __exponent;
+
+  if (__mantissa32.x[1] >= 0x7ff00000)
+  {
+    if (__is_nan)
+    {
+      __mantissa32.x[0] = 0;
+      __mantissa32.x[1] = 0x7fffffff;
+    }
+    else if (__is_inf)
+    {
+      __mantissa32.x[0] = 0;
+      __mantissa32.x[1] = 0x7ff00000;
+    }
+    else
+    {
+      int32_t __sat_exp = 0;
+      __fp64_ovfl_sat<_Rm>(__sign, __sat_exp, __mantissa32);
+      __mantissa32.x[1] |= (uint32_t) __sat_exp << _CCCL_FP64_HI_MANT_SHIFT;
+    }
+  }
+
+  __mantissa32.x[1] += __x.sign;
+  return ::cuda::std::bit_cast<__fpbits64>(__mantissa32);
+}
+} // namespace cuda::experimental
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FP_FPEMU_IMPL_UNPACK_H