[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/test/utils/libcudacxx/__init__.py

@@ -0,0 +1,16 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+"""libcxx python utilities"""
+
+__author__ = "Eric Fiselier"
+__email__ = "eric@efcs.ca"
+__versioninfo__ = (0, 1, 0)
+__version__ = " ".join(str(v) for v in __versioninfo__) + "dev"
+
+__all__ = []

cccl_upstream/libcudacxx/test/utils/libcudacxx/compiler.py

@@ -0,0 +1,527 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import os
+import platform
+
+import libcudacxx.util
+
+
+class CXXCompiler(object):
+    CM_Default = 0
+    CM_PreProcess = 1
+    CM_CheckCompileFlag = 2
+    CM_Compile = 3
+    CM_Link = 4
+
+    def __init__(
+        self,
+        path,
+        first_arg,
+        flags=None,
+        compile_flags=None,
+        link_flags=None,
+        warning_flags=None,
+        verify_supported=None,
+        verify_flags=None,
+        use_verify=False,
+        modules_flags=None,
+        use_modules=False,
+        use_ccache=False,
+        use_warnings=False,
+        compile_env=None,
+        cxx_type=None,
+        cxx_version=None,
+        cuda_path=None,
+    ):
+        self.source_lang = "c++"
+        self.path = path
+        self.first_arg = first_arg or ""
+        self.flags = list(flags or [])
+        self.compile_flags = list(compile_flags or [])
+        self.link_flags = list(link_flags or [])
+        self.warning_flags = list(warning_flags or [])
+        self.verify_supported = verify_supported
+        self.use_verify = use_verify
+        self.verify_flags = list(verify_flags or [])
+        assert not use_verify or verify_supported
+        assert not use_verify or verify_flags is not None
+        self.modules_flags = list(modules_flags or [])
+        self.use_modules = use_modules
+        assert not use_modules or modules_flags is not None
+        self.use_ccache = use_ccache
+        self.use_warnings = use_warnings
+        if compile_env is not None:
+            self.compile_env = dict(compile_env)
+        else:
+            self.compile_env = None
+        self.type = cxx_type
+        self.version = cxx_version
+        self.cuda_path = cuda_path
+        if self.type is None or self.version is None:
+            self._initTypeAndVersion()
+
+    def isVerifySupported(self):
+        if self.verify_supported is None:
+            self.verify_supported = self.hasCompileFlag(
+                ["-Xclang", "-verify-ignore-unexpected"]
+            )
+            if self.verify_supported:
+                self.verify_flags = [
+                    "-Xclang",
+                    "-verify",
+                    "-Xclang",
+                    "-verify-ignore-unexpected=note",
+                    "-ferror-limit=1024",
+                ]
+        return self.verify_supported
+
+    def useVerify(self, value=True):
+        self.use_verify = value
+        assert not self.use_verify or self.verify_flags is not None
+
+    def useModules(self, value=True):
+        self.use_modules = value
+        assert not self.use_modules or self.modules_flags is not None
+
+    def useCCache(self, value=True):
+        self.use_ccache = value
+
+    def useWarnings(self, value=True):
+        self.use_warnings = value
+
+    def _initTypeAndVersion(self):
+        # Get compiler type and version
+        try:
+            macros = self.dumpMacros()
+            compiler_type = None
+            major_ver = minor_ver = patchlevel = None
+
+            if "__NVCC__" in macros.keys():
+                compiler_type = "nvcc"
+                major_ver = int(macros["__CUDACC_VER_MAJOR__"])
+                minor_ver = int(macros["__CUDACC_VER_MINOR__"])
+                patchlevel = int(macros["__CUDACC_VER_BUILD__"])
+            elif "__NVCOMPILER" in macros.keys():
+                compiler_type = "nvhpc"
+                # NVHPC, unfortunately, adds an extra space between the macro name
+                # and macro value in their macro dump mode.
+                major_ver = int(macros["__NVCOMPILER_MAJOR__"].strip())
+                minor_ver = int(macros["__NVCOMPILER_MINOR__"].strip())
+                patchlevel = int(macros["__NVCOMPILER_PATCHLEVEL__"].strip())
+            elif "__clang__" in macros.keys():
+                compiler_type = "clang"
+                major_ver = int(macros["__clang_major__"])
+                minor_ver = int(macros["__clang_minor__"])
+                patchlevel = int(macros["__clang_patchlevel__"])
+            elif "__GNUC__" in macros.keys():
+                compiler_type = "gcc"
+                major_ver = int(macros["__GNUC__"])
+                minor_ver = int(macros["__GNUC_MINOR__"])
+                patchlevel = int(macros["__GNUC_PATCHLEVEL__"])
+            elif "_MSC_VER" in macros.keys():
+                compiler_type = "msvc"
+                major_ver = int(macros["_MSC_FULL_VER"]) // 10000000
+                minor_ver = int(macros["_MSC_FULL_VER"]) // 100000 % 100
+                patchlevel = int(macros["_MSC_FULL_VER"]) % 100000
+
+            if "__cplusplus" in macros.keys():
+                if "_MSVC_LANG" in macros.keys():
+                    msvc_lang = macros["_MSVC_LANG"]
+                    if msvc_lang[-1] == "L":
+                        msvc_lang = msvc_lang[:-1]
+                    msvc_lang = int(msvc_lang)
+                    if msvc_lang <= 201103:
+                        default_dialect = "c++11"
+                    elif msvc_lang <= 201402:
+                        default_dialect = "c++14"
+                    elif msvc_lang <= 201703:
+                        default_dialect = "c++17"
+                    elif msvc_lang > 201703:
+                        default_dialect = "c++20"
+                else:
+                    cplusplus = macros["__cplusplus"]
+                    if cplusplus[-1] == "L":
+                        cplusplus = cplusplus[:-1]
+                    cpp_standard = int(cplusplus)
+                    if cpp_standard <= 199711:
+                        default_dialect = "c++03"
+                    elif cpp_standard <= 201103:
+                        default_dialect = "c++11"
+                    elif cpp_standard <= 201402:
+                        default_dialect = "c++14"
+                    elif cpp_standard <= 201703:
+                        default_dialect = "c++17"
+                    else:
+                        default_dialect = "c++20"
+            else:
+                default_dialect = "c++03"
+
+            self.type = compiler_type
+            self.version = (major_ver, minor_ver, patchlevel)
+            self.default_dialect = default_dialect
+        except Exception:
+            (self.type, self.version, self.default_dialect) = self.dumpVersion()
+
+        if self.type == "nvcc":
+            # Treat C++ as CUDA when the compiler is NVCC.
+            self.source_lang = "cu"
+        elif self.type == "clang":
+            # Treat C++ as clang-cuda when the compiler is Clang.
+            self.source_lang = "cu"
+
+    def _basicCmdCl(
+        self, source_files, out, mode=CM_Default, flags=[], input_is_cxx=False
+    ):
+        cmd = []
+
+        if (
+            self.use_ccache
+            and not mode == self.CM_Link
+            and not mode == self.CM_PreProcess
+            and not mode == self.CM_CheckCompileFlag
+        ):
+            cmd += [os.environ.get("CMAKE_CUDA_COMPILER_LAUNCHER")]
+
+        cmd += [self.path] + ([self.first_arg] if self.first_arg != "" else [])
+
+        if isinstance(source_files, list):
+            cmd += source_files
+        elif isinstance(source_files, str):
+            cmd += [source_files]
+        else:
+            raise TypeError("source_files must be a string or list")
+
+        if mode == self.CM_PreProcess or mode == self.CM_CheckCompileFlag:
+            cmd += ["/Zs", "/options:strict"]
+        elif mode == self.CM_Compile:
+            cmd += ["/c"]
+
+        cmd += self.flags
+        if self.use_verify:
+            cmd += self.verify_flags
+            assert mode in [self.CM_Default, self.CM_Compile]
+        if self.use_modules:
+            cmd += self.modules_flags
+        if mode != self.CM_Link:
+            cmd += self.compile_flags
+            if self.use_warnings:
+                cmd += self.warning_flags
+        if (
+            mode != self.CM_PreProcess
+            and mode != self.CM_Compile
+            and mode != self.CM_CheckCompileFlag
+        ):
+            cmd += self.link_flags
+        cmd += flags
+        if out is not None:
+            cmd += ["/link", '/out:"{}"'.format(out)]
+        return cmd
+
+    def _basicCmd(
+        self, source_files, out, mode=CM_Default, flags=[], input_is_cxx=False
+    ):
+        if self.path.startswith("cl") and not self.path.startswith("clang"):
+            return self._basicCmdCl(source_files, out, mode, flags)
+
+        cmd = []
+
+        if (
+            self.use_ccache
+            and not mode == self.CM_Link
+            and not mode == self.CM_PreProcess
+            and not mode == self.CM_CheckCompileFlag
+        ):
+            cmd += [os.environ.get("CMAKE_CUDA_COMPILER_LAUNCHER")]
+        cmd += [self.path] + ([self.first_arg] if self.first_arg != "" else [])
+        if out is not None:
+            cmd += ["-o", out]
+        if input_is_cxx:
+            cmd += ["-x", self.source_lang]
+        if (
+            self.type == "clang"
+            and self.source_lang == "cu"
+            and self.cuda_path is not None
+        ):
+            cmd += ["--cuda-path=" + self.cuda_path]
+        if isinstance(source_files, list):
+            cmd += source_files
+        elif isinstance(source_files, str):
+            cmd += [source_files]
+        else:
+            raise TypeError("source_files must be a string or list")
+        if mode == self.CM_PreProcess:
+            cmd += ["-E"]
+        elif mode == self.CM_Compile or mode == self.CM_CheckCompileFlag:
+            cmd += ["-c"]
+        cmd += self.flags
+        if self.use_verify:
+            cmd += self.verify_flags
+            assert mode in [self.CM_Default, self.CM_Compile]
+        if self.use_modules:
+            cmd += self.modules_flags
+        if mode != self.CM_Link:
+            cmd += self.compile_flags
+            if self.use_warnings:
+                cmd += self.warning_flags
+        if (
+            mode != self.CM_PreProcess
+            and mode != self.CM_Compile
+            and mode != self.CM_CheckCompileFlag
+        ):
+            cmd += self.link_flags
+        cmd += flags
+        return cmd
+
+    def preprocessCmd(self, source_files, out=None, flags=[]):
+        return self._basicCmd(
+            source_files, out, flags=flags, mode=self.CM_PreProcess, input_is_cxx=True
+        )
+
+    def compileCmd(self, source_files, out=None, flags=[], mode=CM_Compile):
+        return self._basicCmd(
+            source_files, out, flags=flags, mode=mode, input_is_cxx=True
+        ) + ["-c"]
+
+    def linkCmd(self, source_files, out=None, flags=[]):
+        return self._basicCmd(source_files, out, flags=flags, mode=self.CM_Link)
+
+    def compileLinkCmd(self, source_files, out=None, flags=[]):
+        return self._basicCmd(source_files, out, flags=flags)
+
+    def preprocess(self, source_files, out=None, flags=[], cwd=None):
+        cmd = self.preprocessCmd(source_files, out, flags)
+        out, err, rc = libcudacxx.util.executeCommand(
+            cmd, env=self.compile_env, cwd=cwd
+        )
+        return cmd, out, err, rc
+
+    def checkCompileFlag(self, source_files, out=None, flags=[], cwd=None):
+        cmd = self.compileCmd(source_files, out, flags, self.CM_CheckCompileFlag)
+        out, err, rc = libcudacxx.util.executeCommand(
+            cmd, env=self.compile_env, cwd=cwd
+        )
+        return cmd, out, err, rc
+
+    def compile(self, source_files, out=None, flags=[], cwd=None):
+        cmd = self.compileCmd(source_files, out, flags, self.CM_Compile)
+        out, err, rc = libcudacxx.util.executeCommand(
+            cmd, env=self.compile_env, cwd=cwd
+        )
+        return cmd, out, err, rc
+
+    def link(self, source_files, out=None, flags=[], cwd=None):
+        cmd = self.linkCmd(source_files, out, flags)
+        out, err, rc = libcudacxx.util.executeCommand(
+            cmd, env=self.compile_env, cwd=cwd
+        )
+        return cmd, out, err, rc
+
+    def compileLink(self, source_files, out=None, flags=[], cwd=None):
+        cmd = self.compileLinkCmd(source_files, out, flags)
+        out, err, rc = libcudacxx.util.executeCommand(
+            cmd, env=self.compile_env, cwd=cwd
+        )
+        return cmd, out, err, rc
+
+    def compileLinkTwoSteps(
+        self, source_file, out=None, object_file=None, flags=[], cwd=None
+    ):
+        if not isinstance(source_file, str):
+            raise TypeError("This function only accepts a single input file")
+        if object_file is None:
+            # Create, use and delete a temporary object file if none is given.
+            def with_fn():
+                return libcudacxx.util.guardedTempFilename(suffix=".o")
+        else:
+            # Otherwise wrap the filename in a context manager function.
+            def with_fn():
+                return libcudacxx.util.nullContext(object_file)
+
+        with with_fn() as object_file:
+            cc_cmd, cc_stdout, cc_stderr, rc = self.compile(
+                source_file, object_file, flags=flags, cwd=cwd
+            )
+            if rc != 0:
+                return cc_cmd, cc_stdout, cc_stderr, rc
+            link_cmd, link_stdout, link_stderr, rc = self.link(
+                object_file, out=out, flags=flags, cwd=cwd
+            )
+            return (
+                cc_cmd + ["&&"] + link_cmd,
+                cc_stdout + link_stdout,
+                cc_stderr + link_stderr,
+                rc,
+            )
+
+    def dumpVersion(self, flags=[], cwd=None):
+        dumpversion_cpp = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), "dumpversion.cpp"
+        )
+
+        def with_fn():
+            return libcudacxx.util.guardedTempFilename(suffix=".exe")
+
+        with with_fn() as exe:
+            cmd, out, err, rc = self.compileLink(
+                [dumpversion_cpp], out=exe, flags=flags, cwd=cwd
+            )
+            if rc != 0:
+                return ("unknown", (0, 0, 0), "c++03")
+            out, err, rc = libcudacxx.util.executeCommand(
+                exe, env=self.compile_env, cwd=cwd
+            )
+            version = None
+            try:
+                version = eval(out)
+            except Exception:
+                pass
+
+            if not (isinstance(version, tuple) and 3 == len(version)):
+                version = ("unknown", (0, 0, 0), "c++03")
+            return version
+
+    def dumpMacros(self, source_files=None, flags=[], cwd=None):
+        if source_files is None:
+            source_files = os.path.join(
+                os.path.dirname(os.path.abspath(__file__)), "empty.cpp"
+            )
+
+        old_flags = flags
+        # Assume MSVC flags on Windows
+        if platform.system() == "Windows":
+            flags = ["/Zc:preprocessor", "/PD"] + old_flags
+            cmd, out, err, rc = self.preprocess(source_files, flags=flags, cwd=cwd)
+            # Older MSVC does not support dumping macros
+            if err.find("D9002") > 0:
+                raise RuntimeError("Cannot be dumped on old MSVC")
+            if rc != 0:
+                flags = [
+                    "-Xcompiler",
+                    "/Zc:preprocessor",
+                    "-Xcompiler",
+                    "/PD",
+                ] + old_flags
+                cmd, out, err, rc = self.preprocess(source_files, flags=flags, cwd=cwd)
+                if err.find("D9002") > 0:
+                    raise RuntimeError("Cannot be dumped on old MSVC")
+        else:
+            flags = ["-dM"] + flags
+            cmd, out, err, rc = self.preprocess(source_files, flags=flags, cwd=cwd)
+            if rc != 0:
+                flags = ["-Xcompiler"] + flags
+                cmd, out, err, rc = self.preprocess(source_files, flags=flags, cwd=cwd)
+
+        if rc != 0:
+            err_str = "Macros failed to dump:\n\n"
+            err_str += f"Command: {' '.join(cmd)}\n\n"
+            err_str += f"Exit code: {rc}\n\n"
+            err_str += f"stdout:\n{out}\n\n"
+            err_str += f"stderr:\n{err}\n"
+            raise RuntimeError(err_str)
+
+        parsed_macros = {}
+        lines = [line.strip() for line in out.split("\n") if line.strip()]
+        for line in lines:
+            # NVHPC also outputs the file contents from -E -dM for some reason; handle that
+            if not line.startswith("#define "):
+                continue
+            line = line[len("#define ") :]
+            macro, _, value = line.partition(" ")
+            parsed_macros[macro] = value
+        return parsed_macros
+
+    def getTriple(self):
+        if self.type == "msvc":
+            return "x86_64-pc-windows-msvc"
+        cmd = [self.path] + self.flags + ["-dumpmachine"]
+        return libcudacxx.util.capture(cmd).strip()
+
+    def hasCompileFlag(self, flag):
+        if isinstance(flag, list):
+            flags = list(flag)
+        else:
+            flags = [flag]
+
+        # Add -Werror to ensure that an unrecognized flag causes a non-zero
+        # exit code. -Werror is supported on all known non-nvcc compiler types.
+        if self.type is not None and self.type != "nvcc" and self.type != "msvc":
+            flags += ["-Werror", "-fsyntax-only"]
+        if self.type == "clang" and self.source_lang == "cu":
+            flags += ["-Wno-unused-command-line-argument"]
+
+        empty_cpp = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), "empty.cpp"
+        )
+        cmd, out, err, rc = self.checkCompileFlag(
+            empty_cpp, out=os.devnull, flags=flags
+        )
+        if out.find("flag is not supported with the configured host compiler") != -1:
+            return False
+        if err.find("flag is not supported with the configured host compiler") != -1:
+            return False
+        return rc == 0
+
+    def addFlagIfSupported(self, flag):
+        if isinstance(flag, list):
+            flags = list(flag)
+        else:
+            flags = [flag]
+        if self.hasCompileFlag(flags):
+            self.flags += flags
+            return True
+        else:
+            return False
+
+    def addCompileFlagIfSupported(self, flag):
+        if isinstance(flag, list):
+            flags = list(flag)
+        else:
+            flags = [flag]
+        if self.hasCompileFlag(flags):
+            self.compile_flags += flags
+            return True
+        else:
+            return False
+
+    def hasWarningFlag(self, flag):
+        """
+        hasWarningFlag - Test if the compiler supports a given warning flag.
+        Unlike addCompileFlagIfSupported, this function detects when
+        "-Wno-<warning>" flags are unsupported. If flag is a
+        "-Wno-<warning>" GCC will not emit an unknown option diagnostic unless
+        another error is triggered during compilation.
+        """
+        assert isinstance(flag, str)
+        assert flag.startswith("-W")
+        if not flag.startswith("-Wno-"):
+            return self.hasCompileFlag(flag)
+        flags = ["-Werror", flag]
+        old_use_warnings = self.use_warnings
+        self.useWarnings(False)
+        cmd = self.compileCmd("-", os.devnull, flags)
+        self.useWarnings(old_use_warnings)
+        # Remove '-v' because it will cause the command line invocation
+        # to be printed as part of the error output.
+        # TODO(EricWF): Are there other flags we need to worry about?
+        if "-v" in cmd:
+            cmd.remove("-v")
+        out, err, rc = libcudacxx.util.executeCommand(
+            cmd, input=libcudacxx.util.to_bytes("#error\n")
+        )
+        assert rc != 0
+        if flag in err:
+            return False
+        return True
+
+    def addWarningFlagIfSupported(self, flag):
+        if self.hasWarningFlag(flag):
+            if flag not in self.warning_flags:
+                self.warning_flags += [flag]
+            return True
+        return False

cccl_upstream/libcudacxx/test/utils/libcudacxx/dumpversion.cpp

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===##
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===##
+
+extern "C" int printf(char const* format, ...);
+
+int main()
+{
+  char const* compiler_type = "unknown";
+  unsigned major_version    = 0;
+  unsigned minor_version    = 0;
+  unsigned patch_level      = 0;
+  unsigned default_dialect  = 3;
+
+#if defined(__NVCC__)
+  compiler_type = "nvcc";
+  major_version = __CUDACC_VER_MAJOR__;
+  minor_version = __CUDACC_VER_MINOR__;
+  patch_level   = __CUDACC_VER_BUILD__;
+#elif defined(__NVCOMPILER)
+  compiler_type = "nvhpc";
+  major_version = __NVCOMPILER;
+  minor_version = ___NVCOMPILER_MINOR__;
+  patch_level   = ___NVCOMPILER_PATCHLEVEL__;
+#elif defined(__clang__)
+  compiler_type = "clang";
+  major_version = __clang_major__;
+  minor_version = __clang_minor__;
+  patch_level   = __clang_patchlevel__;
+#elif defined(_MSC_VER)
+  compiler_type = "msvc";
+  major_version = _MSC_FULL_VER / 10000000;
+  minor_version = _MSC_FULL_VER / 100000 % 100;
+  patch_level   = _MSC_FULL_VER % 100000;
+#elif defined(__GNUC__)
+  compiler_type = "gcc";
+  major_version = __GNUC__;
+  minor_version = __GNUC_MINOR__;
+  patch_level   = __GNUC_PATCHLEVEL__;
+#endif
+
+#if defined(_MSC_VER)
+#  if !defined(_MSVC_LANG)
+  default_dialect = 3;
+#  elif _MSVC_LANG <= 201103L
+  default_dialect = 11;
+#  elif _MSVC_LANG <= 201402L
+  default_dialect = 14;
+#  elif _MSVC_LANG <= 201703L
+  default_dialect = 17;
+#  else
+  default_dialect = 20;
+#  endif
+#else
+#  if __cplusplus <= 199711L
+  default_dialect = 3;
+#  elif __cplusplus <= 201103L
+  default_dialect = 11;
+#  elif __cplusplus <= 201402L
+  default_dialect = 14;
+#  elif __cplusplus <= 201703L
+  default_dialect = 17;
+#  else
+  default_dialect = 20;
+#  endif
+#endif
+
+  printf(
+    "(\"%s\", (%d, %d, %d), \"c++%02u\")\n", compiler_type, major_version, minor_version, patch_level, default_dialect);
+}

cccl_upstream/libcudacxx/test/utils/libcudacxx/sym_check/__init__.py

@@ -0,0 +1,16 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+"""libcxx abi symbol checker"""
+
+__author__ = "Eric Fiselier"
+__email__ = "eric@efcs.ca"
+__versioninfo__ = (0, 1, 0)
+__version__ = " ".join(str(v) for v in __versioninfo__) + "dev"
+
+__all__ = ["diff", "extract", "util"]

cccl_upstream/libcudacxx/test/utils/libcudacxx/sym_check/diff.py

@@ -0,0 +1,103 @@
+# -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80:
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+"""
+diff - A set of functions for diff-ing two symbol lists.
+"""
+
+from libcudacxx.sym_check import util
+
+
+def _symbol_difference(lhs, rhs):
+    lhs_names = set(((n["name"], n["type"]) for n in lhs))
+    rhs_names = set(((n["name"], n["type"]) for n in rhs))
+    diff_names = lhs_names - rhs_names
+    return [n for n in lhs if (n["name"], n["type"]) in diff_names]
+
+
+def _find_by_key(sym_list, k):
+    for sym in sym_list:
+        if sym["name"] == k:
+            return sym
+    return None
+
+
+def added_symbols(old, new):
+    return _symbol_difference(new, old)
+
+
+def removed_symbols(old, new):
+    return _symbol_difference(old, new)
+
+
+def changed_symbols(old, new):
+    changed = []
+    for old_sym in old:
+        if old_sym in new:
+            continue
+        new_sym = _find_by_key(new, old_sym["name"])
+        if new_sym is not None and new_sym not in old and old_sym != new_sym:
+            changed += [(old_sym, new_sym)]
+    return changed
+
+
+def diff(old, new):
+    added = added_symbols(old, new)
+    removed = removed_symbols(old, new)
+    changed = changed_symbols(old, new)
+    return added, removed, changed
+
+
+def report_diff(
+    added_syms, removed_syms, changed_syms, names_only=False, demangle=True
+):
+    def maybe_demangle(name):
+        return util.demangle_symbol(name) if demangle else name
+
+    report = ""
+    for sym in added_syms:
+        report += "Symbol added: %s\n" % maybe_demangle(sym["name"])
+        if not names_only:
+            report += "    %s\n\n" % sym
+    if added_syms and names_only:
+        report += "\n"
+    for sym in removed_syms:
+        report += "SYMBOL REMOVED: %s\n" % maybe_demangle(sym["name"])
+        if not names_only:
+            report += "    %s\n\n" % sym
+    if removed_syms and names_only:
+        report += "\n"
+    if not names_only:
+        for sym_pair in changed_syms:
+            old_sym, new_sym = sym_pair
+            old_str = "\n    OLD SYMBOL: %s" % old_sym
+            new_str = "\n    NEW SYMBOL: %s" % new_sym
+            report += "SYMBOL CHANGED: %s%s%s\n\n" % (
+                maybe_demangle(old_sym["name"]),
+                old_str,
+                new_str,
+            )
+
+    added = bool(len(added_syms) != 0)
+    abi_break = bool(len(removed_syms))
+    if not names_only:
+        abi_break = abi_break or len(changed_syms)
+    if added or abi_break:
+        report += "Summary\n"
+        report += "    Added:   %d\n" % len(added_syms)
+        report += "    Removed: %d\n" % len(removed_syms)
+        if not names_only:
+            report += "    Changed: %d\n" % len(changed_syms)
+        if not abi_break:
+            report += "Symbols added."
+        else:
+            report += "ABI BREAKAGE: SYMBOLS ADDED OR REMOVED!"
+    else:
+        report += "Symbols match."
+    is_different = abi_break or bool(len(added_syms)) or bool(len(changed_syms))
+    return report, abi_break, is_different

cccl_upstream/libcudacxx/test/utils/libcudacxx/sym_check/extract.py

@@ -0,0 +1,206 @@
+# -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80:
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+"""
+extract - A set of function that extract symbol lists from shared libraries.
+"""
+
+import distutils.spawn
+import os.path
+import sys
+
+import libcudacxx.util
+from libcudacxx.sym_check import util
+
+extract_ignore_names = ["_init", "_fini"]
+
+
+class NMExtractor(object):
+    """
+    NMExtractor - Extract symbol lists from libraries using nm.
+    """
+
+    @staticmethod
+    def find_tool():
+        """
+        Search for the nm executable and return the path.
+        """
+        return distutils.spawn.find_executable("nm")
+
+    def __init__(self, static_lib):
+        """
+        Initialize the nm executable and flags that will be used to extract
+        symbols from shared libraries.
+        """
+        self.nm_exe = self.find_tool()
+        if self.nm_exe is None:
+            # ERROR no NM found
+            print("ERROR: Could not find nm")
+            sys.exit(1)
+        self.static_lib = static_lib
+        self.flags = ["-P", "-g"]
+
+    def extract(self, lib):
+        """
+        Extract symbols from a library and return the results as a dict of
+        parsed symbols.
+        """
+        cmd = [self.nm_exe] + self.flags + [lib]
+        out, _, exit_code = libcudacxx.util.executeCommandVerbose(cmd)
+        if exit_code != 0:
+            raise RuntimeError("Failed to run %s on %s" % (self.nm_exe, lib))
+        fmt_syms = (
+            self._extract_sym(line) for line in out.splitlines() if line.strip()
+        )
+        # Cast symbol to string.
+        final_syms = (repr(s) for s in fmt_syms if self._want_sym(s))
+        # Make unique and sort strings.
+        tmp_list = list(sorted(set(final_syms)))
+        # Cast string back to symbol.
+        return util.read_syms_from_list(tmp_list)
+
+    def _extract_sym(self, sym_str):
+        bits = sym_str.split()
+        # Everything we want has at least two columns.
+        if len(bits) < 2:
+            return None
+        new_sym = {
+            "name": bits[0],
+            "type": bits[1],
+            "is_defined": (bits[1].lower() != "u"),
+        }
+        new_sym["name"] = new_sym["name"].replace("@@", "@")
+        new_sym = self._transform_sym_type(new_sym)
+        # NM types which we want to save the size for.
+        if new_sym["type"] == "OBJECT" and len(bits) > 3:
+            new_sym["size"] = int(bits[3], 16)
+        return new_sym
+
+    @staticmethod
+    def _want_sym(sym):
+        """
+        Check that s is a valid symbol that we want to keep.
+        """
+        if sym is None or len(sym) < 2:
+            return False
+        if sym["name"] in extract_ignore_names:
+            return False
+        bad_types = ["t", "b", "r", "d", "w"]
+        return sym["type"] not in bad_types and sym["name"] not in [
+            "__bss_start",
+            "_end",
+            "_edata",
+        ]
+
+    @staticmethod
+    def _transform_sym_type(sym):
+        """
+        Map the nm single letter output for type to either FUNC or OBJECT.
+        If the type is not recognized it is left unchanged.
+        """
+        func_types = ["T", "W"]
+        obj_types = ["B", "D", "R", "V", "S"]
+        if sym["type"] in func_types:
+            sym["type"] = "FUNC"
+        elif sym["type"] in obj_types:
+            sym["type"] = "OBJECT"
+        return sym
+
+
+class ReadElfExtractor(object):
+    """
+    ReadElfExtractor - Extract symbol lists from libraries using readelf.
+    """
+
+    @staticmethod
+    def find_tool():
+        """
+        Search for the readelf executable and return the path.
+        """
+        return distutils.spawn.find_executable("readelf")
+
+    def __init__(self, static_lib):
+        """
+        Initialize the readelf executable and flags that will be used to
+        extract symbols from shared libraries.
+        """
+        self.tool = self.find_tool()
+        if self.tool is None:
+            # ERROR no NM found
+            print("ERROR: Could not find readelf")
+            sys.exit(1)
+        # TODO: Support readelf for reading symbols from archives
+        assert not static_lib and "RealElf does not yet support static libs"
+        self.flags = ["--wide", "--symbols"]
+
+    def extract(self, lib):
+        """
+        Extract symbols from a library and return the results as a dict of
+        parsed symbols.
+        """
+        cmd = [self.tool] + self.flags + [lib]
+        out, _, exit_code = libcudacxx.util.executeCommandVerbose(cmd)
+        if exit_code != 0:
+            raise RuntimeError("Failed to run %s on %s" % (self.nm_exe, lib))
+        dyn_syms = self.get_dynsym_table(out)
+        return self.process_syms(dyn_syms)
+
+    def process_syms(self, sym_list):
+        new_syms = []
+        for s in sym_list:
+            parts = s.split()
+            if not parts:
+                continue
+            assert len(parts) == 7 or len(parts) == 8 or len(parts) == 9
+            if len(parts) == 7:
+                continue
+            new_sym = {
+                "name": parts[7],
+                "size": int(parts[2]),
+                "type": parts[3],
+                "is_defined": (parts[6] != "UND"),
+            }
+            assert new_sym["type"] in ["OBJECT", "FUNC", "NOTYPE", "TLS"]
+            if new_sym["name"] in extract_ignore_names:
+                continue
+            if new_sym["type"] == "NOTYPE":
+                continue
+            if new_sym["type"] == "FUNC":
+                del new_sym["size"]
+            new_syms += [new_sym]
+        return new_syms
+
+    def get_dynsym_table(self, out):
+        lines = out.splitlines()
+        start = -1
+        end = -1
+        for i in range(len(lines)):
+            if lines[i].startswith("Symbol table '.dynsym'"):
+                start = i + 2
+            if start != -1 and end == -1 and not lines[i].strip():
+                end = i + 1
+        assert start != -1
+        if end == -1:
+            end = len(lines)
+        return lines[start:end]
+
+
+def extract_symbols(lib_file, static_lib=None):
+    """
+    Extract and return a list of symbols extracted from a static or dynamic
+    library. The symbols are extracted using NM or readelf. They are then
+    filtered and formatted. Finally they symbols are made unique.
+    """
+    if static_lib is None:
+        _, ext = os.path.splitext(lib_file)
+        static_lib = True if ext in [".a"] else False
+    if ReadElfExtractor.find_tool() and not static_lib:
+        extractor = ReadElfExtractor(static_lib=static_lib)
+    else:
+        extractor = NMExtractor(static_lib=static_lib)
+    return extractor.extract(lib_file)

cccl_upstream/libcudacxx/test/utils/libcudacxx/sym_check/match.py

@@ -0,0 +1,39 @@
+# -*- Python -*- vim: set syntax=python tabstop=4 expandtab cc=80:
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+"""
+match - A set of functions for matching symbols in a list to a list of regexs
+"""
+
+import re
+
+
+def find_and_report_matching(symbol_list, regex_list):
+    report = ""
+    found_count = 0
+    for regex_str in regex_list:
+        report += 'Matching regex "%s":\n' % regex_str
+        matching_list = find_matching_symbols(symbol_list, regex_str)
+        if not matching_list:
+            report += "    No matches found\n\n"
+            continue
+        # else
+        found_count += len(matching_list)
+        for m in matching_list:
+            report += "    MATCHES: %s\n" % m["name"]
+        report += "\n"
+    return found_count, report
+
+
+def find_matching_symbols(symbol_list, regex_str):
+    regex = re.compile(regex_str)
+    matching_list = []
+    for s in symbol_list:
+        if regex.match(s["name"]):
+            matching_list += [s]
+    return matching_list

cccl_upstream/libcudacxx/test/utils/libcudacxx/sym_check/util.py

@@ -0,0 +1,271 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import ast
+import distutils.spawn
+import re
+import sys
+from pprint import pformat
+
+import libcudacxx.util
+
+
+def read_syms_from_list(slist):
+    """
+    Read a list of symbols from a list of strings.
+    Each string is one symbol.
+    """
+    return [ast.literal_eval(s) for s in slist]
+
+
+def read_syms_from_file(filename):
+    """
+    Read a list of symbols in from a file.
+    """
+    with open(filename, "r") as f:
+        data = f.read()
+    return read_syms_from_list(data.splitlines())
+
+
+def read_blacklist(filename):
+    with open(filename, "r") as f:
+        data = f.read()
+    lines = [line.strip() for line in data.splitlines() if line.strip()]
+    lines = [line for line in lines if not line.startswith("#")]
+    return lines
+
+
+def write_syms(sym_list, out=None, names_only=False, filter=None):
+    """
+    Write a list of symbols to the file named by out.
+    """
+    out_str = ""
+    out_list = sym_list
+    out_list.sort(key=lambda x: x["name"])
+    if filter is not None:
+        out_list = filter(out_list)
+    if names_only:
+        out_list = [sym["name"] for sym in out_list]
+    for sym in out_list:
+        # Use pformat for consistent ordering of keys.
+        out_str += pformat(sym, width=100000) + "\n"
+    if out is None:
+        sys.stdout.write(out_str)
+    else:
+        with open(out, "w") as f:
+            f.write(out_str)
+
+
+_cppfilt_exe = distutils.spawn.find_executable("c++filt")
+
+
+def demangle_symbol(symbol):
+    if _cppfilt_exe is None:
+        return symbol
+    out, _, exit_code = libcudacxx.util.executeCommandVerbose(
+        [_cppfilt_exe], input=symbol
+    )
+    if exit_code != 0:
+        return symbol
+    return out
+
+
+def is_elf(filename):
+    with open(filename, "rb") as f:
+        magic_bytes = f.read(4)
+    return magic_bytes == b"\x7fELF"
+
+
+def is_mach_o(filename):
+    with open(filename, "rb") as f:
+        magic_bytes = f.read(4)
+    return magic_bytes in [
+        "\xfe\xed\xfa\xce",  # MH_MAGIC
+        "\xce\xfa\xed\xfe",  # MH_CIGAM
+        "\xfe\xed\xfa\xcf",  # MH_MAGIC_64
+        "\xcf\xfa\xed\xfe",  # MH_CIGAM_64
+        "\xca\xfe\xba\xbe",  # FAT_MAGIC
+        "\xbe\xba\xfe\xca",  # FAT_CIGAM
+    ]
+
+
+def is_library_file(filename):
+    if sys.platform == "darwin":
+        return is_mach_o(filename)
+    else:
+        return is_elf(filename)
+
+
+def extract_or_load(filename):
+    import libcudacxx.sym_check.extract
+
+    if is_library_file(filename):
+        return libcudacxx.sym_check.extract.extract_symbols(filename)
+    return read_syms_from_file(filename)
+
+
+def adjust_mangled_name(name):
+    if not name.startswith("__Z"):
+        return name
+    return name[1:]
+
+
+new_delete_std_symbols = ["_Znam", "_Znwm", "_ZdaPv", "_ZdaPvm", "_ZdlPv", "_ZdlPvm"]
+
+cxxabi_symbols = [
+    "___dynamic_cast",
+    "___gxx_personality_v0",
+    "_ZTIDi",
+    "_ZTIDn",
+    "_ZTIDs",
+    "_ZTIPDi",
+    "_ZTIPDn",
+    "_ZTIPDs",
+    "_ZTIPKDi",
+    "_ZTIPKDn",
+    "_ZTIPKDs",
+    "_ZTIPKa",
+    "_ZTIPKb",
+    "_ZTIPKc",
+    "_ZTIPKd",
+    "_ZTIPKe",
+    "_ZTIPKf",
+    "_ZTIPKh",
+    "_ZTIPKi",
+    "_ZTIPKj",
+    "_ZTIPKl",
+    "_ZTIPKm",
+    "_ZTIPKs",
+    "_ZTIPKt",
+    "_ZTIPKv",
+    "_ZTIPKw",
+    "_ZTIPKx",
+    "_ZTIPKy",
+    "_ZTIPa",
+    "_ZTIPb",
+    "_ZTIPc",
+    "_ZTIPd",
+    "_ZTIPe",
+    "_ZTIPf",
+    "_ZTIPh",
+    "_ZTIPi",
+    "_ZTIPj",
+    "_ZTIPl",
+    "_ZTIPm",
+    "_ZTIPs",
+    "_ZTIPt",
+    "_ZTIPv",
+    "_ZTIPw",
+    "_ZTIPx",
+    "_ZTIPy",
+    "_ZTIa",
+    "_ZTIb",
+    "_ZTIc",
+    "_ZTId",
+    "_ZTIe",
+    "_ZTIf",
+    "_ZTIh",
+    "_ZTIi",
+    "_ZTIj",
+    "_ZTIl",
+    "_ZTIm",
+    "_ZTIs",
+    "_ZTIt",
+    "_ZTIv",
+    "_ZTIw",
+    "_ZTIx",
+    "_ZTIy",
+    "_ZTSDi",
+    "_ZTSDn",
+    "_ZTSDs",
+    "_ZTSPDi",
+    "_ZTSPDn",
+    "_ZTSPDs",
+    "_ZTSPKDi",
+    "_ZTSPKDn",
+    "_ZTSPKDs",
+    "_ZTSPKa",
+    "_ZTSPKb",
+    "_ZTSPKc",
+    "_ZTSPKd",
+    "_ZTSPKe",
+    "_ZTSPKf",
+    "_ZTSPKh",
+    "_ZTSPKi",
+    "_ZTSPKj",
+    "_ZTSPKl",
+    "_ZTSPKm",
+    "_ZTSPKs",
+    "_ZTSPKt",
+    "_ZTSPKv",
+    "_ZTSPKw",
+    "_ZTSPKx",
+    "_ZTSPKy",
+    "_ZTSPa",
+    "_ZTSPb",
+    "_ZTSPc",
+    "_ZTSPd",
+    "_ZTSPe",
+    "_ZTSPf",
+    "_ZTSPh",
+    "_ZTSPi",
+    "_ZTSPj",
+    "_ZTSPl",
+    "_ZTSPm",
+    "_ZTSPs",
+    "_ZTSPt",
+    "_ZTSPv",
+    "_ZTSPw",
+    "_ZTSPx",
+    "_ZTSPy",
+    "_ZTSa",
+    "_ZTSb",
+    "_ZTSc",
+    "_ZTSd",
+    "_ZTSe",
+    "_ZTSf",
+    "_ZTSh",
+    "_ZTSi",
+    "_ZTSj",
+    "_ZTSl",
+    "_ZTSm",
+    "_ZTSs",
+    "_ZTSt",
+    "_ZTSv",
+    "_ZTSw",
+    "_ZTSx",
+    "_ZTSy",
+]
+
+
+def is_stdlib_symbol_name(name, sym):
+    name = adjust_mangled_name(name)
+    if re.search("@GLIBC|@GCC", name):
+        # Only when symbol is defined do we consider it ours
+        return sym["is_defined"]
+    if re.search("(St[0-9])|(__cxa)|(__cxxabi)", name):
+        return True
+    if name in new_delete_std_symbols:
+        return True
+    if name in cxxabi_symbols:
+        return True
+    if name.startswith("_Z"):
+        return True
+    return False
+
+
+def filter_stdlib_symbols(syms):
+    stdlib_symbols = []
+    other_symbols = []
+    for s in syms:
+        canon_name = adjust_mangled_name(s["name"])
+        if not is_stdlib_symbol_name(canon_name, s):
+            other_symbols += [s]
+        else:
+            stdlib_symbols += [s]
+    return stdlib_symbols, other_symbols

cccl_upstream/libcudacxx/test/utils/libcudacxx/test/executor.py

@@ -0,0 +1,214 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import os
+import platform
+
+from libcudacxx.test import tracing
+from libcudacxx.util import executeCommand
+
+
+class Executor(object):
+    def run(self, exe_path, cmd, local_cwd, file_deps=None, env=None):
+        """Execute a command.
+            Be very careful not to change shared state in this function.
+            Executor objects are shared between python processes in `lit -jN`.
+        Args:
+            exe_path: str:    Local path to the executable to be run
+            cmd: [str]:       subprocess.call style command
+            local_cwd: str:   Local path to the working directory
+            file_deps: [str]: Files required by the test
+            env: {str: str}:  Environment variables to execute under
+        Returns:
+            cmd, out, err, exitCode
+        """
+        raise NotImplementedError
+
+
+class LocalExecutor(Executor):
+    def __init__(self):
+        super(LocalExecutor, self).__init__()
+        self.timeout = 0
+        self.is_windows = platform.system() == "Windows"
+
+    def run(self, exe_path, cmd=None, work_dir=".", file_deps=None, env=None):
+        cmd = cmd or [exe_path]
+        if work_dir == ".":
+            work_dir = os.getcwd()
+        out, err, rc = executeCommand(cmd, cwd=work_dir, env=env, timeout=self.timeout)
+        return (cmd, out, err, rc)
+
+
+class NoopExecutor(Executor):
+    def __init__(self):
+        super(NoopExecutor, self).__init__()
+
+    def run(self, exe_path, cmd=None, work_dir=".", file_deps=None, env=None):
+        cmd = cmd or [exe_path]
+        return (
+            cmd,
+            "",
+            "",
+            1 if (cmd[-1] and cmd[-1].endswith("runfail.cpp.exe")) else 0,
+        )
+
+
+class PrefixExecutor(Executor):
+    """Prefix an executor with some other command wrapper.
+
+    Most useful for setting ulimits on commands, or running an emulator like
+    qemu and valgrind.
+    """
+
+    def __init__(self, commandPrefix, chain):
+        super(PrefixExecutor, self).__init__()
+
+        self.commandPrefix = commandPrefix
+        self.chain = chain
+
+    def run(self, exe_path, cmd=None, work_dir=".", file_deps=None, env=None):
+        cmd = cmd or [exe_path]
+        return self.chain.run(
+            exe_path, self.commandPrefix + cmd, work_dir, file_deps, env=env
+        )
+
+
+class PostfixExecutor(Executor):
+    """Postfix an executor with some args."""
+
+    def __init__(self, commandPostfix, chain):
+        super(PostfixExecutor, self).__init__()
+
+        self.commandPostfix = commandPostfix
+        self.chain = chain
+
+    def run(self, exe_path, cmd=None, work_dir=".", file_deps=None, env=None):
+        cmd = cmd or [exe_path]
+        return self.chain.run(cmd + self.commandPostfix, work_dir, file_deps, env=env)
+
+
+class TimeoutExecutor(PrefixExecutor):
+    """Execute another action under a timeout.
+
+    Deprecated. http://reviews.llvm.org/D6584 adds timeouts to LIT.
+    """
+
+    def __init__(self, duration, chain):
+        super(TimeoutExecutor, self).__init__(["timeout", duration], chain)
+
+
+class RemoteExecutor(Executor):
+    def __init__(self):
+        self.local_run = executeCommand
+
+    def remote_temp_dir(self):
+        return self._remote_temp(True)
+
+    def remote_temp_file(self):
+        return self._remote_temp(False)
+
+    def _remote_temp(self, is_dir):
+        raise NotImplementedError()
+
+    def copy_in(self, local_srcs, remote_dsts):
+        # This could be wrapped up in a tar->scp->untar for performance
+        # if there are lots of files to be copied/moved
+        for src, dst in zip(local_srcs, remote_dsts):
+            self._copy_in_file(src, dst)
+
+    def _copy_in_file(self, src, dst):
+        raise NotImplementedError()
+
+    def delete_remote(self, remote):
+        try:
+            self._execute_command_remote(["rm", "-rf", remote])
+        except OSError:
+            # TODO: Log failure to delete?
+            pass
+
+    def run(self, exe_path, cmd=None, work_dir=".", file_deps=None, env=None):
+        target_exe_path = None
+        target_cwd = None
+        try:
+            target_cwd = self.remote_temp_dir()
+            target_exe_path = os.path.join(target_cwd, "libcxx_test.exe")
+            if cmd:
+                # Replace exe_path with target_exe_path.
+                cmd = [c if c != exe_path else target_exe_path for c in cmd]
+            else:
+                cmd = [target_exe_path]
+
+            srcs = [exe_path]
+            dsts = [target_exe_path]
+            if file_deps is not None:
+                dev_paths = [
+                    os.path.join(target_cwd, os.path.basename(f)) for f in file_deps
+                ]
+                srcs.extend(file_deps)
+                dsts.extend(dev_paths)
+            self.copy_in(srcs, dsts)
+            # TODO(jroelofs): capture the copy_in and delete_remote commands,
+            # and conjugate them with '&&'s around the first tuple element
+            # returned here:
+            return self._execute_command_remote(cmd, target_cwd, env)
+        finally:
+            if target_cwd:
+                self.delete_remote(target_cwd)
+
+    def _execute_command_remote(self, cmd, remote_work_dir=".", env=None):
+        raise NotImplementedError()
+
+
+class SSHExecutor(RemoteExecutor):
+    def __init__(self, host, username=None):
+        super(SSHExecutor, self).__init__()
+
+        self.user_prefix = username + "@" if username else ""
+        self.host = host
+        self.scp_command = "scp"
+        self.ssh_command = "ssh"
+
+        # TODO(jroelofs): switch this on some -super-verbose-debug config flag
+        if False:
+            self.local_run = tracing.trace_function(
+                self.local_run, log_calls=True, log_results=True, label="ssh_local"
+            )
+
+    def _remote_temp(self, is_dir):
+        # TODO: detect what the target system is, and use the correct
+        # mktemp command for it. (linux and darwin differ here, and I'm
+        # sure windows has another way to do it)
+
+        # Not sure how to do suffix on osx yet
+        dir_arg = "-d" if is_dir else ""
+        cmd = "mktemp -q {} /tmp/libcxx.XXXXXXXXXX".format(dir_arg)
+        _, temp_path, err, exitCode = self._execute_command_remote([cmd])
+        temp_path = temp_path.strip()
+        if exitCode != 0:
+            raise RuntimeError(err)
+        return temp_path
+
+    def _copy_in_file(self, src, dst):
+        scp = self.scp_command
+        remote = self.host
+        remote = self.user_prefix + remote
+        cmd = [scp, "-p", src, remote + ":" + dst]
+        self.local_run(cmd)
+
+    def _execute_command_remote(self, cmd, remote_work_dir=".", env=None):
+        remote = self.user_prefix + self.host
+        ssh_cmd = [self.ssh_command, "-oBatchMode=yes", remote]
+        if env:
+            env_cmd = ["env"] + ['%s="%s"' % (k, v) for k, v in env.items()]
+        else:
+            env_cmd = []
+        remote_cmd = " ".join(env_cmd + cmd)
+        if remote_work_dir != ".":
+            remote_cmd = "cd " + remote_work_dir + " && " + remote_cmd
+        out, err, rc = self.local_run(ssh_cmd + [remote_cmd])
+        return (remote_cmd, out, err, rc)

cccl_upstream/libcudacxx/test/utils/libcudacxx/test/format.py

@@ -0,0 +1,370 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import copy
+import errno
+import os
+import time
+
+import lit.Test  # pylint: disable=import-error
+import lit.TestRunner  # pylint: disable=import-error
+from lit.TestRunner import IntegratedTestKeywordParser, ParserKind
+
+import libcudacxx.util
+
+# pylint: disable=import-error
+from libcudacxx.test.executor import LocalExecutor as LocalExecutor
+
+
+class LibcxxTestFormat(object):
+    """
+    Custom test format handler for use with the test format use by libc++.
+
+    Tests fall into two categories:
+      FOO.pass.cpp - Executable test which should compile, run, and exit with
+                     code 0.
+      FOO.fail.cpp - Negative test case which is expected to fail compilation.
+      FOO.runfail.cpp - Negative test case which is expected to compile, run,
+                        and exit with non-zero exit code.
+      FOO.sh.cpp   - A test that uses LIT's ShTest format.
+    """
+
+    def __init__(self, cxx, use_verify_for_fail, execute_external, executor, exec_env):
+        self.cxx = copy.deepcopy(cxx)
+        self.use_verify_for_fail = use_verify_for_fail
+        self.execute_external = execute_external
+        self.executor = executor
+        self.exec_env = dict(exec_env)
+
+    @staticmethod
+    def _make_custom_parsers():
+        return [
+            IntegratedTestKeywordParser(
+                "FLAKY_TEST.", ParserKind.TAG, initial_value=False
+            ),
+            IntegratedTestKeywordParser(
+                "FORCE_ALL_WARNINGS.", ParserKind.TAG, initial_value=False
+            ),
+            IntegratedTestKeywordParser(
+                "MODULES_DEFINES:", ParserKind.LIST, initial_value=[]
+            ),
+            IntegratedTestKeywordParser(
+                "ADDITIONAL_COMPILE_DEFINITIONS:", ParserKind.LIST, initial_value=[]
+            ),
+            IntegratedTestKeywordParser(
+                "ADDITIONAL_COMPILE_OPTIONS_HOST:", ParserKind.LIST, initial_value=[]
+            ),
+            IntegratedTestKeywordParser(
+                "ADDITIONAL_COMPILE_OPTIONS_CUDA:", ParserKind.LIST, initial_value=[]
+            ),
+            IntegratedTestKeywordParser("CONSTEXPR_STEPS:", ParserKind.INTEGER),
+        ]
+
+    @staticmethod
+    def _get_parser(key, parsers):
+        for p in parsers:
+            if p.keyword == key:
+                return p
+        assert False and "parser not found"
+
+    # TODO: Move this into lit's FileBasedTest
+    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
+        source_path = testSuite.getSourcePath(path_in_suite)
+        for filename in os.listdir(source_path):
+            # Ignore dot files and excluded tests.
+            if filename.startswith(".") or filename in localConfig.excludes:
+                continue
+
+            filepath = os.path.join(source_path, filename)
+            if not os.path.isdir(filepath):
+                if any([filename.endswith(ext) for ext in localConfig.suffixes]):
+                    yield lit.Test.Test(
+                        testSuite, path_in_suite + (filename,), localConfig
+                    )
+
+    def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+        yield lit.Test.Test(testSuite, path_in_suite, localConfig)
+
+    def execute(self, test, lit_config):
+        while True:
+            try:
+                return self._execute(test, lit_config)
+            except OSError as oe:
+                if oe.errno != errno.ETXTBSY:
+                    raise
+                time.sleep(0.1)
+
+    def _execute(self, test, lit_config):
+        name = test.path_in_suite[-1]
+        name_root, name_ext = os.path.splitext(name)
+        is_libcxx_test = test.path_in_suite[0] == "libcxx"
+        is_sh_test = name_root.endswith(".sh")
+        is_pass_test = name.endswith(".pass.cpp") or name.endswith(".pass.mm")
+        is_fail_test = name.endswith(".fail.cpp") or name.endswith(".fail.mm")
+        is_runfail_test = name.endswith(".runfail.cpp") or name.endswith(".runfail.mm")
+        assert is_sh_test or name_ext == ".cpp" or name_ext == ".mm", (
+            "non-cpp file must be sh test"
+        )
+
+        if test.config.unsupported:
+            return (lit.Test.UNSUPPORTED, "A lit.local.cfg marked this unsupported")
+
+        parsers = self._make_custom_parsers()
+        script = lit.TestRunner.parseIntegratedTestScript(
+            test, additional_parsers=parsers, require_script=is_sh_test
+        )
+        # Check if a result for the test was returned. If so return that
+        # result.
+        if isinstance(script, lit.Test.Result):
+            return script
+        if lit_config.noExecute:
+            # if we expect the test to fail at runtime, XFAIL is the proper return value if we never run the test
+            if test.xfails:
+                return lit.Test.Result(lit.Test.XFAIL)
+            return lit.Test.Result(lit.Test.PASS)
+
+        # Check that we don't have run lines on tests that don't support them.
+        if not is_sh_test and len(script) != 0:
+            lit_config.fatal("Unsupported RUN line found in test %s" % name)
+
+        tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
+        substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
+        script = lit.TestRunner.applySubstitutions(script, substitutions)
+
+        test_cxx = copy.deepcopy(self.cxx)
+        if is_fail_test:
+            test_cxx.useCCache(False)
+            test_cxx.useWarnings(False)
+
+        force_all_warnings = self._get_parser("FORCE_ALL_WARNINGS.", parsers).getValue()
+
+        if force_all_warnings:
+            test_cxx.useWarnings(True)
+
+        extra_compile_definitions = self._get_parser(
+            "ADDITIONAL_COMPILE_DEFINITIONS:", parsers
+        ).getValue()
+        test_cxx.compile_flags += [
+            ("-D%s" % mdef.strip()) for mdef in extra_compile_definitions
+        ]
+
+        extra_compile_options_host = self._get_parser(
+            "ADDITIONAL_COMPILE_OPTIONS_HOST:", parsers
+        ).getValue()
+        if test_cxx.type == "nvcc":
+            for flag in extra_compile_options_host:
+                if test_cxx.host_cxx.addCompileFlagIfSupported(flag.strip()):
+                    test_cxx.warning_flags += ["-Xcompiler", flag.strip()]
+
+            extra_compile_options_cuda = self._get_parser(
+                "ADDITIONAL_COMPILE_OPTIONS_CUDA:", parsers
+            ).getValue()
+            for flag in extra_compile_options_cuda:
+                if test_cxx.addCompileFlagIfSupported(flag.strip()):
+                    test_cxx.warning_flags += [flag.strip()]
+        else:
+            for flag in extra_compile_options_host:
+                if test_cxx.addCompileFlagIfSupported(flag.strip()):
+                    test_cxx.warning_flags += [flag.strip()]
+
+        extra_modules_defines = self._get_parser("MODULES_DEFINES:", parsers).getValue()
+        if "-fmodules" in test.config.available_features:
+            test_cxx.compile_flags += [
+                ("-D%s" % mdef.strip()) for mdef in extra_modules_defines
+            ]
+            test_cxx.addWarningFlagIfSupported("-Wno-macro-redefined")
+            # FIXME: libc++ debug tests #define _CCCL_ASSERT to override it
+            # If we see this we need to build the test against uniquely built
+            # modules.
+            if is_libcxx_test:
+                with open(test.getSourcePath(), "rb") as f:
+                    contents = f.read()
+                if b"#define _CCCL_ASSERT" in contents:
+                    test_cxx.useModules(False)
+
+        # Handle constexpr steps if specified
+        constexpr_steps = self._get_parser("CONSTEXPR_STEPS:", parsers).getValue()
+        if constexpr_steps is not None:
+            constexpr_steps = constexpr_steps[0]
+            cxx = test_cxx.host_cxx if test_cxx.type == "nvcc" else test_cxx
+            if cxx.type == "msvc":
+                constexpr_steps_opt = f"/constexpr:steps{constexpr_steps}"
+            elif cxx.type == "clang":
+                constexpr_steps_opt = f"-fconstexpr-steps={constexpr_steps}"
+            elif cxx.type == "gcc" and cxx.version[0] >= 9:
+                constexpr_steps_opt = f"-fconstexpr-ops-limit={constexpr_steps}"
+            elif cxx.type == "nvhpc":
+                constexpr_steps_opt = f"-Wc,--max_cost_constexpr_call={constexpr_steps}"
+            else:
+                constexpr_steps_opt = None
+
+            if constexpr_steps_opt is not None:
+                if test_cxx.type == "nvcc":
+                    test_cxx.compile_flags += ["-Xcompiler", f'"{constexpr_steps_opt}"']
+                else:
+                    test_cxx.compile_flags += [constexpr_steps_opt]
+
+        if test.path_in_suite[:2] == ("cuda", "ptx") and test_cxx.type == "nvcc":
+            test_cxx.compile_flags += ["-rdc=true", "-dc"]
+            test_cxx.link_flags += [
+                flag
+                for flag in test_cxx.compile_flags
+                if flag.startswith("-gencode=") or flag.startswith("--generate-code=")
+            ]
+
+        # Dispatch the test based on its suffix.
+        if is_sh_test:
+            if not isinstance(self.executor, LocalExecutor):
+                # We can't run ShTest tests with a executor yet.
+                # For now, bail on trying to run them
+                return lit.Test.UNSUPPORTED, "ShTest format not yet supported"
+            test.config.environment = dict(self.exec_env)
+            return lit.TestRunner._runShTest(
+                test, lit_config, self.execute_external, script, tmpBase
+            )
+        elif is_fail_test:
+            return self._evaluate_fail_test(test, test_cxx, parsers)
+        elif is_pass_test:
+            return self._evaluate_pass_test(
+                test, tmpBase, lit_config, test_cxx, parsers
+            )
+        elif is_runfail_test:
+            return self._evaluate_pass_test(
+                test, tmpBase, lit_config, test_cxx, parsers, run_should_pass=False
+            )
+        else:
+            # No other test type is supported
+            assert False
+
+    def _clean(self, exec_path):  # pylint: disable=no-self-use
+        libcudacxx.util.cleanFile(exec_path)
+
+    def _evaluate_pass_test(
+        self, test, tmpBase, lit_config, test_cxx, parsers, run_should_pass=True
+    ):
+        execDir = os.path.dirname(test.getExecPath())
+        source_path = test.getSourcePath()
+        exec_path = tmpBase + ".exe"
+        object_path = tmpBase + ".o"
+        # Create the output directory if it does not already exist.
+        libcudacxx.util.mkdir_p(os.path.dirname(tmpBase))
+        try:
+            # Compile the test
+            cmd, out, err, rc = test_cxx.compileLinkTwoSteps(
+                source_path, out=exec_path, object_file=object_path, cwd=execDir
+            )
+            compile_cmd = cmd
+            if rc != 0:
+                report = libcudacxx.util.makeReport(cmd, out, err, rc)
+                report += "Compilation failed unexpectedly!"
+                return lit.Test.Result(lit.Test.FAIL, report)
+            # Run the test
+            local_cwd = os.path.dirname(source_path)
+            env = None
+            if self.exec_env:
+                env = self.exec_env
+            # TODO: Only list actually needed files in file_deps.
+            # Right now we just mark all of the .dat files in the same
+            # directory as dependencies, but it's likely less than that. We
+            # should add a `// FILE-DEP: foo.dat` to each test to track this.
+            data_files = [
+                os.path.join(local_cwd, f)
+                for f in os.listdir(local_cwd)
+                if f.endswith(".dat")
+            ]
+            is_flaky = self._get_parser("FLAKY_TEST.", parsers).getValue()
+            max_retry = 3 if is_flaky else 1
+            for retry_count in range(max_retry):
+                cmd, out, err, rc = self.executor.run(
+                    exec_path, [exec_path], local_cwd, data_files, env
+                )
+                report = "Compiled With: '%s'\n" % " ".join(compile_cmd)
+                report += libcudacxx.util.makeReport(cmd, out, err, rc)
+                result_expected = (rc == 0) == run_should_pass
+                if result_expected:
+                    res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
+                    return lit.Test.Result(res, report)
+                # Rarely devices are unavailable, so just restart the test to avoid false negatives.
+                elif (
+                    rc != 0 and "cudaErrorDevicesUnavailable" in out and max_retry <= 5
+                ):
+                    max_retry += 1
+                elif retry_count + 1 == max_retry:
+                    if run_should_pass:
+                        report += "Compiled test failed unexpectedly!"
+                    else:
+                        report += "Compiled test succeeded unexpectedly!"
+                    return lit.Test.Result(lit.Test.FAIL, report)
+
+            assert False  # Unreachable
+        finally:
+            # Note that cleanup of exec_file happens in `_clean()`. If you
+            # override this, cleanup is your responsibility.
+            libcudacxx.util.cleanFile(object_path)
+            self._clean(exec_path)
+
+    def _evaluate_fail_test(self, test, test_cxx, parsers):
+        source_path = test.getSourcePath()
+        # FIXME: lift this detection into LLVM/LIT.
+        with open(source_path, "rb") as f:
+            contents = f.read()
+        verify_tags = [
+            b"expected-note",
+            b"expected-remark",
+            b"expected-warning",
+            b"expected-error",
+            b"expected-no-diagnostics",
+        ]
+        use_verify = self.use_verify_for_fail and any(
+            [tag in contents for tag in verify_tags]
+        )
+        # FIXME(EricWF): GCC 5 does not evaluate static assertions that
+        # are dependant on a template parameter when '-fsyntax-only' is passed.
+        # This is fixed in GCC 6. However for now we only pass "-fsyntax-only"
+        # when using Clang.
+        if test_cxx.type != "gcc" and test_cxx.type != "nvcc":
+            test_cxx.flags += ["-fsyntax-only"]
+        if use_verify:
+            test_cxx.useVerify()
+            test_cxx.useWarnings()
+            if "-Wuser-defined-warnings" in test_cxx.warning_flags:
+                test_cxx.warning_flags += ["-Wno-error=user-defined-warnings"]
+        else:
+            # We still need to enable certain warnings on .fail.cpp test when
+            # -verify isn't enabled. Such as -Werror=unused-result. However,
+            # we don't want it enabled too liberally, which might incorrectly
+            # allow unrelated failure tests to 'pass'.
+            #
+            # Therefore, we check if the test was expected to fail because of
+            # nodiscard before enabling it
+            test_str_list = [b"ignoring return value", b"nodiscard", b"NODISCARD"]
+            if any(test_str in contents for test_str in test_str_list):
+                if test_cxx.type != "nvc++":
+                    test_cxx.flags += [
+                        "-Xcompiler",
+                        "-Werror",
+                        "-Xcompiler",
+                        "-Wunused",
+                    ]
+                else:
+                    test_cxx.flags += ["-Xcompiler", "-Werror=unused-result"]
+        cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
+
+        def check_rc(rc):
+            return rc == 0 if use_verify else rc != 0
+
+        report = libcudacxx.util.makeReport(cmd, out, err, rc)
+        if check_rc(rc):
+            return lit.Test.Result(lit.Test.PASS, report)
+        else:
+            report += (
+                "Expected compilation to fail!\n"
+                if not use_verify
+                else "Expected compilation using verify to pass!\n"
+            )
+            return lit.Test.Result(lit.Test.FAIL, report)

cccl_upstream/libcudacxx/test/utils/libcudacxx/test/googlebenchmark.py

@@ -0,0 +1,126 @@
+from __future__ import absolute_import
+
+import os
+import subprocess
+import sys
+
+import lit.Test
+import lit.TestRunner
+import lit.util
+from lit.formats.base import TestFormat
+
+kIsWindows = sys.platform in ["win32", "cygwin"]
+
+
+class GoogleBenchmark(TestFormat):
+    def __init__(self, test_sub_dirs, test_suffix, benchmark_args=[]):
+        self.benchmark_args = list(benchmark_args)
+        self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(";")
+
+        # On Windows, assume tests will also end in '.exe'.
+        exe_suffix = str(test_suffix)
+        if kIsWindows:
+            exe_suffix += ".exe"
+
+        # Also check for .py files for testing purposes.
+        self.test_suffixes = {exe_suffix, test_suffix + ".py"}
+
+    def getBenchmarkTests(self, path, litConfig, localConfig):
+        """getBenchmarkTests(path) - [name]
+
+        Return the tests available in gtest executable.
+
+        Args:
+          path: String path to a gtest executable
+          litConfig: LitConfig instance
+          localConfig: TestingConfig instance"""
+
+        # TODO: allow splitting tests according to the "benchmark family" so
+        # the output for a single family of tests all belongs to the same test
+        # target.
+        list_test_cmd = [path, "--benchmark_list_tests"]
+        try:
+            output = subprocess.check_output(list_test_cmd, env=localConfig.environment)
+        except subprocess.CalledProcessError as exc:
+            litConfig.warning(
+                "unable to discover google-benchmarks in %r: %s. Process output: %s"
+                % (path, sys.exc_info()[1], exc.output)
+            )
+            raise StopIteration
+
+        nested_tests = []
+        for ln in output.splitlines(False):  # Don't keep newlines.
+            ln = lit.util.to_string(ln)
+            if not ln.strip():
+                continue
+
+            index = 0
+            while ln[index * 2 : index * 2 + 2] == "  ":
+                index += 1
+            while len(nested_tests) > index:
+                nested_tests.pop()
+
+            ln = ln[index * 2 :]
+            if ln.endswith("."):
+                nested_tests.append(ln)
+            elif any([name.startswith("DISABLED_") for name in nested_tests + [ln]]):
+                # Gtest will internally skip these tests. No need to launch a
+                # child process for it.
+                continue
+            else:
+                yield "".join(nested_tests) + ln
+
+    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
+        source_path = testSuite.getSourcePath(path_in_suite)
+        for subdir in self.test_sub_dirs:
+            dir_path = os.path.join(source_path, subdir)
+            if not os.path.isdir(dir_path):
+                continue
+            for fn in lit.util.listdir_files(dir_path, suffixes=self.test_suffixes):
+                # Discover the tests in this executable.
+                execpath = os.path.join(source_path, subdir, fn)
+                testnames = self.getBenchmarkTests(execpath, litConfig, localConfig)
+                for testname in testnames:
+                    testPath = path_in_suite + (subdir, fn, testname)
+                    yield lit.Test.Test(
+                        testSuite, testPath, localConfig, file_path=execpath
+                    )
+
+    def execute(self, test, litConfig):
+        testPath, testName = os.path.split(test.getSourcePath())
+        while not os.path.exists(testPath):
+            # Handle GTest parametrized and typed tests, whose name includes
+            # some '/'s.
+            testPath, namePrefix = os.path.split(testPath)
+            testName = namePrefix + "/" + testName
+
+        cmd = [testPath, "--benchmark_filter=%s$" % testName] + self.benchmark_args
+
+        if litConfig.noExecute:
+            return lit.Test.PASS, ""
+
+        try:
+            out, err, exitCode = lit.util.executeCommand(
+                cmd,
+                env=test.config.environment,
+                timeout=litConfig.maxIndividualTestTime,
+            )
+        except lit.util.ExecuteCommandTimeoutException:
+            return (
+                lit.Test.TIMEOUT,
+                "Reached timeout of {} seconds".format(litConfig.maxIndividualTestTime),
+            )
+
+        if exitCode:
+            return lit.Test.FAIL, ("exit code: %d\n" % exitCode) + out + err
+
+        passing_test_line = testName
+        if passing_test_line not in out:
+            msg = "Unable to find %r in google benchmark output:\n\n%s%s" % (
+                passing_test_line,
+                out,
+                err,
+            )
+            return lit.Test.UNRESOLVED, msg
+
+        return lit.Test.PASS, err + out

cccl_upstream/libcudacxx/test/utils/libcudacxx/test/target_info.py

@@ -0,0 +1,268 @@
+# ===----------------------------------------------------------------------===//
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===//
+
+import importlib
+import locale
+import platform
+import re
+import subprocess
+import sys
+
+from libcudacxx.util import executeCommand
+
+
+class DefaultTargetInfo(object):
+    def __init__(self, full_config):
+        self.full_config = full_config
+
+    def platform(self):
+        return sys.platform.lower().strip()
+
+    def add_locale_features(self, features):
+        self.full_config.lit_config.warning(
+            "No locales entry for target_system: %s" % self.platform()
+        )
+
+    def add_cxx_compile_flags(self, flags):
+        pass
+
+    def add_cxx_link_flags(self, flags):
+        pass
+
+    def configure_env(self, env):
+        pass
+
+    def allow_cxxabi_link(self):
+        return True
+
+    def add_sanitizer_features(self, sanitizer_type, features):
+        pass
+
+    def use_lit_shell_default(self):
+        return False
+
+
+def test_locale(loc):
+    assert loc is not None
+    default_locale = locale.setlocale(locale.LC_ALL)
+    try:
+        locale.setlocale(locale.LC_ALL, loc)
+        return True
+    except locale.Error:
+        return False
+    finally:
+        locale.setlocale(locale.LC_ALL, default_locale)
+
+
+def add_common_locales(features, lit_config, is_windows=False):
+    # A list of locales needed by the test-suite.
+    # The list uses the canonical name for the locale used in the test-suite
+    # TODO: On Linux ISO8859 *may* needs to hyphenated.
+    locales = [
+        ("en_US.UTF-8", "English_United States.1252"),
+        ("fr_FR.UTF-8", "French_France.1252"),
+        ("ru_RU.UTF-8", "Russian_Russia.1251"),
+        ("zh_CN.UTF-8", "Chinese_China.936"),
+        ("fr_CA.ISO8859-1", "French_Canada.1252"),
+        ("cs_CZ.ISO8859-2", "Czech_Czech Republic.1250"),
+    ]
+    for loc_id, windows_loc_name in locales:
+        loc_name = windows_loc_name if is_windows else loc_id
+        if test_locale(loc_name):
+            features.add("locale.{0}".format(loc_id))
+        else:
+            lit_config.warning(
+                "The locale {0} is not supported by "
+                "your platform. Some tests will be "
+                "unsupported.".format(loc_name)
+            )
+
+
+class DarwinLocalTI(DefaultTargetInfo):
+    def __init__(self, full_config):
+        super(DarwinLocalTI, self).__init__(full_config)
+
+    def is_host_macosx(self):
+        name = subprocess.check_output(["sw_vers", "-productName"]).strip()
+        return name == "Mac OS X"
+
+    def get_macosx_version(self):
+        assert self.is_host_macosx()
+        version = subprocess.check_output(["sw_vers", "-productVersion"]).strip()
+        version = re.sub(r"([0-9]+\.[0-9]+)(\..*)?", r"\1", version)
+        return version
+
+    def get_sdk_version(self, name):
+        assert self.is_host_macosx()
+        cmd = ["xcrun", "--sdk", name, "--show-sdk-path"]
+        try:
+            out = subprocess.check_output(cmd).strip()
+        except OSError:
+            pass
+
+        if not out:
+            self.full_config.lit_config.fatal("cannot infer sdk version with: %r" % cmd)
+
+        return re.sub(r".*/[^0-9]+([0-9.]+)\.sdk", r"\1", out)
+
+    def get_platform(self):
+        platform = self.full_config.get_lit_conf("platform")
+        if platform:
+            platform = re.sub(r"([^0-9]+)([0-9\.]*)", r"\1-\2", platform)
+            name, version = tuple(platform.split("-", 1))
+        else:
+            name = "macosx"
+            version = None
+
+        if version:
+            return (False, name, version)
+
+        # Infer the version, either from the SDK or the system itself.  For
+        # macosx, ignore the SDK version; what matters is what's at
+        # /usr/lib/libc++.dylib.
+        if name == "macosx":
+            version = self.get_macosx_version()
+        else:
+            version = self.get_sdk_version(name)
+        return (True, name, version)
+
+    def add_locale_features(self, features):
+        add_common_locales(features, self.full_config.lit_config)
+
+    def add_cxx_compile_flags(self, flags):
+        if self.full_config.use_deployment:
+            _, name, _ = self.full_config.config.deployment
+            cmd = ["xcrun", "--sdk", name, "--show-sdk-path"]
+        else:
+            cmd = ["xcrun", "--show-sdk-path"]
+        out, err, exit_code = executeCommand(cmd)
+        if exit_code != 0:
+            self.full_config.lit_config.warning(
+                "Could not determine macOS SDK path! stderr was " + err
+            )
+        if exit_code == 0 and out:
+            sdk_path = out.strip()
+            self.full_config.lit_config.note("using SDKROOT: %r" % sdk_path)
+            assert isinstance(sdk_path, str)
+            flags += ["-isysroot", sdk_path]
+
+    def add_cxx_link_flags(self, flags):
+        flags += ["-lSystem"]
+
+    def configure_env(self, env):
+        library_paths = []
+        # Configure the library path for libc++
+        if self.full_config.cxx_runtime_root:
+            library_paths += [self.full_config.cxx_runtime_root]
+
+        # Configure the abi library path
+        if self.full_config.abi_library_root:
+            library_paths += [self.full_config.abi_library_root]
+        if library_paths:
+            env["DYLD_LIBRARY_PATH"] = ":".join(library_paths)
+
+    def allow_cxxabi_link(self):
+        # Don't link libc++abi explicitly on OS X because the symbols
+        # should be available in libc++ directly.
+        return False
+
+
+class FreeBSDLocalTI(DefaultTargetInfo):
+    def __init__(self, full_config):
+        super(FreeBSDLocalTI, self).__init__(full_config)
+
+    def add_locale_features(self, features):
+        add_common_locales(features, self.full_config.lit_config)
+
+    def add_cxx_link_flags(self, flags):
+        flags += ["-lc", "-lm", "-lpthread", "-lgcc_s", "-lcxxrt"]
+
+
+class NetBSDLocalTI(DefaultTargetInfo):
+    def __init__(self, full_config):
+        super(NetBSDLocalTI, self).__init__(full_config)
+
+    def add_locale_features(self, features):
+        add_common_locales(features, self.full_config.lit_config)
+
+    def add_cxx_link_flags(self, flags):
+        flags += ["-lc", "-lm", "-lpthread", "-lgcc_s", "-lc++abi", "-lunwind"]
+
+
+class LinuxLocalTI(DefaultTargetInfo):
+    def __init__(self, full_config):
+        super(LinuxLocalTI, self).__init__(full_config)
+
+    def platform(self):
+        return "linux"
+
+    def add_locale_features(self, features):
+        add_common_locales(features, self.full_config.lit_config)
+
+    def add_cxx_compile_flags(self, flags):
+        flags += [
+            "-D__STDC_FORMAT_MACROS",
+            "-D__STDC_LIMIT_MACROS",
+            "-D__STDC_CONSTANT_MACROS",
+        ]
+
+    def add_cxx_link_flags(self, flags):
+        enable_threads = (
+            "libcpp-has-no-threads" not in self.full_config.config.available_features
+        )
+        flags += ["-lm", "-lgcc_s", "-lgcc"]
+        if enable_threads:
+            flags += ["-lpthread", "-lrt"]
+        flags += ["-lc"]
+        builtins_lib = self.full_config.get_lit_conf("builtins_library")
+        if builtins_lib:
+            flags += [builtins_lib]
+        else:
+            flags += ["-lgcc"]
+        san = self.full_config.get_lit_conf("use_sanitizer", "").strip()
+        if san:
+            # The libraries and their order are taken from the
+            # linkSanitizerRuntimeDeps function in
+            # clang/lib/Driver/Tools.cpp
+            flags += ["-lpthread", "-lrt", "-lm", "-ldl"]
+
+
+class WindowsLocalTI(DefaultTargetInfo):
+    def __init__(self, full_config):
+        super(WindowsLocalTI, self).__init__(full_config)
+
+    def add_locale_features(self, features):
+        add_common_locales(features, self.full_config.lit_config, is_windows=True)
+
+    def use_lit_shell_default(self):
+        # Default to the internal shell on Windows, as bash on Windows is
+        # usually very slow.
+        return True
+
+
+def make_target_info(full_config):
+    default = "libcudacxx.test.target_info.LocalTI"
+    info_str = full_config.get_lit_conf("target_info", default)
+    if info_str != default:
+        mod_path, _, info = info_str.rpartition(".")
+        mod = importlib.import_module(mod_path)
+        target_info = getattr(mod, info)(full_config)
+        full_config.lit_config.note("inferred target_info as: %r" % info_str)
+        return target_info
+    target_system = platform.system()
+    if target_system == "Darwin":
+        return DarwinLocalTI(full_config)
+    if target_system == "FreeBSD":
+        return FreeBSDLocalTI(full_config)
+    if target_system == "NetBSD":
+        return NetBSDLocalTI(full_config)
+    if target_system == "Linux":
+        return LinuxLocalTI(full_config)
+    if target_system == "Windows":
+        return WindowsLocalTI(full_config)
+    return DefaultTargetInfo(full_config)

cccl_upstream/libcudacxx/test/utils/libcudacxx/test/tracing.py

@@ -0,0 +1,42 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import inspect
+
+
+def trace_function(function, log_calls, log_results, label=""):
+    def wrapper(*args, **kwargs):
+        kwarg_strs = ["{}={}".format(k, v) for (k, v) in kwargs]
+        arg_str = ", ".join([str(a) for a in args] + kwarg_strs)
+        call_str = "{}({})".format(function.func_name, arg_str)
+
+        # Perform the call itself, logging before, after, and anything thrown.
+        try:
+            if log_calls:
+                print("{}: Calling {}".format(label, call_str))
+            res = function(*args, **kwargs)
+            if log_results:
+                print("{}: {} -> {}".format(label, call_str, res))
+            return res
+        except Exception as ex:
+            if log_results:
+                print("{}: {} raised {}".format(label, call_str, type(ex)))
+            raise ex
+
+    return wrapper
+
+
+def trace_object(obj, log_calls, log_results, label=""):
+    for name, member in inspect.getmembers(obj):
+        if inspect.ismethod(member):
+            # Skip meta-functions, decorate everything else
+            if not member.func_name.startswith("__"):
+                setattr(
+                    obj, name, trace_function(member, log_calls, log_results, label)
+                )
+    return obj

cccl_upstream/libcudacxx/test/utils/libcudacxx/util.py

@@ -0,0 +1,300 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+import errno
+import os
+import platform
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+from contextlib import contextmanager
+
+
+# FIXME: Most of these functions are cribbed from LIT
+def to_bytes(str):
+    # Encode to UTF-8 to get binary data.
+    if isinstance(str, bytes):
+        return str
+    return str.encode("utf-8")
+
+
+def to_string(bytes):
+    if isinstance(bytes, str):
+        return bytes
+    return to_bytes(bytes)
+
+
+def convert_string(bytes):
+    try:
+        return to_string(bytes.decode("utf-8"))
+    except AttributeError:  # 'str' object has no attribute 'decode'.
+        return str(bytes)
+    except UnicodeError:
+        return str(bytes)
+
+
+def cleanFile(filename):
+    try:
+        os.remove(filename)
+    except OSError:
+        pass
+
+
+@contextmanager
+def guardedTempFilename(suffix="", prefix="", dir=None):
+    # Creates and yields a temporary filename within a with statement. The file
+    # is removed upon scope exit.
+    handle, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir)
+    os.close(handle)
+    yield name
+    cleanFile(name)
+
+
+@contextmanager
+def guardedFilename(name):
+    # yields a filename within a with statement. The file is removed upon scope
+    # exit.
+    yield name
+    cleanFile(name)
+
+
+@contextmanager
+def nullContext(value):
+    # yields a variable within a with statement. No action is taken upon scope
+    # exit.
+    yield value
+
+
+def makeReport(cmd, out, err, rc):
+    cmd = " ".join([f'"{a}"' for a in cmd])
+    report = "Command: %s\n" % cmd
+    report += "Exit Code: %d\n" % rc
+    if out:
+        report += "Standard Output:\n--\n%s--\n" % out
+    if err:
+        report += "Standard Error:\n--\n%s--\n" % err
+    report += "\n"
+    return report
+
+
+def capture(args, env=None):
+    """capture(command) - Run the given command (or argv list) in a shell and
+    return the standard output. Raises a CalledProcessError if the command
+    exits with a non-zero status."""
+    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+    out, err = p.communicate()
+    out = convert_string(out)
+    err = convert_string(err)
+    if p.returncode != 0:
+        raise subprocess.CalledProcessError(
+            cmd=args, returncode=p.returncode, output="{}\n{}".format(out, err)
+        )
+    return out
+
+
+def which(command, paths=None):
+    """which(command, [paths]) - Look up the given command in the paths string
+    (or the PATH environment variable, if unspecified)."""
+
+    if paths is None:
+        paths = os.environ.get("PATH", "")
+
+    # Check for absolute match first.
+    if os.path.isfile(command):
+        return command
+
+    # Would be nice if Python had a lib function for this.
+    if not paths:
+        paths = os.defpath
+
+    # Get suffixes to search.
+    # On Cygwin, 'PATHEXT' may exist but it should not be used.
+    if os.pathsep == ";":
+        pathext = os.environ.get("PATHEXT", "").split(";")
+    else:
+        pathext = [""]
+
+    # Search the paths...
+    for path in paths.split(os.pathsep):
+        for ext in pathext:
+            p = os.path.join(path, command + ext)
+            if os.path.exists(p) and not os.path.isdir(p):
+                return p
+
+    return None
+
+
+def checkToolsPath(dir, tools):
+    for tool in tools:
+        if not os.path.exists(os.path.join(dir, tool)):
+            return False
+    return True
+
+
+def whichTools(tools, paths):
+    for path in paths.split(os.pathsep):
+        if checkToolsPath(path, tools):
+            return path
+    return None
+
+
+def mkdir_p(path):
+    """mkdir_p(path) - Make the "path" directory, if it does not exist; this
+    will also make directories for any missing parent directories."""
+    if not path or os.path.exists(path):
+        return
+
+    parent = os.path.dirname(path)
+    if parent != path:
+        mkdir_p(parent)
+
+    try:
+        os.mkdir(path)
+    except OSError:
+        e = sys.exc_info()[1]
+        # Ignore EEXIST, which may occur during a race condition.
+        if e.errno != errno.EEXIST:
+            raise
+
+
+class ExecuteCommandTimeoutException(Exception):
+    def __init__(self, msg, out, err, exitCode):
+        assert isinstance(msg, str)
+        assert isinstance(out, str)
+        assert isinstance(err, str)
+        assert isinstance(exitCode, int)
+        self.msg = msg
+        self.out = out
+        self.err = err
+        self.exitCode = exitCode
+
+
+# Close extra file handles on UNIX (on Windows this cannot be done while
+# also redirecting input).
+kUseCloseFDs = not (platform.system() == "Windows")
+
+
+def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
+    """
+    Execute command ``command`` (list of arguments or string)
+    with
+    * working directory ``cwd`` (str), use None to use the current
+      working directory
+    * environment ``env`` (dict), use None for none
+    * Input to the command ``input`` (str), use string to pass
+      no input.
+    * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
+
+    Returns a tuple (out, err, exitCode) where
+    * ``out`` (str) is the standard output of running the command
+    * ``err`` (str) is the standard error of running the command
+    * ``exitCode`` (int) is the exitCode of running the command
+
+    If the timeout is hit an ``ExecuteCommandTimeoutException``
+    is raised.
+    """
+    if input is not None:
+        input = to_bytes(input)
+    p = subprocess.Popen(
+        command,
+        cwd=cwd,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+        close_fds=kUseCloseFDs,
+    )
+    timerObject = None
+    # FIXME: Because of the way nested function scopes work in Python 2.x we
+    # need to use a reference to a mutable object rather than a plain
+    # bool. In Python 3 we could use the "nonlocal" keyword but we need
+    # to support Python 2 as well.
+    hitTimeOut = [False]
+    try:
+        if timeout > 0:
+
+            def killProcess():
+                # We may be invoking a shell so we need to kill the
+                # process and all its children.
+                hitTimeOut[0] = True
+                killProcessAndChildren(p.pid)
+
+            timerObject = threading.Timer(timeout, killProcess)
+            timerObject.start()
+
+        out, err = p.communicate(input=input)
+        exitCode = p.wait()
+    finally:
+        if timerObject is not None:
+            timerObject.cancel()
+
+    # Ensure the resulting output is always of string type.
+    out = convert_string(out)
+    err = convert_string(err)
+
+    if hitTimeOut[0]:
+        raise ExecuteCommandTimeoutException(
+            msg="Reached timeout of {} seconds".format(timeout),
+            out=out,
+            err=err,
+            exitCode=exitCode,
+        )
+
+    # Detect Ctrl-C in subprocess.
+    if exitCode == -signal.SIGINT:
+        raise KeyboardInterrupt
+
+    return out, err, exitCode
+
+
+def killProcessAndChildren(pid):
+    """
+    This function kills a process with ``pid`` and all its
+    running children (recursively). It is currently implemented
+    using the psutil module which provides a simple platform
+    neutral implementation.
+
+    TODO: Reimplement this without using psutil so we can
+          remove our dependency on it.
+    """
+    if platform.system() == "AIX":
+        subprocess.call("kill -kill $(ps -o pid= -L{})".format(pid), shell=True)
+    else:
+        import psutil
+
+        try:
+            psutilProc = psutil.Process(pid)
+            # Handle the different psutil API versions
+            try:
+                # psutil >= 2.x
+                children_iterator = psutilProc.children(recursive=True)
+            except AttributeError:
+                # psutil 1.x
+                children_iterator = psutilProc.get_children(recursive=True)
+            for child in children_iterator:
+                try:
+                    child.kill()
+                except psutil.NoSuchProcess:
+                    pass
+            psutilProc.kill()
+        except psutil.NoSuchProcess:
+            pass
+
+
+def executeCommandVerbose(cmd, *args, **kwargs):
+    """
+    Execute a command and print its output on failure.
+    """
+    out, err, exitCode = executeCommand(cmd, *args, **kwargs)
+    if exitCode != 0:
+        report = makeReport(cmd, out, err, exitCode)
+        report += "\n\nFailed!"
+        sys.stderr.write("%s\n" % report)
+    return out, err, exitCode