#include #include #include #include #include #include #include #include using namespace unittest; // ensure that we properly support thrust::zip_iterator from cuda::std void TestZipIteratorTraits() { using base_it = thrust::host_vector::iterator; { using it = thrust::zip_iterator>; using traits = cuda::std::iterator_traits; using reference = thrust::detail::tuple_of_iterator_references; static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v>); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v, thrust::random_access_traversal_tag>); static_assert(cuda::std::__has_random_access_traversal); static_assert(!cuda::std::output_iterator); static_assert(cuda::std::input_iterator); static_assert(cuda::std::forward_iterator); static_assert(cuda::std::bidirectional_iterator); static_assert(cuda::std::random_access_iterator); static_assert(!cuda::std::contiguous_iterator); } { // working with proxy iterator cuda::discard_iterator using it = thrust::zip_iterator>; using traits = cuda::std::iterator_traits; using value_type = cuda::std::tuple; using reference = thrust::detail::tuple_of_iterator_references; static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(!cuda::std::output_iterator); static_assert(cuda::std::input_iterator); static_assert(cuda::std::forward_iterator); static_assert(cuda::std::bidirectional_iterator); static_assert(cuda::std::random_access_iterator); static_assert(!cuda::std::contiguous_iterator); } { // working with proxy iterator cuda::tabulate_output_iterator using it = thrust::zip_iterator, short>>>; using traits = cuda::std::iterator_traits; using value_type = cuda::std::tuple, short>>; using reference = thrust::detail::tuple_of_iterator_references, short>>; static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(!cuda::std::output_iterator); static_assert(cuda::std::input_iterator); static_assert(cuda::std::forward_iterator); static_assert(cuda::std::bidirectional_iterator); static_assert(cuda::std::random_access_iterator); static_assert(!cuda::std::contiguous_iterator); } { // working with proxy iterator cuda::transform_output_iterator using it = thrust::zip_iterator, short*>>>; using traits = cuda::std::iterator_traits; using value_type = cuda::std::tuple, short*>>; using reference = thrust::detail::tuple_of_iterator_references, short*>>; static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(!cuda::std::output_iterator); static_assert(cuda::std::input_iterator); static_assert(cuda::std::forward_iterator); static_assert(cuda::std::bidirectional_iterator); static_assert(cuda::std::random_access_iterator); static_assert(!cuda::std::contiguous_iterator); } { // working with proxy iterator cuda::transform_input_output_iterator using it = thrust::zip_iterator< cuda::std::tuple, cuda::std::plus<>, short*>>>; using traits = cuda::std::iterator_traits; using value_type = cuda::std::tuple; using reference = thrust::detail::tuple_of_iterator_references< int&, cuda::__transform_input_output_proxy, cuda::std::plus<>, short*>>; static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(cuda::std::is_same_v); static_assert(!cuda::std::output_iterator); static_assert(cuda::std::input_iterator); static_assert(cuda::std::forward_iterator); static_assert(cuda::std::bidirectional_iterator); static_assert(cuda::std::random_access_iterator); static_assert(!cuda::std::contiguous_iterator); } } DECLARE_UNITTEST(TestZipIteratorTraits); template struct TestZipIteratorConstructionFromIterators { template void test() { Vector v0(4); Vector v1(4); Vector v2(4); // initialize input thrust::sequence(v0.begin(), v0.end()); thrust::sequence(v1.begin(), v1.end()); thrust::sequence(v2.begin(), v2.end()); using IteratorTuple = cuda::std::tuple; using ZipIterator = thrust::zip_iterator; // test construction thrust::zip_iterator iter0(v0.begin(), v1.begin()); ASSERT_EQUAL(true, iter0 == ZipIterator{cuda::std::make_tuple(v0.begin(), v1.begin())}); } void operator()() { test>(); test>(); } }; SimpleUnitTest> TestZipIteratorConstructionFromIteratorsInstance; template struct TestZipIteratorManipulation { template void test() { Vector v0(4); Vector v1(4); Vector v2(4); // initialize input thrust::sequence(v0.begin(), v0.end()); thrust::sequence(v1.begin(), v1.end()); thrust::sequence(v2.begin(), v2.end()); using IteratorTuple = cuda::std::tuple; IteratorTuple t = cuda::std::make_tuple(v0.begin(), v1.begin()); using ZipIterator = thrust::zip_iterator; // test construction from tuple ZipIterator iter0 = thrust::make_zip_iterator(t); ASSERT_EQUAL(true, iter0 == ZipIterator{t}); ASSERT_EQUAL_QUIET(v0.begin(), cuda::std::get<0>(iter0.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin(), cuda::std::get<1>(iter0.get_iterator_tuple())); static_assert(cuda::std::is_same_v); // CTAD // test construction from pack ZipIterator iter0_pack = thrust::make_zip_iterator(v0.begin(), v1.begin()); ASSERT_EQUAL(true, (iter0_pack == ZipIterator{v0.begin(), v1.begin()})); ASSERT_EQUAL_QUIET(v0.begin(), cuda::std::get<0>(iter0_pack.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin(), cuda::std::get<1>(iter0_pack.get_iterator_tuple())); static_assert(cuda::std::is_same_v); // CTAD // test dereference ASSERT_EQUAL(*v0.begin(), cuda::std::get<0>(*iter0)); ASSERT_EQUAL(*v1.begin(), cuda::std::get<1>(*iter0)); // test equality ZipIterator iter1 = iter0; ZipIterator iter2 = thrust::make_zip_iterator(v0.begin(), v2.begin()); ZipIterator iter3 = thrust::make_zip_iterator(v1.begin(), v2.begin()); ASSERT_EQUAL(true, iter0 == iter1); ASSERT_EQUAL(true, iter0 == iter2); ASSERT_EQUAL(false, iter0 == iter3); // test inequality ASSERT_EQUAL(false, iter0 != iter1); ASSERT_EQUAL(false, iter0 != iter2); ASSERT_EQUAL(true, iter0 != iter3); // test advance ZipIterator iter4 = iter0 + 1; ASSERT_EQUAL_QUIET(v0.begin() + 1, cuda::std::get<0>(iter4.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin() + 1, cuda::std::get<1>(iter4.get_iterator_tuple())); // test pre-increment ++iter4; ASSERT_EQUAL_QUIET(v0.begin() + 2, cuda::std::get<0>(iter4.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin() + 2, cuda::std::get<1>(iter4.get_iterator_tuple())); // test post-increment iter4++; ASSERT_EQUAL_QUIET(v0.begin() + 3, cuda::std::get<0>(iter4.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin() + 3, cuda::std::get<1>(iter4.get_iterator_tuple())); // test pre-decrement --iter4; ASSERT_EQUAL_QUIET(v0.begin() + 2, cuda::std::get<0>(iter4.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin() + 2, cuda::std::get<1>(iter4.get_iterator_tuple())); // test post-decrement iter4--; ASSERT_EQUAL_QUIET(v0.begin() + 1, cuda::std::get<0>(iter4.get_iterator_tuple())); ASSERT_EQUAL_QUIET(v1.begin() + 1, cuda::std::get<1>(iter4.get_iterator_tuple())); // test difference ASSERT_EQUAL(1, iter4 - iter0); ASSERT_EQUAL(-1, iter0 - iter4); } void operator()() { test>(); test>(); } }; SimpleUnitTest> TestZipIteratorManipulationInstance; static_assert(cuda::std::is_trivially_copy_constructible>>::value); template struct TestZipIteratorReference { void operator()() { // test host types using Iterator1 = typename thrust::host_vector::iterator; using Iterator2 = typename thrust::host_vector::const_iterator; using IteratorTuple1 = cuda::std::tuple; using ZipIterator1 = thrust::zip_iterator; using zip_iterator_reference_type1 = thrust::detail::it_reference_t; thrust::host_vector h_variable(1); using reference_type1 = cuda::std::tuple; reference_type1 ref1(*h_variable.begin(), *h_variable.cbegin()); zip_iterator_reference_type1 test1(*h_variable.begin(), *h_variable.cbegin()); ASSERT_EQUAL_QUIET(ref1, test1); ASSERT_EQUAL(cuda::std::get<0>(ref1), cuda::std::get<0>(test1)); ASSERT_EQUAL(cuda::std::get<1>(ref1), cuda::std::get<1>(test1)); // test device types using Iterator3 = typename thrust::device_vector::iterator; using Iterator4 = typename thrust::device_vector::const_iterator; using IteratorTuple2 = cuda::std::tuple; using ZipIterator2 = thrust::zip_iterator; using zip_iterator_reference_type2 = thrust::detail::it_reference_t; thrust::device_vector d_variable(1); using reference_type2 = cuda::std::tuple, thrust::device_reference>; reference_type2 ref2(*d_variable.begin(), *d_variable.cbegin()); zip_iterator_reference_type2 test2(*d_variable.begin(), *d_variable.cbegin()); ASSERT_EQUAL_QUIET(ref2, test2); ASSERT_EQUAL(cuda::std::get<0>(ref2), cuda::std::get<0>(test2)); ASSERT_EQUAL(cuda::std::get<1>(ref2), cuda::std::get<1>(test2)); } // end operator()() }; SimpleUnitTest TestZipIteratorReferenceInstance; template void TestZipIteratorCopy() { using T = typename Vector::value_type; Vector input0(4), input1(4); Vector output0(4), output1(4); // initialize input thrust::sequence(input0.begin(), input0.end(), T{0}); thrust::sequence(input1.begin(), input1.end(), T{13}); thrust::copy(thrust::make_zip_iterator(input0.begin(), input1.begin()), thrust::make_zip_iterator(input0.end(), input1.end()), thrust::make_zip_iterator(output0.begin(), output1.begin())); ASSERT_EQUAL(input0, output0); ASSERT_EQUAL(input1, output1); } DECLARE_VECTOR_UNITTEST(TestZipIteratorCopy); struct SumTwoTuple { template _CCCL_HOST_DEVICE cuda::std::remove_reference_t> operator()(Tuple x) const { return cuda::std::get<0>(x) + cuda::std::get<1>(x); } }; // end SumTwoTuple struct SumThreeTuple { template _CCCL_HOST_DEVICE cuda::std::remove_reference_t> operator()(Tuple x) const { return cuda::std::get<0>(x) + cuda::std::get<1>(x) + cuda::std::get<2>(x); } }; // end SumThreeTuple template struct TestZipIteratorTransform { void operator()(const size_t n) { thrust::host_vector h_data0 = unittest::random_samples(n); thrust::host_vector h_data1 = unittest::random_samples(n); thrust::host_vector h_data2 = unittest::random_samples(n); thrust::device_vector d_data0 = h_data0; thrust::device_vector d_data1 = h_data1; thrust::device_vector d_data2 = h_data2; thrust::host_vector h_result(n); thrust::device_vector d_result(n); // Tuples with 2 elements thrust::transform(thrust::make_zip_iterator(h_data0.begin(), h_data1.begin()), thrust::make_zip_iterator(h_data0.end(), h_data1.end()), h_result.begin(), SumTwoTuple()); thrust::transform(thrust::make_zip_iterator(d_data0.begin(), d_data1.begin()), thrust::make_zip_iterator(d_data0.end(), d_data1.end()), d_result.begin(), SumTwoTuple()); ASSERT_EQUAL(h_result, d_result); // Tuples with 3 elements thrust::transform(thrust::make_zip_iterator(h_data0.begin(), h_data1.begin(), h_data2.begin()), thrust::make_zip_iterator(h_data0.end(), h_data1.end(), h_data2.end()), h_result.begin(), SumThreeTuple()); thrust::transform(thrust::make_zip_iterator(d_data0.begin(), d_data1.begin(), d_data2.begin()), thrust::make_zip_iterator(d_data0.end(), d_data1.end(), d_data2.end()), d_result.begin(), SumThreeTuple()); ASSERT_EQUAL(h_result, d_result); } }; VariableUnitTest TestZipIteratorTransformInstance; void TestZipIteratorCopyAoSToSoA() { const size_t n = 1; using structure = cuda::std::tuple; using host_array_of_structures = thrust::host_vector; using device_array_of_structures = thrust::device_vector; using host_structure_of_arrays = thrust::zip_iterator::iterator, thrust::host_vector::iterator>>; using device_structure_of_arrays = thrust::zip_iterator::iterator, thrust::device_vector::iterator>>; host_array_of_structures h_aos(n, cuda::std::make_tuple(7, 13)); device_array_of_structures d_aos(n, cuda::std::make_tuple(7, 13)); // host to host thrust::host_vector h_field0(n), h_field1(n); host_structure_of_arrays h_soa = thrust::make_zip_iterator(h_field0.begin(), h_field1.begin()); thrust::copy(h_aos.begin(), h_aos.end(), h_soa); ASSERT_EQUAL_QUIET(cuda::std::make_tuple(7, 13), h_soa[0]); // host to device thrust::device_vector d_field0(n), d_field1(n); device_structure_of_arrays d_soa = thrust::make_zip_iterator(d_field0.begin(), d_field1.begin()); thrust::copy(h_aos.begin(), h_aos.end(), d_soa); ASSERT_EQUAL_QUIET(cuda::std::make_tuple(7, 13), d_soa[0]); // device to device thrust::fill(d_field0.begin(), d_field0.end(), 0); thrust::fill(d_field1.begin(), d_field1.end(), 0); thrust::copy(d_aos.begin(), d_aos.end(), d_soa); ASSERT_EQUAL_QUIET(cuda::std::make_tuple(7, 13), d_soa[0]); // device to host thrust::fill(h_field0.begin(), h_field0.end(), 0); thrust::fill(h_field1.begin(), h_field1.end(), 0); thrust::copy(d_aos.begin(), d_aos.end(), h_soa); ASSERT_EQUAL_QUIET(cuda::std::make_tuple(7, 13), h_soa[0]); } DECLARE_UNITTEST(TestZipIteratorCopyAoSToSoA); void TestZipIteratorCopySoAToAoS() { const size_t n = 1; using structure = cuda::std::tuple; using host_array_of_structures = thrust::host_vector; using device_array_of_structures = thrust::device_vector; using host_structure_of_arrays = thrust::zip_iterator::iterator, thrust::host_vector::iterator>>; using device_structure_of_arrays = thrust::zip_iterator::iterator, thrust::device_vector::iterator>>; thrust::host_vector h_field0(n, 7), h_field1(n, 13); thrust::device_vector d_field0(n, 7), d_field1(n, 13); host_structure_of_arrays h_soa = thrust::make_zip_iterator(h_field0.begin(), h_field1.begin()); device_structure_of_arrays d_soa = thrust::make_zip_iterator(d_field0.begin(), d_field1.begin()); host_array_of_structures h_aos(n); device_array_of_structures d_aos(n); // host to host thrust::fill(h_aos.begin(), h_aos.end(), cuda::std::make_tuple(0, 0)); thrust::copy(h_soa, h_soa + n, h_aos.begin()); ASSERT_EQUAL_QUIET(7, cuda::std::get<0>(h_soa[0])); ASSERT_EQUAL_QUIET(13, cuda::std::get<1>(h_soa[0])); // host to device thrust::fill(d_aos.begin(), d_aos.end(), cuda::std::make_tuple(0, 0)); thrust::copy(h_soa, h_soa + n, d_aos.begin()); ASSERT_EQUAL_QUIET(7, cuda::std::get<0>(d_soa[0])); ASSERT_EQUAL_QUIET(13, cuda::std::get<1>(d_soa[0])); // device to device thrust::fill(d_aos.begin(), d_aos.end(), cuda::std::make_tuple(0, 0)); thrust::copy(d_soa, d_soa + n, d_aos.begin()); ASSERT_EQUAL_QUIET(7, cuda::std::get<0>(d_soa[0])); ASSERT_EQUAL_QUIET(13, cuda::std::get<1>(d_soa[0])); // device to host thrust::fill(h_aos.begin(), h_aos.end(), cuda::std::make_tuple(0, 0)); thrust::copy(d_soa, d_soa + n, h_aos.begin()); ASSERT_EQUAL_QUIET(7, cuda::std::get<0>(h_soa[0])); ASSERT_EQUAL_QUIET(13, cuda::std::get<1>(h_soa[0])); }; DECLARE_UNITTEST(TestZipIteratorCopySoAToAoS); template void TestZipIteratorDereferenceToValueType(const T& t) { thrust::device_vector data(1, t); // verify that storing the result of dereferencing a zip_iterator and then subsequently converting to its value type // is handled correctly auto a = thrust::make_zip_iterator(data.begin()); static_assert(cuda::std::is_same_v, cuda::std::iter_value_t>); auto b = a[0]; static_assert( cuda::std::is_same_v>, decltype(b)>); // verify that the stored tuple_of_iterator_references> can be cast to tuple auto c = cuda::std::tuple(b); static_assert(cuda::std::is_same_v, decltype(c)>); ASSERT_EQUAL_QUIET(c, cuda::std::make_tuple(t)); } void TestZipIteratorDereferenceToValue() { TestZipIteratorDereferenceToValueType(1); TestZipIteratorDereferenceToValueType(cuda::std::make_tuple(1)); TestZipIteratorDereferenceToValueType(cuda::std::make_tuple(1, cuda::std::make_tuple(1))); TestZipIteratorDereferenceToValueType(cuda::std::make_tuple(1, cuda::std::make_tuple(1, 1))); TestZipIteratorDereferenceToValueType(cuda::std::make_tuple(cuda::std::make_tuple(1), cuda::std::make_tuple(1, 1))); } DECLARE_UNITTEST(TestZipIteratorDereferenceToValue); void TestZipIteratorNestedCopy() { using T = int; { thrust::device_vector a(10, 1); thrust::device_vector>> b(a.size()); thrust::copy_n(thrust::make_zip_iterator(thrust::make_zip_iterator(a.begin())), a.size(), b.begin()); decltype(b) b_expected(b.size(), cuda::std::make_tuple(cuda::std::make_tuple(1))); ASSERT_EQUAL_QUIET(b, b_expected); } { thrust::device_vector a(10, 1); thrust::device_vector, cuda::std::tuple>> b(a.size()); thrust::copy_n(thrust::make_zip_iterator(thrust::make_zip_iterator(a.begin(), a.begin()), thrust::make_zip_iterator(a.begin(), a.begin())), a.size(), b.begin()); decltype(b) b_expected(b.size(), cuda::std::make_tuple(cuda::std::make_tuple(1, 1), cuda::std::make_tuple(1, 1))); ASSERT_EQUAL_QUIET(b, b_expected); } { thrust::device_vector> a(10, cuda::std::make_tuple(1, 1)); thrust::device_vector< cuda::std::tuple, cuda::std::tuple, cuda::std::tuple, cuda::std::tuple>> b(a.size()); thrust::copy_n(thrust::make_zip_iterator(a.begin(), a.begin(), a.begin(), a.begin()), a.size(), b.begin()); decltype(b) b_expected( b.size(), cuda::std::make_tuple(cuda::std::make_tuple(1, 1), cuda::std::make_tuple(1, 1), cuda::std::make_tuple(1, 1), cuda::std::make_tuple(1, 1))); ASSERT_EQUAL_QUIET(b, b_expected); } } DECLARE_UNITTEST(TestZipIteratorNestedCopy); // See https://github.com/NVIDIA/cccl/issues/9773 void TestZipIteratorComparison() { using T = int; thrust::device_vector a{5, 4, 3, 2, 1, 0}; thrust::device_vector b{1, 2, 3, 4, 5, 6}; { static_assert( cuda::std::is_convertible_v, cuda::std::tuple>); auto iter = thrust::make_zip_iterator(a.data(), b.data()); auto pos = thrust::find(iter, iter + 6, cuda::std::tuple{4, 2}); ASSERT_EQUAL_QUIET(pos, iter + 1); } { auto iter = thrust::make_zip_iterator(a.begin(), b.begin()); auto pos = thrust::find(iter, iter + 6, cuda::std::tuple{4, 2}); ASSERT_EQUAL_QUIET(pos, iter + 1); } } DECLARE_UNITTEST(TestZipIteratorComparison);