Add cudf::dictionary::remove_duplicate_keys API

cpp/include/cudf/dictionary/detail/update_keys.hpp

@@ -60,6 +60,14 @@ std::unique_ptr<column> set_keys(dictionary_column_view const& dictionary_column
                                  cuda::stream_ref stream,
                                  rmm::device_async_resource_ref mr);
 
+/**
+ * @copydoc cudf::dictionary::remove_duplicate_keys(dictionary_column_view
+ * const&,rmm::device_async_resource_ref)
+ */
+std::unique_ptr<column> remove_duplicate_keys(dictionary_column_view const& dictionary_column,
+                                              cuda::stream_ref stream,
+                                              rmm::device_async_resource_ref mr);
+
 /**
  * @brief Remap the indices of a dictionary column to a new key set, returning
  * only the remapped index column with its null mask.

cpp/include/cudf/dictionary/update_keys.hpp

@@ -146,6 +146,32 @@ std::unique_ptr<column> set_keys(
   cuda::stream_ref stream           = cudf::get_default_stream(),
   rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
+/**
+ * @brief Create a new dictionary column by removing any duplicate keys.
+ *
+ * Any indices pointing to a duplicate key are remapped to just one of the duplicates.
+ * There is no guarantee which of the duplicate keys will be kept in the output column.
+ *
+ * @code{.pseudo}
+ * d1 = {keys=["b", "a", "b", "c"], indices=[3, 0, 2, 1, 2]}
+ * d2 = remove_duplicate_keys(d1)
+ * d2 is now {keys=["b", "a", "c"], indices=[2, 0, 0, 1, 0]}
+ * @endcode
+ *
+ * The output column will have the same number of rows as the input column.
+ * Null entries from the input column are copied to the output column.
+ * No new null entries are created by this operation.
+ *
+ * @param dictionary_column Existing dictionary column.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used to allocate the returned column's device memory.
+ * @return New dictionary column with unique keys.
+ */
+std::unique_ptr<column> remove_duplicate_keys(
+  dictionary_column_view const& dictionary_column,
+  cuda::stream_ref stream           = cudf::get_default_stream(),
+  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
+
 /**
  * @brief Create new dictionaries that have keys merged from the input dictionaries.
  *

cpp/src/dictionary/remove_keys.cu

@@ -9,7 +9,9 @@
 #include <cudf/detail/indexalator.cuh>
 #include <cudf/detail/nvtx/ranges.hpp>
 #include <cudf/detail/search.hpp>
+#include <cudf/detail/stream_compaction.hpp>
 #include <cudf/detail/valid_if.cuh>
+#include <cudf/dictionary/detail/update_keys.hpp>
 #include <cudf/dictionary/dictionary_column_view.hpp>
 #include <cudf/dictionary/dictionary_factories.hpp>
 #include <cudf/dictionary/update_keys.hpp>
@@ -194,6 +196,23 @@ std::unique_ptr<column> remove_unused_keys(dictionary_column_view const& diction
   return remove_keys_fn(dictionary_column, key_matcher, stream, mr);
 }
 
+std::unique_ptr<column> remove_duplicate_keys(dictionary_column_view const& input,
+                                              cuda::stream_ref stream,
+                                              rmm::device_async_resource_ref mr)
+{
+  if (input.size() == 0) { return std::make_unique<column>(input.parent(), stream, mr); }
+  // deduplicate keys
+  auto unique_keys = cudf::detail::stable_distinct(table_view{{input.keys()}},
+                                                   {0},
+                                                   duplicate_keep_option::KEEP_FIRST,
+                                                   null_equality::EQUAL,
+                                                   nan_equality::UNEQUAL,
+                                                   stream,
+                                                   cudf::get_current_device_resource_ref());
+  // set_keys then remaps all indices into this compacted key set in a single pass
+  return detail::set_keys(input, unique_keys->get_column(0).view(), stream, mr);
+}
+
 }  // namespace detail
 
 // external APIs
@@ -215,5 +234,13 @@ std::unique_ptr<column> remove_unused_keys(dictionary_column_view const& diction
   return detail::remove_unused_keys(dictionary_column, stream, mr);
 }
 
+std::unique_ptr<column> remove_duplicate_keys(dictionary_column_view const& dictionary_column,
+                                              cuda::stream_ref stream,
+                                              rmm::device_async_resource_ref mr)
+{
+  CUDF_FUNC_RANGE();
+  return detail::remove_duplicate_keys(dictionary_column, stream, mr);
+}
+
 }  // namespace dictionary
 }  // namespace cudf

cpp/tests/dictionary/remove_keys_test.cpp

@@ -9,6 +9,7 @@
 
 #include <cudf/copying.hpp>
 #include <cudf/dictionary/dictionary_column_view.hpp>
+#include <cudf/dictionary/dictionary_factories.hpp>
 #include <cudf/dictionary/encode.hpp>
 #include <cudf/dictionary/update_keys.hpp>
 #include <cudf/utilities/error.hpp>
@@ -163,3 +164,101 @@ TEST_F(DictionaryRemoveKeysTest, Errors)
   cudf::test::fixed_width_column_wrapper<int64_t> null_keys{{1, 2, 3}, {true, false, true}};
   EXPECT_THROW(cudf::dictionary::remove_keys(dictionary->view(), null_keys), cudf::logic_error);
 }
+
+TEST_F(DictionaryRemoveKeysTest, RemoveDuplicateKeysStrings)
+{
+  // Build a dictionary with intentionally duplicate keys via set_keys.
+  // keys = ["aaa", "ccc", "eee", "ccc"] (position 1 and 3 are both "ccc")
+  // indices point into that key set
+  auto input =
+    cudf::test::dictionary_column_wrapper<std::string>({"eee", "aaa", "ccc", "ccc", "eee", "aaa"});
+  auto dup_keys        = cudf::test::strings_column_wrapper{"aaa", "ccc", "eee", "ccc"};
+  auto const with_dups = cudf::dictionary::set_keys(input, dup_keys);
+  cudf::dictionary_column_view dv(with_dups->view());
+  EXPECT_EQ(dv.keys_size(), 4);  // confirm we have duplicates
+
+  auto const result = cudf::dictionary::remove_duplicate_keys(dv);
+  cudf::dictionary_column_view rv(result->view());
+
+  EXPECT_EQ(rv.keys_size(), 3);
+  cudf::test::strings_column_wrapper expected_decoded{"eee", "aaa", "ccc", "ccc", "eee", "aaa"};
+  auto const decoded = cudf::dictionary::decode(rv);
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(decoded->view(), expected_decoded);
+}
+
+TEST_F(DictionaryRemoveKeysTest, RemoveDuplicateKeysIntegers)
+{
+  // keys = [10, 20, 10, 30] (10 appears at positions 0 and 2)
+  // All indices pointing to position 2 must be remapped to position 0 after dedup
+  cudf::test::fixed_width_column_wrapper<int32_t> keys_col{10, 20, 10, 30};
+  cudf::test::fixed_width_column_wrapper<int32_t> indices_col{2, 0, 3, 1, 2, 0};
+  auto with_dups = cudf::make_dictionary_column(
+    keys_col.release(), indices_col.release(), rmm::device_buffer{}, 0);
+  cudf::dictionary_column_view dv(with_dups->view());
+  EXPECT_EQ(dv.keys_size(), 4);
+
+  auto const result = cudf::dictionary::remove_duplicate_keys(dv);
+  cudf::dictionary_column_view rv(result->view());
+
+  EXPECT_EQ(rv.keys_size(), 3);
+  cudf::test::fixed_width_column_wrapper<int32_t> expected_decoded{10, 10, 30, 20, 10, 10};
+  auto const decoded = cudf::dictionary::decode(rv);
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(decoded->view(), expected_decoded);
+}
+
+TEST_F(DictionaryRemoveKeysTest, RemoveDuplicateKeysNoDuplicates)
+{
+  // When no duplicates exist the output should be equivalent to the input
+  auto input        = cudf::test::dictionary_column_wrapper<std::string>({"b", "a", "c", "a", "b"});
+  auto const result = cudf::dictionary::remove_duplicate_keys(cudf::dictionary_column_view(input));
+  auto const decoded = cudf::dictionary::decode(cudf::dictionary_column_view(result->view()));
+  cudf::test::strings_column_wrapper expected{"b", "a", "c", "a", "b"};
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(decoded->view(), expected);
+}
+
+TEST_F(DictionaryRemoveKeysTest, RemoveDuplicateKeysEmpty)
+{
+  auto input        = cudf::test::dictionary_column_wrapper<int32_t>({});
+  auto const result = cudf::dictionary::remove_duplicate_keys(cudf::dictionary_column_view(input));
+  EXPECT_EQ(result->size(), 0);
+  EXPECT_EQ(cudf::dictionary_column_view(result->view()).keys_size(), 0);
+}
+
+TEST_F(DictionaryRemoveKeysTest, RemoveDuplicateKeysSliced)
+{
+  // Build a dictionary with duplicate keys then slice it to exercise offset handling.
+  // keys = ["aaa", "ccc", "eee", "ccc"] — "ccc" appears at positions 1 and 3
+  auto input =
+    cudf::test::dictionary_column_wrapper<std::string>({"eee", "aaa", "ccc", "ccc", "eee", "aaa"});
+  auto dup_keys        = cudf::test::strings_column_wrapper{"aaa", "ccc", "eee", "ccc"};
+  auto const with_dups = cudf::dictionary::set_keys(input, dup_keys);
+
+  // Slice to rows [1, 4) → logical values {"aaa", "ccc", "ccc"}
+  auto sliced       = cudf::slice(with_dups->view(), {1, 4}).front();
+  auto const result = cudf::dictionary::remove_duplicate_keys(cudf::dictionary_column_view(sliced));
+  cudf::dictionary_column_view rv(result->view());
+
+  EXPECT_EQ(rv.keys_size(), 3);
+  cudf::test::strings_column_wrapper expected_decoded{"aaa", "ccc", "ccc"};
+  auto const decoded = cudf::dictionary::decode(rv);
+  CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(decoded->view(), expected_decoded);
+}
+
+TEST_F(DictionaryRemoveKeysTest, RemoveDuplicateKeysWithNullRows)
+{
+  // Null rows in the parent must be preserved; no new nulls are introduced
+  auto input = cudf::test::dictionary_column_wrapper<std::string>(
+    {"eee", "aaa", "ccc", "ccc", "eee"}, {true, true, false, true, true});
+  auto dup_keys        = cudf::test::strings_column_wrapper{"aaa", "ccc", "eee", "ccc"};
+  auto const with_dups = cudf::dictionary::set_keys(input, dup_keys);
+  cudf::dictionary_column_view dv(with_dups->view());
+
+  auto const result = cudf::dictionary::remove_duplicate_keys(dv);
+  cudf::dictionary_column_view rv(result->view());
+  EXPECT_EQ(rv.keys_size(), 3);
+
+  cudf::test::strings_column_wrapper expected_decoded({"eee", "aaa", "", "ccc", "eee"},
+                                                      {true, true, false, true, true});
+  auto const decoded = cudf::dictionary::decode(rv);
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(decoded->view(), expected_decoded);
+}

cpp/tests/streams/dictionary_test.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION.
+ * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
 
@@ -14,6 +14,8 @@
 #include <cudf/dictionary/search.hpp>
 #include <cudf/dictionary/update_keys.hpp>
 
+#include <rmm/device_buffer.hpp>
+
 class DictionaryTest : public cudf::test::BaseFixture {};
 
 TEST_F(DictionaryTest, FactoryColumnViews)
@@ -98,6 +100,15 @@ TEST_F(DictionaryTest, RemoveUnsedKeys)
   cudf::dictionary::remove_unused_keys(dict_col_view, cudf::test::get_default_stream());
 }
 
+TEST_F(DictionaryTest, RemoveDuplicateKeys)
+{
+  cudf::test::fixed_width_column_wrapper<int32_t> keys_col{10, 20, 10, 30};
+  cudf::test::fixed_width_column_wrapper<int32_t> indices_col{0, 1, 2, 3, 0};
+  auto dict = cudf::make_dictionary_column(
+    keys_col.release(), indices_col.release(), rmm::device_buffer{}, 0);
+  cudf::dictionary::remove_duplicate_keys(dict->view(), cudf::test::get_default_stream());
+}
+
 TEST_F(DictionaryTest, SetKeys)
 {
   std::vector<int32_t> elements{0, 2, 2, 6, 6};