Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions cpp/include/cuvs/neighbors/common.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -853,14 +853,6 @@ using enable_if_valid_list_t = typename enable_if_valid_list<ListT, T>::type;
* `cuvs::neighbors::ivf_pq::helpers::resize_list` which handle type casting internally.
*/
template <typename ListT>
CUVS_EXPORT void resize_list(raft::resources const& res,
std::shared_ptr<ListT>& orig_list, // NOLINT
const typename ListT::spec_type& spec,
typename ListT::size_type new_used_size,
typename ListT::size_type old_logical_size,
typename ListT::size_type old_used_size);

template <typename ListT>
CUVS_EXPORT void resize_list(raft::resources const& res,
std::shared_ptr<ListT>& orig_list, // NOLINT
const typename ListT::spec_type& spec,
Expand Down
6 changes: 4 additions & 2 deletions cpp/include/cuvs/neighbors/ivf_flat.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand All @@ -11,6 +11,7 @@
#include <cuvs/neighbors/common.hpp>
#include <raft/core/host_mdarray.hpp>
#include <raft/core/host_mdspan.hpp>
#include <raft/util/integer_utils.hpp>
#include <sstream>
#include <string>

Expand Down Expand Up @@ -110,7 +111,8 @@ struct list_spec {
/** Determine the extents of an array enough to hold a given amount of data. */
constexpr auto make_list_extents(SizeT n_rows) const -> list_extents
{
return raft::make_extents<SizeT>(n_rows, dim);
return raft::make_extents<SizeT>(raft::round_up_safe<SizeT>(n_rows, SizeT{kIndexGroupSize}),
dim);
}
};

Expand Down
5 changes: 3 additions & 2 deletions cpp/include/cuvs/neighbors/ivf_sq.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -118,7 +118,8 @@ struct CUVS_EXPORT list_spec {
constexpr auto make_list_extents(SizeT n_rows) const -> list_extents
{
uint32_t padded = raft::round_up_safe<uint32_t>(dim, kVecLen);
return raft::make_extents<SizeT>(n_rows, padded);
return raft::make_extents<SizeT>(raft::round_up_safe<SizeT>(n_rows, SizeT{kIndexGroupSize}),
padded);
}
};

Expand Down
8 changes: 2 additions & 6 deletions cpp/src/neighbors/ivf_flat/ivf_flat_build.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -282,12 +282,8 @@ void extend(raft::resources const& handle,
raft::resource::sync_stream(handle);
auto& lists = index->lists();
for (uint32_t label = 0; label < n_lists; label++) {
ivf::resize_list(handle,
lists[label],
list_device_spec,
new_list_sizes[label],
old_list_sizes[label],
raft::Pow2<kIndexGroupSize>::roundUp(old_list_sizes[label]));
ivf::resize_list(
handle, lists[label], list_device_spec, new_list_sizes[label], old_list_sizes[label]);
}
}
// Update the pointers and the sizes
Expand Down
18 changes: 2 additions & 16 deletions cpp/src/neighbors/ivf_list.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,13 @@ CUVS_EXPORT void resize_list(raft::resources const& res,
std::shared_ptr<ListT>& orig_list, // NOLINT
const typename ListT::spec_type& spec,
typename ListT::size_type new_used_size,
typename ListT::size_type old_logical_size,
typename ListT::size_type old_used_size)
{
// old_logical_size is the previous visible size from this index's list_sizes().
// old_used_size is the old allocation copy extent and may include padded slots
// required by interleaved list layouts.
bool skip_resize = false;
if (orig_list) {
if (new_used_size <= orig_list->indices.extent(0)) {
auto shared_list_size = old_logical_size;
if (new_used_size <= old_logical_size ||
auto shared_list_size = old_used_size;
if (new_used_size <= old_used_size ||
orig_list->size.compare_exchange_strong(shared_list_size, new_used_size)) {
// We don't need to resize the list if:
// 1. The list exists
Expand Down Expand Up @@ -108,16 +104,6 @@ CUVS_EXPORT void resize_list(raft::resources const& res,
new_list.swap(orig_list);
}

template <typename ListT>
CUVS_EXPORT void resize_list(raft::resources const& res,
std::shared_ptr<ListT>& orig_list, // NOLINT
const typename ListT::spec_type& spec,
typename ListT::size_type new_used_size,
typename ListT::size_type old_used_size)
{
resize_list(res, orig_list, spec, new_used_size, old_used_size, old_used_size);
}

template <typename ListT>
enable_if_valid_list_t<ListT> serialize_list(const raft::resources& handle,
std::ostream& os,
Expand Down
8 changes: 2 additions & 6 deletions cpp/src/neighbors/ivf_sq/ivf_sq_build.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -354,12 +354,8 @@ void extend_inplace(raft::resources const& handle,
raft::resource::sync_stream(handle);
auto& lists = index->lists();
for (uint32_t label = 0; label < n_lists; label++) {
ivf::resize_list(handle,
lists[label],
list_device_spec,
new_list_sizes[label],
old_list_sizes[label],
raft::Pow2<kIndexGroupSize>::roundUp(old_list_sizes[label]));
ivf::resize_list(
handle, lists[label], list_device_spec, new_list_sizes[label], old_list_sizes[label]);
}
}
ivf::detail::recompute_internal_state(handle, *index);
Expand Down