Skip to content

Commit 7e0e835

Browse files
yuejiaointelgithub-actions[bot]
yuejiaointel
and
github-actions[bot]
authored
fix: change logging level and add verify function in dynamic index constructor (#150)
Logging level changes in some functions and add verify function in dynamic index ctr --------- Co-authored-by: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com>
1 parent af9bad3 commit 7e0e835

File tree

7 files changed

+58
-52
lines changed

7 files changed

+58
-52
lines changed

‎include/svs/index/vamana/dynamic_index.h‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,7 @@ class MutableVamanaIndex {
225225
, build_parameters_(parameters)
226226
, logger_{std::move(logger)} {
227227
// Verify and set defaults directly on the input parameters
228-
verify_and_set_default_index_parameters(
229-
build_parameters_, distance_function, logger_
230-
);
228+
verify_and_set_default_index_parameters(build_parameters_, distance_function);
231229

232230
// Set graph again as verify function might change graph_max_degree parameter
233231
graph_ = Graph{data_.size(), build_parameters_.graph_max_degree};
@@ -248,7 +246,13 @@ class MutableVamanaIndex {
248246
auto prefetch_parameters =
249247
GreedySearchPrefetchParameters{sp.prefetch_lookahead_, sp.prefetch_step_};
250248
auto builder = VamanaBuilder(
251-
graph_, data_, distance_, build_parameters_, threadpool_, prefetch_parameters
249+
graph_,
250+
data_,
251+
distance_,
252+
build_parameters_,
253+
threadpool_,
254+
prefetch_parameters,
255+
logger_
252256
);
253257
builder.construct(1.0f, entry_point_[0], logging::Level::Trace, logger_);
254258
builder.construct(
@@ -699,7 +703,14 @@ class MutableVamanaIndex {
699703
auto prefetch_parameters =
700704
GreedySearchPrefetchParameters{sp.prefetch_lookahead_, sp.prefetch_step_};
701705
VamanaBuilder builder{
702-
graph_, data_, distance_, parameters, threadpool_, prefetch_parameters
706+
graph_,
707+
data_,
708+
distance_,
709+
parameters,
710+
threadpool_,
711+
prefetch_parameters,
712+
logger_,
713+
logging::Level::Trace
703714
};
704715
builder.construct(alpha_, entry_point(), slots, logging::Level::Trace, logger_);
705716
// Mark all added entries as valid.

‎include/svs/index/vamana/index.h‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -417,16 +417,15 @@ class VamanaIndex {
417417
}
418418
build_parameters_ = parameters;
419419
// verify the parameters before set local var
420-
verify_and_set_default_index_parameters(
421-
build_parameters_, distance_function, logger
422-
);
420+
verify_and_set_default_index_parameters(build_parameters_, distance_function);
423421
auto builder = VamanaBuilder(
424422
graph_,
425423
data_,
426424
distance_,
427425
build_parameters_,
428426
threadpool_,
429-
extensions::estimate_prefetch_parameters(data_)
427+
extensions::estimate_prefetch_parameters(data_),
428+
logger
430429
);
431430

432431
builder.construct(1.0F, entry_point_[0], logging::Level::Trace, logger);
@@ -947,7 +946,7 @@ auto auto_build(
947946

948947
// Default graph.
949948
auto verified_parameters = parameters;
950-
verify_and_set_default_index_parameters(verified_parameters, distance, logger);
949+
verify_and_set_default_index_parameters(verified_parameters, distance);
951950
auto graph =
952951
default_graph(data.size(), verified_parameters.graph_max_degree, graph_allocator);
953952
using I = typename decltype(graph)::index_type;
@@ -1018,9 +1017,7 @@ auto auto_assemble(
10181017
/// @brief Verify parameters and set defaults if needed
10191018
template <typename Dist>
10201019
void verify_and_set_default_index_parameters(
1021-
VamanaBuildParameters& parameters,
1022-
Dist distance_function,
1023-
svs::logging::logger_ptr logger = svs::logging::get()
1020+
VamanaBuildParameters& parameters, Dist distance_function
10241021
) {
10251022
// Set default values
10261023
if (parameters.max_candidate_pool_size == svs::UNSIGNED_INTEGER_PLACEHOLDER) {
@@ -1068,20 +1065,5 @@ void verify_and_set_default_index_parameters(
10681065
if (parameters.prune_to > parameters.graph_max_degree) {
10691066
throw std::invalid_argument("prune_to must be <= graph_max_degree");
10701067
}
1071-
1072-
// Print all parameters
1073-
svs::logging::log(
1074-
logger,
1075-
logging::Level::Info,
1076-
"Vamana Build Parameters: alpha={}, graph_max_degree={}, "
1077-
"max_candidate_pool_size={}, prune_to={}, window_size={}, "
1078-
"use_full_search_history={}",
1079-
parameters.alpha,
1080-
parameters.graph_max_degree,
1081-
parameters.max_candidate_pool_size,
1082-
parameters.prune_to,
1083-
parameters.window_size,
1084-
parameters.use_full_search_history
1085-
);
10861068
}
10871069
} // namespace svs::index::vamana

‎include/svs/index/vamana/vamana_build.h‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,11 @@ template <typename Idx> class BackedgeBuffer {
124124
, bucket_locks_{parameters.num_buckets_} {}
125125

126126
BackedgeBuffer(size_t num_elements, size_t bucket_size)
127-
: BackedgeBuffer(BackedgeBufferParameters{
128-
bucket_size, lib::div_round_up(num_elements, bucket_size)}) {}
127+
: BackedgeBuffer(
128+
BackedgeBufferParameters{
129+
bucket_size, lib::div_round_up(num_elements, bucket_size)
130+
}
131+
) {}
129132

130133
// Add a point.
131134
void add_edge(Idx src, Idx dst) {
@@ -184,7 +187,9 @@ class VamanaBuilder {
184187
Dist distance_function,
185188
const VamanaBuildParameters& params,
186189
Pool& threadpool,
187-
GreedySearchPrefetchParameters prefetch_hint = {}
190+
GreedySearchPrefetchParameters prefetch_hint = {},
191+
svs::logging::logger_ptr logger = svs::logging::get(),
192+
logging::Level level = logging::Level::Debug
188193
)
189194
: graph_{graph}
190195
, data_{data}
@@ -194,6 +199,20 @@ class VamanaBuilder {
194199
, threadpool_{threadpool}
195200
, vertex_locks_(data.size())
196201
, backedge_buffer_{data.size(), 1000} {
202+
// Print all parameters
203+
svs::logging::log(
204+
logger,
205+
level,
206+
"Vamana Build Parameters: alpha={}, graph_max_degree={}, "
207+
"max_candidate_pool_size={}, prune_to={}, window_size={}, "
208+
"use_full_search_history={}",
209+
params.alpha,
210+
params.graph_max_degree,
211+
params.max_candidate_pool_size,
212+
params.prune_to,
213+
params.window_size,
214+
params.use_full_search_history
215+
);
197216
// Check class invariants.
198217
if (graph_.n_nodes() != data_.size()) {
199218
throw ANNEXCEPTION(
@@ -296,12 +315,9 @@ class VamanaBuilder {
296315
}
297316
}
298317
svs::logging::log(
299-
logger,
300-
logging::Level::Debug,
301-
"Completed pass using window size {}.",
302-
params_.window_size
318+
logger, level, "Completed pass using window size {}.", params_.window_size
303319
);
304-
svs::logging::log(logger, logging::Level::Debug, "{}", timer);
320+
svs::logging::log(logger, level, "{}", timer);
305321
}
306322

307323
///
@@ -323,9 +339,7 @@ class VamanaBuilder {
323339
update_type updates{threadpool_.size()};
324340
auto main = timer.push_back("main");
325341
threads::parallel_for(
326-
threadpool_,
327-
range,
328-
[&](const auto& local_indices, uint64_t tid) {
342+
threadpool_, range, [&](const auto& local_indices, uint64_t tid) {
329343
// Thread local variables
330344
auto& thread_local_updates = updates.at(tid);
331345

@@ -476,9 +490,7 @@ class VamanaBuilder {
476490
auto range = threads::StaticPartition{indices};
477491
backedge_buffer_.reset();
478492
threads::parallel_for(
479-
threadpool_,
480-
range,
481-
[&](const auto& is, uint64_t SVS_UNUSED(tid)) {
493+
threadpool_, range, [&](const auto& is, uint64_t SVS_UNUSED(tid)) {
482494
for (auto node_id : is) {
483495
for (auto other_id : graph_.get_node(node_id)) {
484496
std::lock_guard lock{vertex_locks_[other_id]};
@@ -527,7 +539,8 @@ class VamanaBuilder {
527539
i,
528540
distance::compute(
529541
general_distance, src_data, general_accessor(data_, i)
530-
)};
542+
)
543+
};
531544
};
532545

533546
candidates.clear();

‎tests/integration/vamana/index_build.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ CATCH_TEST_CASE("VamanaIndex Logging Tests", "[logging]") {
263263
CATCH_REQUIRE(
264264
captured_logs[0].find("Vamana Build Parameters:") != std::string::npos
265265
);
266-
CATCH_REQUIRE(captured_levels[0] == svs::logging::Level::Info);
266+
CATCH_REQUIRE(captured_levels[0] == svs::logging::Level::Debug);
267267
CATCH_REQUIRE(captured_logs[1].find("Number of syncs:") != std::string::npos);
268268
CATCH_REQUIRE(captured_levels[1] == svs::logging::Level::Trace);
269269
CATCH_REQUIRE(captured_logs[2].find("Batch Size:") != std::string::npos);

‎tests/svs/index/inverted/clustering.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,8 @@ void test_end_to_end_clustering(
288288
});
289289

290290
auto vamana_parameters = svs::index::vamana::VamanaBuildParameters{
291-
construction_alpha, 64, 200, 1000, 60, true};
291+
construction_alpha, 64, 200, 1000, 60, true
292+
};
292293

293294
// Build the index once and reuse it multiple times to help speed up tests.
294295
for (size_t max_replicas : {2, 8}) {
@@ -447,6 +448,6 @@ CATCH_TEST_CASE("Clustering with Logger", "[logging]") {
447448

448449
// Verify the internal log messages
449450
CATCH_REQUIRE(global_captured_logs.empty());
450-
CATCH_REQUIRE(captured_logs[1].find("Vamana Build Parameters:") != std::string::npos);
451-
CATCH_REQUIRE(captured_logs[2].find("Number of syncs") != std::string::npos);
451+
CATCH_REQUIRE(captured_logs[0].find("Vamana Build Parameters:") != std::string::npos);
452+
CATCH_REQUIRE(captured_logs[1].find("Number of syncs") != std::string::npos);
452453
}

‎tests/svs/index/inverted/memory_based.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,6 @@ CATCH_TEST_CASE("InvertedIndex Logging Test", "[logging]") {
7070

7171
// Verify the internal log messages
7272
CATCH_REQUIRE(global_captured_logs.empty());
73-
CATCH_REQUIRE(captured_logs[1].find("Vamana Build Parameters:") != std::string::npos);
74-
CATCH_REQUIRE(captured_logs[2].find("Number of syncs") != std::string::npos);
73+
CATCH_REQUIRE(captured_logs[0].find("Vamana Build Parameters:") != std::string::npos);
74+
CATCH_REQUIRE(captured_logs[1].find("Number of syncs") != std::string::npos);
7575
}

‎tests/svs/index/vamana/dynamic_index_2.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,7 @@ CATCH_TEST_CASE("Testing Graph Index", "[graph_index][dynamic_index]") {
326326
}
327327

328328
svs::index::vamana::VamanaBuildParameters parameters{
329-
1.2, max_degree, 2 * max_degree, 1000, max_degree - 4, true
330-
};
329+
1.2, max_degree, 2 * max_degree, 1000, max_degree - 4, true};
331330

332331
auto tic = svs::lib::now();
333332
auto index = svs::index::vamana::MutableVamanaIndex(
@@ -344,7 +343,7 @@ CATCH_TEST_CASE("Testing Graph Index", "[graph_index][dynamic_index]") {
344343
CATCH_REQUIRE(captured_logs[0].find("Total / % Measured:") != std::string::npos);
345344
CATCH_REQUIRE(captured_levels[0] == svs::logging::Level::Debug);
346345
CATCH_REQUIRE(captured_logs[1].find("Vamana Build Parameters:") != std::string::npos);
347-
CATCH_REQUIRE(captured_levels[1] == svs::logging::Level::Info);
346+
CATCH_REQUIRE(captured_levels[1] == svs::logging::Level::Debug);
348347
CATCH_REQUIRE(captured_logs[2].find("Number of syncs:") != std::string::npos);
349348
CATCH_REQUIRE(captured_levels[2] == svs::logging::Level::Trace);
350349
CATCH_REQUIRE(captured_logs[3].find("Batch Size:") != std::string::npos);

0 commit comments

Comments
 (0)